OpenSearchService.php 51 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445
  1. <?php
  2. // api-v8/app/Services/OpenSearchService.php
  3. namespace App\Services;
  4. use App\Facades\RedisCache;
  5. use Exception;
  6. use GuzzleHttp\Client;
  7. use Illuminate\Support\Facades\Log;
  8. use Illuminate\Support\Str;
  9. use OpenSearch\GuzzleClientFactory;
  10. class OpenSearchService
  11. {
  12. private const EMBEDDING_CACHE_VERSION_KEY = 'embedding/version';
  13. protected $client;
  14. protected $http;
  15. protected $openaiApiKey;
  16. /**
  17. * 默认查询排除字段
  18. *
  19. * @var array
  20. */
  21. private $sourceExcludes = [
  22. 'title.suggest.pali',
  23. 'title.suggest.zh',
  24. 'content.suggest.pali',
  25. 'content.suggest.zh',
  26. 'content.display', // 新增,列表页不返回 HTML
  27. ];
  28. /**
  29. * 默认权重配置
  30. *
  31. * fuzzy / hybrid 两种模式各自的字段权重。
  32. * hybrid 额外包含 fuzzy_ratio / semantic_ratio 用于控制两路得分的混合比例。
  33. *
  34. * 字段名已按新映射结构调整:
  35. * title.text.pali → 原 title.pali.text
  36. * title.text.zh → 原 title.zh
  37. * content.text.pali → 原 content.pali.text
  38. * content.text.zh → 原 content.zh
  39. *
  40. * @var array
  41. */
  42. private $weights = [
  43. 'fuzzy' => [
  44. 'bold_single' => 50,
  45. 'bold_multi' => 10,
  46. 'title.text.pali' => 3,
  47. 'title.text.zh' => 3,
  48. 'summary.text' => 2,
  49. 'content.text.pali' => 1,
  50. 'content.text.zh' => 1,
  51. ],
  52. 'hybrid' => [
  53. 'fuzzy_ratio' => 0.7,
  54. 'semantic_ratio' => 0.3,
  55. 'bold_single' => 50,
  56. 'bold_multi' => 10,
  57. 'title.text.pali' => 3,
  58. 'title.text.zh' => 3,
  59. 'summary.text' => 2,
  60. 'content.text.pali' => 1,
  61. 'content.text.zh' => 1,
  62. ],
  63. ];
  64. /**
  65. * OpenSearch 索引定义(settings + mappings)
  66. *
  67. * 字段结构说明:
  68. *
  69. * title
  70. * ├── text
  71. * │ ├── pali (text) 模糊查询 + exact subfield 精确查询
  72. * │ └── zh (text) 中文分词查询
  73. * ├── vector (knn_vector, dim=1536)
  74. * └── suggest
  75. * ├── pali (completion)
  76. * └── zh (completion)
  77. *
  78. * content(结构与 title 一致,额外包含 tokens nested 字段)
  79. * ├── text
  80. * │ ├── pali (text)
  81. * │ └── zh (text)
  82. * ├── tokens (nested)
  83. * ├── vector (knn_vector, dim=1536)
  84. * └── suggest
  85. * ├── pali (completion)
  86. * └── zh (completion)
  87. *
  88. * summary(中文摘要,结构保持不变)
  89. * ├── text (text)
  90. * └── vector (knn_vector, dim=1536)
  91. *
  92. * 分类维度(resource_id、resource_type、language 等)一律 keyword:它们是枚举值,
  93. * 不该被分词。若声明成 text,'zh-Hans' 会被切成 zh + hans,查 language=zh
  94. * 就会把 zh-Hans 的文档一并捞进来(实测 401 vs 410),而且不报错。
  95. *
  96. * 定义本身在 opensearch/indices/{index}.json,见 indexDefinition()。
  97. */
  98. private ?array $indexDefinition = null;
  99. /**
  100. * 读取索引定义(settings + mappings)
  101. *
  102. * 文件名与索引名一致:{index_definition_path}/{index}.json,
  103. * 默认目录为仓库根目录的 opensearch/indices/。换索引版本时新增一个同名 JSON。
  104. *
  105. * @return array{settings: array, mappings: array}
  106. *
  107. * @throws Exception 文件不存在或不是合法 JSON 时抛出
  108. */
  109. protected function indexDefinition(): array
  110. {
  111. if ($this->indexDefinition !== null) {
  112. return $this->indexDefinition;
  113. }
  114. $path = Str::finish(config('mint.opensearch.index_definition_path'), '/')
  115. .config('mint.opensearch.index').'.json';
  116. if (! is_file($path)) {
  117. throw new Exception("OpenSearch index definition [$path] not found.");
  118. }
  119. return $this->indexDefinition = json_decode(file_get_contents($path), true, flags: JSON_THROW_ON_ERROR);
  120. }
  121. /**
  122. * 创建 OpenSearchService 实例
  123. *
  124. * 从 config('mint.opensearch.config') 读取连接配置,
  125. * 同时初始化 OpenAI HTTP 客户端用于 embedding 调用。
  126. */
  127. public function __construct()
  128. {
  129. $config = config('mint.opensearch.config');
  130. $hostUrl = "{$config['scheme']}://{$config['host']}:{$config['port']}";
  131. $this->client = (new GuzzleClientFactory)->create([
  132. 'base_uri' => $hostUrl,
  133. 'auth' => [$config['username'], $config['password']],
  134. 'verify' => $config['ssl_verification'],
  135. ]);
  136. $this->openaiApiKey = env('OPENAI_API_KEY');
  137. $this->http = new Client([
  138. 'base_uri' => 'https://api.openai.com/v1/',
  139. 'timeout' => 15,
  140. ]);
  141. }
  142. /**
  143. * 动态覆盖指定搜索模式的字段权重
  144. *
  145. * @param string $mode 搜索模式,支持 'fuzzy' | 'hybrid'
  146. * @param array $weights 需要覆盖的权重键值对,例如:['title.text.pali' => 5]
  147. */
  148. public function setWeights(string $mode, array $weights): void
  149. {
  150. if (isset($this->weights[$mode])) {
  151. $this->weights[$mode] = array_merge($this->weights[$mode], $weights);
  152. }
  153. }
  154. /**
  155. * 测试与 OpenSearch 集群的连接状态
  156. *
  157. * @return array{0: bool, 1: string} [连接是否成功, 描述信息]
  158. */
  159. public function testConnection(): array
  160. {
  161. try {
  162. $info = $this->client->info();
  163. $message = 'OpenSearch 连接成功: '.json_encode($info['version']['number']);
  164. Log::info($message);
  165. return [true, $message];
  166. } catch (Exception $e) {
  167. $message = 'OpenSearch 连接失败: '.$e->getMessage();
  168. Log::error($message);
  169. return [false, $message];
  170. }
  171. }
  172. /**
  173. * 检查当前索引是否已存在
  174. */
  175. public function indexExists(): bool
  176. {
  177. $index = config('mint.opensearch.index');
  178. return $this->client->indices()->exists(['index' => $index]);
  179. }
  180. /**
  181. * 创建 OpenSearch 索引
  182. *
  183. * 使用 indexDefinition() 读到的 settings 和 mappings 创建索引。
  184. * 若索引已存在则抛出异常,避免覆盖生产数据。
  185. *
  186. * @return array OpenSearch 响应
  187. *
  188. * @throws Exception 索引已存在时抛出
  189. */
  190. public function createIndex(): array
  191. {
  192. $index = config('mint.opensearch.index');
  193. $exists = $this->client->indices()->exists(['index' => $index]);
  194. if ($exists) {
  195. throw new Exception("Index [$index] already exists.");
  196. }
  197. return $this->client->indices()->create([
  198. 'index' => $index,
  199. 'body' => $this->indexDefinition(),
  200. ]);
  201. }
  202. /**
  203. * 更新已有索引的 settings 和 mappings
  204. *
  205. * 更新 settings 时会临时关闭索引(close → putSettings → open),
  206. * 更新 mappings 支持热更新(新增字段),不可修改已有字段类型。
  207. *
  208. * @return array 包含 'settings' 和/或 'mappings' 的响应数组
  209. */
  210. public function updateIndex(): array
  211. {
  212. $index = config('mint.opensearch.index');
  213. $settings = $this->indexDefinition()['settings'] ?? [];
  214. $mappings = $this->indexDefinition()['mappings'] ?? [];
  215. $response = [];
  216. if (! empty($settings)) {
  217. $this->client->indices()->close(['index' => $index]);
  218. $response['settings'] = $this->client->indices()->putSettings([
  219. 'index' => $index,
  220. 'body' => ['settings' => $settings],
  221. ]);
  222. $this->client->indices()->open(['index' => $index]);
  223. }
  224. if (! empty($mappings)) {
  225. $response['mappings'] = $this->client->indices()->putMapping([
  226. 'index' => $index,
  227. 'body' => $mappings,
  228. ]);
  229. }
  230. return $response;
  231. }
  232. /**
  233. * 切换 pali 同义词文件版本
  234. *
  235. * 把 pali_synonyms filter 的 synonyms_path 指向
  236. * analysis/pali-synonyms-{$version}.txt(文件须已存在于 OpenSearch
  237. * 各节点的 config 目录中,否则索引无法重新打开)。
  238. *
  239. * synonyms_path 属于静态 settings,必须 close → putSettings → open。
  240. * 无论 putSettings 成功与否都会尝试重新打开索引,避免索引停留在 close 状态。
  241. *
  242. * @param string $version 版本号,仅允许 [A-Za-z0-9._-]
  243. * @return array{path: string, settings: array} 新路径与 OpenSearch 响应
  244. *
  245. * @throws Exception 版本号非法、索引不存在或 OpenSearch 拒绝时抛出
  246. *
  247. * @example
  248. * $service->updatePaliSynonymsPath('20260731');
  249. */
  250. public function updatePaliSynonymsPath(string $filename): array
  251. {
  252. $index = config('mint.opensearch.index');
  253. if (! $this->client->indices()->exists(['index' => $index])) {
  254. throw new Exception("Index [$index] does not exist.");
  255. }
  256. $path = Str::finish(config('mint.opensearch.config.synonyms_path'), '/').$filename;
  257. Log::debug('OpenSearchService::updatePaliSynonymsPath', [
  258. 'index' => $index,
  259. 'synonyms_path' => $path,
  260. ]);
  261. // 以代码中的 analysis 定义为准,只替换同义词文件路径
  262. $analysis = $this->indexDefinition()['settings']['analysis'];
  263. $analysis['filter']['pali_synonyms']['synonyms_path'] = $path;
  264. $this->client->indices()->close(['index' => $index]);
  265. try {
  266. $response = $this->client->indices()->putSettings([
  267. 'index' => $index,
  268. 'body' => ['settings' => ['analysis' => $analysis]],
  269. ]);
  270. } finally {
  271. $this->client->indices()->open(['index' => $index]);
  272. }
  273. return ['path' => $path, 'settings' => $response];
  274. }
  275. /**
  276. * 读取当前索引使用的 pali 同义词文件路径
  277. *
  278. * @return string|null 形如 "analysis/pali-synonyms-20260731.txt",未设置时返回 null
  279. */
  280. public function getPaliSynonymsPath(): ?string
  281. {
  282. return $this->getPaliSynonymsSetting()['synonyms_path'] ?? null;
  283. }
  284. /**
  285. * 读取当前索引 pali_synonyms filter 的完整设置
  286. *
  287. * @return array{type?: string, synonyms_path?: string, updateable?: string}|null 未设置时返回 null
  288. */
  289. public function getPaliSynonymsSetting(): ?array
  290. {
  291. $index = config('mint.opensearch.index');
  292. $settings = $this->client->indices()->getSettings(['index' => $index]);
  293. return $settings[$index]['settings']['index']['analysis']['filter']['pali_synonyms'] ?? null;
  294. }
  295. /**
  296. * 删除当前索引
  297. *
  298. * @return array OpenSearch 响应
  299. */
  300. public function deleteIndex(): array
  301. {
  302. $index = config('mint.opensearch.index');
  303. return $this->client->indices()->delete(['index' => $index]);
  304. }
  305. /**
  306. * 统计索引文档数量(支持可选条件过滤)
  307. *
  308. * @param array|null $query OpenSearch DSL query 子句,为 null 时统计全部文档。
  309. * 示例:['term' => ['language' => 'zh']]
  310. * ['exists' => ['field' => 'content.vector']]
  311. * @return int 文档总数
  312. *
  313. * @throws Exception
  314. *
  315. * @example
  316. * $service->count();
  317. * $service->count(['exists' => ['field' => 'content.vector']]);
  318. */
  319. public function count(?array $query = null): int
  320. {
  321. $index = config('mint.opensearch.index');
  322. $params = ['index' => $index];
  323. if (! empty($query)) {
  324. $params['body'] = ['query' => $query];
  325. }
  326. $response = $this->client->count($params);
  327. return (int) ($response['count'] ?? 0);
  328. }
  329. /**
  330. * 写入或覆盖单条文档
  331. *
  332. * @param string $id 文档 ID
  333. * @param array $body 文档内容,字段结构须与 mappings 一致
  334. * @return array OpenSearch 响应
  335. */
  336. public function create(string $id, array $body): array
  337. {
  338. return $this->client->index([
  339. 'index' => config('mint.opensearch.index'),
  340. 'id' => $id,
  341. 'body' => $body,
  342. ]);
  343. }
  344. /**
  345. * 删除单条文档
  346. *
  347. * @param string $id 文档 ID
  348. * @return array OpenSearch 响应
  349. */
  350. public function delete(string $id): array
  351. {
  352. return $this->client->delete([
  353. 'index' => config('mint.opensearch.index'),
  354. 'id' => $id,
  355. ]);
  356. }
  357. /**
  358. * 执行高级搜索
  359. *
  360. * 支持五种搜索模式:
  361. * - fuzzy 多字段模糊查询(默认),巴利文走三层排序
  362. * - exact 精确匹配,使用 markdown_clean analyzer
  363. * - semantic 纯语义向量搜索,需要 OpenAI embedding
  364. * - hybrid fuzzy + semantic 混合,权重由 fuzzy_ratio / semantic_ratio 控制
  365. * - definition 定义查询,黑体字(bold_single/bold_multi)高权重,术语定义排前
  366. *
  367. * 支持的过滤参数:
  368. * resourceType, resourceId, granularity, language, category,
  369. * tags, pageRefs, relatedId, author, channel
  370. *
  371. * @param array $params {
  372. *
  373. * @type string $query 搜索关键词(必填)
  374. * @type string $searchMode 搜索模式,默认 'fuzzy'
  375. * @type int $page 页码,默认 1
  376. * @type int $pageSize 每页条数,默认 20
  377. * @type string $resourceType 按资源类型过滤
  378. * @type string $resourceId 按资源 ID 过滤
  379. * @type string $granularity 按粒度过滤
  380. * @type string $language 按语言过滤
  381. * @type string $category 按分类过滤
  382. * @type array $tags 按标签过滤(terms)
  383. * @type array $pageRefs 按页码引用过滤(terms)
  384. * @type string $relatedId 按关联 ID 过滤
  385. * @type string $author 按作者过滤
  386. * @type string $channel 按频道过滤
  387. * @type array $highlight_pre_tags 高亮前置标签,默认 ['<mark>']
  388. * @type array $highlight_post_tags 高亮后置标签,默认 ['</mark>']
  389. * }
  390. *
  391. * @return array OpenSearch 原始响应
  392. *
  393. * @throws Exception semantic / hybrid 模式下 embedding 调用失败时抛出
  394. */
  395. public function search(array $params): array
  396. {
  397. $page = $params['page'] ?? 1;
  398. $pageSize = $params['pageSize'] ?? 20;
  399. $from = ($page - 1) * $pageSize;
  400. $mode = $params['searchMode'] ?? 'fuzzy';
  401. // 排除字段
  402. if (! empty($params['excludes']) && is_array($params['excludes'])) {
  403. $excludes = array_merge($this->sourceExcludes, $params['excludes']);
  404. } else {
  405. $excludes = $this->sourceExcludes;
  406. }
  407. // ---------- 过滤条件 ----------
  408. $filters = [];
  409. if (! empty($params['resourceType'])) {
  410. $filters[] = ['term' => ['resource_type' => $params['resourceType']]];
  411. }
  412. if (! empty($params['resourceId'])) {
  413. $filters[] = ['term' => ['resource_id' => $params['resourceId']]];
  414. }
  415. if (! empty($params['granularity'])) {
  416. $filters[] = ['term' => ['granularity' => $params['granularity']]];
  417. }
  418. if (! empty($params['language'])) {
  419. $filters[] = ['term' => ['language' => $params['language']]];
  420. }
  421. if (! empty($params['category'])) {
  422. if (is_array($params['category'])) {
  423. $categories = $params['category'];
  424. } else {
  425. $categories = [$params['category']];
  426. }
  427. // 必须匹配全部:为每个 category 创建一个 term 条件
  428. foreach ($categories as $category) {
  429. $filters[] = ['term' => ['category' => $category]];
  430. }
  431. }
  432. if (! empty($params['tags'])) {
  433. $filters[] = ['terms' => ['tags' => $params['tags']]];
  434. }
  435. if (! empty($params['pageRefs'])) {
  436. $filters[] = ['terms' => ['page_refs' => $params['pageRefs']]];
  437. }
  438. if (! empty($params['relatedId'])) {
  439. $filters[] = ['term' => ['related_id' => $params['relatedId']]];
  440. }
  441. if (! empty($params['author'])) {
  442. $filters[] = ['match' => ['metadata.author' => $params['author']]];
  443. }
  444. if (! empty($params['channel'])) {
  445. $filters[] = ['term' => ['metadata.channel' => $params['channel']]];
  446. }
  447. // ---------- 查询部分 ----------
  448. $queryText = trim($params['query'] ?? '');
  449. // fuzzy / definition 的巴利文多词查询按词的罕见度加权;其余模式不需要
  450. $weights = null;
  451. if ($queryText !== '' && ! in_array($mode, ['exact', 'semantic', 'hybrid'], true) && ! $this->isCjkQuery($queryText)) {
  452. $weights = $this->wordRarityWeights($this->scoringWords($this->splitQueryWords($queryText)));
  453. }
  454. if ($queryText === '') {
  455. $query = ['match_all' => new \stdClass];
  456. } else {
  457. switch ($mode) {
  458. case 'exact':
  459. $query = $this->buildExactQuery($queryText);
  460. break;
  461. case 'semantic':
  462. $query = $this->buildSemanticQuery($queryText);
  463. break;
  464. case 'hybrid':
  465. $query = $this->buildHybridQuery($queryText);
  466. break;
  467. case 'definition':
  468. $query = $this->buildDefinitionQuery($queryText, $weights);
  469. break;
  470. case 'fuzzy':
  471. default:
  472. $query = $this->buildFuzzyQuery($queryText, $weights);
  473. break;
  474. }
  475. }
  476. $highlightPreTags = $params['highlight_pre_tags'] ?? ['<mark>'];
  477. $highlightPostTags = $params['highlight_post_tags'] ?? ['</mark>'];
  478. // ---------- 最终 DSL ----------
  479. $dsl = [
  480. 'from' => $from,
  481. 'size' => $pageSize,
  482. '_source' => ['excludes' => $excludes],
  483. 'query' => ! empty($filters)
  484. ? [
  485. 'bool' => [
  486. 'must' => [$query],
  487. 'filter' => $filters,
  488. ],
  489. ]
  490. : $query,
  491. 'aggs' => [
  492. 'resource_type' => [
  493. 'terms' => ['field' => 'resource_type'],
  494. ],
  495. 'language' => [
  496. 'terms' => ['field' => 'language'],
  497. ],
  498. 'category' => [
  499. 'terms' => ['field' => 'category'],
  500. ],
  501. 'granularity' => [
  502. 'terms' => ['field' => 'granularity'],
  503. ],
  504. ],
  505. ];
  506. // 只有有搜索词时才开启高亮;字段按语种路由,避免把巴利文喂给中文分词器
  507. if ($queryText !== '') {
  508. $highlightFields = $this->isCjkQuery($queryText)
  509. ? ['title.text.zh', 'summary.text', 'content.text.zh']
  510. : ['title.text.pali', 'content.text.pali'];
  511. $dsl['highlight'] = [
  512. 'fields' => array_fill_keys($highlightFields, new \stdClass),
  513. 'fragmenter' => 'sentence',
  514. 'fragment_size' => 200,
  515. 'number_of_fragments' => 1,
  516. 'pre_tags' => $highlightPreTags,
  517. 'post_tags' => $highlightPostTags,
  518. ];
  519. }
  520. Log::debug(
  521. 'OpenSearchService::search',
  522. ['dsl' => json_encode($dsl, JSON_UNESCAPED_UNICODE)]
  523. );
  524. return $this->client->search([
  525. 'index' => config('mint.opensearch.index'),
  526. 'body' => $dsl,
  527. ]);
  528. }
  529. /**
  530. * 构建 exact(精确匹配)查询
  531. *
  532. * 使用 markdown_clean analyzer 的 exact subfield 进行匹配,
  533. * 适合巴利文词形精确检索场景。
  534. *
  535. * 查询字段:title.text.pali.exact, content.text.pali.exact, summary.text
  536. *
  537. * @param string $query 搜索关键词
  538. * @return array OpenSearch DSL query 片段
  539. */
  540. protected function buildExactQuery(string $query): array
  541. {
  542. return [
  543. 'multi_match' => [
  544. 'query' => $query,
  545. 'fields' => [
  546. 'title.text.pali.exact',
  547. 'content.text.pali.exact',
  548. 'summary.text',
  549. ],
  550. 'type' => 'best_fields',
  551. ],
  552. ];
  553. }
  554. /**
  555. * 构建 semantic(纯语义向量)查询
  556. *
  557. * 将查询文本通过 OpenAI embedding API 转为向量,
  558. * 同时对 content.vector、summary.vector、title.vector 三个 knn 字段检索,
  559. * 使用 bool should 合并结果。
  560. *
  561. * @param string $query 搜索关键词
  562. * @return array OpenSearch DSL query 片段
  563. *
  564. * @throws Exception embedding 调用失败时抛出
  565. */
  566. protected function buildSemanticQuery(string $query): array
  567. {
  568. $vector = $this->embedText($query);
  569. return [
  570. 'bool' => [
  571. 'should' => [
  572. ['knn' => ['content.vector' => ['vector' => $vector, 'k' => 20]]],
  573. ['knn' => ['summary.vector' => ['vector' => $vector, 'k' => 10]]],
  574. ['knn' => ['title.vector' => ['vector' => $vector, 'k' => 5]]],
  575. ],
  576. 'minimum_should_match' => 1,
  577. ],
  578. ];
  579. }
  580. /**
  581. * 构建 fuzzy(多字段模糊)查询
  582. *
  583. * 按查询语种路由字段,避免巴利文被中文分词器切碎、中文被巴利文 analyzer 误伤:
  584. * - 含汉字 → 只搜 bold_* + *.text.zh + summary.text
  585. * - 巴利文(拉丁字母)→ 走三层排序(见 buildPaliFuzzyQuery)+ bold_* 提权
  586. *
  587. * @param string $query 搜索关键词
  588. * @param array<string, float>|null $weights 巴利文各词的罕见度权重,null 表示都按 1
  589. * @return array OpenSearch DSL query 片段
  590. */
  591. protected function buildFuzzyQuery(string $query, ?array $weights = null): array
  592. {
  593. if ($this->isCjkQuery($query)) {
  594. return [
  595. 'multi_match' => [
  596. 'query' => $query,
  597. 'fields' => [
  598. 'bold_single^50',
  599. 'bold_multi^10',
  600. 'title.text.zh^3',
  601. 'summary.text^2',
  602. 'content.text.zh^1',
  603. ],
  604. 'type' => 'best_fields',
  605. ],
  606. ];
  607. }
  608. return $this->buildPaliFuzzyQuery($query, 1.0, $weights);
  609. }
  610. /**
  611. * 巴利文 fuzzy 的计分表(全部是固定分,逐项累加)
  612. *
  613. * 一个文档的得分 = 命中的每一项相加,于是「部分符合」自然逐级降分:
  614. * - phrase 整句按原顺序、拼写完全相同(≥2 个词才有)
  615. * - bigram 相邻两词按原顺序、拼写相同,每对一次(≥3 个词才有,2 个词时与 phrase 重合)
  616. * - exact 单个原始词拼写相同,每词一次
  617. * - synonym 单个原始词经同义词可替代,每词一次(拼写相同的词也会命中这一项)
  618. *
  619. * 以 3 个词为例:整句原样 66 > 全部拼写相同但乱序 36 > 两词相邻拼写相同 + 一词同义 31
  620. * > 两词拼写相同不相邻 24 > 三词全靠同义词 6 > 一词同义 2。
  621. *
  622. * @var array{phrase: int, bigram: int, exact: int, synonym: int}
  623. */
  624. private const PALI_FUZZY_SCORES = [
  625. 'phrase' => 20,
  626. 'bigram' => 5,
  627. 'exact' => 10,
  628. 'synonym' => 2,
  629. ];
  630. /**
  631. * definition 的计分表(黑体字部分,每个原始词各算一次)
  632. *
  633. * 主分不分词形:同义词词典的一行就是一个词的全部变格变位,所以「任一词形」走同义词,
  634. * 与 v2 search-pali-wbw 把全部词形一视同仁的口径一致;原拼写只给少量加分,排在同档前面。
  635. *
  636. * - single 该词(任一词形)是单独黑体字
  637. * - single_exact 再加:拼写也相同
  638. * - multi 该词(任一词形)出现在多词黑体短语里
  639. * - multi_exact 再加:拼写也相同
  640. * - multi_phrase 整句按原顺序、拼写相同出现在多词黑体短语里(≥2 个词才有)
  641. *
  642. * 每个词:单独黑体 ≥ 100 > 多词黑体 ≤ 35,于是命中的单独黑体字越多排越前。
  643. * 正文得分按 DEFINITION_TEXT_SCALE 缩小后叠加,只用来给黑体同分的文档排先后。
  644. *
  645. * @var array{single: int, single_exact: int, multi: int, multi_exact: int, multi_phrase: int}
  646. */
  647. private const DEFINITION_SCORES = [
  648. 'single' => 100,
  649. 'single_exact' => 10,
  650. 'multi' => 30,
  651. 'multi_exact' => 5,
  652. 'multi_phrase' => 30,
  653. ];
  654. /**
  655. * definition 模式里正文得分的缩放系数,保证正文再怎么命中也压不过一个黑体字
  656. */
  657. private const DEFINITION_TEXT_SCALE = 0.1;
  658. /**
  659. * 同分时的 BM25 微调权重:BM25 一般 < 30,乘上后 < 0.3,小于计分表的最小档差
  660. */
  661. private const TIE_BREAKER_BOOST = 0.01;
  662. /**
  663. * 罕见度权重的下限:再常见的词也保留这么多分量,不至于完全不计
  664. */
  665. private const RARITY_WEIGHT_FLOOR = 0.2;
  666. /**
  667. * 罕见度权重的指数:IDF 取了对数,887 段与 211 段的词只差 0.82 : 1,拉不开;
  668. * 归一后再取幂放大差距
  669. */
  670. private const RARITY_EXPONENT = 4;
  671. /**
  672. * 每个词的文档频率在 RedisCache 里的保存期(秒)。语料按月更新,一天足够新
  673. */
  674. private const DOC_FREQUENCY_TTL = 86400;
  675. /**
  676. * 多词查询里每个词的罕见度权重(类 BM25 的 IDF,按最罕见的词归一到 1)
  677. *
  678. * 按词计的固定分让每个词分量相同,常见词(如 bojjhaṅga,到处都是黑体)会把罕见的
  679. * 目标词(dhammavicayasambojjhaṅga)挤出前排。乘上这个权重后,越罕见的词分越高。
  680. * 文档频率按 title/content 的 .pali(带同义词,即全部词形)统计,一次 _msearch 取齐,
  681. * 逐词缓存。取不到时返回全 1(等于不加权),只记日志不报错。
  682. *
  683. * @param list<string> $words 参与计分的原始词(已去掉停用词)
  684. * @return array<string, float> 原始词 => 权重,范围 [RARITY_WEIGHT_FLOOR, 1]
  685. */
  686. protected function wordRarityWeights(array $words): array
  687. {
  688. $uniqueWords = array_values(array_unique($words));
  689. $flat = array_fill_keys($uniqueWords, 1.0);
  690. if (count($uniqueWords) < 2) {
  691. return $flat;
  692. }
  693. try {
  694. $index = config('mint.opensearch.index');
  695. $total = RedisCache::remember("/opensearch/{$index}/doc-count", self::DOC_FREQUENCY_TTL, fn () => $this->count());
  696. $frequencies = $this->documentFrequencies($index, $uniqueWords);
  697. } catch (Exception $e) {
  698. Log::warning('opensearch word rarity weights failed', ['words' => $uniqueWords, 'error' => $e->getMessage()]);
  699. return $flat;
  700. }
  701. $idf = [];
  702. foreach ($uniqueWords as $word) {
  703. $df = $frequencies[$word] ?? 0;
  704. $idf[$word] = log(1 + ($total - $df + 0.5) / ($df + 0.5));
  705. }
  706. $maxIdf = max($idf);
  707. if ($maxIdf <= 0) {
  708. return $flat;
  709. }
  710. return array_map(
  711. fn (float $value): float => max(self::RARITY_WEIGHT_FLOOR, ($value / $maxIdf) ** self::RARITY_EXPONENT),
  712. $idf,
  713. );
  714. }
  715. /**
  716. * 每个词出现在多少个文档里(含同义词展开的全部词形),逐词缓存,未缓存的一次 _msearch 取齐
  717. *
  718. * @param string $index 索引名
  719. * @param list<string> $words 原始词
  720. * @return array<string, int> 原始词 => 文档数
  721. */
  722. protected function documentFrequencies(string $index, array $words): array
  723. {
  724. $frequencies = [];
  725. $missing = [];
  726. foreach ($words as $word) {
  727. $cached = RedisCache::get($this->documentFrequencyCacheKey($index, $word));
  728. if ($cached === null) {
  729. $missing[] = $word;
  730. } else {
  731. $frequencies[$word] = (int) $cached;
  732. }
  733. }
  734. if ($missing === []) {
  735. return $frequencies;
  736. }
  737. $body = [];
  738. foreach ($missing as $word) {
  739. $body[] = ['index' => $index];
  740. $body[] = [
  741. 'size' => 0,
  742. 'track_total_hits' => true,
  743. 'query' => ['multi_match' => ['query' => $word, 'fields' => ['title.text.pali', 'content.text.pali']]],
  744. ];
  745. }
  746. $responses = $this->client->msearch(['body' => $body])['responses'] ?? [];
  747. foreach ($missing as $i => $word) {
  748. $df = (int) ($responses[$i]['hits']['total']['value'] ?? 0);
  749. $frequencies[$word] = $df;
  750. RedisCache::put($this->documentFrequencyCacheKey($index, $word), $df, self::DOC_FREQUENCY_TTL);
  751. }
  752. return $frequencies;
  753. }
  754. /**
  755. * @param string $index 索引名
  756. * @param string $word 原始词
  757. * @return string 文档频率的缓存 key
  758. */
  759. protected function documentFrequencyCacheKey(string $index, string $word): string
  760. {
  761. return "/opensearch/{$index}/word-df/".md5(mb_strtolower($word));
  762. }
  763. /**
  764. * 巴利文停用词(小写;比较前去掉标点)
  765. *
  766. * 取自 pg 全文检索的 pali.stop(api-v8/public/app/fts/pali.stop),另加 vā。
  767. * 停用词只参与词序打分(整句、相邻两词),不单独计分、不单独召回,见 buildPaliFuzzyQuery。
  768. *
  769. * @var list<string>
  770. */
  771. private const PALI_STOPWORDS = ['na', 'no', 'ca', 'va', 'vā', 'hi', 'pana', 'iva', 'eva'];
  772. /**
  773. * 判断原始词是否巴利文停用词
  774. *
  775. * @param string $word 原始词(可带标点,如「ca,」「‘‘na」)
  776. * @return bool 是停用词返回 true
  777. */
  778. protected function isPaliStopword(string $word): bool
  779. {
  780. $normalized = mb_strtolower(preg_replace('/[^\p{L}\p{M}]/u', '', $word) ?? '');
  781. return in_array($normalized, self::PALI_STOPWORDS, true);
  782. }
  783. /**
  784. * 取出用于单独计分的词:去掉停用词;全是停用词时原样返回,避免查不到东西
  785. *
  786. * @param list<string> $words 原始词
  787. * @return list<string>
  788. */
  789. protected function scoringWords(array $words): array
  790. {
  791. $contentWords = array_values(array_filter(
  792. $words,
  793. fn (string $word): bool => ! $this->isPaliStopword($word),
  794. ));
  795. return $contentWords === [] ? $words : $contentWords;
  796. }
  797. /**
  798. * 把查询切成原始词
  799. *
  800. * 按空白切分,并丢掉不含字母的片段(如「–」「.」):它们经分析器后没有 token,
  801. * 留着会让按词计分的子句永远不命中。
  802. *
  803. * @param string $query 搜索关键词
  804. * @return list<string>
  805. */
  806. protected function splitQueryWords(string $query): array
  807. {
  808. $words = preg_split('/\s+/u', trim($query)) ?: [];
  809. return array_values(array_filter(
  810. $words,
  811. fn (string $word): bool => preg_match('/\p{L}/u', $word) === 1,
  812. ));
  813. }
  814. /**
  815. * 包一层 constant_score:命中即得固定分,不受 BM25 字段长度归一化影响
  816. *
  817. * @param array<string, mixed> $filter 过滤子句
  818. * @param float $boost 命中时的固定分
  819. * @return array{constant_score: array{filter: array<string, mixed>, boost: float}}
  820. */
  821. protected function constantScore(array $filter, float $boost): array
  822. {
  823. return ['constant_score' => ['filter' => $filter, 'boost' => $boost]];
  824. }
  825. /**
  826. * 构建巴利文查询:词序符合、拼写相同得高分,部分符合逐级降分
  827. *
  828. * 命中按「原始词」(空格切分)计,不按同义词展开后的 token 计——每个原始词一个
  829. * constant_score 子句,命中几个同义词都只算一次。计分表见 PALI_FUZZY_SCORES。
  830. * 拼写相同打在 .exact 子字段(markdown_clean,无同义词),同义词打在 .pali。
  831. * 最后加一条极小权重的 BM25 子句,只用来给同分文档排先后。
  832. *
  833. * 停用词(PALI_STOPWORDS)只出现在整句、相邻两词的子句里,保住词序判断;
  834. * 不生成单词子句,也不进 BM25 子句,于是只命中「ca」的文档不会被召回。
  835. * 两个停用词组成的相邻词对同理跳过。查询全是停用词时按普通词处理。
  836. *
  837. * 单词子句(拼写、同义词)乘上该词的罕见度权重(见 wordRarityWeights),
  838. * 常见词分量小、罕见词分量大;词序子句(整句、相邻两词)不乘。
  839. *
  840. * @param string $query 搜索关键词
  841. * @param float $scale 整体缩放(definition 模式用它把正文得分压到黑体字之下)
  842. * @param array<string, float>|null $weights 原始词 => 罕见度权重,null 表示都按 1
  843. * @return array OpenSearch DSL query 片段
  844. */
  845. protected function buildPaliFuzzyQuery(string $query, float $scale = 1.0, ?array $weights = null): array
  846. {
  847. $words = $this->splitQueryWords($query);
  848. $wordCount = count($words);
  849. if ($wordCount === 0) {
  850. return ['match_all' => new \stdClass];
  851. }
  852. $scoringWords = $this->scoringWords($words);
  853. $allStopwords = array_all($words, fn (string $word): bool => $this->isPaliStopword($word));
  854. $exactFields = ['title.text.pali.exact', 'content.text.pali.exact'];
  855. $fuzzyFields = ['title.text.pali', 'content.text.pali'];
  856. $scores = array_map(fn (int $score): float => $score * $scale, self::PALI_FUZZY_SCORES);
  857. $should = [];
  858. // 词序:整句原样
  859. if ($wordCount >= 2) {
  860. $should[] = $this->constantScore([
  861. 'multi_match' => [
  862. 'query' => implode(' ', $words),
  863. 'type' => 'phrase',
  864. 'fields' => $exactFields,
  865. ],
  866. ], $scores['phrase']);
  867. }
  868. // 词序:相邻两词原样(部分顺序符合)
  869. if ($wordCount >= 3) {
  870. for ($i = 0; $i < $wordCount - 1; $i++) {
  871. $bothStopwords = $this->isPaliStopword($words[$i]) && $this->isPaliStopword($words[$i + 1]);
  872. if ($bothStopwords && ! $allStopwords) {
  873. continue;
  874. }
  875. $should[] = $this->constantScore([
  876. 'multi_match' => [
  877. 'query' => $words[$i].' '.$words[$i + 1],
  878. 'type' => 'phrase',
  879. 'fields' => $exactFields,
  880. ],
  881. ], $scores['bigram']);
  882. }
  883. }
  884. // 拼写:每个词各算一次(停用词不算),按罕见度加权
  885. foreach ($scoringWords as $word) {
  886. $weight = $weights[$word] ?? 1.0;
  887. $should[] = $this->constantScore([
  888. 'multi_match' => [
  889. 'query' => $word,
  890. 'fields' => $exactFields,
  891. 'operator' => 'and',
  892. ],
  893. ], $scores['exact'] * $weight);
  894. $should[] = $this->constantScore([
  895. 'multi_match' => [
  896. 'query' => $word,
  897. 'fields' => $fuzzyFields,
  898. ],
  899. ], $scores['synonym'] * $weight);
  900. }
  901. // 同分文档按 BM25 排先后
  902. $should[] = [
  903. 'match' => [
  904. 'content.text.pali' => [
  905. 'query' => implode(' ', $scoringWords),
  906. 'boost' => self::TIE_BREAKER_BOOST * $scale,
  907. ],
  908. ],
  909. ];
  910. return [
  911. 'bool' => [
  912. 'should' => $should,
  913. ],
  914. ];
  915. }
  916. /**
  917. * 判断查询是否含中文(CJK 统一表意文字)
  918. *
  919. * @param string $query 搜索关键词
  920. * @return bool 含汉字返回 true,否则按巴利文(拉丁字母)处理
  921. */
  922. protected function isCjkQuery(string $query): bool
  923. {
  924. return (bool) preg_match('/[\x{4E00}-\x{9FFF}\x{3400}-\x{4DBF}\x{F900}-\x{FAFF}]/u', $query);
  925. }
  926. /**
  927. * 构建 definition(定义查询)
  928. *
  929. * 义注里的黑体字通常就是被解释的词,所以黑体字命中决定排序,正文只做次级排序:
  930. * - 每个原始词单独算:命中的单独黑体字(bold_single)越多分越高
  931. * - 单独黑体字 > 多词黑体短语(bold_multi)里的词
  932. * - 词形不分高下(走同义词,即该词的全部变格变位),原拼写只在同档内加少量分:
  933. * bold_* 建索引用 standard,查询时显式指定 analyzer=standard 即是「拼写相同」,
  934. * 不指定则走带同义词的 pali_query_analyzer
  935. * - 多词黑体也按任一词形算:bold_* 取自 wbw_templates.real(已拆连音),
  936. * 正文里粘了连音(如 vitakkanti)匹配不到的词,往往靠它召回
  937. * - 用 match_phrase 而不是 match:中文在 standard 下按字切分,phrase 要求字连续,
  938. * 避免一个字重合就命中
  939. * - 每个词的黑体分乘上它的罕见度权重(见 wordRarityWeights):常见词(bojjhaṅga)
  940. * 不再与罕见的目标词(dhammavicayasambojjhaṅga)同分,挤占前排
  941. * 计分表见 DEFINITION_SCORES;正文部分复用 fuzzy 的计分,乘 DEFINITION_TEXT_SCALE。
  942. *
  943. * @param string $query 搜索关键词
  944. * @param array<string, float>|null $weights 原始词 => 罕见度权重,null 表示都按 1
  945. * @return array OpenSearch DSL query 片段
  946. */
  947. protected function buildDefinitionQuery(string $query, ?array $weights = null): array
  948. {
  949. $words = $this->splitQueryWords($query);
  950. if ($words === []) {
  951. return ['match_all' => new \stdClass];
  952. }
  953. $should = [];
  954. // 停用词不单独算黑体分(整句的 multi_phrase 仍带上它们)
  955. foreach ($this->scoringWords($words) as $word) {
  956. $weight = $weights[$word] ?? 1.0;
  957. foreach (['single' => 'bold_single', 'multi' => 'bold_multi'] as $tier => $field) {
  958. $should[] = $this->constantScore(
  959. ['match_phrase' => [$field => $word]],
  960. self::DEFINITION_SCORES[$tier] * $weight,
  961. );
  962. $should[] = $this->constantScore(
  963. ['match_phrase' => [$field => ['query' => $word, 'analyzer' => 'standard']]],
  964. self::DEFINITION_SCORES[$tier.'_exact'] * $weight,
  965. );
  966. }
  967. }
  968. if (count($words) >= 2) {
  969. $should[] = $this->constantScore(
  970. ['match_phrase' => ['bold_multi' => ['query' => implode(' ', $words), 'analyzer' => 'standard']]],
  971. self::DEFINITION_SCORES['multi_phrase'],
  972. );
  973. }
  974. // 正文:黑体同分时的次级排序,也保留没有黑体的文档(偈颂等)的召回
  975. if ($this->isCjkQuery($query)) {
  976. $should[] = [
  977. 'multi_match' => [
  978. 'query' => $query,
  979. 'fields' => ['title.text.zh^3', 'content.text.zh'],
  980. 'boost' => self::DEFINITION_TEXT_SCALE,
  981. ],
  982. ];
  983. } else {
  984. $should[] = $this->buildPaliFuzzyQuery($query, self::DEFINITION_TEXT_SCALE, $weights);
  985. }
  986. return [
  987. 'bool' => [
  988. 'should' => $should,
  989. ],
  990. ];
  991. }
  992. /**
  993. * 构建 hybrid(模糊 + 语义混合)查询
  994. *
  995. * 使用 bool should 将 fuzzy(constant_score 包裹)与三路 knn 向量查询合并,
  996. * 权重比例由 $weights['hybrid']['fuzzy_ratio'] 和 'semantic_ratio' 控制。
  997. * title.vector 的语义权重略高(×1.2),以提升标题匹配的排名。
  998. *
  999. * @param string $query 搜索关键词
  1000. * @return array OpenSearch DSL query 片段
  1001. *
  1002. * @throws Exception embedding 调用失败时抛出
  1003. */
  1004. protected function buildHybridQuery(string $query): array
  1005. {
  1006. $fuzzyFields = [];
  1007. foreach ($this->weights['hybrid'] as $field => $weight) {
  1008. if (in_array($field, ['fuzzy_ratio', 'semantic_ratio'])) {
  1009. continue;
  1010. }
  1011. $fuzzyFields[] = $field.'^'.$weight;
  1012. }
  1013. $fuzzyPart = [
  1014. 'multi_match' => [
  1015. 'query' => $query,
  1016. 'fields' => $fuzzyFields,
  1017. 'type' => 'best_fields',
  1018. ],
  1019. ];
  1020. $vector = $this->embedText($query);
  1021. $fuzzyRatio = $this->weights['hybrid']['fuzzy_ratio'];
  1022. $semanticRatio = $this->weights['hybrid']['semantic_ratio'];
  1023. return [
  1024. 'bool' => [
  1025. 'should' => [
  1026. [
  1027. 'constant_score' => [
  1028. 'filter' => $fuzzyPart,
  1029. 'boost' => $fuzzyRatio,
  1030. ],
  1031. ],
  1032. [
  1033. 'knn' => [
  1034. 'content.vector' => [
  1035. 'vector' => $vector,
  1036. 'k' => 20,
  1037. 'boost' => $semanticRatio * 1.0,
  1038. ],
  1039. ],
  1040. ],
  1041. [
  1042. 'knn' => [
  1043. 'summary.vector' => [
  1044. 'vector' => $vector,
  1045. 'k' => 10,
  1046. 'boost' => $semanticRatio * 0.8,
  1047. ],
  1048. ],
  1049. ],
  1050. [
  1051. 'knn' => [
  1052. 'title.vector' => [
  1053. 'vector' => $vector,
  1054. 'k' => 5,
  1055. 'boost' => $semanticRatio * 1.2, // title 权重略高
  1056. ],
  1057. ],
  1058. ],
  1059. ],
  1060. ],
  1061. ];
  1062. }
  1063. /**
  1064. * 调用 OpenAI Embedding API 将文本转为向量
  1065. *
  1066. * 使用 Redis 缓存(TTL 7 天),相同文本不会重复请求 API,
  1067. * 缓存 key 格式见 embeddingCacheKey()。
  1068. *
  1069. * @param string $text 输入文本
  1070. * @return array 1536 维 float 向量
  1071. *
  1072. * @throws Exception 未设置 OPENAI_API_KEY 或 API 返回异常时抛出
  1073. */
  1074. protected function embedText(string $text): array
  1075. {
  1076. if (! $this->openaiApiKey) {
  1077. throw new Exception('请在 .env 设置 OPENAI_API_KEY');
  1078. }
  1079. return RedisCache::remember($this->embeddingCacheKey($text), now()->addDays(7), function () use ($text) {
  1080. $response = $this->http->post('embeddings', [
  1081. 'headers' => [
  1082. 'Authorization' => 'Bearer '.$this->openaiApiKey,
  1083. 'Content-Type' => 'application/json',
  1084. ],
  1085. 'json' => [
  1086. 'model' => 'text-embedding-3-small',
  1087. 'input' => $text,
  1088. ],
  1089. ]);
  1090. $json = json_decode((string) $response->getBody(), true);
  1091. if (empty($json['data'][0]['embedding'])) {
  1092. throw new Exception('OpenAI embedding 返回异常: '.json_encode($json));
  1093. }
  1094. return $json['data'][0]['embedding'];
  1095. });
  1096. }
  1097. /**
  1098. * embedding 缓存键:"embedding/v{版本}/{md5(text)}"。
  1099. *
  1100. * 版本号存在 EMBEDDING_CACHE_VERSION_KEY,clearAllEmbeddingCache() 把它加一,
  1101. * 旧版本的键就再也不会被读到,7 天后自然过期。原来用 Cache::tags(),
  1102. * tag 索引是另外的 key,在 Redis Cluster 上跨 slot。
  1103. */
  1104. private function embeddingCacheKey(string $text): string
  1105. {
  1106. $version = RedisCache::get(self::EMBEDDING_CACHE_VERSION_KEY, 0);
  1107. return "embedding/v{$version}/".md5($text);
  1108. }
  1109. /**
  1110. * 清除指定文本的 embedding 缓存
  1111. *
  1112. * @param string $text 原始文本(与调用 embedText 时一致)
  1113. * @return bool 缓存是否成功删除
  1114. *
  1115. * @example
  1116. * $service->clearEmbeddingCache('sabbe dhammā anattā');
  1117. */
  1118. public function clearEmbeddingCache(string $text): bool
  1119. {
  1120. return RedisCache::forget($this->embeddingCacheKey($text));
  1121. }
  1122. /**
  1123. * 让所有 embedding 缓存失效
  1124. *
  1125. * 不逐条删除,而是把版本号加一;旧键 7 天内自然过期。
  1126. *
  1127. * @example
  1128. * $service->clearAllEmbeddingCache();
  1129. */
  1130. public function clearAllEmbeddingCache(): void
  1131. {
  1132. RedisCache::increment(self::EMBEDDING_CACHE_VERSION_KEY);
  1133. }
  1134. /**
  1135. * 自动建议(Completion Suggest)
  1136. *
  1137. * 基于 completion 字段实现前缀补全,支持同时查询多个语言字段。
  1138. * 结果按 _score 降序排序,跨字段去重。
  1139. *
  1140. * 可用字段标识符($fields 参数):
  1141. * - 'title_pali' → title.suggest.pali
  1142. * - 'title_zh' → title.suggest.zh
  1143. * - 'content_pali' → content.suggest.pali
  1144. * - 'content_zh' → content.suggest.zh
  1145. *
  1146. * @param string $query 查询前缀文本
  1147. * @param array|string|null $fields 要查询的字段标识符,null 表示全部字段
  1148. * @param string|null $language 可选的语言过滤(term query)
  1149. * @param int $limit 每个字段返回的建议数量,默认 10
  1150. * @return array 建议结果列表,每项包含:
  1151. * text, source(字段标识符), score, doc_id, doc_source
  1152. *
  1153. * @throws \InvalidArgumentException $fields 中含无效字段标识符时抛出
  1154. *
  1155. * @example
  1156. * // 查询所有字段
  1157. * $service->suggest('nibb');
  1158. *
  1159. * // 只查询巴利文标题建议
  1160. * $service->suggest('nibb', 'title_pali');
  1161. *
  1162. * // 查询多个字段,限制语言
  1163. * $service->suggest('涅', ['title_zh', 'content_zh'], 'zh', 5);
  1164. */
  1165. public function suggest(
  1166. string $query,
  1167. $fields = null,
  1168. ?string $language = null,
  1169. int $limit = 10
  1170. ): array {
  1171. // 字段标识符 → OpenSearch completion 字段路径
  1172. $fieldMap = [
  1173. 'title_pali' => 'title.suggest.pali',
  1174. 'title_zh' => 'title.suggest.zh',
  1175. 'content_pali' => 'content.suggest.pali',
  1176. 'content_zh' => 'content.suggest.zh',
  1177. ];
  1178. // 处理字段参数
  1179. if ($fields === null) {
  1180. $searchFields = array_keys($fieldMap);
  1181. } elseif (is_string($fields)) {
  1182. $searchFields = [$fields];
  1183. } else {
  1184. $searchFields = $fields;
  1185. }
  1186. // 过滤无效字段
  1187. $searchFields = array_values(array_filter(
  1188. $searchFields,
  1189. fn ($field) => isset($fieldMap[$field])
  1190. ));
  1191. if (empty($searchFields)) {
  1192. throw new \InvalidArgumentException('Invalid fields specified for suggestion');
  1193. }
  1194. // 构建 suggest DSL
  1195. $suggests = [];
  1196. foreach ($searchFields as $field) {
  1197. $suggests[$field.'_suggest'] = [
  1198. 'prefix' => $query,
  1199. 'completion' => [
  1200. 'field' => $fieldMap[$field],
  1201. 'size' => $limit,
  1202. 'skip_duplicates' => true,
  1203. ],
  1204. ];
  1205. }
  1206. $dsl = ['suggest' => $suggests];
  1207. if ($language) {
  1208. $dsl['query'] = ['term' => ['language' => $language]];
  1209. }
  1210. $response = $this->client->search([
  1211. 'index' => config('mint.opensearch.index'),
  1212. 'body' => $dsl,
  1213. ]);
  1214. // 整理结果,附加来源字段
  1215. $results = [];
  1216. foreach ($searchFields as $field) {
  1217. $options = $response['suggest'][$field.'_suggest'][0]['options'] ?? [];
  1218. foreach ($options as $opt) {
  1219. $results[] = [
  1220. 'text' => $opt['text'] ?? '',
  1221. 'source' => $field,
  1222. 'score' => $opt['_score'] ?? 0,
  1223. 'doc_id' => $opt['_id'] ?? null,
  1224. 'doc_source' => $opt['_source'] ?? null,
  1225. ];
  1226. }
  1227. }
  1228. // 按分数降序排序
  1229. usort($results, fn ($a, $b) => $b['score'] <=> $a['score']);
  1230. return $results;
  1231. }
  1232. /**
  1233. * 按文档 ID 获取单条完整文档(包含 content.display)
  1234. *
  1235. * @param string $id 文档 ID,例如 "term_{guid}"
  1236. * @return array OpenSearch 原始响应
  1237. */
  1238. public function get(string $id): array
  1239. {
  1240. return $this->client->get([
  1241. 'index' => config('mint.opensearch.index'),
  1242. 'id' => $id,
  1243. ]);
  1244. }
  1245. /**
  1246. * 校验 pali_synonyms 同义词词典是否对指定词生效
  1247. *
  1248. * 通过 OpenSearch _analyze API,用当前索引的 pali_query_analyzer
  1249. * 对输入文本做实时分析,返回展开后的全部 token(含原词与同义词)。
  1250. * 可用于快速确认 analysis/pali_synonyms.txt 中的某一行是否已生效
  1251. * (例如 dhamma,dharma,法 => dhamma 是否真的展开出 dharma、法)。
  1252. *
  1253. * @param string $text 待检测的巴利词,例如 "dhamma"
  1254. * @return array<int, string> 分析器输出的 token 文本数组(已去重,保留原始顺序)
  1255. *
  1256. * @throws Exception 索引不存在或 OpenSearch 调用失败时抛出
  1257. *
  1258. * @example
  1259. * $service->pali_query_health_check('dhamma');
  1260. * // => ['dhamma', 'dharma', '法']
  1261. */
  1262. public function pali_query_health_check(string $text): array
  1263. {
  1264. $index = config('mint.opensearch.index');
  1265. if (! $this->client->indices()->exists(['index' => $index])) {
  1266. throw new Exception("Index [$index] does not exist.");
  1267. }
  1268. $response = $this->client->indices()->analyze([
  1269. 'index' => $index,
  1270. 'body' => [
  1271. 'analyzer' => 'pali_query_analyzer',
  1272. 'text' => $text,
  1273. ],
  1274. ]);
  1275. $tokens = array_map(
  1276. fn ($token) => $token['token'] ?? '',
  1277. $response['tokens'] ?? []
  1278. );
  1279. return array_values(array_unique(array_filter($tokens, fn ($t) => $t !== '')));
  1280. }
  1281. }