OpenSearchService.php 55 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525
  1. <?php
  2. // api-v8/app/Services/OpenSearchService.php
  3. namespace App\Services;
  4. use App\Facades\RedisCache;
  5. use Exception;
  6. use GuzzleHttp\Client;
  7. use Illuminate\Support\Facades\Log;
  8. use Illuminate\Support\Str;
  9. use OpenSearch\GuzzleClientFactory;
  10. class OpenSearchService
  11. {
  12. private const EMBEDDING_CACHE_VERSION_KEY = 'embedding/version';
  13. protected $client;
  14. protected $http;
  15. protected $openaiApiKey;
  16. /**
  17. * 默认查询排除字段
  18. *
  19. * @var array
  20. */
  21. private $sourceExcludes = [
  22. 'title.suggest.pali',
  23. 'title.suggest.zh',
  24. 'content.suggest.pali',
  25. 'content.suggest.zh',
  26. 'content.display', // 新增,列表页不返回 HTML
  27. ];
  28. /**
  29. * 默认权重配置
  30. *
  31. * fuzzy / hybrid 两种模式各自的字段权重。
  32. * hybrid 额外包含 fuzzy_ratio / semantic_ratio 用于控制两路得分的混合比例。
  33. *
  34. * 字段名已按新映射结构调整:
  35. * title.text.pali → 原 title.pali.text
  36. * title.text.zh → 原 title.zh
  37. * content.text.pali → 原 content.pali.text
  38. * content.text.zh → 原 content.zh
  39. *
  40. * @var array
  41. */
  42. private $weights = [
  43. 'fuzzy' => [
  44. 'bold_single' => 50,
  45. 'bold_multi' => 10,
  46. 'title.text.pali' => 3,
  47. 'title.text.zh' => 3,
  48. 'summary.text' => 2,
  49. 'content.text.pali' => 1,
  50. 'content.text.zh' => 1,
  51. ],
  52. 'hybrid' => [
  53. 'fuzzy_ratio' => 0.7,
  54. 'semantic_ratio' => 0.3,
  55. 'bold_single' => 50,
  56. 'bold_multi' => 10,
  57. 'title.text.pali' => 3,
  58. 'title.text.zh' => 3,
  59. 'summary.text' => 2,
  60. 'content.text.pali' => 1,
  61. 'content.text.zh' => 1,
  62. ],
  63. ];
  64. /**
  65. * OpenSearch 索引定义(settings + mappings)
  66. *
  67. * 字段结构说明:
  68. *
  69. * title
  70. * ├── text
  71. * │ ├── pali (text) 模糊查询 + exact subfield 精确查询
  72. * │ └── zh (text) 中文分词查询
  73. * ├── vector (knn_vector, dim=1536)
  74. * └── suggest
  75. * ├── pali (completion)
  76. * └── zh (completion)
  77. *
  78. * content(结构与 title 一致,额外包含 tokens nested 字段)
  79. * ├── text
  80. * │ ├── pali (text)
  81. * │ └── zh (text)
  82. * ├── tokens (nested)
  83. * ├── vector (knn_vector, dim=1536)
  84. * └── suggest
  85. * ├── pali (completion)
  86. * └── zh (completion)
  87. *
  88. * summary(中文摘要,结构保持不变)
  89. * ├── text (text)
  90. * └── vector (knn_vector, dim=1536)
  91. *
  92. * @var array
  93. */
  94. private $indexDefinition = [
  95. 'settings' => [
  96. 'index' => [
  97. 'knn' => true,
  98. ],
  99. 'analysis' => [
  100. 'analyzer' => [
  101. 'pali_query_analyzer' => [
  102. 'tokenizer' => 'standard',
  103. 'filter' => ['lowercase', 'pali_synonyms'],
  104. ],
  105. 'pali_index_analyzer' => [
  106. 'type' => 'custom',
  107. 'tokenizer' => 'standard',
  108. 'char_filter' => ['markdown_strip'],
  109. 'filter' => ['lowercase'],
  110. ],
  111. 'markdown_clean' => [
  112. 'type' => 'custom',
  113. 'tokenizer' => 'standard',
  114. 'char_filter' => ['markdown_strip'],
  115. 'filter' => ['lowercase'],
  116. ],
  117. // Suggest 专用(忽略大小写 + 变音)
  118. 'pali_suggest_analyzer' => [
  119. 'tokenizer' => 'standard',
  120. 'filter' => ['lowercase', 'asciifolding'],
  121. ],
  122. 'zh_suggest_analyzer' => [
  123. 'tokenizer' => 'ik_max_word',
  124. 'char_filter' => ['tsconvert'],
  125. ],
  126. // 中文简繁统一 (繁 -> 简)
  127. 'zh_index_analyzer' => [
  128. 'tokenizer' => 'ik_max_word',
  129. 'char_filter' => ['tsconvert'],
  130. ],
  131. 'zh_query_analyzer' => [
  132. 'tokenizer' => 'ik_smart',
  133. 'char_filter' => ['tsconvert'],
  134. ],
  135. ],
  136. 'filter' => [
  137. 'pali_synonyms' => [
  138. 'type' => 'synonym_graph',
  139. 'synonyms_path' => 'analysis/pali_synonyms.txt',
  140. 'updateable' => true,
  141. ],
  142. ],
  143. 'char_filter' => [
  144. 'markdown_strip' => [
  145. 'type' => 'pattern_replace',
  146. 'pattern' => '\\*\\*|\\*|_|`|~',
  147. 'replacement' => '',
  148. ],
  149. 'tsconvert' => [
  150. 'type' => 'stconvert',
  151. 'convert_type' => 't2s',
  152. ],
  153. ],
  154. ],
  155. ],
  156. 'mappings' => [
  157. 'properties' => [
  158. 'id' => ['type' => 'keyword'],
  159. // 分类维度一律 keyword:它们是枚举值,不该被分词。
  160. // 若声明成 text,'zh-Hans' 会被切成 zh + hans,查 language=zh 就会把
  161. // zh-Hans 的文档一并捞进来(实测 401 vs 410),而且不报错。
  162. 'resource_id' => ['type' => 'keyword'],
  163. 'resource_type' => ['type' => 'keyword'],
  164. // ----------------------------------------------------------------
  165. // title
  166. // text.pali → 模糊查询(+ exact subfield 精确查询)
  167. // text.zh → 中文查询
  168. // vector → 语义向量
  169. // suggest.pali / suggest.zh → 自动建议
  170. // ----------------------------------------------------------------
  171. 'title' => [
  172. 'properties' => [
  173. 'text' => [
  174. 'properties' => [
  175. 'pali' => [
  176. 'type' => 'text',
  177. 'analyzer' => 'pali_index_analyzer',
  178. 'search_analyzer' => 'pali_query_analyzer',
  179. 'fields' => [
  180. 'exact' => [
  181. 'type' => 'text',
  182. 'analyzer' => 'markdown_clean',
  183. ],
  184. ],
  185. ],
  186. 'zh' => [
  187. 'type' => 'text',
  188. 'analyzer' => 'zh_index_analyzer',
  189. 'search_analyzer' => 'zh_query_analyzer',
  190. ],
  191. ],
  192. ],
  193. 'vector' => [
  194. 'type' => 'knn_vector',
  195. 'dimension' => 1536,
  196. 'method' => [
  197. 'name' => 'hnsw',
  198. 'space_type' => 'innerproduct',
  199. 'engine' => 'faiss',
  200. ],
  201. ],
  202. 'suggest' => [
  203. 'properties' => [
  204. 'pali' => [
  205. 'type' => 'completion',
  206. 'analyzer' => 'pali_suggest_analyzer',
  207. ],
  208. 'zh' => [
  209. 'type' => 'completion',
  210. 'analyzer' => 'zh_suggest_analyzer',
  211. ],
  212. ],
  213. ],
  214. ],
  215. ],
  216. // ----------------------------------------------------------------
  217. // summary(LLM 生成的简体中文摘要,结构保持不变)
  218. // text → 中文查询
  219. // vector → 语义向量
  220. // ----------------------------------------------------------------
  221. 'summary' => [
  222. 'properties' => [
  223. 'text' => [
  224. 'type' => 'text',
  225. 'analyzer' => 'zh_index_analyzer',
  226. 'search_analyzer' => 'zh_query_analyzer',
  227. ],
  228. 'vector' => [
  229. 'type' => 'knn_vector',
  230. 'dimension' => 1536,
  231. 'method' => [
  232. 'name' => 'hnsw',
  233. 'space_type' => 'innerproduct',
  234. 'engine' => 'faiss',
  235. ],
  236. ],
  237. ],
  238. ],
  239. // ----------------------------------------------------------------
  240. // content(结构与 title 对称,额外包含 tokens nested 字段)
  241. // text.pali → 模糊查询(+ exact subfield 精确查询)
  242. // text.zh → 中文查询
  243. // tokens → 词法分析结果(nested)
  244. // vector → 语义向量
  245. // suggest.pali / suggest.zh → 自动建议
  246. // ----------------------------------------------------------------
  247. 'content' => [
  248. 'properties' => [
  249. 'text' => [
  250. 'properties' => [
  251. 'pali' => [
  252. 'type' => 'text',
  253. 'analyzer' => 'pali_index_analyzer',
  254. 'search_analyzer' => 'pali_query_analyzer',
  255. 'fields' => [
  256. 'exact' => [
  257. 'type' => 'text',
  258. 'analyzer' => 'markdown_clean',
  259. ],
  260. ],
  261. ],
  262. 'zh' => [
  263. 'type' => 'text',
  264. 'analyzer' => 'zh_index_analyzer',
  265. 'search_analyzer' => 'zh_query_analyzer',
  266. ],
  267. ],
  268. ],
  269. 'tokens' => [
  270. 'type' => 'nested',
  271. 'properties' => [
  272. 'surface' => ['type' => 'keyword'],
  273. 'lemma' => ['type' => 'keyword'],
  274. 'compound_parts' => ['type' => 'keyword'],
  275. 'case' => ['type' => 'keyword'],
  276. ],
  277. ],
  278. 'vector' => [
  279. 'type' => 'knn_vector',
  280. 'dimension' => 1536,
  281. 'method' => [
  282. 'name' => 'hnsw',
  283. 'space_type' => 'innerproduct',
  284. 'engine' => 'faiss',
  285. ],
  286. ],
  287. 'suggest' => [
  288. 'properties' => [
  289. 'pali' => [
  290. 'type' => 'completion',
  291. 'analyzer' => 'pali_suggest_analyzer',
  292. ],
  293. 'zh' => [
  294. 'type' => 'completion',
  295. 'analyzer' => 'zh_suggest_analyzer',
  296. ],
  297. ],
  298. ],
  299. // 前端展示用,原始 HTML,不参与索引
  300. 'display' => [
  301. 'type' => 'text',
  302. 'index' => false,
  303. ],
  304. ],
  305. ],
  306. 'related_id' => ['type' => 'keyword'],
  307. 'bold_single' => [
  308. 'type' => 'text',
  309. 'analyzer' => 'standard',
  310. 'search_analyzer' => 'pali_query_analyzer',
  311. ],
  312. 'bold_multi' => [
  313. 'type' => 'text',
  314. 'analyzer' => 'standard',
  315. 'search_analyzer' => 'pali_query_analyzer',
  316. ],
  317. 'path' => ['type' => 'text', 'analyzer' => 'standard'],
  318. 'page_refs' => ['type' => 'keyword'],
  319. 'tags' => ['type' => 'keyword'],
  320. 'category' => ['type' => 'keyword'],
  321. 'author' => ['type' => 'text'],
  322. 'language' => ['type' => 'keyword'],
  323. 'updated_at' => ['type' => 'date'],
  324. 'granularity' => ['type' => 'keyword'],
  325. 'metadata' => [
  326. 'properties' => [
  327. 'APA' => ['type' => 'text', 'index' => false],
  328. 'MLA' => ['type' => 'text', 'index' => false],
  329. 'widget' => ['type' => 'text', 'index' => false],
  330. 'author' => ['type' => 'text'],
  331. 'channel' => ['type' => 'text'],
  332. ],
  333. ],
  334. ],
  335. ],
  336. ];
  337. /**
  338. * 创建 OpenSearchService 实例
  339. *
  340. * 从 config('mint.opensearch.config') 读取连接配置,
  341. * 同时初始化 OpenAI HTTP 客户端用于 embedding 调用。
  342. */
  343. public function __construct()
  344. {
  345. $config = config('mint.opensearch.config');
  346. $hostUrl = "{$config['scheme']}://{$config['host']}:{$config['port']}";
  347. $this->client = (new GuzzleClientFactory)->create([
  348. 'base_uri' => $hostUrl,
  349. 'auth' => [$config['username'], $config['password']],
  350. 'verify' => $config['ssl_verification'],
  351. ]);
  352. $this->openaiApiKey = env('OPENAI_API_KEY');
  353. $this->http = new Client([
  354. 'base_uri' => 'https://api.openai.com/v1/',
  355. 'timeout' => 15,
  356. ]);
  357. }
  358. /**
  359. * 动态覆盖指定搜索模式的字段权重
  360. *
  361. * @param string $mode 搜索模式,支持 'fuzzy' | 'hybrid'
  362. * @param array $weights 需要覆盖的权重键值对,例如:['title.text.pali' => 5]
  363. */
  364. public function setWeights(string $mode, array $weights): void
  365. {
  366. if (isset($this->weights[$mode])) {
  367. $this->weights[$mode] = array_merge($this->weights[$mode], $weights);
  368. }
  369. }
  370. /**
  371. * 测试与 OpenSearch 集群的连接状态
  372. *
  373. * @return array{0: bool, 1: string} [连接是否成功, 描述信息]
  374. */
  375. public function testConnection(): array
  376. {
  377. try {
  378. $info = $this->client->info();
  379. $message = 'OpenSearch 连接成功: '.json_encode($info['version']['number']);
  380. Log::info($message);
  381. return [true, $message];
  382. } catch (Exception $e) {
  383. $message = 'OpenSearch 连接失败: '.$e->getMessage();
  384. Log::error($message);
  385. return [false, $message];
  386. }
  387. }
  388. /**
  389. * 检查当前索引是否已存在
  390. */
  391. public function indexExists(): bool
  392. {
  393. $index = config('mint.opensearch.index');
  394. return $this->client->indices()->exists(['index' => $index]);
  395. }
  396. /**
  397. * 创建 OpenSearch 索引
  398. *
  399. * 使用 $indexDefinition 中定义的 settings 和 mappings 创建索引。
  400. * 若索引已存在则抛出异常,避免覆盖生产数据。
  401. *
  402. * @return array OpenSearch 响应
  403. *
  404. * @throws Exception 索引已存在时抛出
  405. */
  406. public function createIndex(): array
  407. {
  408. $index = config('mint.opensearch.index');
  409. $exists = $this->client->indices()->exists(['index' => $index]);
  410. if ($exists) {
  411. throw new Exception("Index [$index] already exists.");
  412. }
  413. return $this->client->indices()->create([
  414. 'index' => $index,
  415. 'body' => $this->indexDefinition,
  416. ]);
  417. }
  418. /**
  419. * 更新已有索引的 settings 和 mappings
  420. *
  421. * 更新 settings 时会临时关闭索引(close → putSettings → open),
  422. * 更新 mappings 支持热更新(新增字段),不可修改已有字段类型。
  423. *
  424. * @return array 包含 'settings' 和/或 'mappings' 的响应数组
  425. */
  426. public function updateIndex(): array
  427. {
  428. $index = config('mint.opensearch.index');
  429. $settings = $this->indexDefinition['settings'] ?? [];
  430. $mappings = $this->indexDefinition['mappings'] ?? [];
  431. $response = [];
  432. if (! empty($settings)) {
  433. $this->client->indices()->close(['index' => $index]);
  434. $response['settings'] = $this->client->indices()->putSettings([
  435. 'index' => $index,
  436. 'body' => ['settings' => $settings],
  437. ]);
  438. $this->client->indices()->open(['index' => $index]);
  439. }
  440. if (! empty($mappings)) {
  441. $response['mappings'] = $this->client->indices()->putMapping([
  442. 'index' => $index,
  443. 'body' => $mappings,
  444. ]);
  445. }
  446. return $response;
  447. }
  448. /**
  449. * 切换 pali 同义词文件版本
  450. *
  451. * 把 pali_synonyms filter 的 synonyms_path 指向
  452. * analysis/pali-synonyms-{$version}.txt(文件须已存在于 OpenSearch
  453. * 各节点的 config 目录中,否则索引无法重新打开)。
  454. *
  455. * synonyms_path 属于静态 settings,必须 close → putSettings → open。
  456. * 无论 putSettings 成功与否都会尝试重新打开索引,避免索引停留在 close 状态。
  457. *
  458. * @param string $version 版本号,仅允许 [A-Za-z0-9._-]
  459. * @return array{path: string, settings: array} 新路径与 OpenSearch 响应
  460. *
  461. * @throws Exception 版本号非法、索引不存在或 OpenSearch 拒绝时抛出
  462. *
  463. * @example
  464. * $service->updatePaliSynonymsPath('20260731');
  465. */
  466. public function updatePaliSynonymsPath(string $filename): array
  467. {
  468. $index = config('mint.opensearch.index');
  469. if (! $this->client->indices()->exists(['index' => $index])) {
  470. throw new Exception("Index [$index] does not exist.");
  471. }
  472. $path = Str::finish(config('mint.opensearch.config.synonyms_path'), '/').$filename;
  473. Log::debug('OpenSearchService::updatePaliSynonymsPath', [
  474. 'index' => $index,
  475. 'synonyms_path' => $path,
  476. ]);
  477. // 以代码中的 analysis 定义为准,只替换同义词文件路径
  478. $analysis = $this->indexDefinition['settings']['analysis'];
  479. $analysis['filter']['pali_synonyms']['synonyms_path'] = $path;
  480. $this->client->indices()->close(['index' => $index]);
  481. try {
  482. $response = $this->client->indices()->putSettings([
  483. 'index' => $index,
  484. 'body' => ['settings' => ['analysis' => $analysis]],
  485. ]);
  486. } finally {
  487. $this->client->indices()->open(['index' => $index]);
  488. }
  489. return ['path' => $path, 'settings' => $response];
  490. }
  491. /**
  492. * 读取当前索引使用的 pali 同义词文件路径
  493. *
  494. * @return string|null 形如 "analysis/pali-synonyms-20260731.txt",未设置时返回 null
  495. */
  496. public function getPaliSynonymsPath(): ?string
  497. {
  498. return $this->getPaliSynonymsSetting()['synonyms_path'] ?? null;
  499. }
  500. /**
  501. * 读取当前索引 pali_synonyms filter 的完整设置
  502. *
  503. * @return array{type?: string, synonyms_path?: string, updateable?: string}|null 未设置时返回 null
  504. */
  505. public function getPaliSynonymsSetting(): ?array
  506. {
  507. $index = config('mint.opensearch.index');
  508. $settings = $this->client->indices()->getSettings(['index' => $index]);
  509. return $settings[$index]['settings']['index']['analysis']['filter']['pali_synonyms'] ?? null;
  510. }
  511. /**
  512. * 删除当前索引
  513. *
  514. * @return array OpenSearch 响应
  515. */
  516. public function deleteIndex(): array
  517. {
  518. $index = config('mint.opensearch.index');
  519. return $this->client->indices()->delete(['index' => $index]);
  520. }
  521. /**
  522. * 统计索引文档数量(支持可选条件过滤)
  523. *
  524. * @param array|null $query OpenSearch DSL query 子句,为 null 时统计全部文档。
  525. * 示例:['term' => ['language' => 'zh']]
  526. * ['exists' => ['field' => 'content.vector']]
  527. * @return int 文档总数
  528. *
  529. * @throws Exception
  530. *
  531. * @example
  532. * $service->count();
  533. * $service->count(['exists' => ['field' => 'content.vector']]);
  534. */
  535. public function count(?array $query = null): int
  536. {
  537. $index = config('mint.opensearch.index');
  538. $params = ['index' => $index];
  539. if (! empty($query)) {
  540. $params['body'] = ['query' => $query];
  541. }
  542. $response = $this->client->count($params);
  543. return (int) ($response['count'] ?? 0);
  544. }
  545. /**
  546. * 写入或覆盖单条文档
  547. *
  548. * @param string $id 文档 ID
  549. * @param array $body 文档内容,字段结构须与 mappings 一致
  550. * @return array OpenSearch 响应
  551. */
  552. public function create(string $id, array $body): array
  553. {
  554. return $this->client->index([
  555. 'index' => config('mint.opensearch.index'),
  556. 'id' => $id,
  557. 'body' => $body,
  558. ]);
  559. }
  560. /**
  561. * 删除单条文档
  562. *
  563. * @param string $id 文档 ID
  564. * @return array OpenSearch 响应
  565. */
  566. public function delete(string $id): array
  567. {
  568. return $this->client->delete([
  569. 'index' => config('mint.opensearch.index'),
  570. 'id' => $id,
  571. ]);
  572. }
  573. /**
  574. * 执行高级搜索
  575. *
  576. * 支持五种搜索模式:
  577. * - fuzzy 多字段模糊查询(默认),巴利文走三层排序
  578. * - exact 精确匹配,使用 markdown_clean analyzer
  579. * - semantic 纯语义向量搜索,需要 OpenAI embedding
  580. * - hybrid fuzzy + semantic 混合,权重由 fuzzy_ratio / semantic_ratio 控制
  581. * - definition 定义查询,黑体字(bold_single/bold_multi)高权重,术语定义排前
  582. *
  583. * 支持的过滤参数:
  584. * resourceType, resourceId, granularity, language, category,
  585. * tags, pageRefs, relatedId, author, channel
  586. *
  587. * @param array $params {
  588. *
  589. * @type string $query 搜索关键词(必填)
  590. * @type string $searchMode 搜索模式,默认 'fuzzy'
  591. * @type int $page 页码,默认 1
  592. * @type int $pageSize 每页条数,默认 20
  593. * @type string $resourceType 按资源类型过滤
  594. * @type string $resourceId 按资源 ID 过滤
  595. * @type string $granularity 按粒度过滤
  596. * @type string $language 按语言过滤
  597. * @type string $category 按分类过滤
  598. * @type array $tags 按标签过滤(terms)
  599. * @type array $pageRefs 按页码引用过滤(terms)
  600. * @type string $relatedId 按关联 ID 过滤
  601. * @type string $author 按作者过滤
  602. * @type string $channel 按频道过滤
  603. * @type array $highlight_pre_tags 高亮前置标签,默认 ['<mark>']
  604. * @type array $highlight_post_tags 高亮后置标签,默认 ['</mark>']
  605. * }
  606. *
  607. * @return array OpenSearch 原始响应
  608. *
  609. * @throws Exception semantic / hybrid 模式下 embedding 调用失败时抛出
  610. */
  611. public function search(array $params): array
  612. {
  613. $page = $params['page'] ?? 1;
  614. $pageSize = $params['pageSize'] ?? 20;
  615. $from = ($page - 1) * $pageSize;
  616. $mode = $params['searchMode'] ?? 'fuzzy';
  617. // 排除字段
  618. if (! empty($params['excludes']) && is_array($params['excludes'])) {
  619. $excludes = array_merge($this->sourceExcludes, $params['excludes']);
  620. } else {
  621. $excludes = $this->sourceExcludes;
  622. }
  623. // ---------- 过滤条件 ----------
  624. $filters = [];
  625. if (! empty($params['resourceType'])) {
  626. $filters[] = ['term' => ['resource_type' => $params['resourceType']]];
  627. }
  628. if (! empty($params['resourceId'])) {
  629. $filters[] = ['term' => ['resource_id' => $params['resourceId']]];
  630. }
  631. if (! empty($params['granularity'])) {
  632. $filters[] = ['term' => ['granularity' => $params['granularity']]];
  633. }
  634. if (! empty($params['language'])) {
  635. $filters[] = ['term' => ['language' => $params['language']]];
  636. }
  637. if (! empty($params['category'])) {
  638. if (is_array($params['category'])) {
  639. $categories = $params['category'];
  640. } else {
  641. $categories = [$params['category']];
  642. }
  643. // 必须匹配全部:为每个 category 创建一个 term 条件
  644. foreach ($categories as $category) {
  645. $filters[] = ['term' => ['category' => $category]];
  646. }
  647. }
  648. if (! empty($params['tags'])) {
  649. $filters[] = ['terms' => ['tags' => $params['tags']]];
  650. }
  651. if (! empty($params['pageRefs'])) {
  652. $filters[] = ['terms' => ['page_refs' => $params['pageRefs']]];
  653. }
  654. if (! empty($params['relatedId'])) {
  655. $filters[] = ['term' => ['related_id' => $params['relatedId']]];
  656. }
  657. if (! empty($params['author'])) {
  658. $filters[] = ['match' => ['metadata.author' => $params['author']]];
  659. }
  660. if (! empty($params['channel'])) {
  661. $filters[] = ['term' => ['metadata.channel' => $params['channel']]];
  662. }
  663. // ---------- 查询部分 ----------
  664. $queryText = trim($params['query'] ?? '');
  665. if ($queryText === '') {
  666. $query = ['match_all' => new \stdClass];
  667. } else {
  668. switch ($mode) {
  669. case 'exact':
  670. $query = $this->buildExactQuery($queryText);
  671. break;
  672. case 'semantic':
  673. $query = $this->buildSemanticQuery($queryText);
  674. break;
  675. case 'hybrid':
  676. $query = $this->buildHybridQuery($queryText);
  677. break;
  678. case 'definition':
  679. $query = $this->buildDefinitionQuery($queryText);
  680. break;
  681. case 'fuzzy':
  682. default:
  683. $query = $this->buildFuzzyQuery($queryText);
  684. break;
  685. }
  686. }
  687. $highlightPreTags = $params['highlight_pre_tags'] ?? ['<mark>'];
  688. $highlightPostTags = $params['highlight_post_tags'] ?? ['</mark>'];
  689. // ---------- 最终 DSL ----------
  690. $dsl = [
  691. 'from' => $from,
  692. 'size' => $pageSize,
  693. '_source' => ['excludes' => $excludes],
  694. 'query' => ! empty($filters)
  695. ? [
  696. 'bool' => [
  697. 'must' => [$query],
  698. 'filter' => $filters,
  699. ],
  700. ]
  701. : $query,
  702. 'aggs' => [
  703. 'resource_type' => [
  704. 'terms' => ['field' => 'resource_type'],
  705. ],
  706. 'language' => [
  707. 'terms' => ['field' => 'language'],
  708. ],
  709. 'category' => [
  710. 'terms' => ['field' => 'category'],
  711. ],
  712. 'granularity' => [
  713. 'terms' => ['field' => 'granularity'],
  714. ],
  715. ],
  716. ];
  717. // 只有有搜索词时才开启高亮;字段按语种路由,避免把巴利文喂给中文分词器
  718. if ($queryText !== '') {
  719. $highlightFields = $this->isCjkQuery($queryText)
  720. ? ['title.text.zh', 'summary.text', 'content.text.zh']
  721. : ['title.text.pali', 'content.text.pali'];
  722. $dsl['highlight'] = [
  723. 'fields' => array_fill_keys($highlightFields, new \stdClass),
  724. 'fragmenter' => 'sentence',
  725. 'fragment_size' => 200,
  726. 'number_of_fragments' => 1,
  727. 'pre_tags' => $highlightPreTags,
  728. 'post_tags' => $highlightPostTags,
  729. ];
  730. }
  731. Log::debug(
  732. 'OpenSearchService::search',
  733. ['dsl' => json_encode($dsl, JSON_UNESCAPED_UNICODE)]
  734. );
  735. return $this->client->search([
  736. 'index' => config('mint.opensearch.index'),
  737. 'body' => $dsl,
  738. ]);
  739. }
  740. /**
  741. * 构建 exact(精确匹配)查询
  742. *
  743. * 使用 markdown_clean analyzer 的 exact subfield 进行匹配,
  744. * 适合巴利文词形精确检索场景。
  745. *
  746. * 查询字段:title.text.pali.exact, content.text.pali.exact, summary.text
  747. *
  748. * @param string $query 搜索关键词
  749. * @return array OpenSearch DSL query 片段
  750. */
  751. protected function buildExactQuery(string $query): array
  752. {
  753. return [
  754. 'multi_match' => [
  755. 'query' => $query,
  756. 'fields' => [
  757. 'title.text.pali.exact',
  758. 'content.text.pali.exact',
  759. 'summary.text',
  760. ],
  761. 'type' => 'best_fields',
  762. ],
  763. ];
  764. }
  765. /**
  766. * 构建 semantic(纯语义向量)查询
  767. *
  768. * 将查询文本通过 OpenAI embedding API 转为向量,
  769. * 同时对 content.vector、summary.vector、title.vector 三个 knn 字段检索,
  770. * 使用 bool should 合并结果。
  771. *
  772. * @param string $query 搜索关键词
  773. * @return array OpenSearch DSL query 片段
  774. *
  775. * @throws Exception embedding 调用失败时抛出
  776. */
  777. protected function buildSemanticQuery(string $query): array
  778. {
  779. $vector = $this->embedText($query);
  780. return [
  781. 'bool' => [
  782. 'should' => [
  783. ['knn' => ['content.vector' => ['vector' => $vector, 'k' => 20]]],
  784. ['knn' => ['summary.vector' => ['vector' => $vector, 'k' => 10]]],
  785. ['knn' => ['title.vector' => ['vector' => $vector, 'k' => 5]]],
  786. ],
  787. 'minimum_should_match' => 1,
  788. ],
  789. ];
  790. }
  791. /**
  792. * 构建 fuzzy(多字段模糊)查询
  793. *
  794. * 按查询语种路由字段,避免巴利文被中文分词器切碎、中文被巴利文 analyzer 误伤:
  795. * - 含汉字 → 只搜 bold_* + *.text.zh + summary.text
  796. * - 巴利文(拉丁字母)→ 走三层排序(见 buildPaliFuzzyQuery)+ bold_* 提权
  797. *
  798. * @param string $query 搜索关键词
  799. * @return array OpenSearch DSL query 片段
  800. */
  801. protected function buildFuzzyQuery(string $query): array
  802. {
  803. if ($this->isCjkQuery($query)) {
  804. return [
  805. 'multi_match' => [
  806. 'query' => $query,
  807. 'fields' => [
  808. 'bold_single^50',
  809. 'bold_multi^10',
  810. 'title.text.zh^3',
  811. 'summary.text^2',
  812. 'content.text.zh^1',
  813. ],
  814. 'type' => 'best_fields',
  815. ],
  816. ];
  817. }
  818. return $this->buildPaliFuzzyQuery($query);
  819. }
  820. /**
  821. * 巴利文 fuzzy 的计分表(全部是固定分,逐项累加)
  822. *
  823. * 一个文档的得分 = 命中的每一项相加,于是「部分符合」自然逐级降分:
  824. * - phrase 整句按原顺序、拼写完全相同(≥2 个词才有)
  825. * - bigram 相邻两词按原顺序、拼写相同,每对一次(≥3 个词才有,2 个词时与 phrase 重合)
  826. * - exact 单个原始词拼写相同,每词一次
  827. * - synonym 单个原始词经同义词可替代,每词一次(拼写相同的词也会命中这一项)
  828. *
  829. * 以 3 个词为例:整句原样 66 > 全部拼写相同但乱序 36 > 两词相邻拼写相同 + 一词同义 31
  830. * > 两词拼写相同不相邻 24 > 三词全靠同义词 6 > 一词同义 2。
  831. *
  832. * @var array{phrase: int, bigram: int, exact: int, synonym: int}
  833. */
  834. private const PALI_FUZZY_SCORES = [
  835. 'phrase' => 20,
  836. 'bigram' => 5,
  837. 'exact' => 10,
  838. 'synonym' => 2,
  839. ];
  840. /**
  841. * definition 的计分表(黑体字部分,每个原始词各算一次)
  842. *
  843. * 主分不分词形:同义词词典的一行就是一个词的全部变格变位,所以「任一词形」走同义词,
  844. * 与 v2 search-pali-wbw 把全部词形一视同仁的口径一致;原拼写只给少量加分,排在同档前面。
  845. *
  846. * - single 该词(任一词形)是单独黑体字
  847. * - single_exact 再加:拼写也相同
  848. * - multi 该词(任一词形)出现在多词黑体短语里
  849. * - multi_exact 再加:拼写也相同
  850. * - multi_phrase 整句按原顺序、拼写相同出现在多词黑体短语里(≥2 个词才有)
  851. *
  852. * 每个词:单独黑体 ≥ 100 > 多词黑体 ≤ 35,于是命中的单独黑体字越多排越前。
  853. * 正文得分按 DEFINITION_TEXT_SCALE 缩小后叠加,只用来给黑体同分的文档排先后。
  854. *
  855. * @var array{single: int, single_exact: int, multi: int, multi_exact: int, multi_phrase: int}
  856. */
  857. private const DEFINITION_SCORES = [
  858. 'single' => 100,
  859. 'single_exact' => 10,
  860. 'multi' => 30,
  861. 'multi_exact' => 5,
  862. 'multi_phrase' => 30,
  863. ];
  864. /**
  865. * definition 模式里正文得分的缩放系数,保证正文再怎么命中也压不过一个黑体字
  866. */
  867. private const DEFINITION_TEXT_SCALE = 0.1;
  868. /**
  869. * 同分时的 BM25 微调权重:BM25 一般 < 30,乘上后 < 0.3,小于计分表的最小档差
  870. */
  871. private const TIE_BREAKER_BOOST = 0.01;
  872. /**
  873. * 巴利文停用词(小写;比较前去掉标点)
  874. *
  875. * 取自 pg 全文检索的 pali.stop(api-v8/public/app/fts/pali.stop),另加 vā。
  876. * 停用词只参与词序打分(整句、相邻两词),不单独计分、不单独召回,见 buildPaliFuzzyQuery。
  877. *
  878. * @var list<string>
  879. */
  880. private const PALI_STOPWORDS = ['na', 'no', 'ca', 'va', 'vā', 'hi', 'pana', 'iva', 'eva'];
  881. /**
  882. * 判断原始词是否巴利文停用词
  883. *
  884. * @param string $word 原始词(可带标点,如「ca,」「‘‘na」)
  885. * @return bool 是停用词返回 true
  886. */
  887. protected function isPaliStopword(string $word): bool
  888. {
  889. $normalized = mb_strtolower(preg_replace('/[^\p{L}\p{M}]/u', '', $word) ?? '');
  890. return in_array($normalized, self::PALI_STOPWORDS, true);
  891. }
  892. /**
  893. * 取出用于单独计分的词:去掉停用词;全是停用词时原样返回,避免查不到东西
  894. *
  895. * @param list<string> $words 原始词
  896. * @return list<string>
  897. */
  898. protected function scoringWords(array $words): array
  899. {
  900. $contentWords = array_values(array_filter(
  901. $words,
  902. fn (string $word): bool => ! $this->isPaliStopword($word),
  903. ));
  904. return $contentWords === [] ? $words : $contentWords;
  905. }
  906. /**
  907. * 把查询切成原始词
  908. *
  909. * 按空白切分,并丢掉不含字母的片段(如「–」「.」):它们经分析器后没有 token,
  910. * 留着会让按词计分的子句永远不命中。
  911. *
  912. * @param string $query 搜索关键词
  913. * @return list<string>
  914. */
  915. protected function splitQueryWords(string $query): array
  916. {
  917. $words = preg_split('/\s+/u', trim($query)) ?: [];
  918. return array_values(array_filter(
  919. $words,
  920. fn (string $word): bool => preg_match('/\p{L}/u', $word) === 1,
  921. ));
  922. }
  923. /**
  924. * 包一层 constant_score:命中即得固定分,不受 BM25 字段长度归一化影响
  925. *
  926. * @param array<string, mixed> $filter 过滤子句
  927. * @param float $boost 命中时的固定分
  928. * @return array{constant_score: array{filter: array<string, mixed>, boost: float}}
  929. */
  930. protected function constantScore(array $filter, float $boost): array
  931. {
  932. return ['constant_score' => ['filter' => $filter, 'boost' => $boost]];
  933. }
  934. /**
  935. * 构建巴利文查询:词序符合、拼写相同得高分,部分符合逐级降分
  936. *
  937. * 命中按「原始词」(空格切分)计,不按同义词展开后的 token 计——每个原始词一个
  938. * constant_score 子句,命中几个同义词都只算一次。计分表见 PALI_FUZZY_SCORES。
  939. * 拼写相同打在 .exact 子字段(markdown_clean,无同义词),同义词打在 .pali。
  940. * 最后加一条极小权重的 BM25 子句,只用来给同分文档排先后。
  941. *
  942. * 停用词(PALI_STOPWORDS)只出现在整句、相邻两词的子句里,保住词序判断;
  943. * 不生成单词子句,也不进 BM25 子句,于是只命中「ca」的文档不会被召回。
  944. * 两个停用词组成的相邻词对同理跳过。查询全是停用词时按普通词处理。
  945. *
  946. * @param string $query 搜索关键词
  947. * @param float $scale 整体缩放(definition 模式用它把正文得分压到黑体字之下)
  948. * @return array OpenSearch DSL query 片段
  949. */
  950. protected function buildPaliFuzzyQuery(string $query, float $scale = 1.0): array
  951. {
  952. $words = $this->splitQueryWords($query);
  953. $wordCount = count($words);
  954. if ($wordCount === 0) {
  955. return ['match_all' => new \stdClass];
  956. }
  957. $scoringWords = $this->scoringWords($words);
  958. $allStopwords = array_all($words, fn (string $word): bool => $this->isPaliStopword($word));
  959. $exactFields = ['title.text.pali.exact', 'content.text.pali.exact'];
  960. $fuzzyFields = ['title.text.pali', 'content.text.pali'];
  961. $scores = array_map(fn (int $score): float => $score * $scale, self::PALI_FUZZY_SCORES);
  962. $should = [];
  963. // 词序:整句原样
  964. if ($wordCount >= 2) {
  965. $should[] = $this->constantScore([
  966. 'multi_match' => [
  967. 'query' => implode(' ', $words),
  968. 'type' => 'phrase',
  969. 'fields' => $exactFields,
  970. ],
  971. ], $scores['phrase']);
  972. }
  973. // 词序:相邻两词原样(部分顺序符合)
  974. if ($wordCount >= 3) {
  975. for ($i = 0; $i < $wordCount - 1; $i++) {
  976. $bothStopwords = $this->isPaliStopword($words[$i]) && $this->isPaliStopword($words[$i + 1]);
  977. if ($bothStopwords && ! $allStopwords) {
  978. continue;
  979. }
  980. $should[] = $this->constantScore([
  981. 'multi_match' => [
  982. 'query' => $words[$i].' '.$words[$i + 1],
  983. 'type' => 'phrase',
  984. 'fields' => $exactFields,
  985. ],
  986. ], $scores['bigram']);
  987. }
  988. }
  989. // 拼写:每个词各算一次(停用词不算)
  990. foreach ($scoringWords as $word) {
  991. $should[] = $this->constantScore([
  992. 'multi_match' => [
  993. 'query' => $word,
  994. 'fields' => $exactFields,
  995. 'operator' => 'and',
  996. ],
  997. ], $scores['exact']);
  998. $should[] = $this->constantScore([
  999. 'multi_match' => [
  1000. 'query' => $word,
  1001. 'fields' => $fuzzyFields,
  1002. ],
  1003. ], $scores['synonym']);
  1004. }
  1005. // 同分文档按 BM25 排先后
  1006. $should[] = [
  1007. 'match' => [
  1008. 'content.text.pali' => [
  1009. 'query' => implode(' ', $scoringWords),
  1010. 'boost' => self::TIE_BREAKER_BOOST * $scale,
  1011. ],
  1012. ],
  1013. ];
  1014. return [
  1015. 'bool' => [
  1016. 'should' => $should,
  1017. ],
  1018. ];
  1019. }
  1020. /**
  1021. * 判断查询是否含中文(CJK 统一表意文字)
  1022. *
  1023. * @param string $query 搜索关键词
  1024. * @return bool 含汉字返回 true,否则按巴利文(拉丁字母)处理
  1025. */
  1026. protected function isCjkQuery(string $query): bool
  1027. {
  1028. return (bool) preg_match('/[\x{4E00}-\x{9FFF}\x{3400}-\x{4DBF}\x{F900}-\x{FAFF}]/u', $query);
  1029. }
  1030. /**
  1031. * 构建 definition(定义查询)
  1032. *
  1033. * 义注里的黑体字通常就是被解释的词,所以黑体字命中决定排序,正文只做次级排序:
  1034. * - 每个原始词单独算:命中的单独黑体字(bold_single)越多分越高
  1035. * - 单独黑体字 > 多词黑体短语(bold_multi)里的词
  1036. * - 词形不分高下(走同义词,即该词的全部变格变位),原拼写只在同档内加少量分:
  1037. * bold_* 建索引用 standard,查询时显式指定 analyzer=standard 即是「拼写相同」,
  1038. * 不指定则走带同义词的 pali_query_analyzer
  1039. * - 多词黑体也按任一词形算:bold_* 取自 wbw_templates.real(已拆连音),
  1040. * 正文里粘了连音(如 vitakkanti)匹配不到的词,往往靠它召回
  1041. * - 用 match_phrase 而不是 match:中文在 standard 下按字切分,phrase 要求字连续,
  1042. * 避免一个字重合就命中
  1043. * 计分表见 DEFINITION_SCORES;正文部分复用 fuzzy 的计分,乘 DEFINITION_TEXT_SCALE。
  1044. *
  1045. * @param string $query 搜索关键词
  1046. * @return array OpenSearch DSL query 片段
  1047. */
  1048. protected function buildDefinitionQuery(string $query): array
  1049. {
  1050. $words = $this->splitQueryWords($query);
  1051. if ($words === []) {
  1052. return ['match_all' => new \stdClass];
  1053. }
  1054. $should = [];
  1055. // 停用词不单独算黑体分(整句的 multi_phrase 仍带上它们)
  1056. foreach ($this->scoringWords($words) as $word) {
  1057. foreach (['single' => 'bold_single', 'multi' => 'bold_multi'] as $tier => $field) {
  1058. $should[] = $this->constantScore(
  1059. ['match_phrase' => [$field => $word]],
  1060. self::DEFINITION_SCORES[$tier],
  1061. );
  1062. $should[] = $this->constantScore(
  1063. ['match_phrase' => [$field => ['query' => $word, 'analyzer' => 'standard']]],
  1064. self::DEFINITION_SCORES[$tier.'_exact'],
  1065. );
  1066. }
  1067. }
  1068. if (count($words) >= 2) {
  1069. $should[] = $this->constantScore(
  1070. ['match_phrase' => ['bold_multi' => ['query' => implode(' ', $words), 'analyzer' => 'standard']]],
  1071. self::DEFINITION_SCORES['multi_phrase'],
  1072. );
  1073. }
  1074. // 正文:黑体同分时的次级排序,也保留没有黑体的文档(偈颂等)的召回
  1075. if ($this->isCjkQuery($query)) {
  1076. $should[] = [
  1077. 'multi_match' => [
  1078. 'query' => $query,
  1079. 'fields' => ['title.text.zh^3', 'content.text.zh'],
  1080. 'boost' => self::DEFINITION_TEXT_SCALE,
  1081. ],
  1082. ];
  1083. } else {
  1084. $should[] = $this->buildPaliFuzzyQuery($query, self::DEFINITION_TEXT_SCALE);
  1085. }
  1086. return [
  1087. 'bool' => [
  1088. 'should' => $should,
  1089. ],
  1090. ];
  1091. }
  1092. /**
  1093. * 构建 hybrid(模糊 + 语义混合)查询
  1094. *
  1095. * 使用 bool should 将 fuzzy(constant_score 包裹)与三路 knn 向量查询合并,
  1096. * 权重比例由 $weights['hybrid']['fuzzy_ratio'] 和 'semantic_ratio' 控制。
  1097. * title.vector 的语义权重略高(×1.2),以提升标题匹配的排名。
  1098. *
  1099. * @param string $query 搜索关键词
  1100. * @return array OpenSearch DSL query 片段
  1101. *
  1102. * @throws Exception embedding 调用失败时抛出
  1103. */
  1104. protected function buildHybridQuery(string $query): array
  1105. {
  1106. $fuzzyFields = [];
  1107. foreach ($this->weights['hybrid'] as $field => $weight) {
  1108. if (in_array($field, ['fuzzy_ratio', 'semantic_ratio'])) {
  1109. continue;
  1110. }
  1111. $fuzzyFields[] = $field.'^'.$weight;
  1112. }
  1113. $fuzzyPart = [
  1114. 'multi_match' => [
  1115. 'query' => $query,
  1116. 'fields' => $fuzzyFields,
  1117. 'type' => 'best_fields',
  1118. ],
  1119. ];
  1120. $vector = $this->embedText($query);
  1121. $fuzzyRatio = $this->weights['hybrid']['fuzzy_ratio'];
  1122. $semanticRatio = $this->weights['hybrid']['semantic_ratio'];
  1123. return [
  1124. 'bool' => [
  1125. 'should' => [
  1126. [
  1127. 'constant_score' => [
  1128. 'filter' => $fuzzyPart,
  1129. 'boost' => $fuzzyRatio,
  1130. ],
  1131. ],
  1132. [
  1133. 'knn' => [
  1134. 'content.vector' => [
  1135. 'vector' => $vector,
  1136. 'k' => 20,
  1137. 'boost' => $semanticRatio * 1.0,
  1138. ],
  1139. ],
  1140. ],
  1141. [
  1142. 'knn' => [
  1143. 'summary.vector' => [
  1144. 'vector' => $vector,
  1145. 'k' => 10,
  1146. 'boost' => $semanticRatio * 0.8,
  1147. ],
  1148. ],
  1149. ],
  1150. [
  1151. 'knn' => [
  1152. 'title.vector' => [
  1153. 'vector' => $vector,
  1154. 'k' => 5,
  1155. 'boost' => $semanticRatio * 1.2, // title 权重略高
  1156. ],
  1157. ],
  1158. ],
  1159. ],
  1160. ],
  1161. ];
  1162. }
  1163. /**
  1164. * 调用 OpenAI Embedding API 将文本转为向量
  1165. *
  1166. * 使用 Redis 缓存(TTL 7 天),相同文本不会重复请求 API,
  1167. * 缓存 key 格式见 embeddingCacheKey()。
  1168. *
  1169. * @param string $text 输入文本
  1170. * @return array 1536 维 float 向量
  1171. *
  1172. * @throws Exception 未设置 OPENAI_API_KEY 或 API 返回异常时抛出
  1173. */
  1174. protected function embedText(string $text): array
  1175. {
  1176. if (! $this->openaiApiKey) {
  1177. throw new Exception('请在 .env 设置 OPENAI_API_KEY');
  1178. }
  1179. return RedisCache::remember($this->embeddingCacheKey($text), now()->addDays(7), function () use ($text) {
  1180. $response = $this->http->post('embeddings', [
  1181. 'headers' => [
  1182. 'Authorization' => 'Bearer '.$this->openaiApiKey,
  1183. 'Content-Type' => 'application/json',
  1184. ],
  1185. 'json' => [
  1186. 'model' => 'text-embedding-3-small',
  1187. 'input' => $text,
  1188. ],
  1189. ]);
  1190. $json = json_decode((string) $response->getBody(), true);
  1191. if (empty($json['data'][0]['embedding'])) {
  1192. throw new Exception('OpenAI embedding 返回异常: '.json_encode($json));
  1193. }
  1194. return $json['data'][0]['embedding'];
  1195. });
  1196. }
  1197. /**
  1198. * embedding 缓存键:"embedding/v{版本}/{md5(text)}"。
  1199. *
  1200. * 版本号存在 EMBEDDING_CACHE_VERSION_KEY,clearAllEmbeddingCache() 把它加一,
  1201. * 旧版本的键就再也不会被读到,7 天后自然过期。原来用 Cache::tags(),
  1202. * tag 索引是另外的 key,在 Redis Cluster 上跨 slot。
  1203. */
  1204. private function embeddingCacheKey(string $text): string
  1205. {
  1206. $version = RedisCache::get(self::EMBEDDING_CACHE_VERSION_KEY, 0);
  1207. return "embedding/v{$version}/".md5($text);
  1208. }
  1209. /**
  1210. * 清除指定文本的 embedding 缓存
  1211. *
  1212. * @param string $text 原始文本(与调用 embedText 时一致)
  1213. * @return bool 缓存是否成功删除
  1214. *
  1215. * @example
  1216. * $service->clearEmbeddingCache('sabbe dhammā anattā');
  1217. */
  1218. public function clearEmbeddingCache(string $text): bool
  1219. {
  1220. return RedisCache::forget($this->embeddingCacheKey($text));
  1221. }
  1222. /**
  1223. * 让所有 embedding 缓存失效
  1224. *
  1225. * 不逐条删除,而是把版本号加一;旧键 7 天内自然过期。
  1226. *
  1227. * @example
  1228. * $service->clearAllEmbeddingCache();
  1229. */
  1230. public function clearAllEmbeddingCache(): void
  1231. {
  1232. RedisCache::increment(self::EMBEDDING_CACHE_VERSION_KEY);
  1233. }
  1234. /**
  1235. * 自动建议(Completion Suggest)
  1236. *
  1237. * 基于 completion 字段实现前缀补全,支持同时查询多个语言字段。
  1238. * 结果按 _score 降序排序,跨字段去重。
  1239. *
  1240. * 可用字段标识符($fields 参数):
  1241. * - 'title_pali' → title.suggest.pali
  1242. * - 'title_zh' → title.suggest.zh
  1243. * - 'content_pali' → content.suggest.pali
  1244. * - 'content_zh' → content.suggest.zh
  1245. *
  1246. * @param string $query 查询前缀文本
  1247. * @param array|string|null $fields 要查询的字段标识符,null 表示全部字段
  1248. * @param string|null $language 可选的语言过滤(term query)
  1249. * @param int $limit 每个字段返回的建议数量,默认 10
  1250. * @return array 建议结果列表,每项包含:
  1251. * text, source(字段标识符), score, doc_id, doc_source
  1252. *
  1253. * @throws \InvalidArgumentException $fields 中含无效字段标识符时抛出
  1254. *
  1255. * @example
  1256. * // 查询所有字段
  1257. * $service->suggest('nibb');
  1258. *
  1259. * // 只查询巴利文标题建议
  1260. * $service->suggest('nibb', 'title_pali');
  1261. *
  1262. * // 查询多个字段,限制语言
  1263. * $service->suggest('涅', ['title_zh', 'content_zh'], 'zh', 5);
  1264. */
  1265. public function suggest(
  1266. string $query,
  1267. $fields = null,
  1268. ?string $language = null,
  1269. int $limit = 10
  1270. ): array {
  1271. // 字段标识符 → OpenSearch completion 字段路径
  1272. $fieldMap = [
  1273. 'title_pali' => 'title.suggest.pali',
  1274. 'title_zh' => 'title.suggest.zh',
  1275. 'content_pali' => 'content.suggest.pali',
  1276. 'content_zh' => 'content.suggest.zh',
  1277. ];
  1278. // 处理字段参数
  1279. if ($fields === null) {
  1280. $searchFields = array_keys($fieldMap);
  1281. } elseif (is_string($fields)) {
  1282. $searchFields = [$fields];
  1283. } else {
  1284. $searchFields = $fields;
  1285. }
  1286. // 过滤无效字段
  1287. $searchFields = array_values(array_filter(
  1288. $searchFields,
  1289. fn ($field) => isset($fieldMap[$field])
  1290. ));
  1291. if (empty($searchFields)) {
  1292. throw new \InvalidArgumentException('Invalid fields specified for suggestion');
  1293. }
  1294. // 构建 suggest DSL
  1295. $suggests = [];
  1296. foreach ($searchFields as $field) {
  1297. $suggests[$field.'_suggest'] = [
  1298. 'prefix' => $query,
  1299. 'completion' => [
  1300. 'field' => $fieldMap[$field],
  1301. 'size' => $limit,
  1302. 'skip_duplicates' => true,
  1303. ],
  1304. ];
  1305. }
  1306. $dsl = ['suggest' => $suggests];
  1307. if ($language) {
  1308. $dsl['query'] = ['term' => ['language' => $language]];
  1309. }
  1310. $response = $this->client->search([
  1311. 'index' => config('mint.opensearch.index'),
  1312. 'body' => $dsl,
  1313. ]);
  1314. // 整理结果,附加来源字段
  1315. $results = [];
  1316. foreach ($searchFields as $field) {
  1317. $options = $response['suggest'][$field.'_suggest'][0]['options'] ?? [];
  1318. foreach ($options as $opt) {
  1319. $results[] = [
  1320. 'text' => $opt['text'] ?? '',
  1321. 'source' => $field,
  1322. 'score' => $opt['_score'] ?? 0,
  1323. 'doc_id' => $opt['_id'] ?? null,
  1324. 'doc_source' => $opt['_source'] ?? null,
  1325. ];
  1326. }
  1327. }
  1328. // 按分数降序排序
  1329. usort($results, fn ($a, $b) => $b['score'] <=> $a['score']);
  1330. return $results;
  1331. }
  1332. /**
  1333. * 按文档 ID 获取单条完整文档(包含 content.display)
  1334. *
  1335. * @param string $id 文档 ID,例如 "term_{guid}"
  1336. * @return array OpenSearch 原始响应
  1337. */
  1338. public function get(string $id): array
  1339. {
  1340. return $this->client->get([
  1341. 'index' => config('mint.opensearch.index'),
  1342. 'id' => $id,
  1343. ]);
  1344. }
  1345. /**
  1346. * 校验 pali_synonyms 同义词词典是否对指定词生效
  1347. *
  1348. * 通过 OpenSearch _analyze API,用当前索引的 pali_query_analyzer
  1349. * 对输入文本做实时分析,返回展开后的全部 token(含原词与同义词)。
  1350. * 可用于快速确认 analysis/pali_synonyms.txt 中的某一行是否已生效
  1351. * (例如 dhamma,dharma,法 => dhamma 是否真的展开出 dharma、法)。
  1352. *
  1353. * @param string $text 待检测的巴利词,例如 "dhamma"
  1354. * @return array<int, string> 分析器输出的 token 文本数组(已去重,保留原始顺序)
  1355. *
  1356. * @throws Exception 索引不存在或 OpenSearch 调用失败时抛出
  1357. *
  1358. * @example
  1359. * $service->pali_query_health_check('dhamma');
  1360. * // => ['dhamma', 'dharma', '法']
  1361. */
  1362. public function pali_query_health_check(string $text): array
  1363. {
  1364. $index = config('mint.opensearch.index');
  1365. if (! $this->client->indices()->exists(['index' => $index])) {
  1366. throw new Exception("Index [$index] does not exist.");
  1367. }
  1368. $response = $this->client->indices()->analyze([
  1369. 'index' => $index,
  1370. 'body' => [
  1371. 'analyzer' => 'pali_query_analyzer',
  1372. 'text' => $text,
  1373. ],
  1374. ]);
  1375. $tokens = array_map(
  1376. fn ($token) => $token['token'] ?? '',
  1377. $response['tokens'] ?? []
  1378. );
  1379. return array_values(array_unique(array_filter($tokens, fn ($t) => $t !== '')));
  1380. }
  1381. }