Преглед изворни кода

✨ (search): 巴利文多词检索按词序与拼写逐项计分,新增 definition 模式

fuzzy(巴利文)改为按原始词逐项累加的固定分:整句原样 20、相邻两词原样 5、
单词拼写相同 10、同义词可替代 2,部分命中逐级降分,命中一个词也召回;
另加极小权重 BM25 只给同分文档排先后。中文查询只搜 zh 字段,高亮字段按语种路由。

巴利文停用词(na no ca va vā hi pana iva eva,取自 pg 的 pali.stop)只参与
整句与相邻词的词序判断,不单独计分、不单独召回;全是停用词时按普通词处理。

新增 search_mode=definition:义注黑体字决定排序,每个词单独计分——单独黑体
拼写相同 100、同义 20,多词黑体里的词 30,整句在多词黑体里 30;正文得分乘 0.1
只作次级排序。规格与 dashboard-v6 类型同步加上 definition。

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
visuddhinanda пре 5 дана
родитељ
комит
fb5a277f12

+ 2 - 2
api-v13/app/Http/Controllers/V3/SearchPlusController.php

@@ -24,7 +24,7 @@ class SearchPlusController extends Controller
      * @unauthenticated
      *
      * @queryParam q string required 搜索关键词
-     * @queryParam search_mode string 检索模式。Enum: fuzzy,exact,semantic,hybrid Default: fuzzy
+     * @queryParam search_mode string 检索模式。Enum: fuzzy,exact,semantic,hybrid,definition Default: fuzzy
      * @queryParam resource_type string 资源类型。
      *             Enum: article,term,dictionary,translation,origin_text,nissaya
      * @queryParam resource_id string 限定某个资源 id
@@ -113,7 +113,7 @@ class SearchPlusController extends Controller
      * @unauthenticated
      *
      * @bodyParam q string required 搜索关键词
-     * @bodyParam search_mode string 检索模式。Enum: fuzzy,exact,semantic,hybrid Default: fuzzy
+     * @bodyParam search_mode string 检索模式。Enum: fuzzy,exact,semantic,hybrid,definition Default: fuzzy
      * @bodyParam resource_type string 资源类型
      * @bodyParam granularity string 文档颗粒度
      * @bodyParam language string 语言

+ 328 - 22
api-v13/app/Services/OpenSearchService.php

@@ -624,11 +624,12 @@ class OpenSearchService
     /**
      * 执行高级搜索
      *
-     * 支持四种搜索模式:
-     *   - fuzzy    多字段模糊查询(默认),基于 BM25
-     *   - exact    精确匹配,使用 markdown_clean analyzer
-     *   - semantic 纯语义向量搜索,需要 OpenAI embedding
-     *   - hybrid   fuzzy + semantic 混合,权重由 fuzzy_ratio / semantic_ratio 控制
+     * 支持五种搜索模式:
+     *   - fuzzy       多字段模糊查询(默认),巴利文走三层排序
+     *   - exact       精确匹配,使用 markdown_clean analyzer
+     *   - semantic    纯语义向量搜索,需要 OpenAI embedding
+     *   - hybrid      fuzzy + semantic 混合,权重由 fuzzy_ratio / semantic_ratio 控制
+     *   - definition  定义查询,黑体字(bold_single/bold_multi)高权重,术语定义排前
      *
      * 支持的过滤参数:
      *   resourceType, resourceId, granularity, language, category,
@@ -743,6 +744,10 @@ class OpenSearchService
                     $query = $this->buildHybridQuery($queryText);
                     break;
 
+                case 'definition':
+                    $query = $this->buildDefinitionQuery($queryText);
+                    break;
+
                 case 'fuzzy':
                 default:
                     $query = $this->buildFuzzyQuery($queryText);
@@ -782,16 +787,14 @@ class OpenSearchService
             ],
         ];
 
-        // 只有有搜索词时才开启高亮
+        // 只有有搜索词时才开启高亮;字段按语种路由,避免把巴利文喂给中文分词器
         if ($queryText !== '') {
+            $highlightFields = $this->isCjkQuery($queryText)
+                ? ['title.text.zh', 'summary.text', 'content.text.zh']
+                : ['title.text.pali', 'content.text.pali'];
+
             $dsl['highlight'] = [
-                'fields' => [
-                    'title.text.pali' => new \stdClass,
-                    'title.text.zh' => new \stdClass,
-                    'summary.text' => new \stdClass,
-                    'content.text.pali' => new \stdClass,
-                    'content.text.zh' => new \stdClass,
-                ],
+                'fields' => array_fill_keys($highlightFields, new \stdClass),
                 'fragmenter' => 'sentence',
                 'fragment_size' => 200,
                 'number_of_fragments' => 1,
@@ -868,24 +871,327 @@ class OpenSearchService
     /**
      * 构建 fuzzy(多字段模糊)查询
      *
-     * 基于 BM25 的 multi_match best_fields 查询,
-     * 字段权重取自 $weights['fuzzy']。
+     * 按查询语种路由字段,避免巴利文被中文分词器切碎、中文被巴利文 analyzer 误伤:
+     *   - 含汉字 → 只搜 bold_* + *.text.zh + summary.text
+     *   - 巴利文(拉丁字母)→ 走三层排序(见 buildPaliFuzzyQuery)+ bold_* 提权
      *
      * @param  string  $query  搜索关键词
      * @return array OpenSearch DSL query 片段
      */
     protected function buildFuzzyQuery(string $query): array
     {
-        $fields = [];
-        foreach ($this->weights['fuzzy'] as $field => $weight) {
-            $fields[] = $field.'^'.$weight;
+        if ($this->isCjkQuery($query)) {
+            return [
+                'multi_match' => [
+                    'query' => $query,
+                    'fields' => [
+                        'bold_single^50',
+                        'bold_multi^10',
+                        'title.text.zh^3',
+                        'summary.text^2',
+                        'content.text.zh^1',
+                    ],
+                    'type' => 'best_fields',
+                ],
+            ];
+        }
+
+        return $this->buildPaliFuzzyQuery($query);
+    }
+
+    /**
+     * 巴利文 fuzzy 的计分表(全部是固定分,逐项累加)
+     *
+     * 一个文档的得分 = 命中的每一项相加,于是「部分符合」自然逐级降分:
+     *   - phrase   整句按原顺序、拼写完全相同(≥2 个词才有)
+     *   - bigram   相邻两词按原顺序、拼写相同,每对一次(≥3 个词才有,2 个词时与 phrase 重合)
+     *   - exact    单个原始词拼写相同,每词一次
+     *   - synonym  单个原始词经同义词可替代,每词一次(拼写相同的词也会命中这一项)
+     *
+     * 以 3 个词为例:整句原样 66 > 全部拼写相同但乱序 36 > 两词相邻拼写相同 + 一词同义 31
+     * > 两词拼写相同不相邻 24 > 三词全靠同义词 6 > 一词同义 2。
+     *
+     * @var array{phrase: int, bigram: int, exact: int, synonym: int}
+     */
+    private const PALI_FUZZY_SCORES = [
+        'phrase' => 20,
+        'bigram' => 5,
+        'exact' => 10,
+        'synonym' => 2,
+    ];
+
+    /**
+     * definition 的计分表(黑体字部分,每个原始词各算一次)
+     *
+     *   - single         该词是单独黑体字,拼写相同
+     *   - single_synonym 该词经同义词可替代地是单独黑体字(拼写相同的也会命中)
+     *   - multi          该词出现在多词黑体短语里,拼写相同
+     *   - multi_phrase   整句按原顺序出现在多词黑体短语里(≥2 个词才有)
+     *
+     * 正文得分按 DEFINITION_TEXT_SCALE 缩小后叠加,只用来给黑体同分的文档排先后。
+     *
+     * @var array{single: int, single_synonym: int, multi: int, multi_phrase: int}
+     */
+    private const DEFINITION_SCORES = [
+        'single' => 100,
+        'single_synonym' => 20,
+        'multi' => 30,
+        'multi_phrase' => 30,
+    ];
+
+    /**
+     * definition 模式里正文得分的缩放系数,保证正文再怎么命中也压不过一个黑体字
+     */
+    private const DEFINITION_TEXT_SCALE = 0.1;
+
+    /**
+     * 同分时的 BM25 微调权重:BM25 一般 < 30,乘上后 < 0.3,小于计分表的最小档差
+     */
+    private const TIE_BREAKER_BOOST = 0.01;
+
+    /**
+     * 巴利文停用词(小写;比较前去掉标点)
+     *
+     * 取自 pg 全文检索的 pali.stop(api-v8/public/app/fts/pali.stop),另加 vā。
+     * 停用词只参与词序打分(整句、相邻两词),不单独计分、不单独召回,见 buildPaliFuzzyQuery。
+     *
+     * @var list<string>
+     */
+    private const PALI_STOPWORDS = ['na', 'no', 'ca', 'va', 'vā', 'hi', 'pana', 'iva', 'eva'];
+
+    /**
+     * 判断原始词是否巴利文停用词
+     *
+     * @param  string  $word  原始词(可带标点,如「ca,」「‘‘na」)
+     * @return bool 是停用词返回 true
+     */
+    protected function isPaliStopword(string $word): bool
+    {
+        $normalized = mb_strtolower(preg_replace('/[^\p{L}\p{M}]/u', '', $word) ?? '');
+
+        return in_array($normalized, self::PALI_STOPWORDS, true);
+    }
+
+    /**
+     * 取出用于单独计分的词:去掉停用词;全是停用词时原样返回,避免查不到东西
+     *
+     * @param  list<string>  $words  原始词
+     * @return list<string>
+     */
+    protected function scoringWords(array $words): array
+    {
+        $contentWords = array_values(array_filter(
+            $words,
+            fn (string $word): bool => ! $this->isPaliStopword($word),
+        ));
+
+        return $contentWords === [] ? $words : $contentWords;
+    }
+
+    /**
+     * 把查询切成原始词
+     *
+     * 按空白切分,并丢掉不含字母的片段(如「–」「.」):它们经分析器后没有 token,
+     * 留着会让按词计分的子句永远不命中。
+     *
+     * @param  string  $query  搜索关键词
+     * @return list<string>
+     */
+    protected function splitQueryWords(string $query): array
+    {
+        $words = preg_split('/\s+/u', trim($query)) ?: [];
+
+        return array_values(array_filter(
+            $words,
+            fn (string $word): bool => preg_match('/\p{L}/u', $word) === 1,
+        ));
+    }
+
+    /**
+     * 包一层 constant_score:命中即得固定分,不受 BM25 字段长度归一化影响
+     *
+     * @param  array<string, mixed>  $filter  过滤子句
+     * @param  float  $boost  命中时的固定分
+     * @return array{constant_score: array{filter: array<string, mixed>, boost: float}}
+     */
+    protected function constantScore(array $filter, float $boost): array
+    {
+        return ['constant_score' => ['filter' => $filter, 'boost' => $boost]];
+    }
+
+    /**
+     * 构建巴利文查询:词序符合、拼写相同得高分,部分符合逐级降分
+     *
+     * 命中按「原始词」(空格切分)计,不按同义词展开后的 token 计——每个原始词一个
+     * constant_score 子句,命中几个同义词都只算一次。计分表见 PALI_FUZZY_SCORES。
+     * 拼写相同打在 .exact 子字段(markdown_clean,无同义词),同义词打在 .pali。
+     * 最后加一条极小权重的 BM25 子句,只用来给同分文档排先后。
+     *
+     * 停用词(PALI_STOPWORDS)只出现在整句、相邻两词的子句里,保住词序判断;
+     * 不生成单词子句,也不进 BM25 子句,于是只命中「ca」的文档不会被召回。
+     * 两个停用词组成的相邻词对同理跳过。查询全是停用词时按普通词处理。
+     *
+     * @param  string  $query  搜索关键词
+     * @param  float  $scale  整体缩放(definition 模式用它把正文得分压到黑体字之下)
+     * @return array OpenSearch DSL query 片段
+     */
+    protected function buildPaliFuzzyQuery(string $query, float $scale = 1.0): array
+    {
+        $words = $this->splitQueryWords($query);
+        $wordCount = count($words);
+
+        if ($wordCount === 0) {
+            return ['match_all' => new \stdClass];
+        }
+
+        $scoringWords = $this->scoringWords($words);
+        $allStopwords = array_all($words, fn (string $word): bool => $this->isPaliStopword($word));
+
+        $exactFields = ['title.text.pali.exact', 'content.text.pali.exact'];
+        $fuzzyFields = ['title.text.pali', 'content.text.pali'];
+        $scores = array_map(fn (int $score): float => $score * $scale, self::PALI_FUZZY_SCORES);
+
+        $should = [];
+
+        // 词序:整句原样
+        if ($wordCount >= 2) {
+            $should[] = $this->constantScore([
+                'multi_match' => [
+                    'query' => implode(' ', $words),
+                    'type' => 'phrase',
+                    'fields' => $exactFields,
+                ],
+            ], $scores['phrase']);
         }
 
+        // 词序:相邻两词原样(部分顺序符合)
+        if ($wordCount >= 3) {
+            for ($i = 0; $i < $wordCount - 1; $i++) {
+                $bothStopwords = $this->isPaliStopword($words[$i]) && $this->isPaliStopword($words[$i + 1]);
+                if ($bothStopwords && ! $allStopwords) {
+                    continue;
+                }
+
+                $should[] = $this->constantScore([
+                    'multi_match' => [
+                        'query' => $words[$i].' '.$words[$i + 1],
+                        'type' => 'phrase',
+                        'fields' => $exactFields,
+                    ],
+                ], $scores['bigram']);
+            }
+        }
+
+        // 拼写:每个词各算一次(停用词不算)
+        foreach ($scoringWords as $word) {
+            $should[] = $this->constantScore([
+                'multi_match' => [
+                    'query' => $word,
+                    'fields' => $exactFields,
+                    'operator' => 'and',
+                ],
+            ], $scores['exact']);
+
+            $should[] = $this->constantScore([
+                'multi_match' => [
+                    'query' => $word,
+                    'fields' => $fuzzyFields,
+                ],
+            ], $scores['synonym']);
+        }
+
+        // 同分文档按 BM25 排先后
+        $should[] = [
+            'match' => [
+                'content.text.pali' => [
+                    'query' => implode(' ', $scoringWords),
+                    'boost' => self::TIE_BREAKER_BOOST * $scale,
+                ],
+            ],
+        ];
+
         return [
-            'multi_match' => [
-                'query' => $query,
-                'fields' => $fields,
-                'type' => 'best_fields',
+            'bool' => [
+                'should' => $should,
+            ],
+        ];
+    }
+
+    /**
+     * 判断查询是否含中文(CJK 统一表意文字)
+     *
+     * @param  string  $query  搜索关键词
+     * @return bool 含汉字返回 true,否则按巴利文(拉丁字母)处理
+     */
+    protected function isCjkQuery(string $query): bool
+    {
+        return (bool) preg_match('/[\x{4E00}-\x{9FFF}\x{3400}-\x{4DBF}\x{F900}-\x{FAFF}]/u', $query);
+    }
+
+    /**
+     * 构建 definition(定义查询)
+     *
+     * 义注里的黑体字通常就是被解释的词,所以黑体字命中决定排序,正文只做次级排序:
+     *   - 每个原始词单独算:命中的单独黑体字(bold_single)越多分越高
+     *   - 单独黑体字 > 多词黑体短语(bold_multi)里的词
+     *   - 拼写相同 > 同义词可替代:bold_* 建索引用 standard,查询时显式指定
+     *     analyzer=standard 即是「拼写相同」,不指定则走带同义词的 pali_query_analyzer
+     *   - 用 match_phrase 而不是 match:中文在 standard 下按字切分,phrase 要求字连续,
+     *     避免一个字重合就命中
+     * 计分表见 DEFINITION_SCORES;正文部分复用 fuzzy 的计分,乘 DEFINITION_TEXT_SCALE。
+     *
+     * @param  string  $query  搜索关键词
+     * @return array OpenSearch DSL query 片段
+     */
+    protected function buildDefinitionQuery(string $query): array
+    {
+        $words = $this->splitQueryWords($query);
+
+        if ($words === []) {
+            return ['match_all' => new \stdClass];
+        }
+
+        $should = [];
+
+        // 停用词不单独算黑体分(整句的 multi_phrase 仍带上它们)
+        foreach ($this->scoringWords($words) as $word) {
+            $should[] = $this->constantScore(
+                ['match_phrase' => ['bold_single' => ['query' => $word, 'analyzer' => 'standard']]],
+                self::DEFINITION_SCORES['single'],
+            );
+            $should[] = $this->constantScore(
+                ['match_phrase' => ['bold_single' => $word]],
+                self::DEFINITION_SCORES['single_synonym'],
+            );
+            $should[] = $this->constantScore(
+                ['match_phrase' => ['bold_multi' => ['query' => $word, 'analyzer' => 'standard']]],
+                self::DEFINITION_SCORES['multi'],
+            );
+        }
+
+        if (count($words) >= 2) {
+            $should[] = $this->constantScore(
+                ['match_phrase' => ['bold_multi' => ['query' => implode(' ', $words), 'analyzer' => 'standard']]],
+                self::DEFINITION_SCORES['multi_phrase'],
+            );
+        }
+
+        // 正文:黑体同分时的次级排序,也保留没有黑体的文档(偈颂等)的召回
+        if ($this->isCjkQuery($query)) {
+            $should[] = [
+                'multi_match' => [
+                    'query' => $query,
+                    'fields' => ['title.text.zh^3', 'content.text.zh'],
+                    'boost' => self::DEFINITION_TEXT_SCALE,
+                ],
+            ];
+        } else {
+            $should[] = $this->buildPaliFuzzyQuery($query, self::DEFINITION_TEXT_SCALE);
+        }
+
+        return [
+            'bool' => [
+                'should' => $should,
             ],
         ];
     }

+ 107 - 0
api-v13/tests/Feature/OpenSearchQueryBuilderTest.php

@@ -0,0 +1,107 @@
+<?php
+
+use App\Services\OpenSearchService;
+
+/**
+ * 调用 OpenSearchService 的 protected 查询构造方法(纯函数,不连 OpenSearch)
+ */
+function buildSearchQuery(string $method, mixed ...$args): array
+{
+    $service = new OpenSearchService;
+
+    return (fn () => $this->{$method}(...$args))->call($service);
+}
+
+/**
+ * 取出 bool.should 里所有 constant_score 的固定分
+ *
+ * @return list<float>
+ */
+function constantScoreBoosts(array $query): array
+{
+    return array_values(array_map(
+        fn (array $clause): float => $clause['constant_score']['boost'],
+        array_filter($query['bool']['should'], fn (array $clause): bool => isset($clause['constant_score'])),
+    ));
+}
+
+it('scores pali fuzzy per word, rewarding word order and exact spelling', function () {
+    $query = buildSearchQuery('buildPaliFuzzyQuery', 'evaṃ me sutaṃ');
+
+    // 整句 20 + 相邻两对各 5 + 每词拼写 10、同义 2
+    expect(constantScoreBoosts($query))->toEqual([20.0, 5.0, 5.0, 10.0, 2.0, 10.0, 2.0, 10.0, 2.0]);
+
+    $phrase = $query['bool']['should'][0]['constant_score']['filter']['multi_match'];
+    expect($phrase['type'])->toBe('phrase')
+        ->and($phrase['query'])->toBe('evaṃ me sutaṃ')
+        ->and($phrase['fields'])->toBe(['title.text.pali.exact', 'content.text.pali.exact']);
+
+    expect($query['bool'])->not->toHaveKey('minimum_should_match');
+});
+
+it('drops punctuation-only words so they cannot block matches', function () {
+    $query = buildSearchQuery('buildPaliFuzzyQuery', 'evaṃ – me .');
+
+    $phrase = $query['bool']['should'][0]['constant_score']['filter']['multi_match'];
+    expect($phrase['query'])->toBe('evaṃ me')
+        ->and(constantScoreBoosts($query))->toEqual([20.0, 10.0, 2.0, 10.0, 2.0]);
+});
+
+it('has no order clauses for a single pali word', function () {
+    $query = buildSearchQuery('buildPaliFuzzyQuery', 'vitakko');
+
+    expect(constantScoreBoosts($query))->toEqual([10.0, 2.0]);
+});
+
+it('ranks definition by each matched single bold word, exact spelling first', function () {
+    $query = buildSearchQuery('buildDefinitionQuery', 'vitakko vicāro');
+    $should = $query['bool']['should'];
+
+    // 每词:单独黑体拼写相同 100、单独黑体同义 20、多词黑体 30;再加整句在多词黑体里 30
+    expect(constantScoreBoosts($query))->toEqual([100.0, 20.0, 30.0, 100.0, 20.0, 30.0, 30.0]);
+
+    expect($should[0]['constant_score']['filter']['match_phrase']['bold_single'])
+        ->toBe(['query' => 'vitakko', 'analyzer' => 'standard']);
+    expect($should[1]['constant_score']['filter']['match_phrase']['bold_single'])->toBe('vitakko');
+
+    // 正文部分整体缩小到 0.1,压不过一个黑体字
+    $text = end($should);
+    expect(constantScoreBoosts($text))->toEqual([2.0, 1.0, 0.2, 1.0, 0.2]);
+});
+
+it('routes cjk definition text to chinese fields', function () {
+    $query = buildSearchQuery('buildDefinitionQuery', '寻');
+    $text = end($query['bool']['should']);
+
+    expect($text['multi_match']['fields'])->toBe(['title.text.zh^3', 'content.text.zh']);
+});
+
+it('keeps pali stopwords in word-order clauses only', function () {
+    $query = buildSearchQuery('buildPaliFuzzyQuery', 'evaṃ ca kho');
+    $should = $query['bool']['should'];
+
+    // 整句 20 + 两对相邻词各 5 + 只有 evaṃ、kho 两个词单独计分
+    expect(constantScoreBoosts($query))->toEqual([20.0, 5.0, 5.0, 10.0, 2.0, 10.0, 2.0]);
+    expect($should[0]['constant_score']['filter']['multi_match']['query'])->toBe('evaṃ ca kho');
+    expect(end($should)['match']['content.text.pali']['query'])->toBe('evaṃ kho');
+});
+
+it('skips adjacent pairs made only of stopwords', function () {
+    $query = buildSearchQuery('buildPaliFuzzyQuery', 'na ca kho');
+
+    // 「na ca」那对跳过,只剩「ca kho」
+    expect(constantScoreBoosts($query))->toEqual([20.0, 5.0, 10.0, 2.0]);
+});
+
+it('treats an all-stopword query as normal words', function () {
+    $query = buildSearchQuery('buildPaliFuzzyQuery', 'Eva');
+
+    expect(constantScoreBoosts($query))->toEqual([10.0, 2.0]);
+});
+
+it('ignores stopwords for definition bold scoring', function () {
+    $query = buildSearchQuery('buildDefinitionQuery', 'vitakko ca');
+
+    // 只有 vitakko 三条 + 整句在多词黑体里 30
+    expect(constantScoreBoosts($query))->toEqual([100.0, 20.0, 30.0, 30.0]);
+});

+ 2 - 2
dashboard-v6/src/api/schema.d.ts

@@ -36195,7 +36195,7 @@ export interface operations {
                 /** @description 每页数量,上限 100 */
                 page_size?: number;
                 /** @description 检索模式 */
-                search_mode?: "fuzzy" | "exact" | "semantic" | "hybrid";
+                search_mode?: "fuzzy" | "exact" | "semantic" | "hybrid" | "definition";
                 /** @description 资源类型 */
                 resource_type?: "article" | "term" | "dictionary" | "translation" | "origin_text" | "nissaya";
                 /** @description 限定某个资源 id */
@@ -36270,7 +36270,7 @@ export interface operations {
                      * @default fuzzy
                      * @enum {string}
                      */
-                    search_mode?: "fuzzy" | "exact" | "semantic" | "hybrid";
+                    search_mode?: "fuzzy" | "exact" | "semantic" | "hybrid" | "definition";
                     /** @description 资源类型 */
                     resource_type?: string;
                     /** @description 文档颗粒度 */

+ 2 - 0
openapi/public/assets/protocol/resources/auto/v3-search.yaml

@@ -46,6 +46,7 @@ get:
           - exact
           - semantic
           - hybrid
+          - definition
         default: fuzzy
       description: 检索模式
     -
@@ -181,6 +182,7 @@ post:
                 - exact
                 - semantic
                 - hybrid
+                - definition
               default: fuzzy
             resource_type:
               type: string