TipitakaChapterService.php 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417
  1. <?php
  2. namespace App\Services\V3;
  3. use App\Facades\RedisCache;
  4. use App\Models\PaliText;
  5. use App\Models\ProgressChapter;
  6. use Illuminate\Contracts\Database\Eloquent\Builder;
  7. use Illuminate\Pagination\LengthAwarePaginator;
  8. /**
  9. * 三藏章节标题:按名字找章节(容错),或列出一本书的全部标题。
  10. *
  11. * 取代 v2 `SearchTitleController@index`(`GET /v2/search-title-index`)。v2 那份原样保留、不共用
  12. * (CLAUDE.md 铁律 2)。与 v2 的差别:
  13. *
  14. * - **返回坐标**(book、para)与体量、路径、上下章;v2 只回标题文字,拿到也定位不了。
  15. * - **容错匹配**:不分大小写与变音符号(`mahaparinibbana` 能找到 Mahāparinibbānasuttaṃ),
  16. * 去掉序号与括号里的丛书缩写,拼错一两个字母也能命中(按编辑距离打分)。v2 是
  17. * `LIKE %key%`,还依赖 `title_en` 列——那一列去变音符号做得不干净(`3. Ānandattheragatha`)。
  18. * - **译名也能搜**:公开 channel 在 `progress_chapters.title` 里写的章节译名(如「大般涅槃经」)。
  19. * - 可按 book(及其中一章)列出全部标题,给 agent「先选书、再从整本书的标题里选章」用。
  20. *
  21. * 实现:3.7 万条标题与规范化后的匹配键缓存一天,匹配在 PHP 里做(库里没有 pg_trgm /
  22. * fuzzystrmatch,又不能装扩展)。先用三字母片段粗筛,再对候选算编辑距离。
  23. *
  24. * **定位(2026-11-11 定)**:本端点是 v2 `search-title-index` 的替代品。`q` 的容错匹配放在 PHP 里,
  25. * 只是为了先验证「按章节名定位」对 agent 的检索效果(评测见 agent-poc
  26. * `agent-tuning/docs/retrieval-recall-2026-11.md` 第七节)。**按名字找章节的正式实现要走 OpenSearch**:
  27. * 插件 `locate` 改调 `/v3/search`(granularity=chapter),计划见 `documents/search-recall.md` 4.6。
  28. * 届时这里的 PHP 匹配只保留给 v2 替代与小规模调用,不再扩展。
  29. */
  30. class TipitakaChapterService
  31. {
  32. /** 只有 level < 8 的段落是标题(与 v2、IndexTipitaka 同口径) */
  33. private const MAX_TITLE_LEVEL = 8;
  34. /** 缓存键带版本号:catalog 的结构变了就改版本号,不用等过期 */
  35. private const CATALOG_KEY = '/v3/tipitaka-chapters/catalog/v1';
  36. private const CATALOG_TTL = 86400;
  37. /** channels.status:全网公开。只有公开 channel 的章节译名才能被搜到、被返回 */
  38. private const STATUS_PUBLIC = 30;
  39. /** 书的 tag → layer。一本书有多个时按这个顺序取第一个 */
  40. private const LAYER_TAGS = ['mūla' => 'mula', 'aṭṭhakathā' => 'atthakatha', 'ṭīkā' => 'tika', 'añña' => 'anna'];
  41. /** 没有任何 layer tag 的书 */
  42. private const DEFAULT_LAYER = 'anna';
  43. private const SCORE_EXACT = 1.0;
  44. private const SCORE_PREFIX = 0.95;
  45. private const SCORE_CONTAINS = 0.9;
  46. /** 编辑距离匹配的最高分(低于「包含」:拼错的永远排在拼对的后面),乘以片段相似度 */
  47. private const SCORE_FUZZY_MAX = 0.85;
  48. /**
  49. * 片段相似度(1 - 距离/查询长度)低于这个就不算命中:20 个字母最多错 4 个。
  50. *
  51. * 0.75 太松:`etadaggapali` 会因为共有词尾 `-vaggapali` 命中 `(DN) Mahāvaggapāḷi`。
  52. */
  53. private const FUZZY_MIN_SIMILARITY = 0.8;
  54. /** 粗筛:查询的三字母片段至少有这么大比例出现在标题里,才算编辑距离 */
  55. private const TRIGRAM_MIN_RATIO = 0.5;
  56. /** 变音符号 → 拉丁字母(巴利罗马转写用到的全部) */
  57. private const FOLD = [
  58. 'ā' => 'a', 'ī' => 'i', 'ū' => 'u', 'ṅ' => 'n', 'ñ' => 'n', 'ṇ' => 'n', 'ṭ' => 't',
  59. 'ḍ' => 'd', 'ḷ' => 'l', 'ṃ' => 'm', 'ṁ' => 'm', 'ŋ' => 'm', 'ḥ' => 'h', 'ṛ' => 'r', 'ś' => 's', 'ṣ' => 's',
  60. ];
  61. /**
  62. * 找章节。给 q 按匹配度排序(同一档里根本典在前,再按与整个标题的接近度、大章在前、经文顺序);
  63. * 不给 q 按经文顺序。
  64. *
  65. * @param array{q?: string, book?: int, para?: int, layer?: string, max_level?: int} $filters
  66. * @return LengthAwarePaginator<int, array<string, mixed>>
  67. */
  68. public function search(array $filters, int $page, int $perPage): LengthAwarePaginator
  69. {
  70. $rows = $this->filter($this->catalog(), $filters);
  71. $query = isset($filters['q']) ? self::normalize($filters['q']) : null;
  72. if ($query !== null) {
  73. $rows = $this->rank($rows, $query);
  74. }
  75. $total = count($rows);
  76. $items = $this->hydrate(array_slice($rows, ($page - 1) * $perPage, $perPage), $query !== null);
  77. return new LengthAwarePaginator($items, $total, $perPage, $page);
  78. }
  79. /**
  80. * 名字规范化成匹配键:小写、去变音符号、去括号里的丛书缩写(`(DN)`)、去序号与一切非字母。
  81. *
  82. * 汉字也是字母,保留;简繁不转换(同一章常有简繁两种译名)。
  83. */
  84. public static function normalize(string $text): string
  85. {
  86. $text = preg_replace('/\([^)]*\)|\[[^\]]*\]/u', '', mb_strtolower($text)) ?? '';
  87. return preg_replace('/[^\p{L}]+/u', '', strtr($text, self::FOLD)) ?? '';
  88. }
  89. /**
  90. * 一个匹配键对一个标题键打分,返回 [匹配度, 接近度];匹配度 0 = 不算命中。
  91. *
  92. * 匹配度只分几档,便于解释也便于排序:相等 1、开头相同 0.95、包含 0.9、拼错的按片段相似度
  93. * 最高 0.85(取两位小数)。接近度是查询与整个标题有多像(0–1),只在同一档、同一层里决定先后——
  94. * 比如拼错的 `anandatheragatha` 对 Ānandattheragāthā 与 Girimānandattheragāthā 片段距离相同,
  95. * 靠它把前者排前。
  96. *
  97. * 非 ASCII 的键(汉字)只做包含匹配:`levenshtein()` 按字节算,对多字节字符没有意义。
  98. *
  99. * @return array{0: float, 1: float}
  100. */
  101. public static function score(string $query, string $title): array
  102. {
  103. if ($query === '' || $title === '') {
  104. return [0.0, 0.0];
  105. }
  106. $position = strpos($title, $query);
  107. if ($position !== false) {
  108. $closeness = strlen($query) / strlen($title);
  109. $grade = match (true) {
  110. $title === $query => self::SCORE_EXACT,
  111. $position === 0 => self::SCORE_PREFIX,
  112. default => self::SCORE_CONTAINS,
  113. };
  114. return [$grade, $closeness];
  115. }
  116. if (! mb_check_encoding($query, 'ASCII') || ! mb_check_encoding($title, 'ASCII')) {
  117. return [0.0, 0.0];
  118. }
  119. return self::fuzzyScore($query, $title);
  120. }
  121. /**
  122. * @return array{0: float, 1: float}
  123. */
  124. private static function fuzzyScore(string $query, string $title): array
  125. {
  126. $miss = [0.0, 0.0];
  127. $queryLength = strlen($query);
  128. $titleLength = strlen($title);
  129. if ($queryLength < 4 || $titleLength < $queryLength * 0.6) {
  130. return $miss;
  131. }
  132. $trigrams = [];
  133. for ($i = 0; $i <= $queryLength - 3; $i++) {
  134. $trigrams[substr($query, $i, 3)] = true;
  135. }
  136. $shared = 0;
  137. foreach (array_keys($trigrams) as $trigram) {
  138. if (str_contains($title, $trigram)) {
  139. $shared++;
  140. }
  141. }
  142. if ($shared < count($trigrams) * self::TRIGRAM_MIN_RATIO) {
  143. return $miss;
  144. }
  145. $best = $queryLength;
  146. $lastStart = max(0, $titleLength - $queryLength + 1);
  147. for ($start = 0; $start <= $lastStart; $start++) {
  148. foreach ([$queryLength - 1, $queryLength, $queryLength + 1] as $width) {
  149. $best = min($best, levenshtein($query, substr($title, $start, $width)));
  150. }
  151. }
  152. $similarity = 1 - $best / $queryLength;
  153. if ($similarity < self::FUZZY_MIN_SIMILARITY) {
  154. return $miss;
  155. }
  156. $closeness = max(0.0, 1 - levenshtein($query, $title) / max($queryLength, $titleLength));
  157. return [round(self::SCORE_FUZZY_MAX * $similarity, 2), $closeness];
  158. }
  159. /**
  160. * @param list<array<string, mixed>> $rows
  161. * @param array{q?: string, book?: int, para?: int, layer?: string, max_level?: int} $filters
  162. * @return list<array<string, mixed>>
  163. */
  164. private function filter(array $rows, array $filters): array
  165. {
  166. $book = $filters['book'] ?? null;
  167. $layers = isset($filters['layer']) ? explode(',', $filters['layer']) : null;
  168. $maxLevel = $filters['max_level'] ?? null;
  169. $range = null;
  170. if ($book !== null && isset($filters['para'])) {
  171. $range = $this->subtreeRange($rows, (int) $book, (int) $filters['para']);
  172. }
  173. return array_values(array_filter($rows, fn (array $row) => ($book === null || $row['book'] === (int) $book)
  174. && ($range === null || ($row['para'] >= $range[0] && $row['para'] < $range[1]))
  175. && ($layers === null || in_array($row['layer'], $layers, true))
  176. && ($maxLevel === null || $row['level'] <= $maxLevel)));
  177. }
  178. /**
  179. * 一章在书里覆盖的段落范围 [起, 止)。章不存在时返回空范围。
  180. *
  181. * @param list<array<string, mixed>> $rows
  182. * @return array{0: int, 1: int}
  183. */
  184. private function subtreeRange(array $rows, int $book, int $para): array
  185. {
  186. foreach ($rows as $row) {
  187. if ($row['book'] === $book && $row['para'] === $para) {
  188. return [$para, $para + max(1, $row['paragraphs'])];
  189. }
  190. }
  191. return [0, 0];
  192. }
  193. /**
  194. * 按匹配度排序,去掉不命中的。巴利名与各个译名取最高分。
  195. *
  196. * @param list<array<string, mixed>> $rows
  197. * @return list<array<string, mixed>>
  198. */
  199. private function rank(array $rows, string $query): array
  200. {
  201. $layerOrder = array_flip(array_values(self::LAYER_TAGS));
  202. $hits = [];
  203. foreach ($rows as $row) {
  204. $best = self::score($query, $row['key']);
  205. foreach ($row['translations'] as $translation) {
  206. $best = max($best, self::score($query, $translation['key']));
  207. }
  208. if ($best[0] > 0) {
  209. [$row['score'], $row['closeness']] = $best;
  210. $hits[] = $row;
  211. }
  212. }
  213. usort($hits, fn (array $a, array $b) => [$b['score'], $layerOrder[$a['layer']], $b['closeness'], $a['level'], $a['book'], $a['para']]
  214. <=> [$a['score'], $layerOrder[$b['layer']], $a['closeness'], $b['level'], $b['book'], $b['para']]);
  215. return $hits;
  216. }
  217. /**
  218. * 给一页补上路径(只有这一页查库,整页一次)。
  219. *
  220. * @param list<array<string, mixed>> $rows
  221. * @return list<array<string, mixed>>
  222. */
  223. private function hydrate(array $rows, bool $withScore): array
  224. {
  225. $paths = PaliText::query()
  226. ->whereKey(array_column($rows, 'id'))
  227. ->pluck('path', 'id');
  228. return array_map(function (array $row) use ($paths, $withScore) {
  229. $item = [
  230. 'book' => $row['book'],
  231. 'para' => $row['para'],
  232. 'level' => $row['level'],
  233. 'title' => $row['title'],
  234. 'translations' => array_map(fn (array $t) => ['lang' => $t['lang'], 'title' => $t['title']], $row['translations']),
  235. 'layer' => $row['layer'],
  236. 'paragraphs' => $row['paragraphs'],
  237. 'chars' => $row['chars'],
  238. 'parent' => $row['parent'],
  239. 'prev' => $row['prev'],
  240. 'next' => $row['next'],
  241. 'path' => self::decodePath($paths[$row['id']] ?? null),
  242. ];
  243. if ($withScore) {
  244. $item['score'] = $row['score'];
  245. }
  246. return $item;
  247. }, $rows);
  248. }
  249. /**
  250. * @return list<array{book: int, para: int, title: string, level: int}>
  251. */
  252. private static function decodePath(?string $json): array
  253. {
  254. $nodes = json_decode($json ?? '[]', true);
  255. if (! is_array($nodes)) {
  256. return [];
  257. }
  258. return array_values(array_map(fn (array $node) => [
  259. 'book' => (int) ($node['book'] ?? 0),
  260. 'para' => (int) ($node['paragraph'] ?? 0),
  261. 'title' => (string) ($node['title'] ?? ''),
  262. 'level' => (int) ($node['level'] ?? 0),
  263. ], array_filter($nodes, 'is_array')));
  264. }
  265. /**
  266. * 全部标题(经文顺序),带匹配键、layer、公开译名。缓存一天。
  267. *
  268. * @return list<array<string, mixed>>
  269. */
  270. private function catalog(): array
  271. {
  272. return RedisCache::remember(self::CATALOG_KEY, self::CATALOG_TTL, function () {
  273. $layers = $this->bookLayers();
  274. $translations = $this->translations();
  275. return PaliText::query()
  276. ->where('level', '<', self::MAX_TITLE_LEVEL)
  277. ->orderBy('book')
  278. ->orderBy('paragraph')
  279. ->get(['id', 'book', 'paragraph', 'level', 'toc', 'chapter_len', 'chapter_strlen', 'parent', 'prev_chapter', 'next_chapter'])
  280. ->map(function (PaliText $row) use ($layers, $translations) {
  281. $title = trim((string) $row->toc);
  282. $key = self::normalize($title);
  283. $own = array_values(array_filter(
  284. $translations["{$row->book}-{$row->paragraph}"] ?? [],
  285. fn (array $t) => $t['key'] !== $key,
  286. ));
  287. return [
  288. 'id' => (int) $row->id,
  289. 'book' => (int) $row->book,
  290. 'para' => (int) $row->paragraph,
  291. 'level' => (int) $row->level,
  292. 'title' => $title,
  293. 'key' => $key,
  294. 'translations' => $own,
  295. 'layer' => $layers[(int) $row->book] ?? self::DEFAULT_LAYER,
  296. 'paragraphs' => (int) $row->chapter_len,
  297. 'chars' => (int) $row->chapter_strlen,
  298. 'parent' => self::chapterOrNull($row->parent),
  299. 'prev' => self::chapterOrNull($row->prev_chapter),
  300. 'next' => self::chapterOrNull($row->next_chapter),
  301. ];
  302. })
  303. ->all();
  304. });
  305. }
  306. /** 上下章、父章的段号;库里用 0 / -1 表示没有 */
  307. private static function chapterOrNull(mixed $para): ?int
  308. {
  309. return $para !== null && (int) $para > 0 ? (int) $para : null;
  310. }
  311. /**
  312. * 每本书属于哪一层:取书(level 1 那一段)的 tag。
  313. *
  314. * @return array<int, string>
  315. */
  316. private function bookLayers(): array
  317. {
  318. $layerTags = fn (Builder $q) => $q->whereIn('name', array_keys(self::LAYER_TAGS));
  319. $books = PaliText::query()
  320. ->where('level', 1)
  321. ->whereHas('tagMaps.tags', $layerTags)
  322. ->with(['tagMaps.tags' => $layerTags])
  323. ->get(['uid', 'book']);
  324. $order = array_flip(array_keys(self::LAYER_TAGS));
  325. $layers = [];
  326. foreach ($books as $book) {
  327. foreach ($book->tagMaps as $map) {
  328. $name = $map->tags?->name;
  329. $current = $layers[(int) $book->book] ?? null;
  330. if ($name !== null && ($current === null || $order[$name] < $order[$current])) {
  331. $layers[(int) $book->book] = $name;
  332. }
  333. }
  334. }
  335. return array_map(fn (string $tag) => self::LAYER_TAGS[$tag], $layers);
  336. }
  337. /**
  338. * 公开 channel 写的章节译名,按「book-para」分组,同一章同一个匹配键只留一条。
  339. *
  340. * 排除巴利原文 channel 的标题(`lang = pali`)、逐词 channel 存进来的 JSON(以 `[` 开头)。
  341. *
  342. * @return array<string, list<array{lang: string, title: string, key: string}>>
  343. */
  344. private function translations(): array
  345. {
  346. $rows = ProgressChapter::query()
  347. ->whereHas('channel', fn (Builder $q) => $q->where('status', self::STATUS_PUBLIC))
  348. ->where('lang', '<>', 'pali')
  349. ->whereRaw("trim(title) <> ''")
  350. ->where('title', 'not like', '[%')
  351. ->orderBy('book')
  352. ->orderBy('para')
  353. ->orderBy('lang')
  354. ->get(['book', 'para', 'lang', 'title']);
  355. $grouped = [];
  356. foreach ($rows as $row) {
  357. $title = trim((string) $row->title);
  358. $key = self::normalize($title);
  359. $chapter = "{$row->book}-{$row->para}";
  360. if ($key === '' || isset($grouped[$chapter][$key])) {
  361. continue;
  362. }
  363. $grouped[$chapter][$key] = ['lang' => (string) $row->lang, 'title' => $title, 'key' => $key];
  364. }
  365. return array_map('array_values', $grouped);
  366. }
  367. }