IndexTipitaka.php 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430
  1. <?php
  2. namespace App\Console\Commands;
  3. use App\Facades\RedisCache;
  4. use App\Models\PaliText;
  5. use App\Models\ProgressChapter;
  6. use App\Models\Sentence;
  7. use App\Services\ChannelService;
  8. use App\Services\OpenSearchService;
  9. use App\Services\PaliContentService;
  10. use App\Services\SearchPaliDataService;
  11. use App\Services\SummaryService;
  12. use App\Services\TagService;
  13. use Illuminate\Console\Command;
  14. use Illuminate\Support\Facades\Log;
  15. class IndexTipitaka extends Command
  16. {
  17. /**
  18. * The name and signature of the console command.
  19. * php artisan opensearch:index-tipitaka 93 --para=6 --granularity=paragraph
  20. *
  21. * @var string
  22. */
  23. protected $signature = 'opensearch:index-tipitaka
  24. {book : The book ID to index data for}
  25. {--para= : index paragraph No. omit to all}
  26. {--channel= : index channel id omit to all}
  27. {--test}
  28. {--summary=on}
  29. {--resume}
  30. {--granularity=all : The granularity to index (paragraph, sutta, sentence; omit to index all)}';
  31. /**
  32. * The console command description.
  33. *
  34. * @var string
  35. */
  36. protected $description = 'Index Pali data into OpenSearch for a specified book and optional granularity (all granularities if not specified)';
  37. private $isTest = false;
  38. private $summary = false;
  39. /**
  40. * Create a new command instance.
  41. *
  42. * @return void
  43. */
  44. public function __construct(
  45. protected ChannelService $channelService,
  46. protected SearchPaliDataService $searchPaliDataService,
  47. protected OpenSearchService $openSearchService,
  48. protected SummaryService $summaryService,
  49. protected TagService $tagService
  50. ) {
  51. parent::__construct();
  52. }
  53. /**
  54. * Execute the console command.
  55. *
  56. * @return int
  57. */
  58. public function handle()
  59. {
  60. $this->line('index tipitaka start');
  61. $book = (int) $this->argument('book');
  62. $paragraph = $this->option('para');
  63. $channel = $this->option('channel');
  64. if ($channel) {
  65. $this->line('channel='.$channel);
  66. }
  67. $granularity = $this->option('granularity');
  68. $this->summary = $this->option('summary') === 'on';
  69. if ($this->option('test')) {
  70. $this->isTest = true;
  71. $this->info('test mode');
  72. }
  73. try {
  74. // Test OpenSearch connection
  75. [$connected, $message] = $this->openSearchService->testConnection();
  76. if (! $connected) {
  77. $this->error($message);
  78. Log::error($message);
  79. return 1;
  80. }
  81. $overallStatus = 0; // Track overall command status (0 for success, 1 for any failure)
  82. $maxBookId = PaliText::max('book');
  83. if ($book === 0) {
  84. $booksId = range(1, $maxBookId);
  85. } elseif ($this->option('resume')) {
  86. $booksId = range($book, $maxBookId);
  87. } else {
  88. $booksId = [$book];
  89. }
  90. foreach ($booksId as $key => $bookId) {
  91. if (
  92. $this->option('granularity') === 'chapter' ||
  93. $this->option('granularity') === 'all'
  94. ) {
  95. $this->indexChapter($bookId, $channel);
  96. }
  97. if (
  98. $this->option('granularity') === 'paragraph' ||
  99. $this->option('granularity') === 'all'
  100. ) {
  101. $this->indexTipitakaParagraph($bookId, $paragraph);
  102. }
  103. }
  104. return $overallStatus;
  105. } catch (\Exception $e) {
  106. $this->error('Failed to index Pali data: '.$e->getMessage());
  107. Log::error("Failed to index Pali data for book: $book, granularity: ".($granularity ?: 'all'), ['error' => $e]);
  108. return 1;
  109. }
  110. }
  111. /**
  112. * Index Pali paragraphs for a given book.
  113. *
  114. * @param int $book
  115. * @return int
  116. */
  117. protected function indexTipitakaParagraph($book, $paragraph = null)
  118. {
  119. $this->info("Starting to index paragraphs for book: $book");
  120. $total = 0;
  121. if ($paragraph) {
  122. $paragraphs = PaliText::where('book', $book)
  123. ->where('paragraph', $paragraph)
  124. ->orderBy('paragraph')->cursor();
  125. } else {
  126. $paragraphs = PaliText::where('book', $book)
  127. ->orderBy('paragraph')->cursor();
  128. }
  129. $bookUid = PaliText::where('book', $book)->where('level', 1)->first()->uid;
  130. $category = $this->tagService->getTagsName($bookUid);
  131. $headings = [];
  132. $currChapterTitle = '';
  133. $commentaryId = '';
  134. $currSession = [];
  135. foreach ($paragraphs as $key => $para) {
  136. $total++;
  137. if ($para->level < 8) {
  138. $currChapterTitle = $para->toc;
  139. }
  140. if ($para->class === 'nikaya') {
  141. $nikaya = $para->text;
  142. }
  143. $paraContent = $this->searchPaliDataService
  144. ->getParaContent($para['book'], $para['paragraph']);
  145. if (! empty($commentaryId)) {
  146. $currSession[] = $paraContent;
  147. }
  148. if (isset($paraContent['commentary'])) {
  149. if (! empty($commentaryId)) {
  150. // 保存 session
  151. $this->indexPaliSession($para->toArray(), $currSession, $currChapterTitle, $commentaryId);
  152. $currSession = [];
  153. }
  154. $commentaryId = $paraContent['commentary'];
  155. }
  156. $this->indexParagraph($para->toArray(), $paraContent, $commentaryId, $category);
  157. $this->info("{$para['book']}-[{$para['paragraph']}]-[{$commentaryId}]");
  158. }
  159. $this->info("Successfully indexed $total paragraphs for book: $book");
  160. Log::info("Indexed $total paragraphs for book: $book");
  161. return 0;
  162. }
  163. protected function indexParagraph($paraInfo, $paraContent, $related_id, array $category)
  164. {
  165. $paraId = $paraInfo['book'].'-'.$paraInfo['paragraph'];
  166. $resource_id = $paraInfo['uid'];
  167. $path = json_decode($paraInfo['path']);
  168. if (is_array($path) && count($path) > 0) {
  169. $title = end($path)->title;
  170. } else {
  171. $title = '';
  172. }
  173. $document = [
  174. 'id' => "tipitaka_paragraph_pi_{$paraId}",
  175. 'resource_id' => $resource_id, // Use uid from getPaliData for resource_id
  176. 'resource_type' => 'tipitaka',
  177. 'title' => [
  178. 'text' => ['pali' => $title],
  179. ],
  180. 'summary' => [
  181. 'text' => $this->summary ? $this->summaryService->summarize($paraContent['markdown']) : '',
  182. ],
  183. 'content' => [
  184. 'text' => ['pali' => $paraContent['text']],
  185. 'suggest' => ['pali' => $paraContent['words']],
  186. ],
  187. 'bold_single' => implode(' ', $paraContent['bold1']),
  188. 'bold_multi' => implode(' ', array_merge($paraContent['bold2'], $paraContent['bold3'])),
  189. 'related_id' => $paraId,
  190. 'category' => $category, // Assuming Pali paragraphs are sutta; adjust as needed
  191. 'language' => 'pi',
  192. 'updated_at' => now()->toIso8601String(),
  193. 'granularity' => 'paragraph',
  194. 'path' => $this->getPathTitle($path),
  195. ];
  196. if ($paraInfo['level'] < 8) {
  197. $document['title']['suggest']['pali'] = $paraContent['words'];
  198. }
  199. if ($this->isTest) {
  200. $this->info($document['title']['text']['pali']);
  201. $this->info($document['summary']['text']);
  202. } else {
  203. $this->openSearchService->create($document['id'], $document);
  204. }
  205. }
  206. protected function indexPaliSession($paraInfo, $contents, $currChapter, $related_id)
  207. {
  208. $markdown = [];
  209. $text = [];
  210. $bold_single = [];
  211. $bold_multi = [];
  212. foreach ($contents as $key => $content) {
  213. $markdown[] = $content['markdown'];
  214. $text[] = $content['text'];
  215. $bold_single = array_merge($bold_single, $content['bold1']);
  216. $bold_multi = array_merge($bold_multi, $content['bold2'], $content['bold3']);
  217. }
  218. $document = [
  219. 'id' => "pali_session_{$related_id}",
  220. 'resource_id' => $paraInfo['uid'], // Use uid from getPaliData for resource_id
  221. 'resource_type' => 'original_text',
  222. 'title' => [
  223. ['text' => ['pali' => "{$currChapter} paragraph {$paraInfo['paragraph']}"]],
  224. ],
  225. 'summary' => [
  226. 'text' => $this->summary ? $this->summaryService->summarize($content['markdown']) : '',
  227. ],
  228. 'content' => [
  229. ['text' => ['pali' => implode("\n\n", $markdown)]],
  230. ],
  231. 'bold_single' => implode(' ', $bold_single),
  232. 'bold_multi' => implode(' ', $bold_multi),
  233. 'related_id' => $related_id,
  234. 'category' => 'pali', // Assuming Pali paragraphs are sutta; adjust as needed
  235. 'language' => 'pi',
  236. 'updated_at' => now()->toIso8601String(),
  237. 'granularity' => 'session',
  238. 'path' => $this->getPathTitle(json_decode($paraInfo['path'])),
  239. ];
  240. if ($this->isTest) {
  241. $this->info($document['title']['pali']);
  242. $this->info($document['summary']['text']);
  243. } else {
  244. $this->openSearchService->create($document['id'], $document);
  245. }
  246. }
  247. /**
  248. * Index Pali suttas for a given book (placeholder for future implementation).
  249. *
  250. * @param int $book
  251. * @param ?string $channel
  252. * @return int
  253. */
  254. protected function indexChapter($book, $channelId = null)
  255. {
  256. $this->info("Starting to index paragraphs for book: $book");
  257. $total = 0;
  258. $chapters = PaliText::where('book', $book)
  259. ->where('level', '<', 8)
  260. ->orderBy('paragraph')->get();
  261. foreach ($chapters as $key => $chapter) {
  262. if ($chapter->level === 1) {
  263. $category = $this->tagService->getTagsName($chapter->uid);
  264. }
  265. /**
  266. * 章节的起始位置算法
  267. * 从章节的标题,到下一个章节的标题之间
  268. */
  269. $start = $chapter->paragraph;
  270. if ($key === count($chapters) - 1) {
  271. // 不能写 ->first()->value('paragraph'):模型实例上的 value() 会新开一个不带条件的查询,
  272. // 拿到全表第一行的段号,最后一章的范围变成空的、从不被索引
  273. $end = PaliText::where('book', $book)->max('paragraph');
  274. } else {
  275. $end = $chapters[$key + 1]->paragraph - 1;
  276. }
  277. // 获取这个段落之间的全部channel
  278. $table = Sentence::where('book_id', $book)
  279. ->whereBetween('paragraph', [$start, $end]);
  280. if ($channelId) {
  281. $table = $table->where('channel_uid', $channelId);
  282. }
  283. $channels = $table->select('channel_uid')
  284. ->groupBy('channel_uid')->get();
  285. $this->info("index chapter start={$start} end={$end}");
  286. foreach ($channels as $channel) {
  287. $display = [];
  288. $content = [];
  289. $channelInfo = $this->channelService->summary($channel->channel_uid);
  290. if (! $channelInfo) {
  291. Log::error('invalid channel', ['id' => $channel->channel_uid]);
  292. continue;
  293. }
  294. $this->info('channel ='.$channelInfo['name']);
  295. if ($channelInfo['type'] === 'wbw') {
  296. $this->info('wbw channel skip');
  297. continue;
  298. }
  299. $paraList = Sentence::where('book_id', $book)
  300. ->whereBetween('paragraph', [$start, $end])
  301. ->where('channel_uid', $channel->channel_uid)
  302. ->orderBy('paragraph')
  303. ->distinct()->pluck('paragraph');
  304. // 生成html数据
  305. $title = '';
  306. foreach ($paraList as $para) {
  307. $para = (int) $para;
  308. $paragraph = app(PaliContentService::class)->readParagraph(
  309. $book,
  310. $para,
  311. $channel->channel_uid,
  312. 'html'
  313. );
  314. if (empty($paragraph['display'])) {
  315. continue;
  316. }
  317. if ($para === $start && empty($title)) {
  318. $title = $paragraph['sentences'][0]['html'];
  319. }
  320. $display[] = $paragraph['display'];
  321. }
  322. $this->chapterSave([
  323. 'book' => $book,
  324. 'para' => $start,
  325. 'level' => $chapter->level,
  326. 'channel' => $channel->channel_uid,
  327. 'content' => implode('', $display),
  328. 'title' => strip_tags($title),
  329. 'cat' => $category ?? null,
  330. ]);
  331. }
  332. }
  333. return 0;
  334. }
  335. protected function chapterSave(array $param)
  336. {
  337. $progress = ProgressChapter::where('book', $param['book'])
  338. ->where('para', $param['para'])
  339. ->where('channel_id', $param['channel'])
  340. ->first();
  341. $channel = $this->channelService->summary($param['channel']);
  342. $docId = "tipitaka_chapter_{$param['book']}-{$param['para']}_{$param['channel']}";
  343. $document = [
  344. 'id' => $docId,
  345. 'resource_id' => $progress ? $progress->uid : "{$param['book']}-{$param['para']}_{$param['channel']}",
  346. 'resource_type' => 'tipitaka',
  347. 'title' => [],
  348. 'summary' => [
  349. 'text' => '',
  350. ],
  351. 'content' => [],
  352. 'related_id' => "{$param['book']}-{$param['para']}",
  353. 'category' => $param['cat'],
  354. 'language' => $channel['lang'],
  355. 'updated_at' => now()->toIso8601String(),
  356. 'granularity' => $param['level'] === 1 ? 'book' : 'chapter',
  357. ];
  358. // TODO: 补充语言判断,将内容放入对应的 text.pali 或 text.zh 字段
  359. $plainText = strip_tags($param['content']);
  360. $title = strip_tags($param['title']);
  361. if (str_contains($channel['lang'], 'zh')) {
  362. $document['content']['text']['zh'] = $plainText;
  363. $document['title']['text']['zh'] = $title;
  364. } else {
  365. $document['content']['text']['pali'] = $plainText;
  366. $document['title']['text']['pali'] = $title;
  367. }
  368. // $document['content']['display'] = $param['content']; // 展示
  369. RedisCache::put($docId, $param['content']);
  370. if ($this->isTest) {
  371. $this->info($param['content']);
  372. } else {
  373. $this->openSearchService->create($document['id'], $document);
  374. $this->info("create index {$document['id']} size=".strlen($param['content']));
  375. }
  376. }
  377. /**
  378. * Index Pali sentences for a given book (placeholder for future implementation).
  379. *
  380. * @param int $book
  381. * @return int
  382. */
  383. protected function indexPaliSentences($book)
  384. {
  385. $this->warn("Sentence indexing is not yet implemented for book: $book");
  386. Log::warning("Sentence indexing not implemented for book: $book");
  387. return 1;
  388. }
  389. private function getPathTitle(array $input)
  390. {
  391. $output = [];
  392. foreach ($input as $key => $node) {
  393. $output[] = $node->title;
  394. }
  395. return implode('/', $output);
  396. }
  397. }