option('output')) { $this->error('please set output file option --output=file'); return 1; } /* //irregular $dictId = ['4d3a0d92-0adc-4052-80f5-512a2603d0e8']; //regular $dictId[] = DictApi::getSysDict('system_regular'); $dictId[] = DictApi::getSysDict('robot_compound'); */ $filename = $this->option('output'); $fp = fopen($filename, 'w') or exit('Unable to open file!'); $parents = UserDict::select('parent') ->whereNotNull('parent') ->where('parent', '<>', '') ->groupBy('parent')->cursor(); $droppedTerms = 0; $droppedLines = 0; foreach ($parents as $parent) { if (str_contains($parent->parent, ' ')) { continue; } $words = UserDict::where('parent', $parent->parent) ->select('word') ->groupBy('word')->get(); $wordsList = []; foreach ($words as $word) { $wordsList[$word->word] = 1; } $teams = DhammaTerm::where('word', $parent->parent) ->select(['meaning'])->get(); foreach ($teams as $term) { $wordsList[$term->meaning] = 1; } $this->info("[{$parent->parent}] ".count($words).' team='.count($teams)); // 合并 $parent->parent, $words->word, $team->meaning 为一个字符串数组 $combinedArray = []; $combinedArray[] = $parent->parent; foreach ($wordsList as $word => $value) { $combinedArray[] = $word; } // 过滤掉会被 analyzer 完全消除的 term,否则建索引时会失败 $termCount = count($combinedArray); $combinedArray = $this->filterSynonymTerms($combinedArray); $droppedTerms += $termCount - count($combinedArray); // 同义词行至少要有两个 term 才有意义 if (count($combinedArray) < 2) { $droppedLines++; continue; } // 将 $combinedArray 写入 CSV 文件 fputcsv($fp, $combinedArray); if ($this->option('test')) { $this->warn('--test 已开启,只导出一行。'); break; } } // 关闭文件 fclose($fp); // 写index.json $info = [ 'index' => config('mint.opensearch.index'), 'pali_synonyms' => app(OpenSearchService::class)->getPaliSynonymsSetting(), ]; file_put_contents( $this->changeExtensionToJson($filename), json_encode($info, JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE) ); $this->info("过滤掉非法 term {$droppedTerms} 个,整行丢弃 {$droppedLines} 行。"); $this->info('done'); return 0; } /** * 词典源码标记,这类 term 不是真正的词条 * * 1. 含 # ‹ › 的:如 "#=cetayati)"、"(#=)(‹paññāpeti)"。 * 其中以 # 开头的 term 若落在行首,整行会被 OpenSearch 当成注释静默忽略。 * 2. 以 ( [ < 开头的:如 "(ku的离格)"、"[ava-hīḷanā<hīḍ]",是词源/变格标注。 * 只匹配开头,保证 "阿拉汉[果]"、"bhesajja[ṃ]" 这类正常释义不受影响。 */ private const DIRTY_TERM_PATTERN = '/[#‹›]|^[(\[<<]/u'; /** * 过滤掉不能用作同义词的 term,并按原顺序去重 * * 丢弃两类 term: * 1. 不含任何字母或数字的(例如 "②"、"——"、"?)")。OpenSearch 的 synonym_graph * filter 会用 analyzer 分析每一个 term,这类 term 分析后被完全消除, * 建索引时会抛出 illegal_argument_exception: Failed to build synonyms。 * 2. 带词典源码标记的脏词条,见 self::DIRTY_TERM_PATTERN。 * * @param array $terms * @return array */ private function filterSynonymTerms(array $terms): array { $seen = []; $filtered = []; foreach ($terms as $term) { $term = trim((string) $term); if ($term === '' || preg_match('/[\p{L}\p{Nd}]/u', $term) !== 1) { continue; } if (preg_match(self::DIRTY_TERM_PATTERN, $term) === 1) { continue; } if (isset($seen[$term])) { continue; } $seen[$term] = true; $filtered[] = $term; } return $filtered; } /** * 将给定文件路径的扩展名替换为 .json * * @param string $filePath 完整的文件路径 * @return string 新的文件路径 */ private function changeExtensionToJson(string $filePath): string { // 获取路径信息 $pathInfo = pathinfo($filePath); // 提取目录、文件名(不含扩展名) $dirname = $pathInfo['dirname'] ?? ''; $filename = $pathInfo['filename'] ?? ''; // 如果目录不是根目录,则添加目录分隔符 $dirname = rtrim($dirname, DIRECTORY_SEPARATOR).DIRECTORY_SEPARATOR; // 构建新路径 return $dirname.$filename.'.json'; } }