MarkupParser.php 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299
  1. <?php
  2. namespace App\Services\Markup\Parsing;
  3. use App\Services\Markup\Annotation\AnnotationSet;
  4. use App\Services\Markup\Annotation\AnnotationToken;
  5. use App\Services\Markup\Annotation\AnnotationTokenParser;
  6. use App\Services\Markup\Annotation\Marker;
  7. use App\Services\Markup\Dialect;
  8. use App\Services\Markup\Ir\Boundary;
  9. use App\Services\Markup\Ir\Call;
  10. use App\Services\Markup\Ir\Document;
  11. use App\Services\Markup\Ir\Element;
  12. use App\Services\Markup\Ir\Node;
  13. use App\Services\Markup\Ir\Params;
  14. use App\Services\Markup\Ir\Scope;
  15. use App\Services\Markup\Ir\Text;
  16. use App\Services\Markup\Templates\SentTemplate;
  17. use App\Services\Markup\Templates\TemplateRegistry;
  18. use Illuminate\Support\Facades\Log;
  19. use League\CommonMark\Environment\Environment;
  20. use League\CommonMark\Extension\CommonMark\CommonMarkCoreExtension;
  21. use League\CommonMark\Extension\GithubFlavoredMarkdownExtension;
  22. use League\CommonMark\Parser\MarkdownParser;
  23. /**
  24. * 源码 → {@see Document}(IR)。纯解析,不查库。
  25. *
  26. * 流程:清洗编码 → nissaya 转换 → {@see Preprocessor}(模板换占位符)→ CommonMark + GFM → {@see IrBuilder}。
  27. * 两种方言共用同一个 CommonMark 环境:旧方言的反引号脚注在预扫描时已经换掉了,剩下的反引号照 CommonMark 处理。
  28. */
  29. final class MarkupParser
  30. {
  31. /** 模板参数里再嵌模板的最大层数 */
  32. private const MAX_NESTING = 8;
  33. private ?Environment $environment = null;
  34. public function __construct(private readonly TemplateRegistry $registry) {}
  35. /**
  36. * @param bool $paragraphMode 巴汉逐段对读,见 {@see self::duplicateSentenceParagraphs()}
  37. * @param AnnotationSet|null $annotations 源码里批注占位符对应的结构化标记(阅读页注入,见 {@see AnnotationSet})
  38. */
  39. public function parse(string $source, Scope $scope, bool $paragraphMode = false, int $nesting = 0, ?AnnotationSet $annotations = null): Document
  40. {
  41. $source = $this->clean($source);
  42. if ($annotations === null) {
  43. // 没有批注时源码里不该有批注占位符;有批注时,下标对不上的占位符解析时自然丢掉
  44. $source = AnnotationToken::strip($source);
  45. }
  46. // 只转换内容本身;模板参数(如 nissaya json 里 note 的正文)不是 nissaya 行
  47. if ($scope->channelType === 'nissaya' && $nesting === 0) {
  48. $source = $this->nissayaToTemplates($source, $scope->contentType);
  49. }
  50. if (trim($source) === '') {
  51. return Document::empty($scope);
  52. }
  53. $calls = [];
  54. $raws = [];
  55. $register = function (string $kind, string $raw, string $inner, string $info) use (&$calls, &$raws, $scope, $nesting): int {
  56. $index = count($calls);
  57. $raws[$index] = $raw;
  58. $calls[$index] = $this->callFromSyntax($kind, $raw, $inner, $info, $scope, $nesting);
  59. return $index;
  60. };
  61. $prepared = (new Preprocessor)->run($source, $scope->dialect, $register);
  62. $ast = (new MarkdownParser($this->environment()))->parse($prepared);
  63. $annotationCalls = [];
  64. $annotation = $annotations === null ? null : function (int $index) use ($annotations, $scope, $nesting, &$annotationCalls): ?Node {
  65. $node = $this->annotationNode($annotations->get($index), $scope, $nesting);
  66. if ($node instanceof Call) {
  67. $annotationCalls[$index] = $node;
  68. }
  69. return $node;
  70. };
  71. $builder = new IrBuilder(
  72. $calls,
  73. $raws,
  74. fn (string $name, array $segments, string $raw) => $this->makeCall($name, $segments, $raw, $scope, $nesting),
  75. $scope,
  76. $source,
  77. $annotation,
  78. );
  79. [$nodes, $inline] = $builder->build($ast);
  80. if ($paragraphMode) {
  81. $blocks = $inline ? [new Element('p', [], $nodes)] : $nodes;
  82. $duplicated = $this->duplicateSentenceParagraphs($blocks);
  83. if (count($duplicated) !== count($blocks)) {
  84. [$nodes, $inline] = [$duplicated, false];
  85. }
  86. }
  87. return new Document($nodes, $inline, $prepared, $calls, $scope, annotationCalls: $annotationCalls);
  88. }
  89. /**
  90. * 批注标记 → IR:点 → 模板调用;区间开 → 带着(待填内容的)调用的边界;区间闭 → 边界。
  91. */
  92. private function annotationNode(?Marker $marker, Scope $scope, int $nesting): ?Node
  93. {
  94. if ($marker === null) {
  95. return null;
  96. }
  97. if ($marker->kind === Marker::CLOSE) {
  98. return new Boundary($marker->pair, false);
  99. }
  100. if ($marker->template === 'note') {
  101. $template = $this->registry->get('note');
  102. // 批注正文的方言跟批注那一行(discussions),不跟句子
  103. $bodyScope = new Scope($scope->channels, Dialect::fromContentType($marker->bodyContentType), $scope->depth, 'translation', $marker->bodyContentType);
  104. $body = (string) $marker->body;
  105. return new Call('note', $template, new Params($marker->params, [1 => $body]), [
  106. 'text' => $this->parse($body, $bodyScope, false, $nesting + 1),
  107. ], $scope, '');
  108. }
  109. $call = new Call($marker->template, $this->registry->internal($marker->template), new Params($marker->params), [], $scope, '');
  110. return $marker->kind === Marker::OPEN ? new Boundary($marker->pair, true, $call) : $call;
  111. }
  112. /**
  113. * @param list<string>|Params $params 参数片段,或已经分好的参数表(```note 的正文不再切 `|`)
  114. */
  115. public function makeCall(string $name, array|Params $params, string $raw, Scope $scope, int $nesting): Call
  116. {
  117. $template = $this->registry->get($name);
  118. if (is_array($params)) {
  119. $params = $template->parseParams($params, $scope->dialect);
  120. }
  121. $slots = [];
  122. foreach ($template->params() as $paramName => $spec) {
  123. if (! $spec->markdown) {
  124. continue;
  125. }
  126. $value = $params->raw($paramName, $spec->position);
  127. if ($value === null) {
  128. continue;
  129. }
  130. $slots[$paramName] = $nesting < self::MAX_NESTING
  131. ? $this->parse($value, $scope, false, $nesting + 1)
  132. : new Document([new Text($value)], true, $value, [], $scope);
  133. }
  134. return new Call($name, $template, $params, $slots, $scope, $raw);
  135. }
  136. private function callFromSyntax(string $kind, string $raw, string $inner, string $info, Scope $scope, int $nesting): Call
  137. {
  138. return match ($kind) {
  139. 'wikilink' => $this->makeCall('term', TemplateSyntax::split('term|'.$inner)[1], $raw, $scope, $nesting),
  140. 'note-backtick' => $this->makeCall('note', TemplateSyntax::split('note|'.$inner)[1], $raw, $scope, $nesting),
  141. 'note-fence' => $this->makeCall('note', $this->fenceParams($inner, $info), $raw, $scope, $nesting),
  142. default => $this->templateCall($inner, $raw, $scope, $nesting),
  143. };
  144. }
  145. private function templateCall(string $inner, string $raw, Scope $scope, int $nesting): Call
  146. {
  147. // 句子简写 {{book-para-start-end}}(可带 @channel)
  148. if (preg_match('/^\s*[0-9]/', $inner) === 1) {
  149. return $this->makeCall('sent', new Params(['id' => trim($inner)]), $raw, $scope, $nesting);
  150. }
  151. [$name, $segments] = TemplateSyntax::split($inner);
  152. return $this->makeCall($name, $segments, $raw, $scope, $nesting);
  153. }
  154. /**
  155. * ```note 的参数:正文整段是第 1 个参数,info string 里的 `name="value"` 是命名参数。
  156. */
  157. private function fenceParams(string $inner, string $info): Params
  158. {
  159. $named = [];
  160. preg_match_all('/([a-z]+)=(?:"([^"]*)"|(\S+))/', $info, $pairs, PREG_SET_ORDER);
  161. foreach ($pairs as $pair) {
  162. $named[$pair[1]] = ($pair[2] ?? '') !== '' ? $pair[2] : ($pair[3] ?? '');
  163. }
  164. return new Params($named, [1 => $inner]);
  165. }
  166. /**
  167. * 只由句子模板组成的段落复制成两段:一段只显示原文、一段只显示译文(v2 的 preprocessingForParagraph)。
  168. *
  169. * @param list<Node> $nodes
  170. * @return list<Node>
  171. */
  172. private function duplicateSentenceParagraphs(array $nodes): array
  173. {
  174. $out = [];
  175. foreach ($nodes as $node) {
  176. if (! $node instanceof Element || $node->tag !== 'p' || ! $this->onlySentences($node->children)) {
  177. $out[] = $node;
  178. continue;
  179. }
  180. foreach (['origin', 'translation'] as $show) {
  181. $out[] = new Element('p', [], array_map(
  182. fn (Node $child) => $child instanceof Call ? $child->withParam('text', $show) : $child,
  183. $node->children,
  184. ));
  185. }
  186. }
  187. return $out;
  188. }
  189. /**
  190. * @param list<Node> $nodes
  191. */
  192. private function onlySentences(array $nodes): bool
  193. {
  194. $found = false;
  195. foreach ($nodes as $node) {
  196. if ($node instanceof Call && $node->template instanceof SentTemplate) {
  197. $found = true;
  198. } elseif (! ($node instanceof Text && trim($node->value, " \t\n>") === '')) {
  199. return false;
  200. }
  201. }
  202. return $found;
  203. }
  204. private function clean(string $source): string
  205. {
  206. $source = (string) iconv('UTF-8', 'UTF-8//IGNORE', $source);
  207. $source = str_replace(["\r\n", "\r"], "\n", $source);
  208. return Placeholder::strip($source);
  209. }
  210. /**
  211. * nissaya channel 的内容转成 nissaya 模板(v2 markdown2wiki 的 nissaya 段,口径不变)。
  212. *
  213. * - json:对齐后的 `{original, translation, note}`(translation 用 `>` 分层)或逐词解析的 wbw json
  214. * - markdown:每行 `巴利=意思`(`$` 也当 `=`)
  215. */
  216. private function nissayaToTemplates(string $source, string $contentType): string
  217. {
  218. if ($contentType === 'json') {
  219. $words = json_decode($source);
  220. if (! is_array($words)) {
  221. Log::error('nissaya json data is not array', ['data' => $source]);
  222. return '';
  223. }
  224. $out = [];
  225. foreach ($words as $word) {
  226. if (isset($word->original)) {
  227. $out[] = '{{nissaya|'.$word->original.'|'.str_replace('>', '=', $word->translation ?? '').'}}';
  228. if (! empty($word->note)) {
  229. $out[] = '{{note|'.$word->note.'|[nt]}}';
  230. }
  231. } elseif (isset($word->sn) && count($word->sn) === 1) {
  232. $out[] = '{{nissaya|'.($word->word->value ?? '').'|'.($word->meaning->value ?? '').'}}';
  233. }
  234. }
  235. return implode('', $out);
  236. }
  237. $lines = [];
  238. foreach (explode("\n", $source) as $line) {
  239. if (! str_contains($line, '=') && ! str_contains($line, '$')) {
  240. $lines[] = $line;
  241. continue;
  242. }
  243. $parts = explode('=', str_replace('$', '=', $line));
  244. $lines[] = '{{nissaya|'.$parts[0].'|'.implode('=', array_slice($parts, 1)).'}}';
  245. }
  246. return implode("\n", $lines);
  247. }
  248. private function environment(): Environment
  249. {
  250. if ($this->environment === null) {
  251. $environment = new Environment(['html_input' => 'allow']);
  252. $environment->addExtension(new CommonMarkCoreExtension);
  253. $environment->addExtension(new GithubFlavoredMarkdownExtension);
  254. $environment->addInlineParser(new PlaceholderParser, 250);
  255. $environment->addInlineParser(new AnnotationTokenParser, 250);
  256. // 抢在 CommonMark 自带的分隔符解析(最低优先级)之前处理 `*`:中文标点旁的粗体
  257. $environment->addInlineParser(new CjkFriendlyEmphasisParser, 0);
  258. $this->environment = $environment;
  259. }
  260. return $this->environment;
  261. }
  262. }