Preprocessor.php 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367
  1. <?php
  2. namespace App\Services\Markup\Parsing;
  3. use App\Services\Markup\Dialect;
  4. use Closure;
  5. /**
  6. * 预扫描:在交给 CommonMark 之前,把模板类语法整段换成占位符。
  7. *
  8. * 换掉的有:
  9. * - `{{…}}`:按 `{{` / `}}` 深度配对,可跨行、可含空行、可嵌套
  10. * - `[[x]]`:术语,单行
  11. * - ```note 围栏(旧方言里没有 info 的 ``` 围栏也是):整块变成脚注,紧挨着的上下文合进同一段(见 {@see self::joinFence()})
  12. * - 旧方言的 `` `x` ``:脚注,单行
  13. *
  14. * 跳过不扫的:代码围栏(含 ```mermaid,交给 CommonMark);E2 方言的行内代码 `` `…` ``。
  15. * 反斜杠转义 `\{{`、`\[[`、`` \` `` 原样保留,由 CommonMark 去掉反斜杠。
  16. *
  17. * 为什么不写成 CommonMark 的 inline parser:inline parser 只看得到一个段落,模板里的空行、
  18. * 行首的 `- ` / `#`、表格里的 `|` 都会先被块解析切开。
  19. */
  20. final class Preprocessor
  21. {
  22. /** `{{` 的最大嵌套深度,与 v2 的 STACK_DEEP 一致 */
  23. private const MAX_BRACE_DEPTH = 8;
  24. private const CJK = '/[\p{Han}\p{Hiragana}\p{Katakana}\p{Hangul}\x{3000}-\x{303F}\x{FF00}-\x{FFEF}]/u';
  25. /** 下一行以这些开头时是新的块,不并进脚注所在段落 */
  26. private const BLOCK_START = '/^ {0,3}(#{1,6}(\s|$)|>|[-*+]\s|\d{1,9}[.)]\s|`{3,}|~{3,}|\||<|([-*_=])[ \t]*(\3[ \t]*){2,}$)/';
  27. private string $src = '';
  28. private int $len = 0;
  29. private string $out = '';
  30. /**
  31. * @param Closure(string $kind, string $raw, string $inner, string $info): int $register
  32. * 登记一次调用并返回下标。kind:template / wikilink / note-backtick / note-fence
  33. */
  34. public function run(string $source, Dialect $dialect, Closure $register): string
  35. {
  36. $this->src = $source;
  37. $this->len = strlen($source);
  38. $this->out = '';
  39. $i = 0;
  40. $lineStart = true;
  41. while ($i < $this->len) {
  42. if ($lineStart) {
  43. $lineStart = false;
  44. $fence = $this->fenceAt($i);
  45. if ($fence !== null) {
  46. [$marker, $info] = $fence;
  47. [$contentStart, $contentEnd, $after] = $this->closingFence($i, $marker);
  48. if ($this->isNoteFence($marker, $info, $dialect)) {
  49. $inner = substr($this->src, $contentStart, max(0, $contentEnd - $contentStart));
  50. $raw = substr($this->src, $i, $after - $i);
  51. $index = $register('note-fence', rtrim($raw, "\n"), rtrim($inner, "\n"), $info);
  52. [$i, $lineStart] = $this->joinFence($index, $after);
  53. } else {
  54. $this->out .= substr($this->src, $i, $after - $i);
  55. $i = $after;
  56. $lineStart = true;
  57. }
  58. continue;
  59. }
  60. }
  61. $char = $this->src[$i];
  62. if ($char === '\\' && $i + 1 < $this->len && in_array($this->src[$i + 1], ['{', '[', '`'], true)) {
  63. $this->out .= substr($this->src, $i, 2);
  64. $i += 2;
  65. continue;
  66. }
  67. if ($char === '{' && substr($this->src, $i, 2) === '{{') {
  68. $end = $this->matchBraces($i);
  69. if ($end !== null) {
  70. $raw = substr($this->src, $i, $end + 2 - $i);
  71. $this->out .= Placeholder::make($register('template', $raw, substr($raw, 2, -2), ''));
  72. $i = $end + 2;
  73. } else {
  74. $this->out .= '{{';
  75. $i += 2;
  76. }
  77. continue;
  78. }
  79. if ($char === '[' && substr($this->src, $i, 2) === '[[') {
  80. $close = $this->sameLine($i + 2, ']]');
  81. if ($close !== null && $close > $i + 2) {
  82. $raw = substr($this->src, $i, $close + 2 - $i);
  83. $this->out .= Placeholder::make($register('wikilink', $raw, substr($raw, 2, -2), ''));
  84. $i = $close + 2;
  85. continue;
  86. }
  87. }
  88. if ($char === '`') {
  89. $i = $dialect === Dialect::MarkdownE2
  90. ? $this->skipCodeSpan($i)
  91. : $this->legacyBacktick($i, $register);
  92. continue;
  93. }
  94. $this->out .= $char;
  95. $i++;
  96. if ($char === "\n") {
  97. $lineStart = true;
  98. }
  99. }
  100. return $this->out;
  101. }
  102. /**
  103. * 行首是否是围栏开头。
  104. *
  105. * @return array{0: string, 1: string}|null [围栏符号串, info string]
  106. */
  107. private function fenceAt(int $i): ?array
  108. {
  109. $line = $this->lineAt($i);
  110. if (preg_match('/^ {0,3}(`{3,}|~{3,})(.*)$/', rtrim($line, "\r"), $m) !== 1) {
  111. return null;
  112. }
  113. // 反引号围栏的 info 里不能再有反引号(CommonMark 规则),否则这行只是行内代码
  114. if ($m[1][0] === '`' && str_contains($m[2], '`')) {
  115. return null;
  116. }
  117. return [$m[1], trim($m[2])];
  118. }
  119. /**
  120. * 找围栏的结束行。没有结束行就到文末(与 CommonMark 一致;v2 旧方言也是补到文末)。
  121. *
  122. * @return array{0: int, 1: int, 2: int} [内容起点, 内容终点(不含), 结束行之后的位置]
  123. */
  124. private function closingFence(int $i, string $marker): array
  125. {
  126. $contentStart = $this->nextLine($i);
  127. $pos = $contentStart;
  128. $closing = '/^ {0,3}'.preg_quote($marker[0], '/').'{'.strlen($marker).',}[ \t]*\r?$/';
  129. while ($pos < $this->len) {
  130. $line = $this->lineAt($pos);
  131. if (preg_match($closing, $line) === 1) {
  132. return [$contentStart, $pos, $this->nextLine($pos)];
  133. }
  134. $pos = $this->nextLine($pos);
  135. }
  136. return [$contentStart, $this->len, $this->len];
  137. }
  138. private function isNoteFence(string $marker, string $info, Dialect $dialect): bool
  139. {
  140. $word = strtolower(explode(' ', $info, 2)[0]);
  141. if ($word === 'note') {
  142. return true;
  143. }
  144. // 旧方言:单独一行的 ``` 就是多行脚注
  145. return $dialect === Dialect::Markdown && $info === '' && $marker === '```';
  146. }
  147. /**
  148. * 脚注围栏的落点。
  149. *
  150. * 上一行有字:去掉换行,占位符直接贴在上一行末尾(角标贴着前面的字)。
  151. * 下一行有字且不是新块:把下一行接到占位符后面;两侧都是 CJK 时不加空格,否则加一个空格。
  152. * 这样「前半句 / ```note … ``` / 后半句」三段合成一个段落,脚注在句中(v2 旧方言就是这个效果)。
  153. * 两侧都是空行时占位符独占一段,由 IR 构建时并到上一段末尾。
  154. *
  155. * @return array{0: int, 1: bool} [继续扫描的位置, 是否在行首]
  156. */
  157. private function joinFence(int $index, int $after): array
  158. {
  159. $left = null;
  160. if (str_ends_with($this->out, "\n") && ! $this->previousLineBlank()) {
  161. $this->out = rtrim($this->out, "\r\n");
  162. $left = mb_substr($this->out, -1);
  163. }
  164. $this->out .= Placeholder::make($index);
  165. if ($after >= $this->len) {
  166. return [$after, false];
  167. }
  168. $next = $this->lineAt($after);
  169. if (trim($next) === '' || preg_match(self::BLOCK_START, $next) === 1) {
  170. $this->out .= "\n";
  171. return [$after, true];
  172. }
  173. $indent = strspn($next, " \t");
  174. $right = mb_substr(substr($next, $indent), 0, 1);
  175. $bothCjk = ($left === null || $this->isCjk($left)) && $this->isCjk($right);
  176. $this->out .= $bothCjk ? '' : ' ';
  177. return [$after + $indent, false];
  178. }
  179. private function previousLineBlank(): bool
  180. {
  181. $body = substr($this->out, 0, -1);
  182. $start = strrpos($body, "\n");
  183. $line = $start === false ? $body : substr($body, $start + 1);
  184. return trim($line) === '';
  185. }
  186. private function isCjk(string $char): bool
  187. {
  188. return $char !== '' && preg_match(self::CJK, $char) === 1;
  189. }
  190. /**
  191. * 从 `{{` 起按深度配对,返回配对的 `}}` 的位置;不配对或超深返回 null。
  192. */
  193. private function matchBraces(int $i): ?int
  194. {
  195. $depth = 0;
  196. $j = $i;
  197. while ($j < $this->len - 1) {
  198. $pair = substr($this->src, $j, 2);
  199. if ($pair === '{{') {
  200. $depth++;
  201. if ($depth > self::MAX_BRACE_DEPTH) {
  202. return null;
  203. }
  204. $j += 2;
  205. continue;
  206. }
  207. if ($pair === '}}') {
  208. $depth--;
  209. if ($depth === 0) {
  210. return $j;
  211. }
  212. $j += 2;
  213. continue;
  214. }
  215. $j++;
  216. }
  217. return null;
  218. }
  219. /**
  220. * E2:行内代码原样跳过(里面的 `{{`、`[[` 不是模板)。反引号串长度 n,找下一个恰好 n 个的串。
  221. */
  222. private function skipCodeSpan(int $i): int
  223. {
  224. $n = strspn($this->src, '`', $i);
  225. $pos = $i + $n;
  226. while (($found = strpos($this->src, str_repeat('`', $n), $pos)) !== false) {
  227. $run = strspn($this->src, '`', $found);
  228. if ($run === $n) {
  229. $end = $found + $n;
  230. $this->out .= substr($this->src, $i, $end - $i);
  231. return $end;
  232. }
  233. $pos = $found + $run;
  234. }
  235. $this->out .= substr($this->src, $i, $n);
  236. return $i + $n;
  237. }
  238. /**
  239. * 旧方言:`` `x` `` 是脚注。配对口径同 v2 的正则 /`(.+?)`/:正文至少一个字符,且可以以反引号开头
  240. * (``` ``x`` ``` 的正文是 `` `x ``,剩下一个反引号是普通字符)。
  241. *
  242. * 与 v2 不同的两点(都按作者本意,v2 在这些写法上配错了对):
  243. * - 可以跨行(不跨空行),反引号后可以直接换行。库里有大量「词`多行注释`」的写法,v2 的正则不跨行,
  244. * 配错了对,剩下的反引号又被 CommonMark 配成跨行的代码 span
  245. * - ``` ``x`` ```(两个以上成对):整对是边界,正文是 x(v2 的正文是「`x」)
  246. * 单行内写对的单反引号,配对结果与 v2 相同。
  247. *
  248. * @param Closure(string, string, string, string): int $register
  249. */
  250. private function legacyBacktick(int $i, Closure $register): int
  251. {
  252. // 两个以上的反引号成对(``x``):作者把整对当作脚注边界,正文是中间的部分
  253. $n = strspn($this->src, '`', $i);
  254. if ($n >= 2 && ($pair = $this->matchingRun($i + $n, $n)) !== null) {
  255. $raw = substr($this->src, $i, $pair + $n - $i);
  256. $this->out .= Placeholder::make($register('note-backtick', $raw, substr($raw, $n, -$n), ''));
  257. return $pair + $n;
  258. }
  259. $close = $i + 2 <= $this->len ? strpos($this->src, '`', $i + 2) : false;
  260. $spansParagraph = $close !== false
  261. && preg_match('/\n[ \t]*\n/', substr($this->src, $i + 1, $close - $i - 1)) === 1;
  262. if ($close === false || $spansParagraph) {
  263. // 配不上对的反引号是普通字符:旧方言里反引号从来不是代码,转义掉免得 CommonMark 配成代码 span
  264. $this->out .= '\\`';
  265. return $i + 1;
  266. }
  267. $raw = substr($this->src, $i, $close + 1 - $i);
  268. $this->out .= Placeholder::make($register('note-backtick', $raw, substr($raw, 1, -1), ''));
  269. return $close + 1;
  270. }
  271. /**
  272. * 从 $from 起找恰好 $n 个反引号的串(不跨空行),返回它的起点。
  273. */
  274. private function matchingRun(int $from, int $n): ?int
  275. {
  276. $pos = $from;
  277. while (($found = strpos($this->src, str_repeat('`', $n), $pos)) !== false) {
  278. if (preg_match('/\n[ \t]*\n/', substr($this->src, $from, $found - $from)) === 1) {
  279. return null;
  280. }
  281. $run = strspn($this->src, '`', $found);
  282. if ($run === $n) {
  283. return $found;
  284. }
  285. $pos = $found + $run;
  286. }
  287. return null;
  288. }
  289. /**
  290. * 在同一行内找 $needle,找不到返回 null。
  291. */
  292. private function sameLine(int $from, string $needle): ?int
  293. {
  294. $found = strpos($this->src, $needle, $from);
  295. if ($found === false) {
  296. return null;
  297. }
  298. $newline = strpos($this->src, "\n", $from);
  299. return ($newline === false || $found < $newline) ? $found : null;
  300. }
  301. private function lineAt(int $i): string
  302. {
  303. $end = strpos($this->src, "\n", $i);
  304. return substr($this->src, $i, ($end === false ? $this->len : $end) - $i);
  305. }
  306. private function nextLine(int $i): int
  307. {
  308. $end = strpos($this->src, "\n", $i);
  309. return $end === false ? $this->len : $end + 1;
  310. }
  311. }