Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

fix(chat): preserve thinking tags inside streamed code fences

dajiaohuang3ad6774717b0
diff --git a/web/lib/think-segments.ts b/web/lib/think-segments.ts
index 201526549..6cea2f1c4 100644
--- a/web/lib/think-segments.ts
+++ b/web/lib/think-segments.ts
@@ -37,7 +37,7 @@ export interface ThinkSegment {
export type ContentSegment = TextSegment | ThinkSegment;
-const FENCED_CODE_REGEX = /```[\s\S]*?```/g;
+const FENCE_LINE_REGEX = /^ {0,3}(`{3,}|~{3,})([^\r\n]*)(?:\r?\n|$)/gm;
const FENCED_PLACEHOLDER_REGEX = /\u0000FENCED_(\d+)\u0000/g;
// `?<\s*think(?:ing)?\b[^>]*>`?
@@ -52,10 +52,37 @@ function closeTagRegex(tag: string): RegExp {
function maskFencedCode(input: string): { masked: string; blocks: string[] } {
const blocks: string[] = [];
- const masked = input.replace(FENCED_CODE_REGEX, (match) => {
- blocks.push(match);
- return `\u0000FENCED_${blocks.length - 1}\u0000`;
- });
+ let masked = "";
+ let cursor = 0;
+ let start = -1;
+ let opening = "";
+ const appendBlock = (end: number) => {
+ masked += input.slice(cursor, start);
+ blocks.push(input.slice(start, end));
+ masked += `\u0000FENCED_${blocks.length - 1}\u0000`;
+ cursor = end;
+ start = -1;
+ opening = "";
+ };
+
+ for (const match of input.matchAll(FENCE_LINE_REGEX)) {
+ const marker = match[1];
+ if (start >= 0) {
+ if (
+ marker[0] === opening[0] &&
+ marker.length >= opening.length &&
+ match[2].trim() === ""
+ ) {
+ appendBlock(match.index + match[0].length);
+ }
+ } else if (marker[0] !== "`" || !match[2].includes("`")) {
+ start = match.index;
+ opening = marker;
+ }
+ }
+ // A streaming code block remains code until its matching fence arrives.
+ if (start >= 0) appendBlock(input.length);
+ masked += input.slice(cursor);
return { masked, blocks };
}
diff --git a/web/tests/thinking-code-fences.test.ts b/web/tests/thinking-code-fences.test.ts
new file mode 100644
index 000000000..8dba6793a
--- /dev/null
+++ b/web/tests/thinking-code-fences.test.ts
@@ -0,0 +1,28 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+
+import { parseModelThinkingSegments } from "../lib/think-segments";
+
+for (const [name, input] of [
+ ["tilde fence", "Before\n~~~html\n<think>literal HTML</think>\n~~~\nAfter"],
+ ["long backtick fence", "Before\n````markdown\n```\n<think>literal example</think>\n```\n````\nAfter"],
+ ["unfinished streaming fence", "Before\n```html\n<think>literal HTML"],
+] as const) {
+ test(`assistant thinking parser preserves ${name}`, () => {
+ assert.deepEqual(parseModelThinkingSegments(input), [
+ { kind: "text", content: input },
+ ]);
+ });
+}
+
+test("thinking after a closed tilde fence remains a thinking segment", () => {
+ const code = "~~~html\n<think>literal HTML</think>\n~~~\n";
+ assert.deepEqual(
+ parseModelThinkingSegments(`${code}<think>actual reasoning</think>Answer`),
+ [
+ { kind: "text", content: code },
+ { kind: "think", content: "actual reasoning", closed: true },
+ { kind: "text", content: "Answer" },
+ ],
+ );
+});