The Open Standard for Generative UI
fix(parser): preserve markdown fences and comments inside strings (#605)
Sayandip Roy96aaed494bfe
diff --git a/packages/lang-core/src/parser/__tests__/parser.test.ts b/packages/lang-core/src/parser/__tests__/parser.test.tsindex 2bd339aa3..b95ed32b2 100644--- a/packages/lang-core/src/parser/__tests__/parser.test.ts+++ b/packages/lang-core/src/parser/__tests__/parser.test.ts@@ -235,3 +235,55 @@ describe("orphaned statements", () => {expect(result.meta.orphaned).toHaveLength(0);});});++describe("markdown fences and multiline comments in strings", () => {+ it("preserves js markdown fences inside strings", () => {+ const code = 'root = Title("```js\\nconsole.log(\\"Hello World\\");\\n```")';+ const result = parse(code, schema);+ expect(result.meta.errors).toHaveLength(0);+ expect(result.root).not.toBeNull();+ expect(result.root?.props.text).toBe('```js\nconsole.log("Hello World");\n```');+ });++ it("preserves python fenced code blocks inside strings", () => {+ const code = "root = Title(\"```python\\nprint('hi')\\n```\")";+ const result = parse(code, schema);+ expect(result.meta.errors).toHaveLength(0);+ expect(result.root).not.toBeNull();+ expect(result.root?.props.text).toBe("```python\nprint('hi')\n```");+ });++ it("preserves // inside multiline strings", () => {+ const code = `root = Title("+Hello // World+https://example.com/+")`;+ const result = parse(code, schema);+ expect(result.meta.errors).toHaveLength(0);+ expect(result.root).not.toBeNull();+ expect(result.root?.props.text).toBe("\nHello // World\nhttps://example.com/\n");+ });++ it("preserves # inside multiline strings", () => {+ const code = `root = Title("+Hello # World+https://example.com/+")`;+ const result = parse(code, schema);+ expect(result.meta.errors).toHaveLength(0);+ expect(result.root).not.toBeNull();+ expect(result.root?.props.text).toBe("\nHello # World\nhttps://example.com/\n");+ });++ it("does not treat apostrophes as string delimiters when stripping fences", () => {+ const code = `Here's the code:++\`\`\`js+root = Title("hello")+\`\`\``;+ const result = parse(code, schema);+ expect(result.meta.errors).toHaveLength(0);+ expect(result.root).not.toBeNull();+ expect(result.root?.props.text).toBe("hello");+ });+});diff --git a/packages/lang-core/src/parser/parser.ts b/packages/lang-core/src/parser/parser.tsindex b76b6240f..9be78733c 100644--- a/packages/lang-core/src/parser/parser.ts+++ b/packages/lang-core/src/parser/parser.ts@@ -246,6 +246,22 @@ function buildResult(// Public API// ─────────────────────────────────────────────────────────────────────────────+function skipString(input: string, start: number): number {+ if (input[start] !== '"') return start;+ let i = start + 1;+ while (i < input.length) {+ const c = input[i];+ if (c === "\\") {+ i += 2; // skip escape character and the escaped character+ } else if (c === '"') {+ return i + 1; // return index after the closing quote+ } else {+ i++;+ }+ }+ return i; // return length if string was unclosed+}+/** Extract code from markdown fences, or return as-is if no fences found.* String-context-aware: skips ``` inside double-quoted strings. */export function stripFences(input: string): string {@@ -253,8 +269,29 @@ export function stripFences(input: string): string {let i = 0;while (i < input.length) {- // Look for opening ```- const fenceStart = input.indexOf("```", i);+ // Scan for opening ``` while tracking string context+ let fenceStart = -1;+ while (i < input.length) {+ const nextI = skipString(input, i);+ if (nextI > i) {+ i = nextI;+ continue;+ }++ const c = input[i];+ if (+ c === "`" &&+ i + 1 < input.length &&+ input[i + 1] === "`" &&+ i + 2 < input.length &&+ input[i + 2] === "`"+ ) {+ fenceStart = i;+ break;+ }+ i++;+ }+if (fenceStart === -1) break;// Skip language tag until newline@@ -269,26 +306,15 @@ export function stripFences(input: string): string {j++; // skip the newline// Scan for closing ``` while tracking string context- let inStr = false;let closePos = -1;let k = j;while (k < input.length) {- const c = input[k];- if (inStr) {- if (c === "\\" && k + 1 < input.length) {- k += 2; // skip escaped character- continue;- }- if (c === '"') inStr = false;- k++;- continue;- }- // Not in string- if (c === '"') {- inStr = true;- k++;+ const nextK = skipString(input, k);+ if (nextK > k) {+ k = nextK;continue;}+ const c = input[k];if (c === "`" &&k + 1 < input.length &&@@ -333,10 +359,10 @@ export function stripFences(input: string): string {/** Strip // and # line comments outside of strings (handles both " and ' delimiters). */function stripComments(input: string): string {+ let inStr: false | '"' | "'" = false;return input.split("\n").map((line) => {- let inStr: false | '"' | "'" = false;for (let i = 0; i < line.length; i++) {const c = line[i];if (inStr) {