Find code by asking what it does. A CLI for coding agents that uses Jev to discover relevant files and source context.
TypeScript ai-sdkclaude-codeclicode-searchcodexcoding-agentscontext-retrievaldeveloper-toolsjevsemantic-searchtypescriptvercel-ai-gateway
Parse long Python comment runs in linear time
taeskimf0715412704c
diff --git a/docs/architecture.md b/docs/architecture.mdindex a8c9974..6615672 100644--- a/docs/architecture.md+++ b/docs/architecture.md@@ -63,7 +63,10 @@ with syntax errors use text fallback. Bare-CR Python also uses text fallbackbecause retrieval coordinates count LF lines. So does generated source nestedbeyond what a parser handles: TypeScript that exhausts the call stack, and Pythonnested more than 50,000 levels, near where Tree-sitter queries stop matching and-slow down sharply. Repository source is never executed.+slow down sharply. Long runs of whole-line Python comments are parsed as blank+lines of the same length, because the grammar's scanner is quadratic in a run's+length; a run that may belong to a string is parsed as written.+Repository source is never executed.See [parser contracts](../test/parser/README.md) and[measurement evidence](../specs/tree-sitter/RESULTS.md).diff --git a/packages/core/src/parser-helpers.mjs b/packages/core/src/parser-helpers.mjsindex ad38213..32db3c8 100644--- a/packages/core/src/parser-helpers.mjs+++ b/packages/core/src/parser-helpers.mjs@@ -4,6 +4,57 @@ import { declarations } from "./parser-declarations.mjs";import { preview } from "./parser-preview.mjs";let initialized;const languages = new Map();+// Tree-sitter's Python scanner rescans the rest of a comment run at every line of it, so parsing is+// quadratic in the run's length: 8,000 consecutive comment lines took 9.5 s. Every helper skips+// comment nodes, so long runs are parsed as blank lines of the same length, which keeps line and+// column coordinates. Only quotes, backslashes and braces can change where strings and f-string+// fields begin or end; if a blanked line containing one lies inside a string, or the substitute+// has an error, the original source is parsed instead.+const longCommentRun = 256;+const commentLine = /^[ \t\f]*#/;+function parsePython(parser, source) {+ const lines = source.split("\n");+ let run = 0,+ longest = 0;+ for (const line of lines) {+ if (commentLine.test(line)) longest = Math.max(longest, ++run);+ else if (line.trim()) run = 0;+ }+ if (longest < longCommentRun) return parser.parse(source);+ const suspect = [];+ const blanked = lines+ .map((line, row) => {+ if (!commentLine.test(line)) return line;+ const end = line.endsWith("\r") ? line.length - 1 : line.length;+ if (/["'\\{}]/.test(line.slice(0, end))) suspect.push(row);+ return " ".repeat(end) + line.slice(end);+ })+ .join("\n");+ const tree = parser.parse(blanked);+ if (tree && !tree.rootNode.hasError) {+ // Queries are unreliable on trees too deep for them; parse() rejects those trees anyway.+ if (deeperThan(tree, maxTreeDepth) || !insideStrings(tree, suspect, lines.length)) return tree;+ }+ tree?.delete();+ return parser.parse(source);+}+let stringSpans;+function insideStrings(tree, rows, lineCount) {+ if (!rows.length) return false;+ stringSpans ??= new Query(languages.get("python"), "(string) @string");+ // Blanked lines hold no quotes, so a string can only span them, never start or end on them.+ const depth = new Int32Array(lineCount + 1);+ for (const { node } of stringSpans.captures(tree.rootNode)) {+ const first = node.startPosition.row + 1,+ last = node.endPosition.row;+ if (first < last) {+ depth[first]++;+ depth[last]--;+ }+ }+ for (let row = 1; row < depth.length; row++) depth[row] += depth[row - 1];+ return rows.some((row) => depth[row] > 0);+}async function parse(source, action, languageName = "python") {// Retrieval coordinates count LF lines; normalizing bare CR would mislabel original bytes.if (languageName === "python" && /\r(?!\n)/.test(source)) return null;@@ -21,7 +72,7 @@ async function parse(source, action, languageName = "python") {let tree;try {parser.setLanguage(languages.get(languageName));- tree = parser.parse(source);+ tree = languageName === "python" ? parsePython(parser, source) : parser.parse(source);if (!tree || tree.rootNode.hasError) return null;// Tree-sitter queries stop reporting matches beyond 65,535 levels of nesting and slow down// sharply near that depth: 70,000 chained `+` took 8 s, 100,000 took 74 s. Treat suchdiff --git a/test/parser/source.test.ts b/test/parser/source.test.tsindex fdc92e4..89ce1f6 100644--- a/test/parser/source.test.ts+++ b/test/parser/source.test.ts@@ -454,3 +454,30 @@ test("class context is marked structurally, not inferred from member names", asy["Widget@1-1", "Widget.context@2-2 [class context]", "Widget.context@3-5"],);});++test("long Python comment runs parse quickly without changing declarations", async () => {+ const run = (indent: string, count: number) =>+ Array.from({ length: count }, (_, i) => `${indent}# n${i} don't "q" {x}`).join("\n");+ // Parsing was quadratic in a comment run: 8,000 lines took 9.5 s.+ const source =+ `class Box:\n${run(" ", 12_000)}\n def first(self):\n return 1\n` ++ `${run("", 12_000)}\ndef second():\n return 2\n`;+ const started = performance.now();+ const result = await inspect(+ { path: "notes.py", source, contentHash: "fixture" },+ { maxUnitBytes: Buffer.byteLength(source) },+ );+ assert.ok(performance.now() - started < 5_000, "comment runs were not parsed quickly");+ assert.deepEqual(+ result.units.map(({ name, range }) => `${name}@${range.startLine}-${range.endLine}`),+ ["Box.context@1-12001", "Box.first@12002-12003", "second@24004-24005"],+ );+ // Comment-looking lines inside a string are string content, even with quotes and braces.+ const inString = `value = f"""\n${run("", 300)}\n"""\n` + 'def after():\n return """#"""\n';+ const parsed = await inspect({ path: "doc.py", source: inString, contentHash: "fixture" });+ assert.equal(parsed.mode, "python");+ assert.deepEqual(+ parsed.units.map(({ name, range }) => `${name}@${range.startLine}-${range.endLine}`),+ ["after@303-304"],+ );+});