Skip to content

Find code by asking what it does. A CLI for coding agents that uses Jev to discover relevant files and source context.

Parse long Python comment runs in linear time

taeskimf0715412704c
diff --git a/docs/architecture.md b/docs/architecture.md
index a8c9974..6615672 100644
--- a/docs/architecture.md
+++ b/docs/architecture.md
@@ -63,7 +63,10 @@ with syntax errors use text fallback. Bare-CR Python also uses text fallback
because retrieval coordinates count LF lines. So does generated source nested
beyond what a parser handles: TypeScript that exhausts the call stack, and Python
nested more than 50,000 levels, near where Tree-sitter queries stop matching and
-slow down sharply. Repository source is never executed.
+slow down sharply. Long runs of whole-line Python comments are parsed as blank
+lines of the same length, because the grammar's scanner is quadratic in a run's
+length; a run that may belong to a string is parsed as written.
+Repository source is never executed.
See [parser contracts](../test/parser/README.md) and
[measurement evidence](../specs/tree-sitter/RESULTS.md).
diff --git a/packages/core/src/parser-helpers.mjs b/packages/core/src/parser-helpers.mjs
index ad38213..32db3c8 100644
--- a/packages/core/src/parser-helpers.mjs
+++ b/packages/core/src/parser-helpers.mjs
@@ -4,6 +4,57 @@ import { declarations } from "./parser-declarations.mjs";
import { preview } from "./parser-preview.mjs";
let initialized;
const languages = new Map();
+// Tree-sitter's Python scanner rescans the rest of a comment run at every line of it, so parsing is
+// quadratic in the run's length: 8,000 consecutive comment lines took 9.5 s. Every helper skips
+// comment nodes, so long runs are parsed as blank lines of the same length, which keeps line and
+// column coordinates. Only quotes, backslashes and braces can change where strings and f-string
+// fields begin or end; if a blanked line containing one lies inside a string, or the substitute
+// has an error, the original source is parsed instead.
+const longCommentRun = 256;
+const commentLine = /^[ \t\f]*#/;
+function parsePython(parser, source) {
+ const lines = source.split("\n");
+ let run = 0,
+ longest = 0;
+ for (const line of lines) {
+ if (commentLine.test(line)) longest = Math.max(longest, ++run);
+ else if (line.trim()) run = 0;
+ }
+ if (longest < longCommentRun) return parser.parse(source);
+ const suspect = [];
+ const blanked = lines
+ .map((line, row) => {
+ if (!commentLine.test(line)) return line;
+ const end = line.endsWith("\r") ? line.length - 1 : line.length;
+ if (/["'\\{}]/.test(line.slice(0, end))) suspect.push(row);
+ return " ".repeat(end) + line.slice(end);
+ })
+ .join("\n");
+ const tree = parser.parse(blanked);
+ if (tree && !tree.rootNode.hasError) {
+ // Queries are unreliable on trees too deep for them; parse() rejects those trees anyway.
+ if (deeperThan(tree, maxTreeDepth) || !insideStrings(tree, suspect, lines.length)) return tree;
+ }
+ tree?.delete();
+ return parser.parse(source);
+}
+let stringSpans;
+function insideStrings(tree, rows, lineCount) {
+ if (!rows.length) return false;
+ stringSpans ??= new Query(languages.get("python"), "(string) @string");
+ // Blanked lines hold no quotes, so a string can only span them, never start or end on them.
+ const depth = new Int32Array(lineCount + 1);
+ for (const { node } of stringSpans.captures(tree.rootNode)) {
+ const first = node.startPosition.row + 1,
+ last = node.endPosition.row;
+ if (first < last) {
+ depth[first]++;
+ depth[last]--;
+ }
+ }
+ for (let row = 1; row < depth.length; row++) depth[row] += depth[row - 1];
+ return rows.some((row) => depth[row] > 0);
+}
async function parse(source, action, languageName = "python") {
// Retrieval coordinates count LF lines; normalizing bare CR would mislabel original bytes.
if (languageName === "python" && /\r(?!\n)/.test(source)) return null;
@@ -21,7 +72,7 @@ async function parse(source, action, languageName = "python") {
let tree;
try {
parser.setLanguage(languages.get(languageName));
- tree = parser.parse(source);
+ tree = languageName === "python" ? parsePython(parser, source) : parser.parse(source);
if (!tree || tree.rootNode.hasError) return null;
// Tree-sitter queries stop reporting matches beyond 65,535 levels of nesting and slow down
// sharply near that depth: 70,000 chained `+` took 8 s, 100,000 took 74 s. Treat such
diff --git a/test/parser/source.test.ts b/test/parser/source.test.ts
index fdc92e4..89ce1f6 100644
--- a/test/parser/source.test.ts
+++ b/test/parser/source.test.ts
@@ -454,3 +454,30 @@ test("class context is marked structurally, not inferred from member names", asy
["Widget@1-1", "Widget.context@2-2 [class context]", "Widget.context@3-5"],
);
});
+
+test("long Python comment runs parse quickly without changing declarations", async () => {
+ const run = (indent: string, count: number) =>
+ Array.from({ length: count }, (_, i) => `${indent}# n${i} don't "q" {x}`).join("\n");
+ // Parsing was quadratic in a comment run: 8,000 lines took 9.5 s.
+ const source =
+ `class Box:\n${run(" ", 12_000)}\n def first(self):\n return 1\n` +
+ `${run("", 12_000)}\ndef second():\n return 2\n`;
+ const started = performance.now();
+ const result = await inspect(
+ { path: "notes.py", source, contentHash: "fixture" },
+ { maxUnitBytes: Buffer.byteLength(source) },
+ );
+ assert.ok(performance.now() - started < 5_000, "comment runs were not parsed quickly");
+ assert.deepEqual(
+ result.units.map(({ name, range }) => `${name}@${range.startLine}-${range.endLine}`),
+ ["Box.context@1-12001", "Box.first@12002-12003", "second@24004-24005"],
+ );
+ // Comment-looking lines inside a string are string content, even with quotes and braces.
+ const inString = `value = f"""\n${run("", 300)}\n"""\n` + 'def after():\n return """#"""\n';
+ const parsed = await inspect({ path: "doc.py", source: inString, contentHash: "fixture" });
+ assert.equal(parsed.mode, "python");
+ assert.deepEqual(
+ parsed.units.map(({ name, range }) => `${name}@${range.startLine}-${range.endLine}`),
+ ["after@303-304"],
+ );
+});