Skip to content

Official Codebase of 3DCodeVerse

Python

languages: one node parses every JS file of a lint, and a file that already parsed is not sent again

gaoypeng13719fd611cf
diff --git a/harness/codeverse3d/languages/_js_lint.py b/harness/codeverse3d/languages/_js_lint.py
index d24251b4..884b45ee 100644
--- a/harness/codeverse3d/languages/_js_lint.py
+++ b/harness/codeverse3d/languages/_js_lint.py
@@ -2,28 +2,31 @@
Replaces two copies of the same algorithm: ``threejs/lint.py:check_syntax`` +
``_lint_imports`` and ``scene_threejs/lint.py:_node_check`` + ``_check_imports``.
-Both ran ``node --input-type=module --check`` over stdin and scraped the same
-``[stdin]:N`` / ``SyntaxError:`` lines; both matched static, re-export and dynamic
-import specifiers with near-identical regexes and applied the same allowlist
-(``three``, ``three/addons/*``, ``three/examples/jsm/*``, relative files under
-``src/``). They differed only in message text (owned by the caller via
-``make_finding``, as in :mod:`_ast_lint`) and in two defects this module fixes:
-threejs's fallback message was a ``list`` (``splitlines()[-1:]``), and scene's
-escape check was a ``str.startswith`` prefix test that let ``src2/`` pass.
-Escape is decided before existence so a symlink/``..`` hop out of ``src/`` is
-reported as an escape, never as "missing".
+Both matched static, re-export and dynamic import specifiers with near-identical
+regexes and applied the same allowlist (``three``, ``three/addons/*``,
+``three/examples/jsm/*``, relative files under ``src/``). They differed only in
+message text (owned by the caller via ``make_finding``, as in :mod:`_ast_lint`) and
+in two defects this module fixes: threejs's fallback message was a ``list``
+(``splitlines()[-1:]``), and scene's escape check was a ``str.startswith`` prefix
+test that let ``src2/`` pass. Escape is decided before existence so a
+symlink/``..`` hop out of ``src/`` is reported as an escape, never as "missing".
+
+Syntax is ``runtime_js/lib/syntax_check.mjs`` (export_glb's checker too) in ONE node
+process per lint, skipping files whose bytes already parsed — it was one ``node --check``
+per file: 51 on a scene, 44 of them unchanged ``src/lib`` modules, ≈ 2.5 s a build call.
"""
from __future__ import annotations
+import hashlib
import re
-import subprocess
-from collections.abc import Callable
+from collections.abc import Callable, Sequence
from dataclasses import dataclass
from enum import StrEnum
from pathlib import Path
from codeverse3d.contracts.artifacts import GateFinding
+from codeverse3d.spatial.node import NodeError, run_node, runtime_js_dir
# threejs's regexes (the superset: side-effect ``import './x.js'``, ``export * from``,
# ``export * as ns from`` and multi-line ``import {\n a,\n b\n} from`` all match)
@@ -31,7 +34,6 @@ _IMPORT_RE = re.compile(
r"""(?:^|\n)\s*(?:import\s+(?:[^'";]*?\s+from\s+)?|export\s+(?:\*(?:\s+as\s+\w+)?|\{[^}]*\})\s+from\s+)['"]([^'"]+)['"]"""
)
_DYN_IMPORT_RE = re.compile(r"""\bimport\s*\(\s*['"]([^'"]+)['"]\s*\)""")
-_STDIN_LINE_RE = re.compile(r"\[stdin\]:(\d+)|:(\d+)\n")
_URL_RE = re.compile(r"https?://")
@@ -45,27 +47,37 @@ class SyntaxProblem:
stderr_tail: str
-def node_check_syntax(path: Path, node_bin: str) -> SyntaxProblem | None:
- """``None`` when ``path`` parses as ESM under ``node --input-type=module --check``.
-
- Stdin is used because ``node --check file.js`` treats a bare ``.js`` as
- CommonJS-or-detect.
- """
+#: sha256 of file contents that parsed — library modules are byte-identical build after build
+_PARSED: set[str] = set()
+
+
+def node_check_syntax(paths: Sequence[Path]) -> dict[Path, SyntaxProblem]:
+ """``{path: problem}`` for every file of ``paths`` that does not parse as an ES
+ module; one ``node`` for the lot, and none at all when every file's content
+ already parsed in this process."""
+ digests: dict[Path, str] = {}
+ out: dict[Path, SyntaxProblem] = {}
+ for p in paths:
+ try:
+ digests[p] = hashlib.sha256(p.read_bytes()).hexdigest()
+ except OSError as e:
+ out[p] = SyntaxProblem(None, f"cannot read the file: {e}", "")
+ todo = [p for p, d in digests.items() if d not in _PARSED]
+ if not todo:
+ return out
try:
- proc = subprocess.run(
- [node_bin, "--input-type=module", "--check"],
- input=path.read_bytes(), capture_output=True, timeout=30,
- )
- except (OSError, subprocess.TimeoutExpired) as e:
- return SyntaxProblem(None, f"node --check could not run: {e}", "")
- if proc.returncode == 0:
- return None
- err = proc.stderr.decode(errors="replace").strip()
- m = _STDIN_LINE_RE.search(err)
- line = int(m.group(1) or m.group(2)) if m else None
- lines = err.splitlines()
- msg = next((ln for ln in lines if "Error" in ln), lines[0] if lines else "syntax error")
- return SyntaxProblem(line, msg, err[-600:])
+ res = run_node(runtime_js_dir() / "lib" / "syntax_check.mjs", [str(p) for p in todo],
+ node_args=["--experimental-vm-modules"], timeout_s=60, check=False)
+ bad = {row["file"]: row for row in (res.last_json or {})["bad"]}
+ except (NodeError, KeyError, TypeError) as e:
+ return out | {p: SyntaxProblem(None, f"the node syntax check could not run: {e}", "") for p in todo}
+ for p in todo:
+ row = bad.get(str(p))
+ if row is None:
+ _PARSED.add(digests[p])
+ else:
+ out[p] = SyntaxProblem(row.get("line"), str(row.get("message") or "syntax error"), str(row.get("stderr", "")))
+ return out
def import_specs(src: str) -> list[tuple[str, int]]:
diff --git a/harness/codeverse3d/languages/scene_threejs/__init__.py b/harness/codeverse3d/languages/scene_threejs/__init__.py
index 7ef10300..8e16f933 100644
--- a/harness/codeverse3d/languages/scene_threejs/__init__.py
+++ b/harness/codeverse3d/languages/scene_threejs/__init__.py
@@ -25,6 +25,7 @@ from codeverse3d.languages._docs import RuntimeDocs
from codeverse3d.languages._js_lint import (
ImportKind,
ImportVerdict,
+ SyntaxProblem,
check_imports,
node_check_syntax,
)
@@ -80,10 +81,7 @@ def _is_lib(path: Path, ws: Workspace) -> bool:
return path.parent == ws.src / "lib"
-def _node_check(path: Path, rel: str) -> GateFinding | None:
- p = node_check_syntax(path, get_settings().binaries.node or "node")
- if p is None:
- return None
+def _syntax_finding(p: SyntaxProblem, rel: str) -> GateFinding:
return _f(Severity.ERROR, f"syntax: {p.message}", target=f"{rel}:{p.line}" if p.line else rel,
hint="fix the syntax error at the quoted line (node --check)", line=p.line, detail=p.stderr_tail)
@@ -124,12 +122,12 @@ def lint(ws: Workspace) -> GateReport:
findings.append(_f(Severity.ERROR, "src/scene.js is missing", target="src/scene.js",
hint="create src/scene.js exporting createScene({THREE, renderer, loaders})"))
big: list[str] = []
+ syntax = node_check_syntax(files)
for path in files:
rel = path.relative_to(ws.root).as_posix()
text = path.read_text(errors="replace")
- syntax = _node_check(path, rel)
- if syntax:
- findings.append(syntax)
+ if path in syntax:
+ findings.append(_syntax_finding(syntax[path], rel))
continue
if _is_lib(path, ws):
continue
@@ -723,8 +721,6 @@ class SceneThreeJsRuntime(RuntimeDocs):
"""Probe + shader preflight (one browser boot); ok iff the module loads
and no shader errors. External contract unchanged: the same two
GateReports land under ``artifacts/gates/``."""
- from codeverse3d.config import get_settings
-
t0 = time.time()
tmo = min(float(timeout_s or get_settings().limits.build_timeout_s), 120.0)
ws.artifacts.mkdir(parents=True, exist_ok=True)
diff --git a/harness/codeverse3d/languages/threejs/__init__.py b/harness/codeverse3d/languages/threejs/__init__.py
index 7de983c8..b2fc6ba6 100644
--- a/harness/codeverse3d/languages/threejs/__init__.py
+++ b/harness/codeverse3d/languages/threejs/__init__.py
@@ -211,9 +211,9 @@ def lint_workspace(ws: Workspace) -> GateReport:
"""Run every static check and return the ``lint:threejs`` GateReport."""
t0 = time.time()
findings: list[GateFinding] = []
- node_bin = get_settings().binaries.node
entry = ws.src / "object.js"
sources = list_sources(ws)
+ syntax = check_syntax(sources)
if not entry.is_file():
findings.append(GateFinding(gate=GATE, severity=Severity.ERROR, target="src/object.js",
@@ -223,7 +223,7 @@ def lint_workspace(ws: Workspace) -> GateReport:
for path in sources:
src = path.read_text(errors="replace")
rel = _rel(ws, path)
- syn = check_syntax(path, node_bin)
+ syn = syntax.get(path)
if syn is not None:
findings.append(GateFinding(gate=GATE, severity=Severity.ERROR, target=rel,
message=f"{rel}:{syn.line or '?'}: {syn.message}", fix_hint="fix the syntax error at that line"))
diff --git a/harness/runtime_js/README.md b/harness/runtime_js/README.md
index ba7ed5b1..82eaa03f 100644
--- a/harness/runtime_js/README.md
+++ b/harness/runtime_js/README.md
@@ -76,7 +76,9 @@ lib/node_polyfills.mjs FileReader/self shims so GLTFExporter writes binary GLB w
lib/census.mjs per-part tri counts, world bboxes, materials, NaN check naming mesh + part (node + browser)
lib/instances.mjs bakeInstancedMeshes(THREE, root): InstancedMesh → Group of named plain meshes
lib/stack.mjs Error → {type,message,file,line,frames} with workspace-relative src/ paths
-lib/syntax_check.mjs `node --input-type=module --check` per file to locate ESM SyntaxErrors
+lib/syntax_check.mjs THE JS syntax check: every file parsed as an ES module in ONE node
+ (vm.SourceTextModule, never linked or run; `node --check` only for a failing
+ file's line). CLI for the python lints, findSyntaxError() for export_glb
lib/browser/*.js page-side ESM (served through /__runtime/): renderer.js (THE WebGLRenderer factory —
sRGB + ACES + PCF shadows + pixel ratio 1, used by the object rig AND the scene host),
camera_fit.js (azimuth/elevation → tight bbox fit; the distance math itself is
diff --git a/harness/runtime_js/lib/syntax_check.mjs b/harness/runtime_js/lib/syntax_check.mjs
index 000eef16..59b5f516 100644
--- a/harness/runtime_js/lib/syntax_check.mjs
+++ b/harness/runtime_js/lib/syntax_check.mjs
@@ -1,25 +1,42 @@
-// Locate ESM syntax errors file-by-file. V8 SyntaxErrors thrown by a dynamic
-// import() carry no file/line, so we re-check each candidate source file with
-// `node --input-type=module --check` (stdin) and parse "[stdin]:LINE".
-//
-// import { checkSyntax, findSyntaxError } from './syntax_check.mjs';
+// THE JS syntax check: every file parsed as an ES module in ONE node process
+// (vm.SourceTextModule: compiled, never linked or run). That parse error has no line, so
+// only a file that fails pays a second process, `node --input-type=module --check`.
+// node --experimental-vm-modules lib/syntax_check.mjs FILE... → {"bad": [{file, line, message, stderr}]}
+// import { findSyntaxError, listJsFiles } from './syntax_check.mjs';
import fs from 'node:fs';
import path from 'node:path';
+import vm from 'node:vm';
import { spawnSync } from 'node:child_process';
+import { fileURLToPath } from 'node:url';
-/** Check one file; returns null when it parses, else {file, line, message}. */
-export function checkSyntax(file) {
- const src = fs.readFileSync(file);
- const r = spawnSync(process.execPath, ['--input-type=module', '--check'], { input: src, encoding: 'utf8', timeout: 20000 });
+const SELF = fileURLToPath(import.meta.url);
+
+/** `node --check`'s report on one file: null when it parses, else {file, line, message, stderr}. */
+function checkOne(file) {
+ const r = spawnSync(process.execPath, ['--input-type=module', '--check'], { input: fs.readFileSync(file), encoding: 'utf8', timeout: 20000 });
if (r.status === 0) return null;
- const err = r.stderr || '';
+ const err = (r.stderr || '').trim();
const m = /\[stdin\]:(\d+)/.exec(err);
- const msg = (/^(SyntaxError:.*)$/m.exec(err) || [])[1] || err.trim().split('\n').pop() || 'syntax error';
- return { file, line: m ? Number(m[1]) : null, message: msg };
+ const msg = (/^(SyntaxError:.*)$/m.exec(err) || [])[1] || err.split('\n').pop() || 'syntax error';
+ return { file, line: m ? Number(m[1]) : null, message: msg, stderr: err.slice(-600) };
}
-/** Recursively list *.js / *.mjs under dir (skipping node_modules). */
+/** The files of `files` that do not parse as ES modules, in order. */
+export function checkFiles(files) {
+ const bad = [];
+ for (const file of files) {
+ try {
+ new vm.SourceTextModule(fs.readFileSync(file, 'utf8'), { identifier: file });
+ } catch (e) {
+ const hit = e instanceof SyntaxError ? checkOne(file) : { file, line: null, message: String(e), stderr: '' };
+ if (hit) bad.push(hit);
+ }
+ }
+ return bad;
+}
+
+/** Recursively list *.js / *.mjs under dir (skipping node_modules and dot entries). */
export function listJsFiles(dir) {
const out = [];
if (!fs.existsSync(dir)) return out;
@@ -35,11 +52,17 @@ export function listJsFiles(dir) {
return out.sort();
}
-/** First file under srcDir that fails to parse, or null. */
+/** First file under srcDir that fails to parse, or null (one child process for all of them:
+ * vm modules need the flag this process was not started with). */
export function findSyntaxError(srcDir) {
- for (const f of listJsFiles(srcDir)) {
- const r = checkSyntax(f);
- if (r) return r;
+ const r = spawnSync(process.execPath, ['--experimental-vm-modules', SELF, ...listJsFiles(srcDir)], { encoding: 'utf8', timeout: 60000 });
+ try {
+ return JSON.parse((r.stdout || '').trim().split('\n').pop()).bad[0] || null;
+ } catch (_e) {
+ return null;
}
- return null;
+}
+
+if (process.argv[1] && fs.realpathSync(process.argv[1]) === SELF) {
+ process.stdout.write(JSON.stringify({ bad: checkFiles(process.argv.slice(2)) }) + '\n');
}
diff --git a/harness/tests/threejs_render/test_lint.py b/harness/tests/threejs_render/test_lint.py
index 11483e28..1867d069 100644
--- a/harness/tests/threejs_render/test_lint.py
+++ b/harness/tests/threejs_render/test_lint.py
@@ -16,7 +16,7 @@ from codeverse3d.workspace import Workspace
@pytest.fixture
def no_node_syntax(monkeypatch):
"""Run lint offline: pretend every file parses."""
- monkeypatch.setattr(lint_mod, "check_syntax", lambda path, node_bin: None)
+ monkeypatch.setattr(lint_mod, "check_syntax", lambda paths: {})
def _msgs(rep, sev=None):
@@ -98,3 +98,22 @@ def test_syntax_error_reported_with_line(stool_ws: Workspace):
@pytest.mark.node
def test_real_syntax_check_on_clean_stool(stool_ws: Workspace):
assert lint_workspace(stool_ws).passed
+
+
+def test_one_node_checks_every_file_and_a_parsed_file_is_not_sent_again(tmp_path: Path, monkeypatch):
+ import codeverse3d.languages._js_lint as js_lint
+
+ good = [tmp_path / f"ok_{i}.js" for i in range(3)]
+ for i, p in enumerate(good):
+ p.write_text(f"import * as THREE from 'three';\nexport const n{i} = await Promise.resolve({i});\n")
+ bad = tmp_path / "bad.js"
+ bad.write_text("export function f() {\n return 1;\n}\nconst = 2;\n")
+ calls = []
+ real = js_lint.run_node
+ monkeypatch.setattr(js_lint, "run_node", lambda *a, **k: calls.append(a) or real(*a, **k))
+ problems = js_lint.node_check_syntax([*good, bad])
+ assert list(problems) == [bad] and problems[bad].line == 4 and "SyntaxError" in problems[bad].message
+ assert len(calls) == 1 # one process for four files
+ assert js_lint.node_check_syntax(good) == {} and len(calls) == 1 # all three parsed already: no node at all
+ assert list(js_lint.node_check_syntax([bad])) == [bad] and len(calls) == 2 # a failure is never cached