Official Codebase of 3DCodeVerse
Python
languages: one node parses every JS file of a lint, and a file that already parsed is not sent again
gaoypeng13719fd611cf
diff --git a/harness/codeverse3d/languages/_js_lint.py b/harness/codeverse3d/languages/_js_lint.pyindex d24251b4..884b45ee 100644--- a/harness/codeverse3d/languages/_js_lint.py+++ b/harness/codeverse3d/languages/_js_lint.py@@ -2,28 +2,31 @@Replaces two copies of the same algorithm: ``threejs/lint.py:check_syntax`` +``_lint_imports`` and ``scene_threejs/lint.py:_node_check`` + ``_check_imports``.-Both ran ``node --input-type=module --check`` over stdin and scraped the same-``[stdin]:N`` / ``SyntaxError:`` lines; both matched static, re-export and dynamic-import specifiers with near-identical regexes and applied the same allowlist-(``three``, ``three/addons/*``, ``three/examples/jsm/*``, relative files under-``src/``). They differed only in message text (owned by the caller via-``make_finding``, as in :mod:`_ast_lint`) and in two defects this module fixes:-threejs's fallback message was a ``list`` (``splitlines()[-1:]``), and scene's-escape check was a ``str.startswith`` prefix test that let ``src2/`` pass.-Escape is decided before existence so a symlink/``..`` hop out of ``src/`` is-reported as an escape, never as "missing".+Both matched static, re-export and dynamic import specifiers with near-identical+regexes and applied the same allowlist (``three``, ``three/addons/*``,+``three/examples/jsm/*``, relative files under ``src/``). They differed only in+message text (owned by the caller via ``make_finding``, as in :mod:`_ast_lint`) and+in two defects this module fixes: threejs's fallback message was a ``list``+(``splitlines()[-1:]``), and scene's escape check was a ``str.startswith`` prefix+test that let ``src2/`` pass. Escape is decided before existence so a+symlink/``..`` hop out of ``src/`` is reported as an escape, never as "missing".++Syntax is ``runtime_js/lib/syntax_check.mjs`` (export_glb's checker too) in ONE node+process per lint, skipping files whose bytes already parsed — it was one ``node --check``+per file: 51 on a scene, 44 of them unchanged ``src/lib`` modules, ≈ 2.5 s a build call."""from __future__ import annotations+import hashlibimport re-import subprocess-from collections.abc import Callable+from collections.abc import Callable, Sequencefrom dataclasses import dataclassfrom enum import StrEnumfrom pathlib import Pathfrom codeverse3d.contracts.artifacts import GateFinding+from codeverse3d.spatial.node import NodeError, run_node, runtime_js_dir# threejs's regexes (the superset: side-effect ``import './x.js'``, ``export * from``,# ``export * as ns from`` and multi-line ``import {\n a,\n b\n} from`` all match)@@ -31,7 +34,6 @@ _IMPORT_RE = re.compile(r"""(?:^|\n)\s*(?:import\s+(?:[^'";]*?\s+from\s+)?|export\s+(?:\*(?:\s+as\s+\w+)?|\{[^}]*\})\s+from\s+)['"]([^'"]+)['"]""")_DYN_IMPORT_RE = re.compile(r"""\bimport\s*\(\s*['"]([^'"]+)['"]\s*\)""")-_STDIN_LINE_RE = re.compile(r"\[stdin\]:(\d+)|:(\d+)\n")_URL_RE = re.compile(r"https?://")@@ -45,27 +47,37 @@ class SyntaxProblem:stderr_tail: str-def node_check_syntax(path: Path, node_bin: str) -> SyntaxProblem | None:- """``None`` when ``path`` parses as ESM under ``node --input-type=module --check``.-- Stdin is used because ``node --check file.js`` treats a bare ``.js`` as- CommonJS-or-detect.- """+#: sha256 of file contents that parsed — library modules are byte-identical build after build+_PARSED: set[str] = set()+++def node_check_syntax(paths: Sequence[Path]) -> dict[Path, SyntaxProblem]:+ """``{path: problem}`` for every file of ``paths`` that does not parse as an ES+ module; one ``node`` for the lot, and none at all when every file's content+ already parsed in this process."""+ digests: dict[Path, str] = {}+ out: dict[Path, SyntaxProblem] = {}+ for p in paths:+ try:+ digests[p] = hashlib.sha256(p.read_bytes()).hexdigest()+ except OSError as e:+ out[p] = SyntaxProblem(None, f"cannot read the file: {e}", "")+ todo = [p for p, d in digests.items() if d not in _PARSED]+ if not todo:+ return outtry:- proc = subprocess.run(- [node_bin, "--input-type=module", "--check"],- input=path.read_bytes(), capture_output=True, timeout=30,- )- except (OSError, subprocess.TimeoutExpired) as e:- return SyntaxProblem(None, f"node --check could not run: {e}", "")- if proc.returncode == 0:- return None- err = proc.stderr.decode(errors="replace").strip()- m = _STDIN_LINE_RE.search(err)- line = int(m.group(1) or m.group(2)) if m else None- lines = err.splitlines()- msg = next((ln for ln in lines if "Error" in ln), lines[0] if lines else "syntax error")- return SyntaxProblem(line, msg, err[-600:])+ res = run_node(runtime_js_dir() / "lib" / "syntax_check.mjs", [str(p) for p in todo],+ node_args=["--experimental-vm-modules"], timeout_s=60, check=False)+ bad = {row["file"]: row for row in (res.last_json or {})["bad"]}+ except (NodeError, KeyError, TypeError) as e:+ return out | {p: SyntaxProblem(None, f"the node syntax check could not run: {e}", "") for p in todo}+ for p in todo:+ row = bad.get(str(p))+ if row is None:+ _PARSED.add(digests[p])+ else:+ out[p] = SyntaxProblem(row.get("line"), str(row.get("message") or "syntax error"), str(row.get("stderr", "")))+ return outdef import_specs(src: str) -> list[tuple[str, int]]:diff --git a/harness/codeverse3d/languages/scene_threejs/__init__.py b/harness/codeverse3d/languages/scene_threejs/__init__.pyindex 7ef10300..8e16f933 100644--- a/harness/codeverse3d/languages/scene_threejs/__init__.py+++ b/harness/codeverse3d/languages/scene_threejs/__init__.py@@ -25,6 +25,7 @@ from codeverse3d.languages._docs import RuntimeDocsfrom codeverse3d.languages._js_lint import (ImportKind,ImportVerdict,+ SyntaxProblem,check_imports,node_check_syntax,)@@ -80,10 +81,7 @@ def _is_lib(path: Path, ws: Workspace) -> bool:return path.parent == ws.src / "lib"-def _node_check(path: Path, rel: str) -> GateFinding | None:- p = node_check_syntax(path, get_settings().binaries.node or "node")- if p is None:- return None+def _syntax_finding(p: SyntaxProblem, rel: str) -> GateFinding:return _f(Severity.ERROR, f"syntax: {p.message}", target=f"{rel}:{p.line}" if p.line else rel,hint="fix the syntax error at the quoted line (node --check)", line=p.line, detail=p.stderr_tail)@@ -124,12 +122,12 @@ def lint(ws: Workspace) -> GateReport:findings.append(_f(Severity.ERROR, "src/scene.js is missing", target="src/scene.js",hint="create src/scene.js exporting createScene({THREE, renderer, loaders})"))big: list[str] = []+ syntax = node_check_syntax(files)for path in files:rel = path.relative_to(ws.root).as_posix()text = path.read_text(errors="replace")- syntax = _node_check(path, rel)- if syntax:- findings.append(syntax)+ if path in syntax:+ findings.append(_syntax_finding(syntax[path], rel))continueif _is_lib(path, ws):continue@@ -723,8 +721,6 @@ class SceneThreeJsRuntime(RuntimeDocs):"""Probe + shader preflight (one browser boot); ok iff the module loadsand no shader errors. External contract unchanged: the same twoGateReports land under ``artifacts/gates/``."""- from codeverse3d.config import get_settings-t0 = time.time()tmo = min(float(timeout_s or get_settings().limits.build_timeout_s), 120.0)ws.artifacts.mkdir(parents=True, exist_ok=True)diff --git a/harness/codeverse3d/languages/threejs/__init__.py b/harness/codeverse3d/languages/threejs/__init__.pyindex 7de983c8..b2fc6ba6 100644--- a/harness/codeverse3d/languages/threejs/__init__.py+++ b/harness/codeverse3d/languages/threejs/__init__.py@@ -211,9 +211,9 @@ def lint_workspace(ws: Workspace) -> GateReport:"""Run every static check and return the ``lint:threejs`` GateReport."""t0 = time.time()findings: list[GateFinding] = []- node_bin = get_settings().binaries.nodeentry = ws.src / "object.js"sources = list_sources(ws)+ syntax = check_syntax(sources)if not entry.is_file():findings.append(GateFinding(gate=GATE, severity=Severity.ERROR, target="src/object.js",@@ -223,7 +223,7 @@ def lint_workspace(ws: Workspace) -> GateReport:for path in sources:src = path.read_text(errors="replace")rel = _rel(ws, path)- syn = check_syntax(path, node_bin)+ syn = syntax.get(path)if syn is not None:findings.append(GateFinding(gate=GATE, severity=Severity.ERROR, target=rel,message=f"{rel}:{syn.line or '?'}: {syn.message}", fix_hint="fix the syntax error at that line"))diff --git a/harness/runtime_js/README.md b/harness/runtime_js/README.mdindex ba7ed5b1..82eaa03f 100644--- a/harness/runtime_js/README.md+++ b/harness/runtime_js/README.md@@ -76,7 +76,9 @@ lib/node_polyfills.mjs FileReader/self shims so GLTFExporter writes binary GLB wlib/census.mjs per-part tri counts, world bboxes, materials, NaN check naming mesh + part (node + browser)lib/instances.mjs bakeInstancedMeshes(THREE, root): InstancedMesh → Group of named plain mesheslib/stack.mjs Error → {type,message,file,line,frames} with workspace-relative src/ paths-lib/syntax_check.mjs `node --input-type=module --check` per file to locate ESM SyntaxErrors+lib/syntax_check.mjs THE JS syntax check: every file parsed as an ES module in ONE node+ (vm.SourceTextModule, never linked or run; `node --check` only for a failing+ file's line). CLI for the python lints, findSyntaxError() for export_glblib/browser/*.js page-side ESM (served through /__runtime/): renderer.js (THE WebGLRenderer factory —sRGB + ACES + PCF shadows + pixel ratio 1, used by the object rig AND the scene host),camera_fit.js (azimuth/elevation → tight bbox fit; the distance math itself isdiff --git a/harness/runtime_js/lib/syntax_check.mjs b/harness/runtime_js/lib/syntax_check.mjsindex 000eef16..59b5f516 100644--- a/harness/runtime_js/lib/syntax_check.mjs+++ b/harness/runtime_js/lib/syntax_check.mjs@@ -1,25 +1,42 @@-// Locate ESM syntax errors file-by-file. V8 SyntaxErrors thrown by a dynamic-// import() carry no file/line, so we re-check each candidate source file with-// `node --input-type=module --check` (stdin) and parse "[stdin]:LINE".-//-// import { checkSyntax, findSyntaxError } from './syntax_check.mjs';+// THE JS syntax check: every file parsed as an ES module in ONE node process+// (vm.SourceTextModule: compiled, never linked or run). That parse error has no line, so+// only a file that fails pays a second process, `node --input-type=module --check`.+// node --experimental-vm-modules lib/syntax_check.mjs FILE... → {"bad": [{file, line, message, stderr}]}+// import { findSyntaxError, listJsFiles } from './syntax_check.mjs';import fs from 'node:fs';import path from 'node:path';+import vm from 'node:vm';import { spawnSync } from 'node:child_process';+import { fileURLToPath } from 'node:url';-/** Check one file; returns null when it parses, else {file, line, message}. */-export function checkSyntax(file) {- const src = fs.readFileSync(file);- const r = spawnSync(process.execPath, ['--input-type=module', '--check'], { input: src, encoding: 'utf8', timeout: 20000 });+const SELF = fileURLToPath(import.meta.url);++/** `node --check`'s report on one file: null when it parses, else {file, line, message, stderr}. */+function checkOne(file) {+ const r = spawnSync(process.execPath, ['--input-type=module', '--check'], { input: fs.readFileSync(file), encoding: 'utf8', timeout: 20000 });if (r.status === 0) return null;- const err = r.stderr || '';+ const err = (r.stderr || '').trim();const m = /\[stdin\]:(\d+)/.exec(err);- const msg = (/^(SyntaxError:.*)$/m.exec(err) || [])[1] || err.trim().split('\n').pop() || 'syntax error';- return { file, line: m ? Number(m[1]) : null, message: msg };+ const msg = (/^(SyntaxError:.*)$/m.exec(err) || [])[1] || err.split('\n').pop() || 'syntax error';+ return { file, line: m ? Number(m[1]) : null, message: msg, stderr: err.slice(-600) };}-/** Recursively list *.js / *.mjs under dir (skipping node_modules). */+/** The files of `files` that do not parse as ES modules, in order. */+export function checkFiles(files) {+ const bad = [];+ for (const file of files) {+ try {+ new vm.SourceTextModule(fs.readFileSync(file, 'utf8'), { identifier: file });+ } catch (e) {+ const hit = e instanceof SyntaxError ? checkOne(file) : { file, line: null, message: String(e), stderr: '' };+ if (hit) bad.push(hit);+ }+ }+ return bad;+}++/** Recursively list *.js / *.mjs under dir (skipping node_modules and dot entries). */export function listJsFiles(dir) {const out = [];if (!fs.existsSync(dir)) return out;@@ -35,11 +52,17 @@ export function listJsFiles(dir) {return out.sort();}-/** First file under srcDir that fails to parse, or null. */+/** First file under srcDir that fails to parse, or null (one child process for all of them:+ * vm modules need the flag this process was not started with). */export function findSyntaxError(srcDir) {- for (const f of listJsFiles(srcDir)) {- const r = checkSyntax(f);- if (r) return r;+ const r = spawnSync(process.execPath, ['--experimental-vm-modules', SELF, ...listJsFiles(srcDir)], { encoding: 'utf8', timeout: 60000 });+ try {+ return JSON.parse((r.stdout || '').trim().split('\n').pop()).bad[0] || null;+ } catch (_e) {+ return null;}- return null;+}++if (process.argv[1] && fs.realpathSync(process.argv[1]) === SELF) {+ process.stdout.write(JSON.stringify({ bad: checkFiles(process.argv.slice(2)) }) + '\n');}diff --git a/harness/tests/threejs_render/test_lint.py b/harness/tests/threejs_render/test_lint.pyindex 11483e28..1867d069 100644--- a/harness/tests/threejs_render/test_lint.py+++ b/harness/tests/threejs_render/test_lint.py@@ -16,7 +16,7 @@ from codeverse3d.workspace import Workspace@pytest.fixturedef no_node_syntax(monkeypatch):"""Run lint offline: pretend every file parses."""- monkeypatch.setattr(lint_mod, "check_syntax", lambda path, node_bin: None)+ monkeypatch.setattr(lint_mod, "check_syntax", lambda paths: {})def _msgs(rep, sev=None):@@ -98,3 +98,22 @@ def test_syntax_error_reported_with_line(stool_ws: Workspace):@pytest.mark.nodedef test_real_syntax_check_on_clean_stool(stool_ws: Workspace):assert lint_workspace(stool_ws).passed+++def test_one_node_checks_every_file_and_a_parsed_file_is_not_sent_again(tmp_path: Path, monkeypatch):+ import codeverse3d.languages._js_lint as js_lint++ good = [tmp_path / f"ok_{i}.js" for i in range(3)]+ for i, p in enumerate(good):+ p.write_text(f"import * as THREE from 'three';\nexport const n{i} = await Promise.resolve({i});\n")+ bad = tmp_path / "bad.js"+ bad.write_text("export function f() {\n return 1;\n}\nconst = 2;\n")+ calls = []+ real = js_lint.run_node+ monkeypatch.setattr(js_lint, "run_node", lambda *a, **k: calls.append(a) or real(*a, **k))+ problems = js_lint.node_check_syntax([*good, bad])+ assert list(problems) == [bad] and problems[bad].line == 4 and "SyntaxError" in problems[bad].message+ assert len(calls) == 1 # one process for four files+ assert js_lint.node_check_syntax(good) == {} and len(calls) == 1 # all three parsed already: no node at all+ assert list(js_lint.node_check_syntax([bad])) == [bad] and len(calls) == 2 # a failure is never cached