Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: validate a recipe job's OXC batches in one Node process (#15925)
Daniel Han9215e4d4bf64
diff --git a/studio/backend/core/data_recipe/jobs/worker.py b/studio/backend/core/data_recipe/jobs/worker.pyindex a7bfbb8b97f..37400865e91 100644--- a/studio/backend/core/data_recipe/jobs/worker.py+++ b/studio/backend/core/data_recipe/jobs/worker.py@@ -16,6 +16,7 @@ from pathlib import Pathfrom typing import Anyfrom ..jsonable import to_jsonable, to_preview_jsonable_row+from ..local_callable_validators import oxc_validator_sessionfrom .constants import EVENT_JOB_COMPLETED, EVENT_JOB_ERROR, EVENT_JOB_STARTEDfrom ..service import (build_config_builder,@@ -159,7 +160,8 @@ def run_job_process(*, event_queue, recipe: dict[str, Any], run: dict[str, Any])designer.set_run_config(RunConfig.model_validate(run_config_raw))if execution_type == "preview":- results = designer.preview(builder, num_records = rows)+ with oxc_validator_session():+ results = designer.preview(builder, num_records = rows)analysis = (Noneif results.analysis is None@@ -187,7 +189,8 @@ def run_job_process(*, event_queue, recipe: dict[str, Any], run: dict[str, Any])})else:- results = designer.create(builder, num_records = rows, dataset_name = dataset_name)+ with oxc_validator_session():+ results = designer.create(builder, num_records = rows, dataset_name = dataset_name)analysis = to_jsonable(results.load_analysis().model_dump(mode = "json"))_unify_batch_schemas(results.artifact_storage.base_dataset_path)if merge_batches:diff --git a/studio/backend/core/data_recipe/local_callable_validators.py b/studio/backend/core/data_recipe/local_callable_validators.pyindex e6abd84be96..ad016e63357 100644--- a/studio/backend/core/data_recipe/local_callable_validators.py+++ b/studio/backend/core/data_recipe/local_callable_validators.py@@ -5,8 +5,12 @@ from __future__ import annotationsimport jsonimport os+import queueimport structlogimport subprocess+import threading+import time+from contextlib import contextmanagerfrom copy import deepcopyfrom dataclasses import dataclassfrom functools import lru_cache@@ -218,6 +222,189 @@ def _build_oxc_validation_function(lang: str, validation_mode: str, code_shape:return _validator+# One ``validate.mjs --serve`` per job, not per batch: Node startup was ~99% of a syntax batch.+_OXC_WORKER_MAX_LINE_BYTES = 64 * 1024 * 1024+_OXC_WORKER_MAX_FAILURES = 3+_OXC_WORKER_ENV = "UNSLOTH_OXC_PERSISTENT_WORKER"+_WORKER_UNUSED = object()+_WORKER_TIMED_OUT = object()+++class _OxcWorkerFailed(Exception):+ pass+++class _OxcWorker:+ def __init__(self, node_executable: str, env: dict[str, str], key: Any) -> None:+ self.key = key+ self._next_id = 0+ self._replies: queue.Queue = queue.Queue()+ self.proc = subprocess.Popen(+ [node_executable, str(_OXC_RUNNER_PATH), "--serve"],+ cwd = str(_OXC_TOOL_DIR),+ stdin = subprocess.PIPE,+ stdout = subprocess.PIPE,+ stderr = subprocess.DEVNULL,+ env = env,+ **_windows_hidden_subprocess_kwargs(),+ )+ self._reader = threading.Thread(+ target = self._read_replies, name = "oxc-validator-reader", daemon = True+ )+ self._reader.start()++ def _read_replies(self) -> None:+ stdout = self.proc.stdout+ try:+ while True:+ line = stdout.readline(_OXC_WORKER_MAX_LINE_BYTES + 1)+ if not line.endswith(b"\n"):+ break+ self._replies.put(line)+ except (OSError, ValueError):+ pass+ self._replies.put(None)++ def _write_request(self, line: bytes, errors: list[BaseException]) -> None:+ try:+ self.proc.stdin.write(line)+ self.proc.stdin.flush()+ except (OSError, ValueError) as exc:+ errors.append(exc)++ def request(self, request_text: str) -> Any:+ self._next_id += 1+ request_id = self._next_id+ line = f'{{"id": {request_id}, "payload": {request_text}}}\n'.encode("utf-8")+ # Bounded by the deadline: a Node that never reads would hold the session lock forever.+ deadline = time.monotonic() + _OXC_TIMEOUT_S+ write_errors: list[BaseException] = []+ writer = threading.Thread(+ target = self._write_request,+ args = (line, write_errors),+ name = "oxc-validator-writer",+ daemon = True,+ )+ writer.start()+ writer.join(_OXC_TIMEOUT_S)+ if writer.is_alive():+ return _WORKER_TIMED_OUT+ if write_errors:+ raise write_errors[0]+ try:+ reply = self._replies.get(timeout = max(0.0, deadline - time.monotonic()))+ except queue.Empty:+ return _WORKER_TIMED_OUT+ if reply is None:+ raise _OxcWorkerFailed("OXC worker exited")+ message = json.loads(reply.decode("utf-8", errors = "replace"))+ if (+ not isinstance(message, dict)+ or message.get("id") != request_id+ or "result" not in message+ ):+ raise _OxcWorkerFailed("OXC worker could not answer")+ return message["result"]++ def close(self) -> None:+ proc = self.proc+ try:+ if proc.stdin is not None:+ proc.stdin.close()+ except OSError:+ pass+ try:+ proc.wait(timeout = 2)+ except subprocess.TimeoutExpired:+ proc.kill()+ proc.wait()+ self._reader.join(timeout = 2)+ if proc.stdout is not None and not self._reader.is_alive():+ proc.stdout.close()++ def kill(self) -> None:+ self.proc.kill()+ self.proc.wait()+ self.close()+++class _OxcWorkerSession:+ def __init__(self) -> None:+ self._lock = threading.Lock()+ self._worker: _OxcWorker | None = None+ self._failures = 0+ self._closed = False++ def run(self, node_executable: str, env: dict[str, str], request_text: str) -> Any:+ if (+ self._closed+ or self._failures >= _OXC_WORKER_MAX_FAILURES+ or len(request_text) > _OXC_WORKER_MAX_LINE_BYTES // 2+ ):+ return _WORKER_UNUSED+ # Concurrent callers run one-shot instead of queueing.+ if not self._lock.acquire(blocking = False):+ return _WORKER_UNUSED+ try:+ if self._closed:+ return _WORKER_UNUSED+ # The env carries the account's temp dir.+ key = (node_executable, tuple(sorted(env.items())))+ if self._worker is not None and self._worker.key != key:+ self._drop_worker(kill = False)+ if self._worker is None:+ self._worker = _OxcWorker(node_executable, env, key)+ result = self._worker.request(request_text)+ if result is _WORKER_TIMED_OUT:+ self._drop_worker(kill = True)+ return _WORKER_TIMED_OUT+ self._failures = 0+ return result+ except (OSError, ValueError, _OxcWorkerFailed) as exc:+ self._failures += 1+ logger.debug("OXC worker unavailable, running the batch one-shot: %s", exc)+ self._drop_worker(kill = True)+ return _WORKER_UNUSED+ finally:+ self._lock.release()++ def _drop_worker(self, *, kill: bool) -> None:+ worker, self._worker = self._worker, None+ if worker is None:+ return+ try:+ worker.kill() if kill else worker.close()+ except OSError:+ pass++ def close(self) -> None:+ with self._lock:+ self._closed = True+ self._drop_worker(kill = False)+++_OXC_SESSION: _OxcWorkerSession | None = None+++@contextmanager+def oxc_validator_session():+ """OXC batches inside this block share one Node process (off: UNSLOTH_OXC_PERSISTENT_WORKER=0).++ A killed caller closes the worker's stdin, which ends it on every platform.+ """+ global _OXC_SESSION+ if os.environ.get(_OXC_WORKER_ENV, "1").strip() == "0":+ yield+ return+ session = _OxcWorkerSession()+ previous, _OXC_SESSION = _OXC_SESSION, session+ try:+ yield+ finally:+ _OXC_SESSION = previous+ session.close()++def _run_oxc_batch(*, node_lang: str, validation_mode: str, code_shape: str, code_values: list[str]) -> list[dict[str, Any]]:@@ -254,19 +441,13 @@ def _run_oxc_batch(if node_bin_dir:env["PATH"] = node_bin_dir + os.pathsep + env.get("PATH", "")env.pop("NODE_PATH", None)- proc = subprocess.run(- [node_executable, str(_OXC_RUNNER_PATH)],- cwd = str(_OXC_TOOL_DIR),- input = json.dumps(payload),- text = True,- encoding = "utf-8",- errors = "replace",- capture_output = True,- check = False,- env = env,- timeout = _OXC_TIMEOUT_S,- **_windows_hidden_subprocess_kwargs(),- )+ request_text = json.dumps(payload)+ session = _OXC_SESSION+ raw = _WORKER_UNUSED if session is None else session.run(node_executable, env, request_text)+ if raw is _WORKER_TIMED_OUT:+ raise subprocess.TimeoutExpired(cmd = node_executable, timeout = _OXC_TIMEOUT_S)+ if raw is _WORKER_UNUSED:+ raw = _run_oxc_one_shot(node_executable, env, request_text)except subprocess.TimeoutExpired:logger.warning("OXC validation timed out after %ss", _OXC_TIMEOUT_S)return _fallback_results(len(code_values), "OXC validation timed out")@@ -274,19 +455,45 @@ def _run_oxc_batch(logger.warning("OXC subprocess launch failed: %s", exc)return _fallback_results(len(code_values), f"OXC launch failed: {exc}")+ if isinstance(raw, _OneShotFailure):+ return _fallback_results(len(code_values), raw.message)+ return _normalize_oxc_results(raw, len(code_values))+++class _OneShotFailure:+ def __init__(self, message: str) -> None:+ self.message = message+++def _run_oxc_one_shot(node_executable: str, env: dict[str, str], request_text: str) -> Any:+ proc = subprocess.run(+ [node_executable, str(_OXC_RUNNER_PATH)],+ cwd = str(_OXC_TOOL_DIR),+ input = request_text,+ text = True,+ encoding = "utf-8",+ errors = "replace",+ capture_output = True,+ check = False,+ env = env,+ timeout = _OXC_TIMEOUT_S,+ **_windows_hidden_subprocess_kwargs(),+ )if proc.returncode != 0:message = (proc.stderr or proc.stdout or "unknown error").strip()if len(message) > 300:message = f"{message[:300]}..."- return _fallback_results(len(code_values), f"OXC failed: {message}")+ return _OneShotFailure(f"OXC failed: {message}")try:- raw = json.loads(proc.stdout)+ return json.loads(proc.stdout)except json.JSONDecodeError:- return _fallback_results(len(code_values), "OXC output parse failed.")+ return _OneShotFailure("OXC output parse failed.")++def _normalize_oxc_results(raw: Any, row_count: int) -> list[dict[str, Any]]:if not isinstance(raw, list):- return _fallback_results(len(code_values), "OXC output must be an array.")+ return _fallback_results(row_count, "OXC output must be an array.")out: list[dict[str, Any]] = []for item in raw:diff --git a/studio/backend/core/data_recipe/oxc-validator/validate.mjs b/studio/backend/core/data_recipe/oxc-validator/validate.mjsindex 66ca0e71409..ef98a833e05 100644--- a/studio/backend/core/data_recipe/oxc-validator/validate.mjs+++ b/studio/backend/core/data_recipe/oxc-validator/validate.mjs@@ -392,7 +392,7 @@ function fallbackLintResults(entries, message) {);}-function runLintBatch(entries, budgetMs) {+function runLintBatch(entries, budgetMs, startedAtMs) {if (entries.length === 0) {return new Map();}@@ -429,7 +429,9 @@ function runLintBatch(entries, budgetMs) {];const rawDiagnostics = [];for (const names of chunks) {- const timeoutMs = Math.floor(budgetMs - OXLINT_BUDGET_MARGIN_MS - performance.now());+ const timeoutMs = Math.floor(+ budgetMs - OXLINT_BUDGET_MARGIN_MS - (performance.now() - startedAtMs),+ );if (timeoutMs < OXLINT_MIN_TIMEOUT_MS) {return fallbackLintResults(entries, "oxlint skipped: validation budget exhausted");}@@ -539,7 +541,7 @@ function readStdin() {});}-function runValidation({ codes, lang, mode, codeShape, oxlintBudgetMs }) {+function runValidation({ codes, lang, mode, codeShape, oxlintBudgetMs, startedAtMs }) {if (mode === "syntax") {return codes.map((code, index) =>validateSyntaxOne({ code, lang, index, codeShape }).result,@@ -550,7 +552,7 @@ function runValidation({ codes, lang, mode, codeShape, oxlintBudgetMs }) {const entries = codes.map((code, index) =>resolveLintEntry({ code, lang, index, codeShape }),);- const lintMap = runLintBatch(entries, oxlintBudgetMs);+ const lintMap = runLintBatch(entries, oxlintBudgetMs, startedAtMs);return entries.map((entry) =>lintMap.get(entry.index) ??@@ -568,7 +570,7 @@ function runValidation({ codes, lang, mode, codeShape, oxlintBudgetMs }) {const lintTargets = syntaxRuns.filter((run) => run.result.is_valid === true).map((run) => run.lintEntry);- const lintMap = runLintBatch(lintTargets, oxlintBudgetMs);+ const lintMap = runLintBatch(lintTargets, oxlintBudgetMs, startedAtMs);return syntaxRuns.map((run) => {if (run.result.is_valid !== true) {@@ -585,6 +587,17 @@ function runValidation({ codes, lang, mode, codeShape, oxlintBudgetMs }) {});}+function validatePayload(payload, startedAtMs) {+ const lang = mapLang(payload?.lang);+ const mode = mapMode(payload?.mode);+ const codeShape = mapCodeShape(payload?.code_shape);+ const codes = Array.isArray(payload?.codes) ? payload.codes : [];+ const oxlintBudgetMs = mapBudgetMs(payload?.timeout_ms);+ return JSON.stringify(+ runValidation({ codes, lang, mode, codeShape, oxlintBudgetMs, startedAtMs }),+ );+}+async function main() {const raw = await readStdin();let payload;@@ -604,17 +617,71 @@ async function main() {);return;}+ process.stdout.write(validatePayload(payload, 0));+}- const lang = mapLang(payload?.lang);- const mode = mapMode(payload?.mode);- const codeShape = mapCodeShape(payload?.code_shape);- const codes = Array.isArray(payload?.codes) ? payload.codes : [];- const oxlintBudgetMs = mapBudgetMs(payload?.timeout_ms);- const out = runValidation({ codes, lang, mode, codeShape, oxlintBudgetMs });- process.stdout.write(JSON.stringify(out));+// --serve: one {"id", "payload"} per line, answered in order with {"id", "result"|"error"}.+const SERVE_MAX_LINE_CHARS = 64 * 1024 * 1024;++let servedRequests = 0;++function serveLine(line) {+ // The caller's deadline started before Node did, so the first request pays for startup.+ const startedAtMs = servedRequests++ === 0 ? 0 : performance.now();+ let id = null;+ let reply;+ try {+ const request = JSON.parse(line);+ id = Number.isSafeInteger(request?.id) ? request.id : null;+ reply = `{"id":${JSON.stringify(id)},"result":${validatePayload(request?.payload, startedAtMs)}}`;+ } catch (error) {+ reply = JSON.stringify({ id, error: String(error?.stack || error) });+ }+ process.stdout.write(`${reply}\n`);}-main().catch((error) => {- process.stderr.write(String(error?.stack || error));- process.exit(1);-});+function serve() {+ // A closed stdin (caller done or dead) ends the process.+ const parentPid = process.ppid;+ if (process.platform !== "win32") {+ setInterval(() => {+ if (process.ppid !== parentPid) {+ process.exit(0);+ }+ }, 1000).unref();+ }+ // Only each new chunk is scanned: rescanning the whole pending line made large requests quadratic.+ let pending = [];+ let pendingChars = 0;+ process.stdin.setEncoding("utf8");+ process.stdin.on("data", (chunk) => {+ let start = 0;+ let newline = chunk.indexOf("\n");+ while (newline !== -1) {+ pending.push(chunk.slice(start, newline));+ const line = pending.join("");+ pending = [];+ pendingChars = 0;+ serveLine(line);+ start = newline + 1;+ newline = chunk.indexOf("\n", start);+ }+ if (start < chunk.length) {+ pending.push(chunk.slice(start));+ pendingChars += chunk.length - start;+ }+ if (pendingChars > SERVE_MAX_LINE_CHARS) {+ process.exit(2);+ }+ });+ process.stdin.on("error", () => process.exit(0));+}++if (process.argv.includes("--serve")) {+ serve();+} else {+ main().catch((error) => {+ process.stderr.write(String(error?.stack || error));+ process.exit(1);+ });+}diff --git a/studio/backend/tests/test_local_callable_validators.py b/studio/backend/tests/test_local_callable_validators.pyindex 01d027cfce9..55774a7af08 100644--- a/studio/backend/tests/test_local_callable_validators.py+++ b/studio/backend/tests/test_local_callable_validators.py@@ -166,3 +166,190 @@ def test_oxlint_lints_a_batch_longer_than_one_command_line(monkeypatch, tmp_path)assert [result["code"] for result in results] == ["eslint(no-debugger)", None] * 1500+++_HAS_REAL_PARSER = (+ validators.resolve_node_executable() is not None+ and (validators._OXC_TOOL_DIR / "node_modules" / "oxc-parser" / "package.json").is_file()+)+real_parser = pytest.mark.skipif(+ not _HAS_REAL_PARSER, reason = "needs Node and oxc-validator/node_modules/oxc-parser"+)++_PARITY_CODES = [+ "const value = 1;\n",+ "debugger;\n",+ "let broken = ;\n",+ "return 5",+ "const s = 'café \U0001f600';\nfoo(",+ "<div>{value}</div>",+ "",+ "var a = 1; a == null; if (a = 2) {}\n",+ "function f(x: number): string { return `${x}`; }\n",+ "export default function App() { return <p>{' '}</p>; }\n",+ "import x from 'y';\nx(\n",+ "\ud800 lone surrogate",+ "/* unterminated",+ "class A { #p = 1; m() { return this.#p; } }\n",+]+++def _parity_batches(rng):+ batches = []+ for lang in ("js", "ts", "jsx", "tsx"):+ for mode in ("syntax", "lint", "syntax+lint"):+ for shape in ("auto", "module", "snippet"):+ size = rng.randint(1, 6)+ batches.append((lang, mode, shape, tuple(rng.sample(_PARITY_CODES, size))))+ return batches+++def _run_batch(batch):+ lang, mode, shape, codes = batch+ return validators._run_oxc_batch(+ node_lang = lang, validation_mode = mode, code_shape = shape, code_values = list(codes)+ )+++def _count_launches(monkeypatch):+ launches = {"popen": 0, "run": 0}+ real_popen, real_run = validators.subprocess.Popen, validators.subprocess.run++ def counting_popen(*args, **kwargs):+ # subprocess.run goes through Popen too.+ if "--serve" in args[0]:+ launches["popen"] += 1+ return real_popen(*args, **kwargs)++ def counting_run(*args, **kwargs):+ launches["run"] += 1+ return real_run(*args, **kwargs)++ monkeypatch.setattr(validators.subprocess, "Popen", counting_popen)+ monkeypatch.setattr(validators.subprocess, "run", counting_run)+ return launches+++@real_oxlint[email protected]("seed", [0, 1, 2])+def test_one_worker_answers_every_batch_exactly_like_a_process_per_batch(monkeypatch, seed):+ import random++ rng = random.Random(seed)+ batches = _parity_batches(rng)+ expected = {batch: json.dumps(_run_batch(batch)) for batch in batches}+ order = batches + rng.sample(batches, len(batches) // 2)+ rng.shuffle(order)++ launches = _count_launches(monkeypatch)+ with validators.oxc_validator_session():+ for batch in order:+ assert json.dumps(_run_batch(batch)) == expected[batch], batch+ assert launches == {"popen": 1, "run": 0}+++@real_parser+def test_a_dead_worker_costs_one_batch_a_one_shot_run_and_is_replaced(monkeypatch):+ batch = ("ts", "syntax", "auto", ("let x: number = ;", "const ok = 1;"))+ expected = _run_batch(batch)+ launches = _count_launches(monkeypatch)+ with validators.oxc_validator_session():+ assert _run_batch(batch) == expected+ first = validators._OXC_SESSION._worker.proc+ first.kill()+ first.wait()+ assert _run_batch(batch) == expected+ assert launches == {"popen": 1, "run": 1}+ assert _run_batch(batch) == expected+ assert launches == {"popen": 2, "run": 1}+ assert validators._OXC_SESSION._worker.proc is not first+++@real_parser+def test_the_worker_ends_with_its_session():+ with validators.oxc_validator_session():+ _run_batch(("js", "syntax", "auto", ("const a = 1;",)))+ proc = validators._OXC_SESSION._worker.proc+ assert proc.poll() is None+ assert proc.poll() is not None+ assert validators._OXC_SESSION is None+++ validators.resolve_node_executable() is None, reason = "needs Node to start the stub worker"+)+def test_a_worker_past_its_deadline_is_killed_and_the_batch_times_out(monkeypatch, tmp_path):+ silent = tmp_path / "silent.mjs"+ silent.write_text("process.stdin.resume();\n", encoding = "utf-8")+ monkeypatch.setattr(validators, "_OXC_RUNNER_PATH", silent)+ monkeypatch.setattr(validators, "_OXC_TIMEOUT_S", 1)+ monkeypatch.setattr(validators, "oxc_validator_tmp_root", lambda: tmp_path)+ with validators.oxc_validator_session():+ results = _run_batch(("js", "syntax", "auto", ("a", "b")))+ assert validators._OXC_SESSION._worker is None+ assert [result["error_message"] for result in results] == ["OXC validation timed out"] * 2+++def test_the_kill_switch_keeps_one_process_per_batch(monkeypatch):+ monkeypatch.setenv("UNSLOTH_OXC_PERSISTENT_WORKER", "0")+ with validators.oxc_validator_session():+ assert validators._OXC_SESSION is None+++@real_parser+def test_the_worker_exits_when_its_caller_is_killed(tmp_path):+ psutil = pytest.importorskip("psutil")+ import sys+ import time++ backend = Path(validators.__file__).resolve().parents[2]+ script = (+ "import sys, time\n"+ f"sys.path.insert(0, {str(backend)!r})\n"+ "from core.data_recipe import local_callable_validators as v\n"+ "with v.oxc_validator_session():\n"+ " v._run_oxc_batch(node_lang='js', validation_mode='syntax', code_shape='auto',"+ " code_values=['const a = 1;'])\n"+ " print(v._OXC_SESSION._worker.proc.pid, flush=True)\n"+ " time.sleep(120)\n"+ )+ caller = subprocess.Popen([sys.executable, "-c", script], stdout = subprocess.PIPE, text = True)+ try:+ worker_pid = int(caller.stdout.readline())+ finally:+ caller.kill()+ caller.wait()++ deadline = time.monotonic() + 15+ while time.monotonic() < deadline:+ try:+ if psutil.Process(worker_pid).status() == psutil.STATUS_ZOMBIE:+ break+ except psutil.NoSuchProcess:+ break+ time.sleep(0.1)+ else:+ psutil.Process(worker_pid).kill()+ pytest.fail("the OXC worker outlived the process that started it")+++@real_parser+def test_a_large_batch_costs_the_worker_about_what_it_costs_one_shot(monkeypatch):+ import time++ # A JSON line carries no raw newline until its end, so rescanning the buffer per stdin chunk+ # was quadratic: 18x one-shot for 32 MB, past the deadline on slower hosts.+ batch = (+ "js",+ "syntax",+ "auto",+ tuple("/*" + "a" * 3_200_000 + "*/\nconst x = 1;\n" for _ in range(10)),+ )+ started = time.monotonic()+ expected = _run_batch(batch)+ monkeypatch.setattr(validators, "_OXC_TIMEOUT_S", max(3.0, 5 * (time.monotonic() - started)))+ launches = _count_launches(monkeypatch)+ with validators.oxc_validator_session():+ assert _run_batch(batch) == expected+ assert launches == {"popen": 1, "run": 0}+ assert all(result["is_valid"] for result in expected)