Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(verify): isolate the soak concurrency burst in a helper process
aashish2547d00cd3cd633
diff --git a/verify/soak_check.py b/verify/soak_check.pyindex 171bfa55..2dffbc08 100644--- a/verify/soak_check.py+++ b/verify/soak_check.py@@ -12,13 +12,16 @@ Three questions this answers, none of which the other suites ask:3. Is a loaded agent safe to call from several threads, and what does that buy in throughput?Exits non-zero if a check fails. Concurrency is attempted on the selected device and retried on-CPU if the device rejects it; that outcome is reported rather than hidden.+CPU if the device rejects it; a device-level crash (SIGABRT/SIGSEGV from a GPU driver) is+isolated in a short-lived helper process so it becomes a reported FAIL with a CPU retry,+not an aborted run with no verdict."""import argparseimport gcimport jsonimport osimport statistics+import subprocessimport sysimport threadingimport time@@ -74,6 +77,80 @@ def _live(cls):return sum(1 for o in gc.get_objects() if isinstance(o, cls))+def _concurrency_child(models_dir, call_device, ref, n_threads, per_thread):+ """The threading burst from section 3, running in a disposable helper process.++ Same script, same agent loader, same threaded `predict` storm, same single-threaded+ reference as the in-process loop it replaces; only the outcome travels back (one JSON+ verdict line on stdout). A device-level crash (SIGABRT/SIGSEGV from a GPU driver)+ then kills this helper, not the process owed the leak verdicts -- the parent sees a+ bad exit code and runs the same CPU retry the script always had.+ """+ agent = _concurrency_load(models_dir, call_device)+ here_ref = json.dumps(agent.predict(STATE, QUESTIONS)["answers"], sort_keys=True)+ if here_ref != ref:+ print(json.dumps({"status": "reference-mismatch",+ "detail": "child reference differs from the parent one"}), flush=True)+ return 3++ def worker(results, errors):+ """Every worker asks the same questions about the same state, so any cross-talk or+ corruption shows up as an answer that differs from the single-threaded reference.+ The agent is closed over because there is exactly one burst in exactly one process,+ and no retry can happen underneath running threads."""+ try:+ for _ in range(per_thread):+ out = agent.predict(STATE, QUESTIONS)+ results.append(json.dumps(out["answers"], sort_keys=True))+ except Exception as e: # noqa: BLE001+ errors.append("%s: %s" % (type(e).__name__, str(e)[:120]))++ results, errors = [], []+ threads = [threading.Thread(target=worker, args=(results, errors)) for _ in range(n_threads)]+ start = time.perf_counter()+ for t in threads:+ t.start()+ for t in threads:+ t.join()+ elapsed = time.perf_counter() - start+ if errors:+ print(json.dumps({"status": "errors", "errors": errors[:5], "elapsed": elapsed,+ "done": len(results)}), flush=True)+ return 4+ mismatches = sum(1 for r in results if r != here_ref)+ print(json.dumps({"status": "ok", "total": len(results), "mismatches": mismatches,+ "elapsed": elapsed}), flush=True)+ return 0 if mismatches == 0 else 5+++def _concurrency_load(models_dir, call_device):+ """The multilingual agent section 3 bursts against, loaded one way in both processes."""+ return laya.load(os.path.join(models_dir, "laya-multilingual"), device=call_device)+++def _concurrency_shim():+ """`python verify/soak_check.py --concurrency-child ...`: section 3's burst, on its+ own argv. Prints one JSON verdict line on stdout and exits with a code the parent+ reads; argparse learns the extra flag through `parse_known_args`, so `main` is+ untouched."""+ if "--concurrency-child" not in sys.argv:+ return None+ ap = argparse.ArgumentParser()+ ap.add_argument("--models-dir", required=True)+ ap.add_argument("--device", required=True)+ ap.add_argument("--ref", required=True)+ ap.add_argument("--threads", type=int, required=True)+ ap.add_argument("--per-thread", type=int, required=True)+ known, _ = ap.parse_known_args()+ sys.exit(_concurrency_child(known.models_dir, known.device,+ known.ref, known.threads, known.per_thread))+++_CONCURRENCY_CHILD_RC = _concurrency_shim()+if _CONCURRENCY_CHILD_RC is not None: # pragma: no cover - child entry only+ sys.exit(_CONCURRENCY_CHILD_RC)++def report(name, ok, detail=""):print(" %s %-52s %s" % ("PASS" if ok else "FAIL", name, detail if not ok else ""))if not ok:@@ -197,51 +274,96 @@ def main():% rss_mb())# ------------------------------------------------------------ 3. concurrency+ # ------------------------------------------------------------ 3. concurrency+ # Threads hitting an MPS (or CUDA/XPU) model at once are exactly what SIGABRTed this+ # script's interpreter on Apple silicon (torch 2.14, Metal assertion+ # `_status < MTLCommandBufferStatusCommitted`): the driver aborts below Python, so a+ # worker pool's try/except never sees it and a threaded runner that mimics the parent+ # loop dies with no verdict either. Run the burst in a disposable helper process+ # instead -- same loader, same threads, same reference, one JSON verdict -- whose+ # death the parent converts into the CPU retry the script already promised. Leak+ # sections 1-2 stay in-process: they measure the serving process, not a throwaway+ # helper.head("3. Concurrent calls from %d threads" % args.threads)per_thread = max(1, args.calls // (args.threads * 4))total = args.threads * per_thread- def worker(ag, results, errors):- """Every worker asks the same questions about the same state, so any cross-talk or- corruption shows up as an answer that differs from the single-threaded reference.- The agent is passed in rather than closed over, so a device retry cannot change it- underneath running threads."""+ def _report_verdict(verdict, device_name, expected_total):+ status = verdict["status"]+ if status == "reference-mismatch":+ report("concurrency on %s" % device_name, False, verdict["detail"])+ return device_name == "cpu"+ if status == "errors":+ errs = verdict["errors"] or ["no detail"]+ report("concurrency on %s" % device_name, False,+ "%d error(s), first: %s" % (len(errs), errs[0]))+ return device_name == "cpu"+ done = verdict.get("total", 0)+ report("all %d concurrent calls returned (%s)" % (expected_total, device_name),+ done == expected_total,+ "expected %d, got %d" % (expected_total, done))+ mismatches = verdict.get("mismatches", 1)+ report("concurrent answers match the single-threaded reference",+ mismatches == 0,+ "%d of %d differ" % (mismatches, done))+ elapsed = verdict.get("elapsed") or 0.0+ print(" %d calls in %.1f s across %d threads (%.0f calls/s)"+ % (done, elapsed, args.threads, done / elapsed if elapsed else 0.0))+ return True++ def _last_verdict(lines):+ """The child's one JSON line, read back-to-front past warnings and progress."""+ for line in reversed(lines):+ try:+ verdict = json.loads(line)+ except ValueError:+ continue+ if isinstance(verdict, dict) and "status" in verdict:+ return verdict+ return None++ def _child_timeout():+ """Bound the helper's life: 300 s floor, rising with --calls like the work does."""+ return max(300.0, 300.0 * args.calls / 200.0 if args.calls else 0.0)++ def burst_in_child(call_device, expected_total):+ """Run section 3's burst for one device inside the helper and report its verdict.++ Returns True when the parent should stop retrying -- either the burst succeeded,+ or this was the CPU pass itself and there is nothing left to fall back to. The+ burst keeps the original report names, so a passing machine prints exactly what+ the in-process loop used to, including the note when the CPU fallback is the pass+ that won.+ """+ argv = [sys.executable, os.path.join(ROOT, "verify", "soak_check.py"),+ "--concurrency-child", "--models-dir", args.models, "--device", call_device,+ "--ref", ref, "--threads", str(args.threads), "--per-thread", str(per_thread)]try:- for _ in range(per_thread):- out = ag.predict(STATE, QUESTIONS)- results.append(json.dumps(out["answers"], sort_keys=True))- except Exception as e: # noqa: BLE001- errors.append("%s: %s" % (type(e).__name__, str(e)[:120]))-- primary = laya.load(models["multilingual"], device=device)+ child = subprocess.run(argv, capture_output=True, text=True, timeout=_child_timeout())+ except subprocess.TimeoutExpired:+ report("concurrency on %s" % call_device, False,+ "helper timed out after %ds" % _child_timeout())+ return call_device == "cpu"+ verdict = _last_verdict((child.stdout or "").splitlines())+ if verdict is None or child.returncode != 0:+ report("concurrency on %s" % call_device, False,+ "helper died without a verdict (exit %s)" % child.returncode)+ return call_device == "cpu"+ return _report_verdict(verdict, call_device, expected_total)++ primary = _concurrency_load(args.models, device)ref = json.dumps(primary.predict(STATE, QUESTIONS)["answers"], sort_keys=True)attempts = [primary.device.type] + (["cpu"] if primary.device.type != "cpu" else [])for attempt_device in attempts:if primary.device.type != attempt_device:print(" the %s pass failed; retrying on cpu" % primary.device.type)- primary = laya.load(models["multilingual"], device="cpu")- results, errors = [], []- threads = [threading.Thread(target=worker, args=(primary, results, errors))- for _ in range(args.threads)]- t0 = time.perf_counter()- for t in threads:- t.start()- for t in threads:- t.join()- elapsed = time.perf_counter() - t0- if not errors:- report("all %d concurrent calls returned (%s)" % (total, attempt_device), len(results) == total,- "expected %d, got %d" % (total, len(results)))- report("concurrent answers match the single-threaded reference",- all(r == ref for r in results),- "%d of %d differ" % (sum(1 for r in results if r != ref), len(results)))- print(" %d calls in %.1f s across %d threads (%.0f calls/s)"- % (total, elapsed, args.threads, total / elapsed))+ primary = _concurrency_load(args.models, "cpu")+ done = burst_in_child(attempt_device, total)+ if done:if attempt_device != primary.device.type:print(" note: the %s pass is what succeeded, not the default device"% attempt_device)break- report("concurrency on %s" % attempt_device, False, "%d error(s), first: %s" % (len(errors), errors[0]))if attempt_device == "cpu":break