Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(agent): synchronize GPU OOM fallback to protect concurrent in-flight inference (#649)
Swatantra Yadav93e855edd387
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex 39843b20..af8176fc 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -129,6 +129,7 @@ jobs:python tests/test_question_token_reuse.pypython tests/test_lazy_import.pypython tests/test_runtime_fixes.py+ python tests/test_agent_gate.pypython tests/test_option_collapse.pypython tests/test_lang_guess.pypython tests/test_identifier_complexity.pydiff --git a/.github/workflows/release.yml b/.github/workflows/release.ymlindex ef9eec7a..562aec32 100644--- a/.github/workflows/release.yml+++ b/.github/workflows/release.yml@@ -66,6 +66,7 @@ jobs:python tests/test_tokenizer_concurrency.pypython tests/test_lazy_import.pypython tests/test_runtime_fixes.py+ python tests/test_agent_gate.pypython tests/test_lang_guess.pypython tests/test_option_collapse.pypython tests/test_identifier_complexity.pydiff --git a/laya/agent.py b/laya/agent.pyindex c98e6859..b376706c 100644--- a/laya/agent.py+++ b/laya/agent.py@@ -5,7 +5,7 @@ import tempfileimport threadingimport timeimport warnings-from contextlib import ExitStack, nullcontext+from contextlib import ExitStack, contextmanager, nullcontextfrom contextvars import ContextVarfrom typing import Any, Dict, List, Optional, Union@@ -155,6 +155,54 @@ def _verify_compatibility(model: torch.nn.Module, cfg: Dict, weights: Dict[str,_TOKENIZERS: Dict[tuple, Any] = {}_TOKENIZERS_LOCK = threading.Lock()+class _InferenceGate:+ """Read-write synchronization gate protecting shared model device placement (#649).++ Normal inference acquires the read lock so multiple threads can evaluate the model+ concurrently without serialization. When a thread encounters GPU OOM and needs+ to temporarily migrate the model to CPU for a scoped fallback, it acquires the write+ lock. This ensures all in-flight requests finish safely before the model is moved,+ and blocks new requests until the model and device state are restored.+ """++ def __init__(self):+ self._cond = threading.Condition(threading.Lock())+ self._readers = 0+ self._writers = 0+ self._waiting_writers = 0++ @contextmanager+ def read_lock(self):+ with self._cond:+ while self._writers > 0 or self._waiting_writers > 0:+ self._cond.wait()+ self._readers += 1+ try:+ yield+ finally:+ with self._cond:+ self._readers -= 1+ if self._readers == 0:+ self._cond.notify_all()++ @contextmanager+ def write_lock(self):+ with self._cond:+ self._waiting_writers += 1+ try:+ while self._readers > 0 or self._writers > 0:+ self._cond.wait()+ self._writers = 1+ finally:+ self._waiting_writers -= 1+ try:+ yield+ finally:+ with self._cond:+ self._writers = 0+ self._cond.notify_all()++# The per-inference CPU fallback rewrites shared runtime state (device, dtype, amp) and moves the# model while other threads may be running their own forward, so the demotion and the restore are# serialised. A second request that hits OOM waits here and re-demotes only if it needs to.@@ -323,6 +371,7 @@ class Agent(HookRegistry):last_fallback_reason = None# Set when `compile=True` wrapped the model in torch.compile._compiled = False+ _gate = _InferenceGate()def __init__(self,@@ -568,6 +617,7 @@ class Agent(HookRegistry):self.dtype = torch.bfloat16self._fast = None+ self._gate = _InferenceGate()# 2. Place on device with graceful fallback to CPU on memory errorfell_back_from = fell_back_why = None@@ -956,7 +1006,8 @@ class Agent(HookRegistry):)try:- out = run()+ with self._gate.read_lock():+ out = run()except (RuntimeError, torch.cuda.OutOfMemoryError) as e:low = str(e).lower()# An OOM is `torch.cuda.OutOfMemoryError` or says so in its message. The bare@@ -966,13 +1017,16 @@ class Agent(HookRegistry):print("Warning: GPU memory exceeded during inference. Retrying this request on CPU...")# Scoped, not permanent: the demotion used to rewrite device/dtype/amp and move# the model for the life of the process, so one oversized request left every- # later call ~10-15x slower on CPU. Demote under the lock, answer this request- # on CPU, then put the runtime back the way it was.- with _OOM_FALLBACK_LOCK:+ # later call ~10-15x slower on CPU. Demote under the write lock (#649) so any+ # concurrent in-flight GPU requests finish before the model is moved to CPU,+ # and put the runtime back before allowing new requests to start.+ with self._gate.write_lock(), _OOM_FALLBACK_LOCK:# Recorded on entry, under the same lock as the demotion: the count and# reason must be readable by /health without racing a concurrent fallback.self.cpu_fallback_count += 1self.last_fallback_reason = str(e)+ if self.device.type == "cpu":+ return run()held_device, held_dtype, held_amp = self.device, self.dtype, self.amp_enabledhad_fast = self._fast is not None# Only our batch scope can retain BF16 copies after this failed forward.@@ -998,15 +1052,16 @@ class Agent(HookRegistry):# request in full precision. One miss must not turn AMP off; a build that# lacks the op fails the same way every time, so after a short streak the# process drops to full precision instead of paying for two forwards (#351).- try:- return run(enabled=False)- finally:- self._amp_failures += 1- if self._amp_failures >= _AMP_FAIL_LIMIT:- print("Warning: autocast failed %d times in a row. Disabling mixed precision."- % self._amp_failures)- self.amp_enabled = False- self.dtype = torch.float32+ with self._gate.read_lock():+ try:+ return run(enabled=False)+ finally:+ self._amp_failures += 1+ if self._amp_failures >= _AMP_FAIL_LIMIT:+ print("Warning: autocast failed %d times in a row. Disabling mixed precision."+ % self._amp_failures)+ self.amp_enabled = False+ self.dtype = torch.float32raiseelse:self._amp_failures = 0diff --git a/tests/test_agent_gate.py b/tests/test_agent_gate.pynew file mode 100644index 00000000..fbe256aa--- /dev/null+++ b/tests/test_agent_gate.py@@ -0,0 +1,248 @@+"""Tests for _InferenceGate read-write synchronization gate (#649).++Ensures that:+1. Normal inference allows concurrent readers without mutual exclusion.+2. An OOM fallback (writer) waits for in-flight readers to finish before migrating the model.+3. While fallback is active (writer lock held), incoming requests (readers) are queued.+4. Writers have preference over new readers so OOM fallback does not starve under load.+5. Exceptions inside read or write contexts cleanly release state without deadlocking.+"""+import os+import sys+import threading+import time+import types++ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))+sys.path.insert(0, ROOT)++if "torch" not in sys.modules:+ t = types.ModuleType("torch")+ t.cuda = types.ModuleType("torch.cuda")+ t.cuda.OutOfMemoryError = type("OutOfMemoryError", (RuntimeError,), {})+ t.nn = types.ModuleType("torch.nn")+ t.nn.Module = type("Module", (), {})+ t.nn.MultiheadAttention = type("MultiheadAttention", (), {})+ t.nn.functional = types.ModuleType("torch.nn.functional")+ t.utils = types.ModuleType("torch.utils")+ t.utils.checkpoint = types.ModuleType("torch.utils.checkpoint")+ t.utils.checkpoint.checkpoint = lambda f, *a, **kw: f(*a, **kw)+ t.device = lambda d: d+ t.float32 = "float32"+ t.float16 = "float16"+ t.bfloat16 = "bfloat16"+ t.no_grad = lambda: (lambda f: f)+ sys.modules["torch"] = t+ sys.modules["torch.cuda"] = t.cuda+ sys.modules["torch.nn"] = t.nn+ sys.modules["torch.nn.functional"] = t.nn.functional+ sys.modules["torch.utils"] = t.utils+ sys.modules["torch.utils.checkpoint"] = t.utils.checkpoint++from laya.agent import _InferenceGate++PASS, FAIL = [], []+++def check(name, got, want):+ if got == want:+ PASS.append(name)+ else:+ FAIL.append("%s: got %r, want %r" % (name, got, want))+++def test_concurrent_readers():+ gate = _InferenceGate()+ active_readers = []+ max_concurrent = [0]+ lock = threading.Lock()+ all_in = threading.Event()+ entered = 0+ entered_lock = threading.Lock()++ def reader():+ nonlocal entered+ with gate.read_lock():+ with lock:+ active_readers.append(1)+ max_concurrent[0] = max(max_concurrent[0], len(active_readers))+ with entered_lock:+ entered += 1+ if entered == 5:+ all_in.set()+ all_in.wait(5)+ with lock:+ active_readers.pop()++ threads = [threading.Thread(target=reader) for _ in range(5)]+ for t in threads:+ t.start()+ for t in threads:+ t.join()++ check("concurrent_readers/all ran concurrently", max_concurrent[0], 5)+ check("concurrent_readers/all finished", len(active_readers), 0)+++def test_writer_mutual_exclusion():+ gate = _InferenceGate()+ events = []+ r_in, r_go = threading.Event(), threading.Event()++ def reader():+ with gate.read_lock():+ events.append("reader_start")+ r_in.set()+ r_go.wait(5)+ events.append("reader_end")++ def writer():+ with gate.write_lock():+ events.append("writer_start")+ events.append("writer_end")++ t_reader = threading.Thread(target=reader)+ t_writer = threading.Thread(target=writer)++ t_reader.start()+ r_in.wait(5) # ensure reader holds lock+ t_writer.start() # writer must wait until reader finishes+ while gate._waiting_writers == 0:+ time.sleep(0.001)+ r_go.set()++ t_reader.join()+ t_writer.join()++ check("writer_exclusion/order", events, [+ "reader_start",+ "reader_end",+ "writer_start",+ "writer_end",+ ])+++def test_writer_preference():+ gate = _InferenceGate()+ events = []+ r1_in, r1_go = threading.Event(), threading.Event()+ w1_in, w1_go = threading.Event(), threading.Event()++ def reader_1():+ with gate.read_lock():+ events.append("r1_start")+ r1_in.set()+ r1_go.wait(5)+ events.append("r1_end")++ def writer_1():+ with gate.write_lock():+ events.append("w1_start")+ w1_in.set()+ w1_go.wait(5)+ events.append("w1_end")++ def reader_2():+ with gate.read_lock():+ events.append("r2_start")+ events.append("r2_end")++ t_r1 = threading.Thread(target=reader_1)+ t_w1 = threading.Thread(target=writer_1)+ t_r2 = threading.Thread(target=reader_2)++ t_r1.start()+ r1_in.wait(5) # r1 active and holding read lock+ t_w1.start()+ while gate._waiting_writers == 0:+ time.sleep(0.001) # w1 registered as waiting writer+ t_r2.start() # r2 arrives after w1, must wait for w1+ time.sleep(0.02) # r2 is blocked behind w1+ r1_go.set() # release r1 so w1 can enter++ w1_in.wait(5) # w1 is now active+ w1_go.set() # release w1 so r2 can enter++ t_r1.join()+ t_w1.join()+ t_r2.join()++ check("writer_preference/order", events, [+ "r1_start",+ "r1_end",+ "w1_start",+ "w1_end",+ "r2_start",+ "r2_end",+ ])+++def test_exception_safety():+ gate = _InferenceGate()++ try:+ with gate.read_lock():+ raise RuntimeError("reader failed")+ except RuntimeError:+ pass++ check("exception_safety/readers cleared after error", gate._readers, 0)++ try:+ with gate.write_lock():+ raise RuntimeError("writer failed")+ except RuntimeError:+ pass++ check("exception_safety/writers cleared after error", gate._writers, 0)+ check("exception_safety/waiting writers cleared", gate._waiting_writers, 0)++ # Next lock acquisitions should succeed immediately without hanging+ acquired_read = False+ with gate.read_lock():+ acquired_read = True+ check("exception_safety/subsequent read succeeds", acquired_read, True)++ acquired_write = False+ with gate.write_lock():+ acquired_write = True+ check("exception_safety/subsequent write succeeds", acquired_write, True)+++def test_multiple_concurrent_writers():+ gate = _InferenceGate()+ active_writers = []+ max_writers = [0]+ lock = threading.Lock()++ def writer(duration):+ with gate.write_lock():+ with lock:+ active_writers.append(1)+ max_writers[0] = max(max_writers[0], len(active_writers))+ time.sleep(duration)+ with lock:+ active_writers.pop()++ threads = [threading.Thread(target=writer, args=(0.02,)) for _ in range(4)]+ for t in threads:+ t.start()+ for t in threads:+ t.join()++ check("concurrent_writers/strictly serialized", max_writers[0], 1)+ check("concurrent_writers/all finished", len(active_writers), 0)+++test_concurrent_readers()+test_writer_mutual_exclusion()+test_writer_preference()+test_exception_safety()+test_multiple_concurrent_writers()++print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))+for f in FAIL:+ print(" FAIL", f)+if not FAIL:+ print("all agent gate tests passed")+sys.exit(1 if FAIL else 0)