Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(agent): synchronize GPU OOM fallback to protect concurrent in-flight inference (#649)

Swatantra Yadav93e855edd387
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 39843b20..af8176fc 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -129,6 +129,7 @@ jobs:
python tests/test_question_token_reuse.py
python tests/test_lazy_import.py
python tests/test_runtime_fixes.py
+ python tests/test_agent_gate.py
python tests/test_option_collapse.py
python tests/test_lang_guess.py
python tests/test_identifier_complexity.py
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index ef9eec7a..562aec32 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -66,6 +66,7 @@ jobs:
python tests/test_tokenizer_concurrency.py
python tests/test_lazy_import.py
python tests/test_runtime_fixes.py
+ python tests/test_agent_gate.py
python tests/test_lang_guess.py
python tests/test_option_collapse.py
python tests/test_identifier_complexity.py
diff --git a/laya/agent.py b/laya/agent.py
index c98e6859..b376706c 100644
--- a/laya/agent.py
+++ b/laya/agent.py
@@ -5,7 +5,7 @@ import tempfile
import threading
import time
import warnings
-from contextlib import ExitStack, nullcontext
+from contextlib import ExitStack, contextmanager, nullcontext
from contextvars import ContextVar
from typing import Any, Dict, List, Optional, Union
@@ -155,6 +155,54 @@ def _verify_compatibility(model: torch.nn.Module, cfg: Dict, weights: Dict[str,
_TOKENIZERS: Dict[tuple, Any] = {}
_TOKENIZERS_LOCK = threading.Lock()
+class _InferenceGate:
+ """Read-write synchronization gate protecting shared model device placement (#649).
+
+ Normal inference acquires the read lock so multiple threads can evaluate the model
+ concurrently without serialization. When a thread encounters GPU OOM and needs
+ to temporarily migrate the model to CPU for a scoped fallback, it acquires the write
+ lock. This ensures all in-flight requests finish safely before the model is moved,
+ and blocks new requests until the model and device state are restored.
+ """
+
+ def __init__(self):
+ self._cond = threading.Condition(threading.Lock())
+ self._readers = 0
+ self._writers = 0
+ self._waiting_writers = 0
+
+ @contextmanager
+ def read_lock(self):
+ with self._cond:
+ while self._writers > 0 or self._waiting_writers > 0:
+ self._cond.wait()
+ self._readers += 1
+ try:
+ yield
+ finally:
+ with self._cond:
+ self._readers -= 1
+ if self._readers == 0:
+ self._cond.notify_all()
+
+ @contextmanager
+ def write_lock(self):
+ with self._cond:
+ self._waiting_writers += 1
+ try:
+ while self._readers > 0 or self._writers > 0:
+ self._cond.wait()
+ self._writers = 1
+ finally:
+ self._waiting_writers -= 1
+ try:
+ yield
+ finally:
+ with self._cond:
+ self._writers = 0
+ self._cond.notify_all()
+
+
# The per-inference CPU fallback rewrites shared runtime state (device, dtype, amp) and moves the
# model while other threads may be running their own forward, so the demotion and the restore are
# serialised. A second request that hits OOM waits here and re-demotes only if it needs to.
@@ -323,6 +371,7 @@ class Agent(HookRegistry):
last_fallback_reason = None
# Set when `compile=True` wrapped the model in torch.compile.
_compiled = False
+ _gate = _InferenceGate()
def __init__(
self,
@@ -568,6 +617,7 @@ class Agent(HookRegistry):
self.dtype = torch.bfloat16
self._fast = None
+ self._gate = _InferenceGate()
# 2. Place on device with graceful fallback to CPU on memory error
fell_back_from = fell_back_why = None
@@ -956,7 +1006,8 @@ class Agent(HookRegistry):
)
try:
- out = run()
+ with self._gate.read_lock():
+ out = run()
except (RuntimeError, torch.cuda.OutOfMemoryError) as e:
low = str(e).lower()
# An OOM is `torch.cuda.OutOfMemoryError` or says so in its message. The bare
@@ -966,13 +1017,16 @@ class Agent(HookRegistry):
print("Warning: GPU memory exceeded during inference. Retrying this request on CPU...")
# Scoped, not permanent: the demotion used to rewrite device/dtype/amp and move
# the model for the life of the process, so one oversized request left every
- # later call ~10-15x slower on CPU. Demote under the lock, answer this request
- # on CPU, then put the runtime back the way it was.
- with _OOM_FALLBACK_LOCK:
+ # later call ~10-15x slower on CPU. Demote under the write lock (#649) so any
+ # concurrent in-flight GPU requests finish before the model is moved to CPU,
+ # and put the runtime back before allowing new requests to start.
+ with self._gate.write_lock(), _OOM_FALLBACK_LOCK:
# Recorded on entry, under the same lock as the demotion: the count and
# reason must be readable by /health without racing a concurrent fallback.
self.cpu_fallback_count += 1
self.last_fallback_reason = str(e)
+ if self.device.type == "cpu":
+ return run()
held_device, held_dtype, held_amp = self.device, self.dtype, self.amp_enabled
had_fast = self._fast is not None
# Only our batch scope can retain BF16 copies after this failed forward.
@@ -998,15 +1052,16 @@ class Agent(HookRegistry):
# request in full precision. One miss must not turn AMP off; a build that
# lacks the op fails the same way every time, so after a short streak the
# process drops to full precision instead of paying for two forwards (#351).
- try:
- return run(enabled=False)
- finally:
- self._amp_failures += 1
- if self._amp_failures >= _AMP_FAIL_LIMIT:
- print("Warning: autocast failed %d times in a row. Disabling mixed precision."
- % self._amp_failures)
- self.amp_enabled = False
- self.dtype = torch.float32
+ with self._gate.read_lock():
+ try:
+ return run(enabled=False)
+ finally:
+ self._amp_failures += 1
+ if self._amp_failures >= _AMP_FAIL_LIMIT:
+ print("Warning: autocast failed %d times in a row. Disabling mixed precision."
+ % self._amp_failures)
+ self.amp_enabled = False
+ self.dtype = torch.float32
raise
else:
self._amp_failures = 0
diff --git a/tests/test_agent_gate.py b/tests/test_agent_gate.py
new file mode 100644
index 00000000..fbe256aa
--- /dev/null
+++ b/tests/test_agent_gate.py
@@ -0,0 +1,248 @@
+"""Tests for _InferenceGate read-write synchronization gate (#649).
+
+Ensures that:
+1. Normal inference allows concurrent readers without mutual exclusion.
+2. An OOM fallback (writer) waits for in-flight readers to finish before migrating the model.
+3. While fallback is active (writer lock held), incoming requests (readers) are queued.
+4. Writers have preference over new readers so OOM fallback does not starve under load.
+5. Exceptions inside read or write contexts cleanly release state without deadlocking.
+"""
+import os
+import sys
+import threading
+import time
+import types
+
+ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.insert(0, ROOT)
+
+if "torch" not in sys.modules:
+ t = types.ModuleType("torch")
+ t.cuda = types.ModuleType("torch.cuda")
+ t.cuda.OutOfMemoryError = type("OutOfMemoryError", (RuntimeError,), {})
+ t.nn = types.ModuleType("torch.nn")
+ t.nn.Module = type("Module", (), {})
+ t.nn.MultiheadAttention = type("MultiheadAttention", (), {})
+ t.nn.functional = types.ModuleType("torch.nn.functional")
+ t.utils = types.ModuleType("torch.utils")
+ t.utils.checkpoint = types.ModuleType("torch.utils.checkpoint")
+ t.utils.checkpoint.checkpoint = lambda f, *a, **kw: f(*a, **kw)
+ t.device = lambda d: d
+ t.float32 = "float32"
+ t.float16 = "float16"
+ t.bfloat16 = "bfloat16"
+ t.no_grad = lambda: (lambda f: f)
+ sys.modules["torch"] = t
+ sys.modules["torch.cuda"] = t.cuda
+ sys.modules["torch.nn"] = t.nn
+ sys.modules["torch.nn.functional"] = t.nn.functional
+ sys.modules["torch.utils"] = t.utils
+ sys.modules["torch.utils.checkpoint"] = t.utils.checkpoint
+
+from laya.agent import _InferenceGate
+
+PASS, FAIL = [], []
+
+
+def check(name, got, want):
+ if got == want:
+ PASS.append(name)
+ else:
+ FAIL.append("%s: got %r, want %r" % (name, got, want))
+
+
+def test_concurrent_readers():
+ gate = _InferenceGate()
+ active_readers = []
+ max_concurrent = [0]
+ lock = threading.Lock()
+ all_in = threading.Event()
+ entered = 0
+ entered_lock = threading.Lock()
+
+ def reader():
+ nonlocal entered
+ with gate.read_lock():
+ with lock:
+ active_readers.append(1)
+ max_concurrent[0] = max(max_concurrent[0], len(active_readers))
+ with entered_lock:
+ entered += 1
+ if entered == 5:
+ all_in.set()
+ all_in.wait(5)
+ with lock:
+ active_readers.pop()
+
+ threads = [threading.Thread(target=reader) for _ in range(5)]
+ for t in threads:
+ t.start()
+ for t in threads:
+ t.join()
+
+ check("concurrent_readers/all ran concurrently", max_concurrent[0], 5)
+ check("concurrent_readers/all finished", len(active_readers), 0)
+
+
+def test_writer_mutual_exclusion():
+ gate = _InferenceGate()
+ events = []
+ r_in, r_go = threading.Event(), threading.Event()
+
+ def reader():
+ with gate.read_lock():
+ events.append("reader_start")
+ r_in.set()
+ r_go.wait(5)
+ events.append("reader_end")
+
+ def writer():
+ with gate.write_lock():
+ events.append("writer_start")
+ events.append("writer_end")
+
+ t_reader = threading.Thread(target=reader)
+ t_writer = threading.Thread(target=writer)
+
+ t_reader.start()
+ r_in.wait(5) # ensure reader holds lock
+ t_writer.start() # writer must wait until reader finishes
+ while gate._waiting_writers == 0:
+ time.sleep(0.001)
+ r_go.set()
+
+ t_reader.join()
+ t_writer.join()
+
+ check("writer_exclusion/order", events, [
+ "reader_start",
+ "reader_end",
+ "writer_start",
+ "writer_end",
+ ])
+
+
+def test_writer_preference():
+ gate = _InferenceGate()
+ events = []
+ r1_in, r1_go = threading.Event(), threading.Event()
+ w1_in, w1_go = threading.Event(), threading.Event()
+
+ def reader_1():
+ with gate.read_lock():
+ events.append("r1_start")
+ r1_in.set()
+ r1_go.wait(5)
+ events.append("r1_end")
+
+ def writer_1():
+ with gate.write_lock():
+ events.append("w1_start")
+ w1_in.set()
+ w1_go.wait(5)
+ events.append("w1_end")
+
+ def reader_2():
+ with gate.read_lock():
+ events.append("r2_start")
+ events.append("r2_end")
+
+ t_r1 = threading.Thread(target=reader_1)
+ t_w1 = threading.Thread(target=writer_1)
+ t_r2 = threading.Thread(target=reader_2)
+
+ t_r1.start()
+ r1_in.wait(5) # r1 active and holding read lock
+ t_w1.start()
+ while gate._waiting_writers == 0:
+ time.sleep(0.001) # w1 registered as waiting writer
+ t_r2.start() # r2 arrives after w1, must wait for w1
+ time.sleep(0.02) # r2 is blocked behind w1
+ r1_go.set() # release r1 so w1 can enter
+
+ w1_in.wait(5) # w1 is now active
+ w1_go.set() # release w1 so r2 can enter
+
+ t_r1.join()
+ t_w1.join()
+ t_r2.join()
+
+ check("writer_preference/order", events, [
+ "r1_start",
+ "r1_end",
+ "w1_start",
+ "w1_end",
+ "r2_start",
+ "r2_end",
+ ])
+
+
+def test_exception_safety():
+ gate = _InferenceGate()
+
+ try:
+ with gate.read_lock():
+ raise RuntimeError("reader failed")
+ except RuntimeError:
+ pass
+
+ check("exception_safety/readers cleared after error", gate._readers, 0)
+
+ try:
+ with gate.write_lock():
+ raise RuntimeError("writer failed")
+ except RuntimeError:
+ pass
+
+ check("exception_safety/writers cleared after error", gate._writers, 0)
+ check("exception_safety/waiting writers cleared", gate._waiting_writers, 0)
+
+ # Next lock acquisitions should succeed immediately without hanging
+ acquired_read = False
+ with gate.read_lock():
+ acquired_read = True
+ check("exception_safety/subsequent read succeeds", acquired_read, True)
+
+ acquired_write = False
+ with gate.write_lock():
+ acquired_write = True
+ check("exception_safety/subsequent write succeeds", acquired_write, True)
+
+
+def test_multiple_concurrent_writers():
+ gate = _InferenceGate()
+ active_writers = []
+ max_writers = [0]
+ lock = threading.Lock()
+
+ def writer(duration):
+ with gate.write_lock():
+ with lock:
+ active_writers.append(1)
+ max_writers[0] = max(max_writers[0], len(active_writers))
+ time.sleep(duration)
+ with lock:
+ active_writers.pop()
+
+ threads = [threading.Thread(target=writer, args=(0.02,)) for _ in range(4)]
+ for t in threads:
+ t.start()
+ for t in threads:
+ t.join()
+
+ check("concurrent_writers/strictly serialized", max_writers[0], 1)
+ check("concurrent_writers/all finished", len(active_writers), 0)
+
+
+test_concurrent_readers()
+test_writer_mutual_exclusion()
+test_writer_preference()
+test_exception_safety()
+test_multiple_concurrent_writers()
+
+print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))
+for f in FAIL:
+ print(" FAIL", f)
+if not FAIL:
+ print("all agent gate tests passed")
+sys.exit(1 if FAIL else 0)