Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
perf(agent): reuse CUDA autocast weight cache across batches
周诚宇(Zhou Chengyu)06fe81422a13
diff --git a/laya/agent.py b/laya/agent.pyindex 5ca3861c..ffea4743 100644--- a/laya/agent.py+++ b/laya/agent.py@@ -5,7 +5,8 @@ import tempfileimport threadingimport timeimport warnings-from contextlib import nullcontext+from contextlib import ExitStack, nullcontext+from contextvars import ContextVarfrom typing import Any, Dict, List, Optional, Unionimport numpy as np@@ -149,6 +150,7 @@ _TOKENIZERS_LOCK = threading.Lock()# model while other threads may be running their own forward, so the demotion and the restore are# serialised. A second request that hits OOM waits here and re-demotes only if it needs to._OOM_FALLBACK_LOCK = threading.Lock()+_BATCH_AUTOCAST_CACHE = ContextVar("laya_batch_autocast_cache", default=False)def _load_tokenizer(tok_dir: str, cfg: Dict) -> Any:@@ -839,6 +841,10 @@ class Agent(HookRegistry):self.last_fallback_reason = str(e)held_device, held_dtype, held_amp = self.device, self.dtype, self.amp_enabledhad_fast = self._fast is not None+ # Only our batch scope can retain BF16 copies after this failed forward.+ # Release them before moving the model and retrying on CPU.+ if self.device.type == "cuda" and _BATCH_AUTOCAST_CACHE.get():+ torch.clear_autocast_cache()# FastLaya keeps copied CUDA weights and replaces model.forward. Move# the model first without that replacement, or the retry would still# execute on the failed CUDA fast path.@@ -975,6 +981,7 @@ class Agent(HookRegistry):timeout = self.hooks_timeout if hooks_timeout is None else validate_timeout(hooks_timeout)ctx = PredictContext(states=states, questions=questions, model=self.model_id, agent=self,max_len=max_len, head_max_len=head_max_len)+ amp_stack = ExitStack()try:dispatch(active, "on_predict_start", ctx, raise_errors=raise_errors, lock=self._hooks_lock, timeout=timeout)states, questions = ctx.states, ctx.questions@@ -1014,6 +1021,18 @@ class Agent(HookRegistry):internal = {qid: self._to_internal(questions[qid]) for qid in ids}chunk = batch_size if (batch_size and batch_size > 0) else len(states)+ # Keep inner _infer weight casts cached across CUDA eager forwards.+ # A disabled outer scope leaves encoding/decoding in their usual+ # precision and inherits the caller's cache_enabled setting. If the+ # caller already owns a CUDA autocast scope, its cache is sufficient.+ if (chunk < len(states) and getattr(self, "device", None) is not None+ and self.device.type == "cuda" and self.amp_enabled+ and self._fast is None and not self._compiled+ and not torch.is_autocast_enabled()):+ amp_stack.enter_context(torch.autocast(device_type="cuda", dtype=self.dtype,+ enabled=False))+ amp_stack.callback(_BATCH_AUTOCAST_CACHE.reset, _BATCH_AUTOCAST_CACHE.set(True))+# Per-call token-budget overrides (a start hook may have set them).overrides: Dict[str, int] = {}if ctx.max_len is not None:@@ -1065,6 +1084,7 @@ class Agent(HookRegistry):results.extend(window_results)ctx.results = resultsexcept BaseException as exc:+ amp_stack.close()ctx.error = exctry:dispatch(active, "on_error", ctx, raise_errors=raise_errors, lock=self._hooks_lock, timeout=timeout)@@ -1073,6 +1093,7 @@ class Agent(HookRegistry):exc.__context__ = hook_excraisefinally:+ amp_stack.close()ctx.elapsed_ms = (time.perf_counter() - ctx.started_at) * 1000.0if ctx.results is not None:ctx.usage = aggregate_usage(ctx.results)diff --git a/tests/test_batch.py b/tests/test_batch.pyindex 47b3b845..a62383c9 100644--- a/tests/test_batch.py+++ b/tests/test_batch.py@@ -12,11 +12,14 @@ These tests need no model weights. They cover:import inspectimport osimport sys+from contextlib import contextmanager+from types import SimpleNamespacefrom unittest.mock import patchsys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))import numpy as np # noqa: E402+import torch # noqa: E402from laya import agent as _agent # noqa: E402from laya.agent import Agent # noqa: E402@@ -305,6 +308,66 @@ check_raises("min_confidence/bool rejected", ValueError,lambda: make_confidence_fake().predict_batch(["s0"], QUESTIONS, min_confidence=True))+# --------------------------------------------------------------- CUDA batch autocast lifetime+# Fake CUDA device and autocast so this orchestration test also runs in CPU-only CI.+def run_amp_scope_case(batch_size=2, device="cuda", amp=True, fast=None, compiled=False, fail=False):+ fake = make_fake()+ fake.device = torch.device(device)+ fake.dtype = torch.bfloat16+ fake.amp_enabled = amp+ fake._fast = fast+ fake._compiled = compiled+ events = []++ @contextmanager+ def record_autocast(**kwargs):+ events.append(("enter", kwargs))+ try:+ yield+ finally:+ events.append(("exit", kwargs))++ original_forward = fake._forward++ def forward(batch):+ events.append(("forward", _agent._BATCH_AUTOCAST_CACHE.get()))+ if fail:+ raise ValueError("forward failed")+ return original_forward(batch)++ fake._forward = forward+ with patch.object(torch, "autocast", side_effect=record_autocast):+ try:+ fake.predict_batch(["s0", "s1", "s2"], QUESTIONS, batch_size=batch_size,+ on_predict_end=lambda ctx: events.append(("end", None)),+ hooks=SimpleNamespace(on_error=lambda ctx: events.append(("error", None))))+ except ValueError:+ if not fail:+ raise+ return events+++scoped = run_amp_scope_case()+check("autocast/multiple forwards use one enclosing scope",+ [name for name, _ in scoped], ["enter", "forward", "forward", "exit", "end"])+check("autocast/outer scope disables autocast between forwards",+ scoped[0][1], {"device_type": "cuda", "dtype": torch.bfloat16, "enabled": False})+check("autocast/batch marker only active during forwards",+ [value for name, value in scoped if name == "forward"], [True, True])+check("autocast/batch marker reset before end hook", _agent._BATCH_AUTOCAST_CACHE.get(), False)+check("autocast/single forward skips extra scope",+ [name for name, _ in run_amp_scope_case(batch_size=None)], ["forward", "end"])+for label, settings in (("CPU", {"device": "cpu"}), ("disabled AMP", {"amp": False}),+ ("fast path", {"fast": object()}), ("compiled path", {"compiled": True})):+ check("autocast/%s skips extra scope" % label,+ [name for name, _ in run_amp_scope_case(**settings)], ["forward", "forward", "end"])+with patch.object(torch, "is_autocast_enabled", return_value=True):+ check("autocast/caller-owned scope needs no extra scope",+ [name for name, _ in run_amp_scope_case()], ["forward", "forward", "end"])+check("autocast/error closes scope before error and end hooks",+ [name for name, _ in run_amp_scope_case(fail=True)], ["enter", "forward", "exit", "error", "end"])++# --------------------------------------------------------------------------- reportprint("\n%d passed, %d failed" % (len(PASS), len(FAIL)))for f_ in FAIL:diff --git a/tests/test_runtime_fixes.py b/tests/test_runtime_fixes.pyindex a89e0890..16f802d6 100644--- a/tests/test_runtime_fixes.py+++ b/tests/test_runtime_fixes.py@@ -9,13 +9,16 @@ import osimport sysfrom contextlib import nullcontextfrom types import SimpleNamespace+from unittest.mock import patchimport torchimport torch.nn as nnsys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))-from laya.agent import MPS_AMP_MIN_ROWS_DEFAULT, Agent, _amp_context, _cuda_amp_dtype, _mps_amp_min_rows # noqa: E402+from laya.agent import ( # noqa: E402+ MPS_AMP_MIN_ROWS_DEFAULT, Agent, _BATCH_AUTOCAST_CACHE, _amp_context, _cuda_amp_dtype, _mps_amp_min_rows,+)from laya.common import DecisionModel, build_sequence, serialize_state # noqa: E402PASS, FAIL = [], []@@ -280,14 +283,33 @@ oom.device = torch.device("cuda") # the OOM branch only reads .typecheck("oom-fallback/count starts at 0", oom.cpu_fallback_count, 0)check("oom-fallback/reason starts None", oom.last_fallback_reason, None)-out = oom._infer(oom_batch) # first forward raises OOM -> scoped CPU retry+fallback_events = []+original_to = oom.model.to+++def record_move(device):+ fallback_events.append("move-%s" % device)+ return original_to(device)+++oom.model.to = record_move+with patch.object(torch, "clear_autocast_cache", side_effect=lambda: fallback_events.append("clear")):+ out = oom._infer(oom_batch) # first forward raises OOM -> scoped CPU retrycheck("oom-fallback/retry answered", isinstance(out, tuple), True)+check("oom-fallback/no batch scope leaves other caches alone", fallback_events[:1], ["move-cpu"])check("oom-fallback/count recorded", oom.cpu_fallback_count, 1)check("oom-fallback/reason recorded","out of memory" in (oom.last_fallback_reason or ""), True)check("oom-fallback/scoped: device restored", oom.device.type, "cuda")-oom._infer(oom_batch) # a second OOM accumulates+fallback_events.clear()+token = _BATCH_AUTOCAST_CACHE.set(True)+try:+ with patch.object(torch, "clear_autocast_cache", side_effect=lambda: fallback_events.append("clear")):+ oom._infer(oom_batch) # a second OOM inside the batch scope clears before CPU move+finally:+ _BATCH_AUTOCAST_CACHE.reset(token)+check("oom-fallback/batch copies cleared before CPU move", fallback_events[:2], ["clear", "move-cpu"])check("oom-fallback/second OOM counts too", oom.cpu_fallback_count, 2)# a plain forward never touches the counters