Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

perf(agent): reuse CUDA autocast weight cache across batches

周诚宇(Zhou Chengyu)06fe81422a13
diff --git a/laya/agent.py b/laya/agent.py
index 5ca3861c..ffea4743 100644
--- a/laya/agent.py
+++ b/laya/agent.py
@@ -5,7 +5,8 @@ import tempfile
import threading
import time
import warnings
-from contextlib import nullcontext
+from contextlib import ExitStack, nullcontext
+from contextvars import ContextVar
from typing import Any, Dict, List, Optional, Union
import numpy as np
@@ -149,6 +150,7 @@ _TOKENIZERS_LOCK = threading.Lock()
# model while other threads may be running their own forward, so the demotion and the restore are
# serialised. A second request that hits OOM waits here and re-demotes only if it needs to.
_OOM_FALLBACK_LOCK = threading.Lock()
+_BATCH_AUTOCAST_CACHE = ContextVar("laya_batch_autocast_cache", default=False)
def _load_tokenizer(tok_dir: str, cfg: Dict) -> Any:
@@ -839,6 +841,10 @@ class Agent(HookRegistry):
self.last_fallback_reason = str(e)
held_device, held_dtype, held_amp = self.device, self.dtype, self.amp_enabled
had_fast = self._fast is not None
+ # Only our batch scope can retain BF16 copies after this failed forward.
+ # Release them before moving the model and retrying on CPU.
+ if self.device.type == "cuda" and _BATCH_AUTOCAST_CACHE.get():
+ torch.clear_autocast_cache()
# FastLaya keeps copied CUDA weights and replaces model.forward. Move
# the model first without that replacement, or the retry would still
# execute on the failed CUDA fast path.
@@ -975,6 +981,7 @@ class Agent(HookRegistry):
timeout = self.hooks_timeout if hooks_timeout is None else validate_timeout(hooks_timeout)
ctx = PredictContext(states=states, questions=questions, model=self.model_id, agent=self,
max_len=max_len, head_max_len=head_max_len)
+ amp_stack = ExitStack()
try:
dispatch(active, "on_predict_start", ctx, raise_errors=raise_errors, lock=self._hooks_lock, timeout=timeout)
states, questions = ctx.states, ctx.questions
@@ -1014,6 +1021,18 @@ class Agent(HookRegistry):
internal = {qid: self._to_internal(questions[qid]) for qid in ids}
chunk = batch_size if (batch_size and batch_size > 0) else len(states)
+ # Keep inner _infer weight casts cached across CUDA eager forwards.
+ # A disabled outer scope leaves encoding/decoding in their usual
+ # precision and inherits the caller's cache_enabled setting. If the
+ # caller already owns a CUDA autocast scope, its cache is sufficient.
+ if (chunk < len(states) and getattr(self, "device", None) is not None
+ and self.device.type == "cuda" and self.amp_enabled
+ and self._fast is None and not self._compiled
+ and not torch.is_autocast_enabled()):
+ amp_stack.enter_context(torch.autocast(device_type="cuda", dtype=self.dtype,
+ enabled=False))
+ amp_stack.callback(_BATCH_AUTOCAST_CACHE.reset, _BATCH_AUTOCAST_CACHE.set(True))
+
# Per-call token-budget overrides (a start hook may have set them).
overrides: Dict[str, int] = {}
if ctx.max_len is not None:
@@ -1065,6 +1084,7 @@ class Agent(HookRegistry):
results.extend(window_results)
ctx.results = results
except BaseException as exc:
+ amp_stack.close()
ctx.error = exc
try:
dispatch(active, "on_error", ctx, raise_errors=raise_errors, lock=self._hooks_lock, timeout=timeout)
@@ -1073,6 +1093,7 @@ class Agent(HookRegistry):
exc.__context__ = hook_exc
raise
finally:
+ amp_stack.close()
ctx.elapsed_ms = (time.perf_counter() - ctx.started_at) * 1000.0
if ctx.results is not None:
ctx.usage = aggregate_usage(ctx.results)
diff --git a/tests/test_batch.py b/tests/test_batch.py
index 47b3b845..a62383c9 100644
--- a/tests/test_batch.py
+++ b/tests/test_batch.py
@@ -12,11 +12,14 @@ These tests need no model weights. They cover:
import inspect
import os
import sys
+from contextlib import contextmanager
+from types import SimpleNamespace
from unittest.mock import patch
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
import numpy as np # noqa: E402
+import torch # noqa: E402
from laya import agent as _agent # noqa: E402
from laya.agent import Agent # noqa: E402
@@ -305,6 +308,66 @@ check_raises("min_confidence/bool rejected", ValueError,
lambda: make_confidence_fake().predict_batch(["s0"], QUESTIONS, min_confidence=True))
+# --------------------------------------------------------------- CUDA batch autocast lifetime
+# Fake CUDA device and autocast so this orchestration test also runs in CPU-only CI.
+def run_amp_scope_case(batch_size=2, device="cuda", amp=True, fast=None, compiled=False, fail=False):
+ fake = make_fake()
+ fake.device = torch.device(device)
+ fake.dtype = torch.bfloat16
+ fake.amp_enabled = amp
+ fake._fast = fast
+ fake._compiled = compiled
+ events = []
+
+ @contextmanager
+ def record_autocast(**kwargs):
+ events.append(("enter", kwargs))
+ try:
+ yield
+ finally:
+ events.append(("exit", kwargs))
+
+ original_forward = fake._forward
+
+ def forward(batch):
+ events.append(("forward", _agent._BATCH_AUTOCAST_CACHE.get()))
+ if fail:
+ raise ValueError("forward failed")
+ return original_forward(batch)
+
+ fake._forward = forward
+ with patch.object(torch, "autocast", side_effect=record_autocast):
+ try:
+ fake.predict_batch(["s0", "s1", "s2"], QUESTIONS, batch_size=batch_size,
+ on_predict_end=lambda ctx: events.append(("end", None)),
+ hooks=SimpleNamespace(on_error=lambda ctx: events.append(("error", None))))
+ except ValueError:
+ if not fail:
+ raise
+ return events
+
+
+scoped = run_amp_scope_case()
+check("autocast/multiple forwards use one enclosing scope",
+ [name for name, _ in scoped], ["enter", "forward", "forward", "exit", "end"])
+check("autocast/outer scope disables autocast between forwards",
+ scoped[0][1], {"device_type": "cuda", "dtype": torch.bfloat16, "enabled": False})
+check("autocast/batch marker only active during forwards",
+ [value for name, value in scoped if name == "forward"], [True, True])
+check("autocast/batch marker reset before end hook", _agent._BATCH_AUTOCAST_CACHE.get(), False)
+check("autocast/single forward skips extra scope",
+ [name for name, _ in run_amp_scope_case(batch_size=None)], ["forward", "end"])
+for label, settings in (("CPU", {"device": "cpu"}), ("disabled AMP", {"amp": False}),
+ ("fast path", {"fast": object()}), ("compiled path", {"compiled": True})):
+ check("autocast/%s skips extra scope" % label,
+ [name for name, _ in run_amp_scope_case(**settings)], ["forward", "forward", "end"])
+with patch.object(torch, "is_autocast_enabled", return_value=True):
+ check("autocast/caller-owned scope needs no extra scope",
+ [name for name, _ in run_amp_scope_case()], ["forward", "forward", "end"])
+check("autocast/error closes scope before error and end hooks",
+ [name for name, _ in run_amp_scope_case(fail=True)], ["enter", "forward", "exit", "error", "end"])
+
+
# --------------------------------------------------------------------------- report
print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))
for f_ in FAIL:
diff --git a/tests/test_runtime_fixes.py b/tests/test_runtime_fixes.py
index a89e0890..16f802d6 100644
--- a/tests/test_runtime_fixes.py
+++ b/tests/test_runtime_fixes.py
@@ -9,13 +9,16 @@ import os
import sys
from contextlib import nullcontext
from types import SimpleNamespace
+from unittest.mock import patch
import torch
import torch.nn as nn
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-from laya.agent import MPS_AMP_MIN_ROWS_DEFAULT, Agent, _amp_context, _cuda_amp_dtype, _mps_amp_min_rows # noqa: E402
+from laya.agent import ( # noqa: E402
+ MPS_AMP_MIN_ROWS_DEFAULT, Agent, _BATCH_AUTOCAST_CACHE, _amp_context, _cuda_amp_dtype, _mps_amp_min_rows,
+)
from laya.common import DecisionModel, build_sequence, serialize_state # noqa: E402
PASS, FAIL = [], []
@@ -280,14 +283,33 @@ oom.device = torch.device("cuda") # the OOM branch only reads .type
check("oom-fallback/count starts at 0", oom.cpu_fallback_count, 0)
check("oom-fallback/reason starts None", oom.last_fallback_reason, None)
-out = oom._infer(oom_batch) # first forward raises OOM -> scoped CPU retry
+fallback_events = []
+original_to = oom.model.to
+
+
+def record_move(device):
+ fallback_events.append("move-%s" % device)
+ return original_to(device)
+
+
+oom.model.to = record_move
+with patch.object(torch, "clear_autocast_cache", side_effect=lambda: fallback_events.append("clear")):
+ out = oom._infer(oom_batch) # first forward raises OOM -> scoped CPU retry
check("oom-fallback/retry answered", isinstance(out, tuple), True)
+check("oom-fallback/no batch scope leaves other caches alone", fallback_events[:1], ["move-cpu"])
check("oom-fallback/count recorded", oom.cpu_fallback_count, 1)
check("oom-fallback/reason recorded",
"out of memory" in (oom.last_fallback_reason or ""), True)
check("oom-fallback/scoped: device restored", oom.device.type, "cuda")
-oom._infer(oom_batch) # a second OOM accumulates
+fallback_events.clear()
+token = _BATCH_AUTOCAST_CACHE.set(True)
+try:
+ with patch.object(torch, "clear_autocast_cache", side_effect=lambda: fallback_events.append("clear")):
+ oom._infer(oom_batch) # a second OOM inside the batch scope clears before CPU move
+finally:
+ _BATCH_AUTOCAST_CACHE.reset(token)
+check("oom-fallback/batch copies cleared before CPU move", fallback_events[:2], ["clear", "move-cpu"])
check("oom-fallback/second OOM counts too", oom.cpu_fallback_count, 2)
# a plain forward never touches the counters