Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix: warn on CPU fallback only when one actually happened
Nandakishore630a68f93aa
diff --git a/laya/agent.py b/laya/agent.pyindex cc244626..ae03574d 100644--- a/laya/agent.py+++ b/laya/agent.py@@ -201,30 +201,30 @@ class Agent:self.dtype = torch.float32# 2. Place on device with graceful fallback to CPU on memory error+ fell_back_from = fell_back_why = Nonetry:self.model.to(self.device).eval()except (RuntimeError, torch.cuda.OutOfMemoryError) as e:if self.device.type != "cpu":+ # Record what actually went wrong: the reason matters more than the symptom,+ # and it is the only place the underlying exception is ever surfaced.+ fell_back_from, fell_back_why = self.device, eself.device = torch.device("cpu")self.dtype = torch.float32self.model.to(self.device).eval()else:raise e- # 3. Warn on CPU fallback if CUDA was expected or available (fixes Issue #6)- actual_device = next(self.model.parameters()).device- if actual_device.type == "cpu" and (- torch.cuda.is_available() or getattr(torch.version, "cuda", None) is not None- ):+ if fell_back_from is not None:print(- "\n[laya] ⚠️ Warning: Model loaded on CPU despite CUDA being present in the environment.\n"- " Inference will be ~10-15x slower (~200-500 ms vs ~35 ms on GPU).\n"- " If you have a newer NVIDIA GPU (e.g. Blackwell/RTX 50-series), your current\n"- " PyTorch build may not support your CUDA architecture.\n"- " Fix: pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cu128\n"- " See: https://pytorch.org/get-started/locally/\n",- flush=True,- )+ "\n[laya] Warning: could not place the model on %s, so it is running on CPU.\n"+ " Reason: %s\n"+ " Inference will be roughly 10-15x slower (~200-500 ms rather than ~35 ms).\n"+ " If this is a newer NVIDIA GPU (Blackwell / RTX 50-series), your PyTorch build\n"+ " may not support its CUDA architecture:\n"+ " pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cu128\n"+ " See https://pytorch.org/get-started/locally/\n"+ % (fell_back_from, fell_back_why), flush=True)@staticmethoddef _to_internal(qdef: Dict) -> Dict:diff --git a/tests/test_criteria.py b/tests/test_criteria.pyindex 76a8d6fa..42eb6407 100644--- a/tests/test_criteria.py+++ b/tests/test_criteria.py@@ -99,6 +99,23 @@ parsed = json.loads(render_options({"t": "noul", "ins": "x", "crit": {"true": {"a": 1}, "false": {"b": 2}}})[1].split("true: ", 1)[1])check("emitted json round-trips", parsed, {"a": 1})++# --------------------------------------------------------------- CPU-fallback warning (#9 follow-up)+# The warning must fire only when a fallback actually happened -- not merely because the machine+# has CUDA. `laya.load(path, device="cpu")` on a GPU box is a deliberate choice, not a problem.+import inspect # noqa: E402++from laya import agent as _agent # noqa: E402++_src = inspect.getsource(_agent.Agent.__init__)+check_true("fallback/flag is initialised", "fell_back_from = fell_back_why = None" in _src)+check_true("fallback/warns only on a real fallback", "if fell_back_from is not None:" in _src)+check_true("fallback/reports the underlying reason", "Reason: %s" in _src)+check_true("fallback/keeps the actionable advice", "download.pytorch.org/whl/nightly" in _src)+check_true("fallback/no bare cuda probe for the warning",+ "torch.cuda.is_available() or getattr(torch.version" not in _src)++print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))for f in FAIL:print(" FAIL " + f)