Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
Merge pull request #700 from RagingSilence/perf/autocast-cache
Nandakishor280dff8579b7
diff --cc tests/test_batch.pyindex fdbd5dff,a62383c9..788e7b71--- a/tests/test_batch.py+++ b/tests/test_batch.py@@@ -303,35 -306,68 +306,95 @@@ check_raises("min_confidence/out-of-ranlambda: make_confidence_fake().predict_batch(["s0"], QUESTIONS, min_confidence=1.5))check_raises("min_confidence/bool rejected", ValueError,lambda: make_confidence_fake().predict_batch(["s0"], QUESTIONS, min_confidence=True))+# A legend maps a level index to the TEXT of that level. Its keys are already strings, and+# `structured` stringifies every level it builds, so a numeric scale written directly used to be+# the one path that echoed the caller's own type back: `{"0": 1, "1": 2}` instead of+# `{"0": "1", "1": "2"}`. That made the response's JSON types depend on the input's types, so a+# client that reads a level as a string had to handle a number as well.+#+# `render_criterion` is what renders the text, so a structured level comes back as the JSON the+# model was shown rather than a Python repr, and bool/None follow JSON spelling (`true`, `null`).+for label, crit, want in (+ ("int levels", [1, 2, 3], {"0": "1", "1": "2", "2": "3"}),+ ("float levels", [1.5, 2.5], {"0": "1.5", "1": "2.5"}),+ ("bool levels", [True, False], {"0": "true", "1": "false"}),+ ("string levels", ["low", "high"], {"0": "low", "1": "high"}),+ ("dict level", [{"a": 1}], {"0": '{"a": 1}'}),+ ("list level", [[1, 2]], {"0": "[1, 2]"}),+):+ internal = {"level": {"t": "score", "crit": crit}}+ k = len(crit)+ logits = np.tile(np.log([1.0 / k] * k), (k, 1))+ act = np.array([[0.25, 0.75]] * k)+ result = decoder._decode_answers(+ logits, act, [{"markers": [0, 1]}] * k, ["level"] * k, internal, 0+ )+ check("decode/score legend values are str (%s)" % label, result["level"]["legend"], want)+ check_true("decode/score legend values have no non-str (%s)" % label,+ all(isinstance(v, str) for v in result["level"]["legend"].values()),+ result["level"]["legend"])+ # --------------------------------------------------------------- CUDA batch autocast lifetime+ # Fake CUDA device and autocast so this orchestration test also runs in CPU-only CI.+ def run_amp_scope_case(batch_size=2, device="cuda", amp=True, fast=None, compiled=False, fail=False):+ fake = make_fake()+ fake.device = torch.device(device)+ fake.dtype = torch.bfloat16+ fake.amp_enabled = amp+ fake._fast = fast+ fake._compiled = compiled+ events = []++ @contextmanager+ def record_autocast(**kwargs):+ events.append(("enter", kwargs))+ try:+ yield+ finally:+ events.append(("exit", kwargs))++ original_forward = fake._forward++ def forward(batch):+ events.append(("forward", _agent._BATCH_AUTOCAST_CACHE.get()))+ if fail:+ raise ValueError("forward failed")+ return original_forward(batch)++ fake._forward = forward+ with patch.object(torch, "autocast", side_effect=record_autocast):+ try:+ fake.predict_batch(["s0", "s1", "s2"], QUESTIONS, batch_size=batch_size,+ on_predict_end=lambda ctx: events.append(("end", None)),+ hooks=SimpleNamespace(on_error=lambda ctx: events.append(("error", None))))+ except ValueError:+ if not fail:+ raise+ return events+++ scoped = run_amp_scope_case()+ check("autocast/multiple forwards use one enclosing scope",+ [name for name, _ in scoped], ["enter", "forward", "forward", "exit", "end"])+ check("autocast/outer scope disables autocast between forwards",+ scoped[0][1], {"device_type": "cuda", "dtype": torch.bfloat16, "enabled": False})+ check("autocast/batch marker only active during forwards",+ [value for name, value in scoped if name == "forward"], [True, True])+ check("autocast/batch marker reset before end hook", _agent._BATCH_AUTOCAST_CACHE.get(), False)+ check("autocast/single forward skips extra scope",+ [name for name, _ in run_amp_scope_case(batch_size=None)], ["forward", "end"])+ for label, settings in (("CPU", {"device": "cpu"}), ("disabled AMP", {"amp": False}),+ ("fast path", {"fast": object()}), ("compiled path", {"compiled": True})):+ check("autocast/%s skips extra scope" % label,+ [name for name, _ in run_amp_scope_case(**settings)], ["forward", "forward", "end"])+ with patch.object(torch, "is_autocast_enabled", return_value=True):+ check("autocast/caller-owned scope needs no extra scope",+ [name for name, _ in run_amp_scope_case()], ["forward", "forward", "end"])+ check("autocast/error closes scope before error and end hooks",+ [name for name, _ in run_amp_scope_case(fail=True)], ["enter", "forward", "exit", "error", "end"])++# --------------------------------------------------------------------------- reportprint("\n%d passed, %d failed" % (len(PASS), len(FAIL)))for f_ in FAIL: