Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

Merge pull request #700 from RagingSilence/perf/autocast-cache

Nandakishor280dff8579b7
diff --cc tests/test_batch.py
index fdbd5dff,a62383c9..788e7b71
--- a/tests/test_batch.py
+++ b/tests/test_batch.py
@@@ -303,35 -306,68 +306,95 @@@ check_raises("min_confidence/out-of-ran
lambda: make_confidence_fake().predict_batch(["s0"], QUESTIONS, min_confidence=1.5))
check_raises("min_confidence/bool rejected", ValueError,
lambda: make_confidence_fake().predict_batch(["s0"], QUESTIONS, min_confidence=True))
+# A legend maps a level index to the TEXT of that level. Its keys are already strings, and
+# `structured` stringifies every level it builds, so a numeric scale written directly used to be
+# the one path that echoed the caller's own type back: `{"0": 1, "1": 2}` instead of
+# `{"0": "1", "1": "2"}`. That made the response's JSON types depend on the input's types, so a
+# client that reads a level as a string had to handle a number as well.
+#
+# `render_criterion` is what renders the text, so a structured level comes back as the JSON the
+# model was shown rather than a Python repr, and bool/None follow JSON spelling (`true`, `null`).
+for label, crit, want in (
+ ("int levels", [1, 2, 3], {"0": "1", "1": "2", "2": "3"}),
+ ("float levels", [1.5, 2.5], {"0": "1.5", "1": "2.5"}),
+ ("bool levels", [True, False], {"0": "true", "1": "false"}),
+ ("string levels", ["low", "high"], {"0": "low", "1": "high"}),
+ ("dict level", [{"a": 1}], {"0": '{"a": 1}'}),
+ ("list level", [[1, 2]], {"0": "[1, 2]"}),
+):
+ internal = {"level": {"t": "score", "crit": crit}}
+ k = len(crit)
+ logits = np.tile(np.log([1.0 / k] * k), (k, 1))
+ act = np.array([[0.25, 0.75]] * k)
+ result = decoder._decode_answers(
+ logits, act, [{"markers": [0, 1]}] * k, ["level"] * k, internal, 0
+ )
+ check("decode/score legend values are str (%s)" % label, result["level"]["legend"], want)
+ check_true("decode/score legend values have no non-str (%s)" % label,
+ all(isinstance(v, str) for v in result["level"]["legend"].values()),
+ result["level"]["legend"])
+ # --------------------------------------------------------------- CUDA batch autocast lifetime
+ # Fake CUDA device and autocast so this orchestration test also runs in CPU-only CI.
+ def run_amp_scope_case(batch_size=2, device="cuda", amp=True, fast=None, compiled=False, fail=False):
+ fake = make_fake()
+ fake.device = torch.device(device)
+ fake.dtype = torch.bfloat16
+ fake.amp_enabled = amp
+ fake._fast = fast
+ fake._compiled = compiled
+ events = []
+
+ @contextmanager
+ def record_autocast(**kwargs):
+ events.append(("enter", kwargs))
+ try:
+ yield
+ finally:
+ events.append(("exit", kwargs))
+
+ original_forward = fake._forward
+
+ def forward(batch):
+ events.append(("forward", _agent._BATCH_AUTOCAST_CACHE.get()))
+ if fail:
+ raise ValueError("forward failed")
+ return original_forward(batch)
+
+ fake._forward = forward
+ with patch.object(torch, "autocast", side_effect=record_autocast):
+ try:
+ fake.predict_batch(["s0", "s1", "s2"], QUESTIONS, batch_size=batch_size,
+ on_predict_end=lambda ctx: events.append(("end", None)),
+ hooks=SimpleNamespace(on_error=lambda ctx: events.append(("error", None))))
+ except ValueError:
+ if not fail:
+ raise
+ return events
+
+
+ scoped = run_amp_scope_case()
+ check("autocast/multiple forwards use one enclosing scope",
+ [name for name, _ in scoped], ["enter", "forward", "forward", "exit", "end"])
+ check("autocast/outer scope disables autocast between forwards",
+ scoped[0][1], {"device_type": "cuda", "dtype": torch.bfloat16, "enabled": False})
+ check("autocast/batch marker only active during forwards",
+ [value for name, value in scoped if name == "forward"], [True, True])
+ check("autocast/batch marker reset before end hook", _agent._BATCH_AUTOCAST_CACHE.get(), False)
+ check("autocast/single forward skips extra scope",
+ [name for name, _ in run_amp_scope_case(batch_size=None)], ["forward", "end"])
+ for label, settings in (("CPU", {"device": "cpu"}), ("disabled AMP", {"amp": False}),
+ ("fast path", {"fast": object()}), ("compiled path", {"compiled": True})):
+ check("autocast/%s skips extra scope" % label,
+ [name for name, _ in run_amp_scope_case(**settings)], ["forward", "forward", "end"])
+ with patch.object(torch, "is_autocast_enabled", return_value=True):
+ check("autocast/caller-owned scope needs no extra scope",
+ [name for name, _ in run_amp_scope_case()], ["forward", "forward", "end"])
+ check("autocast/error closes scope before error and end hooks",
+ [name for name, _ in run_amp_scope_case(fail=True)], ["enter", "forward", "exit", "error", "end"])
+
+
# --------------------------------------------------------------------------- report
print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))
for f_ in FAIL: