Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(shortlist): refresh embedding device on each call (#145)
Alphin Tom42a086a831c9
diff --git a/laya/shortlist.py b/laya/shortlist.pyindex b37638e9..71a51bd5 100644--- a/laya/shortlist.py+++ b/laya/shortlist.py@@ -122,6 +122,7 @@ def embed_fn_from_agent(Padding positions are excluded from the mean. The encoder's train/eval flag isleft as the caller set it (a loaded ``Agent`` is already in eval).+ Each call uses the current ``agent.device``, including after CPU fallback."""if isinstance(max_length, bool) or not isinstance(max_length, int) or max_length < 1:raise ValueError("max_length must be a positive integer, got %r" % (max_length,))@@ -132,13 +133,13 @@ def embed_fn_from_agent(tok = agent.tokencoder = agent.model.encoder- device = agent.devicedef embed_fn(texts: Sequence[str]) -> np.ndarray:rows = ["" if text is None else str(text) for text in texts]hidden = _hidden_size(encoder)if not rows:return np.zeros((0, hidden), dtype=np.float32)+ device = agent.deviceparts: List[np.ndarray] = []for start in range(0, len(rows), batch_size):chunk = rows[start : start + batch_size]diff --git a/tests/test_shortlist.py b/tests/test_shortlist.pyindex 362df649..5cb17758 100644--- a/tests/test_shortlist.py+++ b/tests/test_shortlist.py@@ -468,6 +468,51 @@ check_raises("encoder/bad max_length", lambda: embed_fn_from_agent(tiny, max_lencheck_raises("encoder/bad batch_size", lambda: embed_fn_from_agent(tiny, batch_size=True))+# ---------------------------------------------------------------- device changes, simulated entirely on CPU+class DeviceRecordingTensor:+ def __init__(self, tensor, requests):+ self.tensor = tensor+ self.requests = requests++ def to(self, device):+ # Record placement requests, but never allocate a CUDA tensor in these tests.+ self.requests.append(str(device))+ return self.tensor+++class DeviceRecordingTok(TinyTok):+ def __init__(self):+ super().__init__()+ self.requests = []++ def __call__(self, *args, **kwargs):+ encoded = super().__call__(*args, **kwargs)+ return {key: DeviceRecordingTensor(value, self.requests) for key, value in encoded.items()}+++moving = TinyAgent()+moving.tok = DeviceRecordingTok()+moving.device = torch.device("cuda")+moving_fn = embed_fn_from_agent(moving, batch_size=2)+# A callback can outlive Agent.system_one's GPU-to-CPU fallback, even before its+# first use. Later invocations must also follow the current device, not hardcode CPU.+for call, target in enumerate(("cpu", "cuda:1", "cpu")):+ moving.device = torch.device(target)+ moving.encoder.train(call == 1)+ moving.tok.requests.clear()+ pooled = moving_fn(["ab", "a", "abc"])+ check("device/call %d places both tensors in every batch" % call, moving.tok.requests, [target] * 4)+ check("device/call %d keeps batch boundaries" % call, moving.tok.batches[-2:], [["ab", "a"], ["abc"]])+ check("device/call %d keeps masked pooling" % call, pooled.tolist(), [[3.5, 1.0], [3.0, 1.0], [4.0, 1.0]])+ check("device/call %d keeps output dtype" % call, pooled.dtype, np.dtype("float32"))+ check("device/call %d preserves encoder mode" % call, moving.encoder.training, call == 1)++before_empty = (len(moving.tok.batches), moving.encoder.forwards, len(moving.tok.requests))+check("device/empty input shape", moving_fn([]).shape, (0, 2))+check("device/empty input skips tokenization, forward and placement",+ (len(moving.tok.batches), moving.encoder.forwards, len(moving.tok.requests)), before_empty)++# ---------------------------------------------------------------- end-to-end helper: shortlist then the real question normalizer only# Confirms a reduced choice still has one criterion per kept label, which is what# system_one would score. No weights, no forward.