Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(shortlist): refresh embedding device on each call (#145)

Alphin Tom42a086a831c9
diff --git a/laya/shortlist.py b/laya/shortlist.py
index b37638e9..71a51bd5 100644
--- a/laya/shortlist.py
+++ b/laya/shortlist.py
@@ -122,6 +122,7 @@ def embed_fn_from_agent(
Padding positions are excluded from the mean. The encoder's train/eval flag is
left as the caller set it (a loaded ``Agent`` is already in eval).
+ Each call uses the current ``agent.device``, including after CPU fallback.
"""
if isinstance(max_length, bool) or not isinstance(max_length, int) or max_length < 1:
raise ValueError("max_length must be a positive integer, got %r" % (max_length,))
@@ -132,13 +133,13 @@ def embed_fn_from_agent(
tok = agent.tok
encoder = agent.model.encoder
- device = agent.device
def embed_fn(texts: Sequence[str]) -> np.ndarray:
rows = ["" if text is None else str(text) for text in texts]
hidden = _hidden_size(encoder)
if not rows:
return np.zeros((0, hidden), dtype=np.float32)
+ device = agent.device
parts: List[np.ndarray] = []
for start in range(0, len(rows), batch_size):
chunk = rows[start : start + batch_size]
diff --git a/tests/test_shortlist.py b/tests/test_shortlist.py
index 362df649..5cb17758 100644
--- a/tests/test_shortlist.py
+++ b/tests/test_shortlist.py
@@ -468,6 +468,51 @@ check_raises("encoder/bad max_length", lambda: embed_fn_from_agent(tiny, max_len
check_raises("encoder/bad batch_size", lambda: embed_fn_from_agent(tiny, batch_size=True))
+# ---------------------------------------------------------------- device changes, simulated entirely on CPU
+class DeviceRecordingTensor:
+ def __init__(self, tensor, requests):
+ self.tensor = tensor
+ self.requests = requests
+
+ def to(self, device):
+ # Record placement requests, but never allocate a CUDA tensor in these tests.
+ self.requests.append(str(device))
+ return self.tensor
+
+
+class DeviceRecordingTok(TinyTok):
+ def __init__(self):
+ super().__init__()
+ self.requests = []
+
+ def __call__(self, *args, **kwargs):
+ encoded = super().__call__(*args, **kwargs)
+ return {key: DeviceRecordingTensor(value, self.requests) for key, value in encoded.items()}
+
+
+moving = TinyAgent()
+moving.tok = DeviceRecordingTok()
+moving.device = torch.device("cuda")
+moving_fn = embed_fn_from_agent(moving, batch_size=2)
+# A callback can outlive Agent.system_one's GPU-to-CPU fallback, even before its
+# first use. Later invocations must also follow the current device, not hardcode CPU.
+for call, target in enumerate(("cpu", "cuda:1", "cpu")):
+ moving.device = torch.device(target)
+ moving.encoder.train(call == 1)
+ moving.tok.requests.clear()
+ pooled = moving_fn(["ab", "a", "abc"])
+ check("device/call %d places both tensors in every batch" % call, moving.tok.requests, [target] * 4)
+ check("device/call %d keeps batch boundaries" % call, moving.tok.batches[-2:], [["ab", "a"], ["abc"]])
+ check("device/call %d keeps masked pooling" % call, pooled.tolist(), [[3.5, 1.0], [3.0, 1.0], [4.0, 1.0]])
+ check("device/call %d keeps output dtype" % call, pooled.dtype, np.dtype("float32"))
+ check("device/call %d preserves encoder mode" % call, moving.encoder.training, call == 1)
+
+before_empty = (len(moving.tok.batches), moving.encoder.forwards, len(moving.tok.requests))
+check("device/empty input shape", moving_fn([]).shape, (0, 2))
+check("device/empty input skips tokenization, forward and placement",
+ (len(moving.tok.batches), moving.encoder.forwards, len(moving.tok.requests)), before_empty)
+
+
# ---------------------------------------------------------------- end-to-end helper: shortlist then the real question normalizer only
# Confirms a reduced choice still has one criterion per kept label, which is what
# system_one would score. No weights, no forward.