Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(calibration): validate fitting inputs and save maps atomically

Antoniod853cf54e45f
diff --git a/laya/agent.py b/laya/agent.py
index 82342ade..885f79f6 100644
--- a/laya/agent.py
+++ b/laya/agent.py
@@ -1824,9 +1824,27 @@ class Agent(HookRegistry):
subfolder=getattr(self, "subfolder", None),
config=getattr(self, "cfg", None),
)
- with open(path, "w") as f:
- json.dump(payload, f, indent=2)
- f.write("\n")
+ destination = os.path.realpath(path)
+ fd, temporary = tempfile.mkstemp(dir=os.path.dirname(destination), prefix=".calibration.", suffix=".tmp")
+ try:
+ with os.fdopen(fd, "w") as f:
+ json.dump(payload, f, indent=2)
+ f.write("\n")
+ f.flush()
+ os.fsync(f.fileno())
+ try:
+ mode = os.stat(destination).st_mode & 0o777
+ except FileNotFoundError:
+ pass
+ else:
+ os.chmod(temporary, mode)
+ os.replace(temporary, destination)
+ except BaseException:
+ try:
+ os.unlink(temporary)
+ except OSError:
+ pass
+ raise
def load_calibration(self, path: str) -> None:
"""Read a JSON map written by `save_calibration` onto this agent.
diff --git a/laya/calibrate.py b/laya/calibrate.py
index 3906a5fe..a1df00f3 100644
--- a/laya/calibrate.py
+++ b/laya/calibrate.py
@@ -17,6 +17,7 @@ from `laya.common`, the same guard checkpoint load uses. There is no second pair
bounds in this module.
"""
import warnings
+from numbers import Integral
from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple
import numpy as np
@@ -55,6 +56,35 @@ def _vec(x) -> np.ndarray:
return np.asarray(x, dtype=np.float32).reshape(-1)
+def _validated_pair(logits, target, k=None):
+ vectors = []
+ for name, value in (("logits", logits), ("target", target)):
+ if isinstance(value, torch.Tensor):
+ value = value.detach().cpu().numpy()
+ try:
+ vector = np.asarray(value)
+ if vector.ndim != 1 or vector.dtype.kind not in "iuf":
+ raise ValueError("%s must be a one-dimensional numeric vector" % name)
+ with np.errstate(over="ignore", invalid="ignore"):
+ vector = vector.astype(np.float32)
+ except (TypeError, ValueError, OverflowError) as exc:
+ raise ValueError("%s must be a one-dimensional numeric vector" % name) from exc
+ if not np.isfinite(vector).all():
+ raise ValueError("%s must contain only finite values" % name)
+ vectors.append(vector)
+ logits, target = vectors
+ if len(logits) != len(target):
+ raise ValueError("logits and target must have the same length")
+ if k is None:
+ k = len(logits)
+ if isinstance(k, bool) or not isinstance(k, Integral) or k < 1 or k > len(logits):
+ raise ValueError("k must be an integer between 1 and the vector length")
+ logits, target = logits[:k], target[:k]
+ if (target < 0).any() or not np.isclose(target.sum(), 1.0, rtol=1e-5, atol=1e-6):
+ raise ValueError("target must be a nonnegative probability vector summing to 1")
+ return logits, target, int(k)
+
+
def _pairs_to_tensors(pairs: Sequence) -> Tuple[torch.Tensor, torch.Tensor]:
kmax = max(len(_vec(z)) for z, _ in pairs)
z_mat = torch.full((len(pairs), kmax), -1e4)
@@ -62,7 +92,7 @@ def _pairs_to_tensors(pairs: Sequence) -> Tuple[torch.Tensor, torch.Tensor]:
for i, (z, t) in enumerate(pairs):
z = _vec(z)
t = _vec(t)
- n = min(len(z), len(t))
+ n = len(z)
z_mat[i, :n] = torch.from_numpy(np.ascontiguousarray(z[:n]))
t_mat[i, :n] = torch.from_numpy(np.ascontiguousarray(t[:n]))
return z_mat, t_mat
@@ -80,7 +110,16 @@ def fit_one_temperature(pairs: Sequence, min_n: Optional[int] = None) -> float:
"""
if min_n is None:
min_n = MIN_BUCKET_N
- sel = list(pairs)
+ if isinstance(min_n, bool) or not isinstance(min_n, Integral) or min_n < 1:
+ raise ValueError("min_n must be a positive integer")
+ sel = []
+ for pair in pairs:
+ try:
+ z, t = pair
+ except (TypeError, ValueError) as exc:
+ raise ValueError("pair must be (logits, target)") from exc
+ z, t, _k = _validated_pair(z, t)
+ sel.append((z, t))
if len(sel) < min_n:
return 1.0
z_mat, t_mat = _pairs_to_tensors(sel)
@@ -102,19 +141,19 @@ def fit_one_temperature(pairs: Sequence, min_n: Optional[int] = None) -> float:
def _iter_records(records: Iterable) -> List[Tuple[int, np.ndarray, np.ndarray, int]]:
out = []
for rec in records:
+ if not isinstance(rec, (list, tuple)) or len(rec) not in (3, 4):
+ raise ValueError("record must be (qtype, logits, target[, k])")
if len(rec) == 4:
qtype, logits, target, k = rec
- elif len(rec) == 3:
- qtype, logits, target = rec
- k = len(_vec(logits))
+ if k is None:
+ raise ValueError("k must be an integer between 1 and the vector length")
else:
- raise ValueError("record must be (qtype, logits, target[, k])")
- logits = _vec(logits)
- target = _vec(target)
- k = int(k)
- if k < 1:
- raise ValueError("k must be >= 1")
- out.append((int(qtype), logits[:k], target[:k], k))
+ qtype, logits, target = rec
+ k = None
+ if isinstance(qtype, bool) or not isinstance(qtype, Integral) or not 0 <= qtype < N_QTYPES:
+ raise ValueError("qtype must be an integer between 0 and %d" % (N_QTYPES - 1))
+ logits, target, k = _validated_pair(logits, target, k)
+ out.append((int(qtype), logits, target, k))
return out
diff --git a/tests/test_calibrate.py b/tests/test_calibrate.py
index e4adb12e..ddb072c6 100644
--- a/tests/test_calibrate.py
+++ b/tests/test_calibrate.py
@@ -10,6 +10,10 @@ import os
import sys
import tempfile
import warnings
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+from threading import Barrier
+from unittest.mock import patch
import numpy as np
@@ -33,6 +37,7 @@ from laya.calibrate import ( # noqa: E402
)
from laya.common import QTYPES, TEMP_MAX, TEMP_MIN, temp_bucket # noqa: E402
import laya.calibrate as _calibrate # noqa: E402
+import laya.agent as _agent_module # noqa: E402
PASS, FAIL = [], []
@@ -211,6 +216,59 @@ check("ece/fit deterministic", again_fit_map["report"], fitted["report"])
# --------------------------------------------------------------- live Agent map + JSON round-trip
+
+def _refuses_records(name, records, fragment):
+ try:
+ fit_temperature_map(records)
+ except ValueError as exc:
+ check_true("records/" + name, fragment in str(exc), str(exc))
+ except Exception as exc:
+ FAIL.append("records/%s: raised %r, want ValueError" % (name, exc))
+ else:
+ FAIL.append("records/%s: invalid input was accepted" % name)
+
+
+for name, record, field in (
+ ("scalar record", 3, "record"),
+ ("wrong record length", (0, [1, 2]), "record"),
+ ("bool type", (True, [1, 2], [1, 0]), "qtype"),
+ ("fractional type", (0.9, [1, 2], [1, 0]), "qtype"),
+ ("unknown type", (3, [1, 2], [1, 0]), "qtype"),
+ ("negative type", (-1, [1, 2], [1, 0]), "qtype"),
+ ("bool width", (0, [1, 2], [1, 0], True), "k"),
+ ("null width", (0, [1, 2], [1, 0], None), "k"),
+ ("fractional width", (0, [1, 2], [1, 0], 1.9), "k"),
+ ("zero width", (0, [1, 2], [1, 0], 0), "k"),
+ ("oversized width", (0, [1, 2], [1, 0], 3), "k"),
+ ("empty vectors", (0, [], []), "k"),
+ ("matrix logits", (0, [[1, 2]], [1, 0]), "logits"),
+ ("matrix target", (0, [1, 2], [[1, 0]]), "target"),
+ ("text logits", (0, ["1", "2"], [1, 0]), "logits"),
+ ("nan logits", (0, [np.nan, 2], [1, 0]), "logits"),
+ ("infinite target", (0, [1, 2], [np.inf, 0]), "target"),
+ ("unequal vectors", (0, [1, 2], [1]), "same length"),
+ ("negative target", (0, [1, 2], [-0.1, 1.1]), "probability"),
+ ("unnormalized target", (0, [1, 2], [1, 1]), "probability"),
+):
+ _refuses_records(name, [record], field)
+
+soft_record = _iter_records([(np.int64(0), [1, 2, 0], [0.3, 0.7, 0], np.int64(2))])[0]
+check("records/numpy integer and explicit padding", soft_record[3], 2)
+check_true("records/soft target kept", np.allclose(soft_record[2], [0.3, 0.7]))
+check_true("records/soft target fits", np.isfinite(fit_one_temperature([(soft_record[1], soft_record[2])], min_n=1)))
+check("records/empty dataset still neutral", fit_temperature_map([])["temperature"], [1.0] * 3)
+for name, pairs, min_n in (
+ ("mismatched lengths", [([1, 2], [1])], None),
+ ("invalid below sample floor", [([np.nan, 2], [1, 0])], None),
+ ("invalid min_n", [], 0),
+):
+ try:
+ fit_one_temperature(pairs, min_n=min_n)
+ except ValueError:
+ PASS.append("pairs/" + name)
+ else:
+ FAIL.append("pairs/%s: invalid input was accepted" % name)
+
_CFG = {
"encoder": "answerdotai/ModernBERT-large",
"head_layers": 2,
@@ -256,6 +314,77 @@ check_true(
)
check_true("save/no weights key", "model.safetensors" not in json.dumps(payload))
+
+# --------------------------------------------------------------- failed saves preserve the old map
+with tempfile.TemporaryDirectory() as atomic_dir:
+ atomic_path = Path(atomic_dir) / "calibration.json"
+ agent.save_calibration(atomic_path)
+ original_bytes = atomic_path.read_bytes()
+ original_replace = os.replace
+
+ def _partial_dump(body, stream, **kwargs):
+ stream.write('{"temperature": [')
+ raise OSError("injected write failure")
+
+ for failure, context in (
+ ("write", patch.object(_agent_module.json, "dump", _partial_dump)),
+ ("sync", patch.object(_agent_module.os, "fsync", side_effect=OSError("injected sync failure"))),
+ ("replace", patch.object(_agent_module.os, "replace", side_effect=OSError("injected replace failure"))),
+ ):
+ with context:
+ try:
+ agent.save_calibration(atomic_path)
+ except OSError as exc:
+ check_true("atomic/%s error propagates" % failure, "injected" in str(exc))
+ else:
+ FAIL.append("atomic/%s error did not propagate" % failure)
+ check("atomic/%s keeps prior bytes" % failure, atomic_path.read_bytes(), original_bytes)
+ check("atomic/%s cleans temp" % failure, sorted(os.listdir(atomic_dir)), ["calibration.json"])
+
+ missing_path = Path(atomic_dir) / "new.json"
+ with patch.object(_agent_module.json, "dump", _partial_dump):
+ try:
+ agent.save_calibration(missing_path)
+ except OSError:
+ pass
+ else:
+ FAIL.append("atomic/first-save error did not propagate")
+ check_true("atomic/failed first save leaves no destination", not missing_path.exists())
+ check("atomic/failed first save cleans temp", sorted(os.listdir(atomic_dir)), ["calibration.json"])
+
+ # Successful updates keep the bytes/format and the existing readers' permissions.
+ agent.save_calibration(atomic_path)
+ check("atomic/success preserves JSON format", atomic_path.read_bytes(), original_bytes)
+ if os.name != "nt":
+ os.chmod(atomic_path, 0o640)
+ agent.save_calibration(atomic_path)
+ check("atomic/preserves mode", atomic_path.stat().st_mode & 0o777, 0o640)
+ link_path = Path(atomic_dir) / "linked.json"
+ link_path.symlink_to(atomic_path.name)
+ agent.save_calibration(link_path)
+ check_true("atomic/keeps symlink", link_path.is_symlink())
+ check("atomic/writes symlink target", atomic_path.read_bytes(), original_bytes)
+ link_path.unlink()
+
+ rival = Agent.__new__(Agent)
+ rival.temperature = [1.1, 1.2, 1.3]
+ rival.temperature_by_options = {"choice:2": 1.4}
+ barrier = Barrier(2)
+
+ def _together_replace(src, dst):
+ barrier.wait(timeout=10)
+ original_replace(src, dst)
+
+ with patch.object(_agent_module.os, "replace", _together_replace):
+ with ThreadPoolExecutor(max_workers=2) as pool:
+ writes = [pool.submit(writer.save_calibration, atomic_path) for writer in (agent, rival)]
+ for write in writes:
+ write.result()
+ concurrent_payload = json.loads(atomic_path.read_text())
+ check_true("atomic/concurrent writers leave one complete payload",
+ concurrent_payload in (payload, calibration_payload(rival.temperature, rival.temperature_by_options)))
+ check("atomic/concurrent writers clean temps", sorted(os.listdir(atomic_dir)), ["calibration.json"])
+
other = Agent.__new__(Agent)
other.model_id_or_path = agent.model_id_or_path
other.subfolder = agent.subfolder