Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
feat(calibrate): wire histogram binning into answers and the calibration payload
Bruce-Yii2555c8830241
diff --git a/docs/reference/helpers.md b/docs/reference/helpers.mdindex 697ba4be..79f0c5f5 100644--- a/docs/reference/helpers.md+++ b/docs/reference/helpers.md@@ -71,6 +71,8 @@::: laya.calibrate.apply_binning_map+::: laya.agent.Agent.fit_binning+::: laya.common.render_options::: laya.common.proper_rewarddiff --git a/laya/agent.py b/laya/agent.pyindex 83b18da4..1bcd47ab 100644--- a/laya/agent.py+++ b/laya/agent.py@@ -16,7 +16,10 @@ import torchfrom ._compile import compile_model, independent_dimsfrom .calibrate import (_install_temperatures,+ apply_binning_map,apply_calibration_payload,+ MIN_BINNING_BUCKET_N,+ fit_binning_map,calibration_payload,fit_temperature_map,)@@ -662,6 +665,9 @@ class Agent(HookRegistry):self.temperature = [clamp_temperature(t) for t in self.temperature_raw]self.temperature_by_options = {k: clamp_temperature(v)for k, v in self.temperature_by_options_raw.items()}+ # Optional histogram-binning map installed by `load_calibration` or fit via+ # `fit_binning`; absent/None means answer_confidence is the temperature-scaled one.+ self.binning_map = None# Shared with `ONNXAgent` so both backends accept the same option and produce the same# confidences; see `common.resolve_lang_temperatures` for why the shape is checked before@@ -1212,6 +1218,10 @@ class Agent(HookRegistry):# callers, report both: `answer_confidence` is the calibrated one, on every# question type, so a caller can gate across types on a single number.ans_conf = round(answer_confidence(p, k), 4)+ if getattr(self, "binning_map", None):+ ans_conf = round(+ apply_binning_map(ans_conf, temp_bucket(qt, k), self.binning_map), 4+ )ext = {"act_probability": round(float(act[r, 0]), 4)}if q["t"] == "choice":@@ -1832,6 +1842,18 @@ class Agent(HookRegistry):_install_temperatures(self, result["temperature"], result["temperature_by_options"], warn=False)return result+ def fit_binning(self, records, min_bucket_n: int = MIN_BINNING_BUCKET_N) -> Dict[str, Any]:+ """Fit a histogram-binning map on top of this agent's fitted temperatures and store it.++ `records` are the same `(qtype, logits, target[, k])` tuples as `fit_temperatures`+ consumed. The map is keyed exactly like `temperature_by_options`, composes on top of+ the current temperatures, and is written out by `save_calibration` as `binning_map`.+ """+ self.binning_map = fit_binning_map(+ records, self.temperature, self.temperature_by_options, min_bucket_n=min_bucket_n+ )+ return self.binning_map+def save_calibration(self, path: str) -> None:"""Write temperatures and the checkpoint they were fitted for. Does not write weights."""payload = calibration_payload(@@ -1840,6 +1862,7 @@ class Agent(HookRegistry):model_id_or_path=getattr(self, "model_id_or_path", None),subfolder=getattr(self, "subfolder", None),config=getattr(self, "cfg", None),+ binning_map=getattr(self, "binning_map", None),)destination = os.path.realpath(path)fd, temporary = tempfile.mkstemp(dir=os.path.dirname(destination), prefix=".calibration.", suffix=".tmp")diff --git a/laya/calibrate.py b/laya/calibrate.pyindex 443d642a..a9ad656c 100644--- a/laya/calibrate.py+++ b/laya/calibrate.py@@ -488,6 +488,7 @@ def calibration_payload(model_id_or_path=None,subfolder=None,config=None,+ binning_map=None,) -> Dict[str, Any]:"""JSON body written by `Agent.save_calibration` (no weights).@@ -497,8 +498,11 @@ def calibration_payload(`model_id_or_path`, `subfolder`, and `config` say which checkpoint the map was fittedagainst. `config` is `rl_agent_config.json` without `temperature` /`temperature_by_options` (those live at the top of this payload).++ `binning_map` is the optional histogram-binning recalibration map fitted by+ `fit_binning_map`; the key is omitted when no map is installed."""- return {+ payload = {"version": CALIBRATION_VERSION,"temperature": [float(x) for x in temperature],"temperature_by_options": {@@ -508,6 +512,9 @@ def calibration_payload("subfolder": subfolder,"config": _config_identity(config),}+ if binning_map is not None:+ payload["binning_map"] = binning_map+ return payloaddef _warn_if_identity_mismatch(obj, payload) -> None:@@ -611,6 +618,32 @@ def apply_calibration_payload(obj, payload: Dict[str, Any]) -> None:raise ValueError("calibration JSON temperature_by_options must be an object of bucket -> float, ""got %s" % type(by_options).__name__)+ binning = payload.get("binning_map")+ if binning is not None:+ if not isinstance(binning, dict):+ raise ValueError(+ "calibration JSON binning_map must be an object of bucket -> {bins, values}, "+ "got %s" % type(binning).__name__)+ for name, entry in binning.items():+ if not isinstance(entry, dict):+ raise ValueError(+ "calibration JSON binning_map[%r] must be an object with \"bins\" and \"values\", "+ "got %s" % (name, type(entry).__name__))+ n_bins = entry.get("bins")+ values = entry.get("values")+ if isinstance(n_bins, bool) or not isinstance(n_bins, int) or n_bins < 1:+ raise ValueError(+ "calibration JSON binning_map[%r] must have an integer \"bins\" >= 1, got %r" % (name, n_bins))+ if not isinstance(values, (list, tuple)) or len(values) != n_bins:+ raise ValueError(+ "calibration JSON binning_map[%r] must have \"values\" of length \"bins\" (%d)"+ % (name, n_bins))+ try:+ [float(v) for v in values]+ except (TypeError, ValueError) as exc:+ raise ValueError(+ "calibration JSON binning_map[%r] values must be numbers, got %r" % (name, values)) from exc+ obj.binning_map = binningif version >= CALIBRATION_VERSION:_warn_if_identity_mismatch(obj, payload)_install_temperatures(obj, temps, by_options)diff --git a/laya/onnx_agent.py b/laya/onnx_agent.pyindex 96c21dcd..899c2b48 100644--- a/laya/onnx_agent.py+++ b/laya/onnx_agent.py@@ -187,6 +187,9 @@ class ONNXAgent(HookRegistry):self.temperature = [clamp_temperature(t) for t in self.temperature_raw]self.temperature_by_options = {k: clamp_temperature(v)for k, v in self.temperature_by_options_raw.items()}+ # Optional histogram-binning map installed by `load_calibration`; absent/None means+ # answer_confidence is the temperature-scaled one, same as the PyTorch Agent.+ self.binning_map = None# Per-language temperature overrides, through the same helper the PyTorch Agent uses, so a# caller can hand the same `lang_temperatures` to either backend and read the same# confidence -- including the same error for the same malformed input.@@ -644,6 +647,9 @@ class ONNXAgent(HookRegistry):lang: Optional[str] = None) -> Dict[str, Any]:"""Decode the `len(ids)` head rows starting at `offset` into this state's answers."""answers = {}+ binning_map = getattr(self, "binning_map", None)+ if binning_map:+ from .calibrate import apply_binning_mapfor r, qid in enumerate(ids):q = internal[qid]k = len(items[r]["markers"])@@ -664,6 +670,10 @@ class ONNXAgent(HookRegistry):# type so a caller can gate across types on one number -- matching the PyTorch Agent,# whose output ONNX callers otherwise cannot read (KeyError on cross-backend swap).ans_conf = round(answer_confidence(p, k), 4)+ if binning_map:+ ans_conf = round(+ apply_binning_map(ans_conf, temp_bucket(qt, k), binning_map), 4+ )ext = {"act_probability": round(float(act[offset + r, 0]), 4)}if q["t"] == "choice":diff --git a/tests/test_calibrate.py b/tests/test_calibrate.pyindex 62769ee0..8d7e0541 100644--- a/tests/test_calibrate.py+++ b/tests/test_calibrate.py@@ -454,6 +454,33 @@ apply_calibration_payload(stub, calibration_payload([1.2, 1.1, 1.3], {"choice:2"check("stub/temperature", stub.temperature, [1.2, 1.1, 1.3])check("stub/by_options", stub.temperature_by_options, {"choice:2": 1.4})+# Optional histogram-binning map round-trips through the payload and lands on the agent.+bm = {"choice:2": {"bins": 2, "values": [0.9, 0.2]}}+with_bm = calibration_payload([1.2, 1.1, 1.3], {"choice:2": 1.4}, binning_map=bm)+check("payload/no-binning omission", "binning_map" not in calibration_payload([1.2, 1.1, 1.3], {}), True)+check_true("payload/binning key present", "binning_map" in with_bm, with_bm.keys())+apply_calibration_payload(stub, with_bm)+check("stub/binning_map", stub.binning_map, bm)+stub_no_bm = type("Stub", (), {})()+apply_calibration_payload(stub_no_bm, {"temperature": [1.0, 1.0, 1.0]})+check("stub/binning_map cleared when absent", stub_no_bm.binning_map, None)++# The same `_decode_answers` stub test_batch uses proves the map is selectable: no map,+# the temperature-scaled confidence; a map on this bucket, the remapped one.+bin_decoder = Agent.__new__(Agent)+bin_decoder.temperature = [1.0, 1.0, 1.0]+bin_decoder.temperature_by_options = {}+bin_ids = ["pick"]+bin_internal = {"pick": {"t": "choice", "crit": {"left": "left", "right": "right"}}}+bin_items = [{"markers": [0, 1]}]+bin_logits = np.log([[0.5, 0.5]]) * 1.0+bin_act = np.array([[0.2, 0.8]])+plain = bin_decoder._decode_answers(bin_logits, bin_act, bin_items, bin_ids, bin_internal, 0)+check("decode/no binning keeps scaled confidence", plain["pick"]["answer_confidence"], 0.5)+bin_decoder.binning_map = bm+remapped = bin_decoder._decode_answers(bin_logits, bin_act, bin_items, bin_ids, bin_internal, 0)+check("decode/binning remaps confidence", remapped["pick"]["answer_confidence"], 0.2)+# A payload with no version is the original schema and must still load, even onto an# agent that has its own identity. No warning: there is no recorded checkpoint to disagree with.legacy = {"temperature": [1.4, 1.2, 1.1], "temperature_by_options": {"noul:2": 1.5}}@@ -526,6 +553,9 @@ def _refuses(name, payload, fragment=None):_refuses("shape/non-object payload", ["temperature", [1.0, 1.0, 1.0]], "must be an object")+_refuses("shape/binning not object", {"temperature": [1.0, 1.0, 1.0], "binning_map": [1, 2]}, "binning_map")+_refuses("shape/binning wrong entry", {"temperature": [1.0, 1.0, 1.0], "binning_map": {"a": 1}}, "bins")+_refuses("shape/binning values length", {"temperature": [1.0, 1.0, 1.0], "binning_map": {"a": {"bins": 2, "values": [0.5]}}}, "values")_refuses("shape/scalar temperature", {"temperature": 5}, "[3 floats]")_refuses("shape/string temperature", {"temperature": "abc"}, "[3 floats]")_refuses("shape/dict temperature", {"temperature": {"a": 1, "b": 2, "c": 3}}, "[3 floats]")