Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(calibrate): wire histogram binning into answers and the calibration payload

Bruce-Yii2555c8830241
diff --git a/docs/reference/helpers.md b/docs/reference/helpers.md
index 697ba4be..79f0c5f5 100644
--- a/docs/reference/helpers.md
+++ b/docs/reference/helpers.md
@@ -71,6 +71,8 @@
::: laya.calibrate.apply_binning_map
+::: laya.agent.Agent.fit_binning
+
::: laya.common.render_options
::: laya.common.proper_reward
diff --git a/laya/agent.py b/laya/agent.py
index 83b18da4..1bcd47ab 100644
--- a/laya/agent.py
+++ b/laya/agent.py
@@ -16,7 +16,10 @@ import torch
from ._compile import compile_model, independent_dims
from .calibrate import (
_install_temperatures,
+ apply_binning_map,
apply_calibration_payload,
+ MIN_BINNING_BUCKET_N,
+ fit_binning_map,
calibration_payload,
fit_temperature_map,
)
@@ -662,6 +665,9 @@ class Agent(HookRegistry):
self.temperature = [clamp_temperature(t) for t in self.temperature_raw]
self.temperature_by_options = {k: clamp_temperature(v)
for k, v in self.temperature_by_options_raw.items()}
+ # Optional histogram-binning map installed by `load_calibration` or fit via
+ # `fit_binning`; absent/None means answer_confidence is the temperature-scaled one.
+ self.binning_map = None
# Shared with `ONNXAgent` so both backends accept the same option and produce the same
# confidences; see `common.resolve_lang_temperatures` for why the shape is checked before
@@ -1212,6 +1218,10 @@ class Agent(HookRegistry):
# callers, report both: `answer_confidence` is the calibrated one, on every
# question type, so a caller can gate across types on a single number.
ans_conf = round(answer_confidence(p, k), 4)
+ if getattr(self, "binning_map", None):
+ ans_conf = round(
+ apply_binning_map(ans_conf, temp_bucket(qt, k), self.binning_map), 4
+ )
ext = {"act_probability": round(float(act[r, 0]), 4)}
if q["t"] == "choice":
@@ -1832,6 +1842,18 @@ class Agent(HookRegistry):
_install_temperatures(self, result["temperature"], result["temperature_by_options"], warn=False)
return result
+ def fit_binning(self, records, min_bucket_n: int = MIN_BINNING_BUCKET_N) -> Dict[str, Any]:
+ """Fit a histogram-binning map on top of this agent's fitted temperatures and store it.
+
+ `records` are the same `(qtype, logits, target[, k])` tuples as `fit_temperatures`
+ consumed. The map is keyed exactly like `temperature_by_options`, composes on top of
+ the current temperatures, and is written out by `save_calibration` as `binning_map`.
+ """
+ self.binning_map = fit_binning_map(
+ records, self.temperature, self.temperature_by_options, min_bucket_n=min_bucket_n
+ )
+ return self.binning_map
+
def save_calibration(self, path: str) -> None:
"""Write temperatures and the checkpoint they were fitted for. Does not write weights."""
payload = calibration_payload(
@@ -1840,6 +1862,7 @@ class Agent(HookRegistry):
model_id_or_path=getattr(self, "model_id_or_path", None),
subfolder=getattr(self, "subfolder", None),
config=getattr(self, "cfg", None),
+ binning_map=getattr(self, "binning_map", None),
)
destination = os.path.realpath(path)
fd, temporary = tempfile.mkstemp(dir=os.path.dirname(destination), prefix=".calibration.", suffix=".tmp")
diff --git a/laya/calibrate.py b/laya/calibrate.py
index 443d642a..a9ad656c 100644
--- a/laya/calibrate.py
+++ b/laya/calibrate.py
@@ -488,6 +488,7 @@ def calibration_payload(
model_id_or_path=None,
subfolder=None,
config=None,
+ binning_map=None,
) -> Dict[str, Any]:
"""JSON body written by `Agent.save_calibration` (no weights).
@@ -497,8 +498,11 @@ def calibration_payload(
`model_id_or_path`, `subfolder`, and `config` say which checkpoint the map was fitted
against. `config` is `rl_agent_config.json` without `temperature` /
`temperature_by_options` (those live at the top of this payload).
+
+ `binning_map` is the optional histogram-binning recalibration map fitted by
+ `fit_binning_map`; the key is omitted when no map is installed.
"""
- return {
+ payload = {
"version": CALIBRATION_VERSION,
"temperature": [float(x) for x in temperature],
"temperature_by_options": {
@@ -508,6 +512,9 @@ def calibration_payload(
"subfolder": subfolder,
"config": _config_identity(config),
}
+ if binning_map is not None:
+ payload["binning_map"] = binning_map
+ return payload
def _warn_if_identity_mismatch(obj, payload) -> None:
@@ -611,6 +618,32 @@ def apply_calibration_payload(obj, payload: Dict[str, Any]) -> None:
raise ValueError(
"calibration JSON temperature_by_options must be an object of bucket -> float, "
"got %s" % type(by_options).__name__)
+ binning = payload.get("binning_map")
+ if binning is not None:
+ if not isinstance(binning, dict):
+ raise ValueError(
+ "calibration JSON binning_map must be an object of bucket -> {bins, values}, "
+ "got %s" % type(binning).__name__)
+ for name, entry in binning.items():
+ if not isinstance(entry, dict):
+ raise ValueError(
+ "calibration JSON binning_map[%r] must be an object with \"bins\" and \"values\", "
+ "got %s" % (name, type(entry).__name__))
+ n_bins = entry.get("bins")
+ values = entry.get("values")
+ if isinstance(n_bins, bool) or not isinstance(n_bins, int) or n_bins < 1:
+ raise ValueError(
+ "calibration JSON binning_map[%r] must have an integer \"bins\" >= 1, got %r" % (name, n_bins))
+ if not isinstance(values, (list, tuple)) or len(values) != n_bins:
+ raise ValueError(
+ "calibration JSON binning_map[%r] must have \"values\" of length \"bins\" (%d)"
+ % (name, n_bins))
+ try:
+ [float(v) for v in values]
+ except (TypeError, ValueError) as exc:
+ raise ValueError(
+ "calibration JSON binning_map[%r] values must be numbers, got %r" % (name, values)) from exc
+ obj.binning_map = binning
if version >= CALIBRATION_VERSION:
_warn_if_identity_mismatch(obj, payload)
_install_temperatures(obj, temps, by_options)
diff --git a/laya/onnx_agent.py b/laya/onnx_agent.py
index 96c21dcd..899c2b48 100644
--- a/laya/onnx_agent.py
+++ b/laya/onnx_agent.py
@@ -187,6 +187,9 @@ class ONNXAgent(HookRegistry):
self.temperature = [clamp_temperature(t) for t in self.temperature_raw]
self.temperature_by_options = {k: clamp_temperature(v)
for k, v in self.temperature_by_options_raw.items()}
+ # Optional histogram-binning map installed by `load_calibration`; absent/None means
+ # answer_confidence is the temperature-scaled one, same as the PyTorch Agent.
+ self.binning_map = None
# Per-language temperature overrides, through the same helper the PyTorch Agent uses, so a
# caller can hand the same `lang_temperatures` to either backend and read the same
# confidence -- including the same error for the same malformed input.
@@ -644,6 +647,9 @@ class ONNXAgent(HookRegistry):
lang: Optional[str] = None) -> Dict[str, Any]:
"""Decode the `len(ids)` head rows starting at `offset` into this state's answers."""
answers = {}
+ binning_map = getattr(self, "binning_map", None)
+ if binning_map:
+ from .calibrate import apply_binning_map
for r, qid in enumerate(ids):
q = internal[qid]
k = len(items[r]["markers"])
@@ -664,6 +670,10 @@ class ONNXAgent(HookRegistry):
# type so a caller can gate across types on one number -- matching the PyTorch Agent,
# whose output ONNX callers otherwise cannot read (KeyError on cross-backend swap).
ans_conf = round(answer_confidence(p, k), 4)
+ if binning_map:
+ ans_conf = round(
+ apply_binning_map(ans_conf, temp_bucket(qt, k), binning_map), 4
+ )
ext = {"act_probability": round(float(act[offset + r, 0]), 4)}
if q["t"] == "choice":
diff --git a/tests/test_calibrate.py b/tests/test_calibrate.py
index 62769ee0..8d7e0541 100644
--- a/tests/test_calibrate.py
+++ b/tests/test_calibrate.py
@@ -454,6 +454,33 @@ apply_calibration_payload(stub, calibration_payload([1.2, 1.1, 1.3], {"choice:2"
check("stub/temperature", stub.temperature, [1.2, 1.1, 1.3])
check("stub/by_options", stub.temperature_by_options, {"choice:2": 1.4})
+# Optional histogram-binning map round-trips through the payload and lands on the agent.
+bm = {"choice:2": {"bins": 2, "values": [0.9, 0.2]}}
+with_bm = calibration_payload([1.2, 1.1, 1.3], {"choice:2": 1.4}, binning_map=bm)
+check("payload/no-binning omission", "binning_map" not in calibration_payload([1.2, 1.1, 1.3], {}), True)
+check_true("payload/binning key present", "binning_map" in with_bm, with_bm.keys())
+apply_calibration_payload(stub, with_bm)
+check("stub/binning_map", stub.binning_map, bm)
+stub_no_bm = type("Stub", (), {})()
+apply_calibration_payload(stub_no_bm, {"temperature": [1.0, 1.0, 1.0]})
+check("stub/binning_map cleared when absent", stub_no_bm.binning_map, None)
+
+# The same `_decode_answers` stub test_batch uses proves the map is selectable: no map,
+# the temperature-scaled confidence; a map on this bucket, the remapped one.
+bin_decoder = Agent.__new__(Agent)
+bin_decoder.temperature = [1.0, 1.0, 1.0]
+bin_decoder.temperature_by_options = {}
+bin_ids = ["pick"]
+bin_internal = {"pick": {"t": "choice", "crit": {"left": "left", "right": "right"}}}
+bin_items = [{"markers": [0, 1]}]
+bin_logits = np.log([[0.5, 0.5]]) * 1.0
+bin_act = np.array([[0.2, 0.8]])
+plain = bin_decoder._decode_answers(bin_logits, bin_act, bin_items, bin_ids, bin_internal, 0)
+check("decode/no binning keeps scaled confidence", plain["pick"]["answer_confidence"], 0.5)
+bin_decoder.binning_map = bm
+remapped = bin_decoder._decode_answers(bin_logits, bin_act, bin_items, bin_ids, bin_internal, 0)
+check("decode/binning remaps confidence", remapped["pick"]["answer_confidence"], 0.2)
+
# A payload with no version is the original schema and must still load, even onto an
# agent that has its own identity. No warning: there is no recorded checkpoint to disagree with.
legacy = {"temperature": [1.4, 1.2, 1.1], "temperature_by_options": {"noul:2": 1.5}}
@@ -526,6 +553,9 @@ def _refuses(name, payload, fragment=None):
_refuses("shape/non-object payload", ["temperature", [1.0, 1.0, 1.0]], "must be an object")
+_refuses("shape/binning not object", {"temperature": [1.0, 1.0, 1.0], "binning_map": [1, 2]}, "binning_map")
+_refuses("shape/binning wrong entry", {"temperature": [1.0, 1.0, 1.0], "binning_map": {"a": 1}}, "bins")
+_refuses("shape/binning values length", {"temperature": [1.0, 1.0, 1.0], "binning_map": {"a": {"bins": 2, "values": [0.5]}}}, "values")
_refuses("shape/scalar temperature", {"temperature": 5}, "[3 floats]")
_refuses("shape/string temperature", {"temperature": "abc"}, "[3 floats]")
_refuses("shape/dict temperature", {"temperature": {"a": 1, "b": 2, "c": 3}}, "[3 floats]")