Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(ts): port histogram-binning recalibration to laya-ts

Swatantra Yadava76dc39d9162
diff --git a/laya-ts/src/agent.ts b/laya-ts/src/agent.ts
index 5bbdad94..cd368278 100644
--- a/laya-ts/src/agent.ts
+++ b/laya-ts/src/agent.ts
@@ -1,7 +1,9 @@
import {
TEMP_MAX,
TEMP_MIN,
+ applyBinningMap,
buildQuestionPrefix,
+ checkBinningMap,
clampTemperature,
collateItems,
collapsedOptions,
@@ -16,7 +18,7 @@ import {
softmax,
tempBucket,
} from "./common.js";
-import type { MinConfidence, OptionStats, SequenceStats } from "./common.js";
+import type { BinningMap, MinConfidence, OptionStats, SequenceStats } from "./common.js";
import type { Batch, SessionProvider } from "./providers.js";
import { encodeWithData, parseTokenizerJson, type TokenizerLike } from "./tokenizer.js";
import { decide, type DecideOptions, type DecisionResult } from "./structured.js";
@@ -103,6 +105,7 @@ export interface AgentCfg {
head_max_len?: number;
temperature?: unknown;
temperature_by_options?: Record<string, unknown>;
+ binning_map?: BinningMap | null;
[k: string]: unknown;
}
@@ -116,6 +119,9 @@ export interface AgentOptions {
head_max_len?: number;
temperature?: unknown;
temperature_by_options?: Record<string, unknown>;
+ /** Optional histogram-binning map applied to answer_confidence. */
+ binning_map?: BinningMap | null;
+ binningMap?: BinningMap | null;
/**
* Per-language temperature overrides, keyed by language code; keys are normalised to
* their base subtag (`de-AT` -> `de`), matching Python `Agent(lang_temperatures=...)`.
@@ -385,6 +391,7 @@ export class Agent extends HookRegistry {
temperatureByOptionsRaw: Record<string, unknown>;
temperature: number[];
temperatureByOptions: Record<string, number>;
+ readonly binningMap: BinningMap | null;
langTemperatures: Record<
string,
{ temperature: number[]; temperatureByOptions: Record<string, number> }
@@ -403,10 +410,14 @@ export class Agent extends HookRegistry {
if (opts.head_max_len !== undefined) cfg.head_max_len = opts.head_max_len;
if (opts.temperature !== undefined) cfg.temperature = opts.temperature;
if (opts.temperature_by_options !== undefined) cfg.temperature_by_options = opts.temperature_by_options;
+ if (opts.binning_map !== undefined) cfg.binning_map = opts.binning_map;
+ if (opts.binningMap !== undefined) cfg.binning_map = opts.binningMap;
this.cfg = cfg;
this.maxLen = Number(cfg.max_len ?? 512);
this.headMaxLen = Number(cfg.head_max_len ?? 192);
this.tok = opts.tok ?? defaultTokenizer();
+ const bmapRaw = cfg.binning_map;
+ this.binningMap = bmapRaw !== undefined && bmapRaw !== null ? checkBinningMap(bmapRaw) : null;
const raw = (cfg.temperature ?? [1.0, 1.0, 1.0]) as unknown;
this.temperatureRaw = raw;
// The decode indexes this by question type, so refuse any other shape here, as Python's
@@ -724,7 +735,15 @@ export class Agent extends HookRegistry {
const ext = { act_probability: r4(actP[0]) };
// Same quantity on every question type (max(p)), so callers can gate across types on
// one number; `confidence` stays as-is for existing callers (entropy for choice/score).
- const ansConf = r4(answerConfidence(p));
+ // Histogram-binning recalibration (issue #871, Python parity) recalibrates ans_raw
+ // per temp_bucket unless a language override is active.
+ const ansRaw = answerConfidence(p);
+ const langOverride = Boolean(lang && lang.split("-")[0].toLowerCase() in this.langTemperatures);
+ const ansConf = r4(
+ this.binningMap && !langOverride
+ ? applyBinningMap(ansRaw, bucket, this.binningMap)
+ : ansRaw,
+ );
if (q.t === "choice") {
const keys = Object.keys(q.crit as Record<string, unknown>);
let best = 0;
diff --git a/laya-ts/src/common.ts b/laya-ts/src/common.ts
index 89c52335..38f170ed 100644
--- a/laya-ts/src/common.ts
+++ b/laya-ts/src/common.ts
@@ -355,6 +355,74 @@ export function tempBucket(qtype: number, k: number): string {
const size = k <= 2 ? "2" : k <= 5 ? "3-5" : k <= 10 ? "6-10" : "11+";
return `${["choice", "score", "noul"][qtype]}:${size}`;
}
+
+export interface BinningEntry {
+ bins: number;
+ values: number[];
+}
+
+export type BinningMap = Record<string, BinningEntry>;
+
+/**
+ * Validate a histogram-binning recalibration map.
+ *
+ * Keys are option-count bucket strings like "choice:2", "choice:3-5", "score:6-10", "noul:2".
+ * Values are objects with an integer `bins >= 1` and a `values` array of length `bins`
+ * where each number is in [0.0, 1.0].
+ */
+export function checkBinningMap(m: unknown): BinningMap {
+ if (!m || typeof m !== "object" || Array.isArray(m)) {
+ throw new Error(`binning_map must be an object of bucket -> {bins, values}, got ${pyRepr(m)}`);
+ }
+ const out: BinningMap = {};
+ for (const [name, entry] of Object.entries(m as Record<string, unknown>)) {
+ if (!BUCKET_KEY.test(name)) {
+ throw new Error(`binning_map key ${pyRepr(name)} is not a bucket like "choice:2" or "score:3-5"`);
+ }
+ if (!entry || typeof entry !== "object" || Array.isArray(entry)) {
+ throw new Error(`binning_map[${pyRepr(name)}] must be an object with "bins" and "values", got ${pyRepr(entry)}`);
+ }
+ const rec = entry as Record<string, unknown>;
+ const nBins = rec.bins;
+ const values = rec.values;
+ if (typeof nBins === "boolean" || typeof nBins !== "number" || !Number.isInteger(nBins) || nBins < 1) {
+ throw new Error(`binning_map[${pyRepr(name)}] must have an integer "bins" >= 1, got ${pyRepr(nBins)}`);
+ }
+ if (!Array.isArray(values) || values.length !== nBins) {
+ throw new Error(`binning_map[${pyRepr(name)}] must have "values" of length "bins" (${nBins})`);
+ }
+ const parsedValues: number[] = [];
+ for (const v of values) {
+ if (typeof v === "boolean" || typeof v !== "number" || !Number.isFinite(v) || v < 0.0 || v > 1.0) {
+ throw new Error(`binning_map[${pyRepr(name)}] values must be numbers in [0, 1], got ${pyRepr(v)}`);
+ }
+ parsedValues.push(v);
+ }
+ out[name] = { bins: nBins, values: parsedValues };
+ }
+ return out;
+}
+
+/**
+ * Recalibrate one `answer_confidence` for its option-count `bucket` (tempBucket).
+ *
+ * Returns the confidence unchanged when the map has no entry for the bucket, so a bucket the map
+ * was not fit for passes through rather than being forced to a wrong value.
+ */
+export function applyBinningMap(
+ confidence: number,
+ bucket: string,
+ binningMap?: BinningMap | null,
+): number {
+ if (!Number.isFinite(confidence)) return confidence;
+ const entry = binningMap?.[bucket];
+ if (!entry) {
+ return confidence;
+ }
+ const bins = entry.bins;
+ const b = Math.min(bins - 1, Math.max(0, Math.floor(confidence * bins)));
+ return entry.values[b];
+}
/** Max of a length list without spread (Math.max(...arr) throws RangeError past ~100k args). */
export function maxOf(values: ArrayLike<number>, fallback = 0): number {
let m = fallback;
diff --git a/laya-ts/src/index.ts b/laya-ts/src/index.ts
index c670a6ee..517cfeed 100644
--- a/laya-ts/src/index.ts
+++ b/laya-ts/src/index.ts
@@ -66,6 +66,8 @@ export {
resolveMinConfidence,
clampTemperature,
tempBucket,
+ applyBinningMap,
+ checkBinningMap,
collateItems,
TEMP_MIN,
TEMP_MAX,
@@ -80,6 +82,8 @@ export type {
OptionStats,
MinConfidence,
MinConfidenceMap,
+ BinningEntry,
+ BinningMap,
} from "./common.js";
export { q, best, topK, isConfident } from "./dx.js";
export { bpeEncode, metaspaceEncode, encodeWithData, parseTokenizerJson, loadTokenizerJson, CHECKPOINT_IDS, SPECIAL_ALIASES, METASPACE_REPLACEMENT } from "./tokenizer.js";
diff --git a/laya-ts/tests/histogram-binning.test.ts b/laya-ts/tests/histogram-binning.test.ts
new file mode 100644
index 00000000..3e83216b
--- /dev/null
+++ b/laya-ts/tests/histogram-binning.test.ts
@@ -0,0 +1,269 @@
+import { describe, expect, it } from "vitest";
+import {
+ applyBinningMap,
+ checkBinningMap,
+ type BinningMap,
+} from "../src/common.js";
+import { Agent, type ChoiceAnswer, type ScoreAnswer, type NoulAnswer } from "../src/agent.js";
+
+const fakeProvider = (logits: number[][] = [[2.0, 0.0]], act: number[][] = [[3.0, 0.0]]) => ({
+ async runEncoder(_b: any) {
+ return { lastHidden: [[1, 0], [0, 1]] };
+ },
+ async runHead(_h: any) {
+ return {
+ logits,
+ act,
+ };
+ },
+});
+
+describe("checkBinningMap", () => {
+ it("validates well-formed binning maps", () => {
+ const valid: BinningMap = {
+ "choice:2": { bins: 5, values: [0.1, 0.3, 0.5, 0.7, 0.9] },
+ "score:3-5": { bins: 2, values: [0.2, 0.8] },
+ };
+ expect(checkBinningMap(valid)).toEqual(valid);
+ });
+
+ it("accepts an empty binning map", () => {
+ expect(checkBinningMap({})).toEqual({});
+ });
+
+ it("rejects non-objects, arrays, null and primitives", () => {
+ const bads = [null, undefined, 123, "not-a-map", true, false, [1, 2, 3]];
+ for (const bad of bads) {
+ expect(() => checkBinningMap(bad)).toThrowError(/binning_map must be an object of bucket -> \{bins, values\}/);
+ }
+ });
+
+ it("rejects invalid bucket keys that cannot match", () => {
+ const badKeys = [
+ { "choice:3to5": { bins: 1, values: [0.5] } },
+ { "invalid:2": { bins: 1, values: [0.5] } },
+ { "choice:1": { bins: 1, values: [0.5] } },
+ { "default": { bins: 1, values: [0.5] } },
+ ];
+ for (const bad of badKeys) {
+ expect(() => checkBinningMap(bad)).toThrowError(/is not a bucket like "choice:2" or "score:3-5"/);
+ }
+ });
+
+ it("rejects non-object bucket entries", () => {
+ const bads = [
+ { "choice:2": null },
+ { "choice:2": "invalid" },
+ { "choice:2": [1, 2, 3] },
+ { "choice:2": 42 },
+ ];
+ for (const bad of bads) {
+ expect(() => checkBinningMap(bad)).toThrowError(/must be an object with "bins" and "values"/);
+ }
+ });
+
+ it("rejects invalid bins count", () => {
+ const badBins = [
+ { "choice:2": { bins: 0, values: [] } },
+ { "choice:2": { bins: -1, values: [] } },
+ { "choice:2": { bins: 2.5, values: [0.1, 0.2] } },
+ { "choice:2": { bins: "3", values: [0.1, 0.2, 0.3] } },
+ { "choice:2": { bins: true, values: [0.1] } },
+ { "choice:2": { values: [0.1] } },
+ ];
+ for (const bad of badBins) {
+ expect(() => checkBinningMap(bad)).toThrowError(/must have an integer "bins" >= 1/);
+ }
+ });
+
+ it("rejects mismatched values length", () => {
+ const mismatches = [
+ { "choice:2": { bins: 3, values: [0.1, 0.2] } },
+ { "choice:2": { bins: 2, values: [0.1, 0.2, 0.3] } },
+ { "choice:2": { bins: 2, values: "not-an-array" } },
+ ];
+ for (const bad of mismatches) {
+ expect(() => checkBinningMap(bad)).toThrowError(/must have "values" of length "bins"/);
+ }
+ });
+
+ it("rejects non-numbers and out of range [0, 1] numbers in values", () => {
+ const badValues = [
+ { "choice:2": { bins: 2, values: [0.1, "0.5"] } },
+ { "choice:2": { bins: 2, values: [0.1, true] } },
+ { "choice:2": { bins: 2, values: [0.1, false] } },
+ { "choice:2": { bins: 2, values: [0.1, NaN] } },
+ { "choice:2": { bins: 2, values: [0.1, Infinity] } },
+ { "choice:2": { bins: 2, values: [-0.01, 0.5] } },
+ { "choice:2": { bins: 2, values: [0.5, 1.01] } },
+ ];
+ for (const bad of badValues) {
+ expect(() => checkBinningMap(bad)).toThrowError(/values must be numbers in \[0, 1\]/);
+ }
+ });
+});
+
+describe("applyBinningMap", () => {
+ const map: BinningMap = {
+ "choice:2": { bins: 4, values: [0.15, 0.35, 0.65, 0.95] },
+ "score:2": { bins: 2, values: [0.2, 0.8] },
+ };
+
+ it("passes through confidence when map is null/undefined or bucket is missing", () => {
+ expect(applyBinningMap(0.75, "choice:2", null)).toBe(0.75);
+ expect(applyBinningMap(0.75, "choice:2", undefined)).toBe(0.75);
+ expect(applyBinningMap(0.75, "choice:3-5", map)).toBe(0.75);
+ expect(applyBinningMap(0.75, "unknown:bucket", map)).toBe(0.75);
+ });
+
+ it("passes through non-finite confidence", () => {
+ expect(applyBinningMap(NaN, "choice:2", map)).toBeNaN();
+ expect(applyBinningMap(Infinity, "choice:2", map)).toBe(Infinity);
+ });
+
+ it("correctly indexes bins for valid inputs", () => {
+ // bins = 4: bin 0: [0, 0.25), bin 1: [0.25, 0.50), bin 2: [0.50, 0.75), bin 3: [0.75, 1.0]
+ expect(applyBinningMap(0.0, "choice:2", map)).toBe(0.15);
+ expect(applyBinningMap(0.1, "choice:2", map)).toBe(0.15);
+ expect(applyBinningMap(0.24, "choice:2", map)).toBe(0.15);
+ expect(applyBinningMap(0.25, "choice:2", map)).toBe(0.35);
+ expect(applyBinningMap(0.49, "choice:2", map)).toBe(0.35);
+ expect(applyBinningMap(0.50, "choice:2", map)).toBe(0.65);
+ expect(applyBinningMap(0.74, "choice:2", map)).toBe(0.65);
+ expect(applyBinningMap(0.75, "choice:2", map)).toBe(0.95);
+ expect(applyBinningMap(0.99, "choice:2", map)).toBe(0.95);
+ expect(applyBinningMap(1.0, "choice:2", map)).toBe(0.95);
+ });
+
+ it("clamps values outside [0, 1] to first and last bins", () => {
+ expect(applyBinningMap(-0.5, "choice:2", map)).toBe(0.15);
+ expect(applyBinningMap(1.5, "choice:2", map)).toBe(0.95);
+ });
+});
+
+describe("Agent binning_map integration", () => {
+ const binningMap: BinningMap = {
+ "choice:2": { bins: 4, values: [0.1, 0.4, 0.7, 0.99] },
+ "score:2": { bins: 2, values: [0.05, 0.95] },
+ "noul:2": { bins: 2, values: [0.12, 0.88] },
+ };
+
+ it("accepts binning_map via cfg or options", () => {
+ const a1 = new Agent({ provider: fakeProvider() as any, cfg: { binning_map: binningMap } });
+ expect(a1.binningMap).toEqual(binningMap);
+
+ const a2 = new Agent({ provider: fakeProvider() as any, binning_map: binningMap });
+ expect(a2.binningMap).toEqual(binningMap);
+
+ const a3 = new Agent({ provider: fakeProvider() as any, binningMap });
+ expect(a3.binningMap).toEqual(binningMap);
+
+ const a4 = new Agent({ provider: fakeProvider() as any });
+ expect(a4.binningMap).toBeNull();
+ });
+
+ it("rejects invalid binning_map during Agent construction", () => {
+ expect(
+ () => new Agent({ provider: fakeProvider() as any, binning_map: "not-a-map" as any }),
+ ).toThrowError(/binning_map must be an object/);
+ });
+
+ it("recalibrates answer_confidence on choice question", async () => {
+ // logits: [2.0, 0.0] -> softmax([2, 0]) ~ [0.8808, 0.1192] -> ans_raw ~ 0.8808
+ // With 4 bins: floor(0.8808 * 4) = floor(3.5232) = 3 -> values[3] = 0.99
+ const agentWithBinning = new Agent({
+ provider: fakeProvider([[2.0, 0.0]]) as any,
+ binning_map: binningMap,
+ });
+ const agentWithout = new Agent({
+ provider: fakeProvider([[2.0, 0.0]]) as any,
+ });
+
+ const resWith = await agentWithBinning.predict("dummy state", {
+ q1: { type: "choice", instructions: "Pick one", criteria: ["A", "B"] },
+ });
+ const ansWith = resWith.answers.q1 as ChoiceAnswer;
+ expect(ansWith.answer_confidence).toBe(0.99);
+
+ const resWithout = await agentWithout.predict("dummy state", {
+ q1: { type: "choice", instructions: "Pick one", criteria: ["A", "B"] },
+ });
+ const ansWithout = resWithout.answers.q1 as ChoiceAnswer;
+ expect(ansWithout.answer_confidence).toBe(0.8808);
+ // Uncalibrated entropy confidence remains untouched
+ expect(ansWith.confidence).toBe(ansWithout.confidence);
+ });
+
+ it("recalibrates answer_confidence on score question", async () => {
+ const agentWithBinning = new Agent({
+ provider: fakeProvider([[2.0, 0.0]]) as any,
+ binning_map: binningMap,
+ });
+ const res = await agentWithBinning.predict("dummy state", {
+ q1: { type: "score", instructions: "Score this", criteria: ["Low", "High"] },
+ });
+ const ans = res.answers.q1 as ScoreAnswer;
+ // max(p) ~ 0.8808 in bucket "score:2", bins=2 -> floor(0.8808 * 2) = 1 -> values[1] = 0.95
+ expect(ans.answer_confidence).toBe(0.95);
+ });
+
+ it("recalibrates answer_confidence on noul question", async () => {
+ const agentWithBinning = new Agent({
+ provider: fakeProvider([[2.0, 0.0]]) as any,
+ binning_map: binningMap,
+ });
+ const res = await agentWithBinning.predict("dummy state", {
+ q1: { type: "noul", instructions: "Is this valid?" },
+ });
+ const ans = res.answers.q1 as NoulAnswer;
+ // bucket "noul:2", bins=2 -> floor(0.8808 * 2) = 1 -> values[1] = 0.88
+ expect(ans.answer_confidence).toBe(0.88);
+ });
+
+ it("bypasses binning recalibration when language override is active (Python parity)", async () => {
+ const agent = new Agent({
+ provider: fakeProvider([[2.0, 0.0]]) as any,
+ binning_map: binningMap,
+ lang_temperatures: {
+ fr: { temperature: [1.0, 1.0, 1.0] },
+ },
+ });
+
+ // Without lang override (or non-matching lang), binning recalibration applies
+ const resEn = await agent.predict(
+ "dummy state",
+ { q1: { type: "choice", instructions: "Pick one", criteria: ["A", "B"] } },
+ { lang: "en" },
+ );
+ expect((resEn.answers.q1 as ChoiceAnswer).answer_confidence).toBe(0.99);
+
+ // With matching lang override "fr", binning recalibration is bypassed
+ const resFr = await agent.predict(
+ "dummy state",
+ { q1: { type: "choice", instructions: "Pick one", criteria: ["A", "B"] } },
+ { lang: "fr" },
+ );
+ expect((resFr.answers.q1 as ChoiceAnswer).answer_confidence).toBe(0.8808);
+ });
+
+ it("gates low confidence using recalibrated answer_confidence", async () => {
+ // Without binning: ans_raw = 0.8808 >= 0.95 threshold -> low_confidence would be false.
+ // With binning mapping into 0.70: 0.70 < 0.95 threshold -> low_confidence is flagged true!
+ const mappingDown: BinningMap = {
+ "choice:2": { bins: 1, values: [0.70] },
+ };
+ const agent = new Agent({
+ provider: fakeProvider([[2.0, 0.0]]) as any,
+ binning_map: mappingDown,
+ });
+
+ const res = await agent.predict(
+ "dummy state",
+ { q1: { type: "choice", instructions: "Pick one", criteria: ["A", "B"] } },
+ { minConfidence: 0.85 },
+ );
+ const ans = res.answers.q1 as ChoiceAnswer;
+ expect(ans.answer_confidence).toBe(0.70);
+ expect(ans.low_confidence).toBe(true);
+ });
+});