Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
feat(ts): port histogram-binning recalibration to laya-ts
Swatantra Yadava76dc39d9162
diff --git a/laya-ts/src/agent.ts b/laya-ts/src/agent.tsindex 5bbdad94..cd368278 100644--- a/laya-ts/src/agent.ts+++ b/laya-ts/src/agent.ts@@ -1,7 +1,9 @@import {TEMP_MAX,TEMP_MIN,+ applyBinningMap,buildQuestionPrefix,+ checkBinningMap,clampTemperature,collateItems,collapsedOptions,@@ -16,7 +18,7 @@ import {softmax,tempBucket,} from "./common.js";-import type { MinConfidence, OptionStats, SequenceStats } from "./common.js";+import type { BinningMap, MinConfidence, OptionStats, SequenceStats } from "./common.js";import type { Batch, SessionProvider } from "./providers.js";import { encodeWithData, parseTokenizerJson, type TokenizerLike } from "./tokenizer.js";import { decide, type DecideOptions, type DecisionResult } from "./structured.js";@@ -103,6 +105,7 @@ export interface AgentCfg {head_max_len?: number;temperature?: unknown;temperature_by_options?: Record<string, unknown>;+ binning_map?: BinningMap | null;[k: string]: unknown;}@@ -116,6 +119,9 @@ export interface AgentOptions {head_max_len?: number;temperature?: unknown;temperature_by_options?: Record<string, unknown>;+ /** Optional histogram-binning map applied to answer_confidence. */+ binning_map?: BinningMap | null;+ binningMap?: BinningMap | null;/*** Per-language temperature overrides, keyed by language code; keys are normalised to* their base subtag (`de-AT` -> `de`), matching Python `Agent(lang_temperatures=...)`.@@ -385,6 +391,7 @@ export class Agent extends HookRegistry {temperatureByOptionsRaw: Record<string, unknown>;temperature: number[];temperatureByOptions: Record<string, number>;+ readonly binningMap: BinningMap | null;langTemperatures: Record<string,{ temperature: number[]; temperatureByOptions: Record<string, number> }@@ -403,10 +410,14 @@ export class Agent extends HookRegistry {if (opts.head_max_len !== undefined) cfg.head_max_len = opts.head_max_len;if (opts.temperature !== undefined) cfg.temperature = opts.temperature;if (opts.temperature_by_options !== undefined) cfg.temperature_by_options = opts.temperature_by_options;+ if (opts.binning_map !== undefined) cfg.binning_map = opts.binning_map;+ if (opts.binningMap !== undefined) cfg.binning_map = opts.binningMap;this.cfg = cfg;this.maxLen = Number(cfg.max_len ?? 512);this.headMaxLen = Number(cfg.head_max_len ?? 192);this.tok = opts.tok ?? defaultTokenizer();+ const bmapRaw = cfg.binning_map;+ this.binningMap = bmapRaw !== undefined && bmapRaw !== null ? checkBinningMap(bmapRaw) : null;const raw = (cfg.temperature ?? [1.0, 1.0, 1.0]) as unknown;this.temperatureRaw = raw;// The decode indexes this by question type, so refuse any other shape here, as Python's@@ -724,7 +735,15 @@ export class Agent extends HookRegistry {const ext = { act_probability: r4(actP[0]) };// Same quantity on every question type (max(p)), so callers can gate across types on// one number; `confidence` stays as-is for existing callers (entropy for choice/score).- const ansConf = r4(answerConfidence(p));+ // Histogram-binning recalibration (issue #871, Python parity) recalibrates ans_raw+ // per temp_bucket unless a language override is active.+ const ansRaw = answerConfidence(p);+ const langOverride = Boolean(lang && lang.split("-")[0].toLowerCase() in this.langTemperatures);+ const ansConf = r4(+ this.binningMap && !langOverride+ ? applyBinningMap(ansRaw, bucket, this.binningMap)+ : ansRaw,+ );if (q.t === "choice") {const keys = Object.keys(q.crit as Record<string, unknown>);let best = 0;diff --git a/laya-ts/src/common.ts b/laya-ts/src/common.tsindex 89c52335..38f170ed 100644--- a/laya-ts/src/common.ts+++ b/laya-ts/src/common.ts@@ -355,6 +355,74 @@ export function tempBucket(qtype: number, k: number): string {const size = k <= 2 ? "2" : k <= 5 ? "3-5" : k <= 10 ? "6-10" : "11+";return `${["choice", "score", "noul"][qtype]}:${size}`;}++export interface BinningEntry {+ bins: number;+ values: number[];+}++export type BinningMap = Record<string, BinningEntry>;++/**+ * Validate a histogram-binning recalibration map.+ *+ * Keys are option-count bucket strings like "choice:2", "choice:3-5", "score:6-10", "noul:2".+ * Values are objects with an integer `bins >= 1` and a `values` array of length `bins`+ * where each number is in [0.0, 1.0].+ */+export function checkBinningMap(m: unknown): BinningMap {+ if (!m || typeof m !== "object" || Array.isArray(m)) {+ throw new Error(`binning_map must be an object of bucket -> {bins, values}, got ${pyRepr(m)}`);+ }+ const out: BinningMap = {};+ for (const [name, entry] of Object.entries(m as Record<string, unknown>)) {+ if (!BUCKET_KEY.test(name)) {+ throw new Error(`binning_map key ${pyRepr(name)} is not a bucket like "choice:2" or "score:3-5"`);+ }+ if (!entry || typeof entry !== "object" || Array.isArray(entry)) {+ throw new Error(`binning_map[${pyRepr(name)}] must be an object with "bins" and "values", got ${pyRepr(entry)}`);+ }+ const rec = entry as Record<string, unknown>;+ const nBins = rec.bins;+ const values = rec.values;+ if (typeof nBins === "boolean" || typeof nBins !== "number" || !Number.isInteger(nBins) || nBins < 1) {+ throw new Error(`binning_map[${pyRepr(name)}] must have an integer "bins" >= 1, got ${pyRepr(nBins)}`);+ }+ if (!Array.isArray(values) || values.length !== nBins) {+ throw new Error(`binning_map[${pyRepr(name)}] must have "values" of length "bins" (${nBins})`);+ }+ const parsedValues: number[] = [];+ for (const v of values) {+ if (typeof v === "boolean" || typeof v !== "number" || !Number.isFinite(v) || v < 0.0 || v > 1.0) {+ throw new Error(`binning_map[${pyRepr(name)}] values must be numbers in [0, 1], got ${pyRepr(v)}`);+ }+ parsedValues.push(v);+ }+ out[name] = { bins: nBins, values: parsedValues };+ }+ return out;+}++/**+ * Recalibrate one `answer_confidence` for its option-count `bucket` (tempBucket).+ *+ * Returns the confidence unchanged when the map has no entry for the bucket, so a bucket the map+ * was not fit for passes through rather than being forced to a wrong value.+ */+export function applyBinningMap(+ confidence: number,+ bucket: string,+ binningMap?: BinningMap | null,+): number {+ if (!Number.isFinite(confidence)) return confidence;+ const entry = binningMap?.[bucket];+ if (!entry) {+ return confidence;+ }+ const bins = entry.bins;+ const b = Math.min(bins - 1, Math.max(0, Math.floor(confidence * bins)));+ return entry.values[b];+}/** Max of a length list without spread (Math.max(...arr) throws RangeError past ~100k args). */export function maxOf(values: ArrayLike<number>, fallback = 0): number {let m = fallback;diff --git a/laya-ts/src/index.ts b/laya-ts/src/index.tsindex c670a6ee..517cfeed 100644--- a/laya-ts/src/index.ts+++ b/laya-ts/src/index.ts@@ -66,6 +66,8 @@ export {resolveMinConfidence,clampTemperature,tempBucket,+ applyBinningMap,+ checkBinningMap,collateItems,TEMP_MIN,TEMP_MAX,@@ -80,6 +82,8 @@ export type {OptionStats,MinConfidence,MinConfidenceMap,+ BinningEntry,+ BinningMap,} from "./common.js";export { q, best, topK, isConfident } from "./dx.js";export { bpeEncode, metaspaceEncode, encodeWithData, parseTokenizerJson, loadTokenizerJson, CHECKPOINT_IDS, SPECIAL_ALIASES, METASPACE_REPLACEMENT } from "./tokenizer.js";diff --git a/laya-ts/tests/histogram-binning.test.ts b/laya-ts/tests/histogram-binning.test.tsnew file mode 100644index 00000000..3e83216b--- /dev/null+++ b/laya-ts/tests/histogram-binning.test.ts@@ -0,0 +1,269 @@+import { describe, expect, it } from "vitest";+import {+ applyBinningMap,+ checkBinningMap,+ type BinningMap,+} from "../src/common.js";+import { Agent, type ChoiceAnswer, type ScoreAnswer, type NoulAnswer } from "../src/agent.js";++const fakeProvider = (logits: number[][] = [[2.0, 0.0]], act: number[][] = [[3.0, 0.0]]) => ({+ async runEncoder(_b: any) {+ return { lastHidden: [[1, 0], [0, 1]] };+ },+ async runHead(_h: any) {+ return {+ logits,+ act,+ };+ },+});++describe("checkBinningMap", () => {+ it("validates well-formed binning maps", () => {+ const valid: BinningMap = {+ "choice:2": { bins: 5, values: [0.1, 0.3, 0.5, 0.7, 0.9] },+ "score:3-5": { bins: 2, values: [0.2, 0.8] },+ };+ expect(checkBinningMap(valid)).toEqual(valid);+ });++ it("accepts an empty binning map", () => {+ expect(checkBinningMap({})).toEqual({});+ });++ it("rejects non-objects, arrays, null and primitives", () => {+ const bads = [null, undefined, 123, "not-a-map", true, false, [1, 2, 3]];+ for (const bad of bads) {+ expect(() => checkBinningMap(bad)).toThrowError(/binning_map must be an object of bucket -> \{bins, values\}/);+ }+ });++ it("rejects invalid bucket keys that cannot match", () => {+ const badKeys = [+ { "choice:3to5": { bins: 1, values: [0.5] } },+ { "invalid:2": { bins: 1, values: [0.5] } },+ { "choice:1": { bins: 1, values: [0.5] } },+ { "default": { bins: 1, values: [0.5] } },+ ];+ for (const bad of badKeys) {+ expect(() => checkBinningMap(bad)).toThrowError(/is not a bucket like "choice:2" or "score:3-5"/);+ }+ });++ it("rejects non-object bucket entries", () => {+ const bads = [+ { "choice:2": null },+ { "choice:2": "invalid" },+ { "choice:2": [1, 2, 3] },+ { "choice:2": 42 },+ ];+ for (const bad of bads) {+ expect(() => checkBinningMap(bad)).toThrowError(/must be an object with "bins" and "values"/);+ }+ });++ it("rejects invalid bins count", () => {+ const badBins = [+ { "choice:2": { bins: 0, values: [] } },+ { "choice:2": { bins: -1, values: [] } },+ { "choice:2": { bins: 2.5, values: [0.1, 0.2] } },+ { "choice:2": { bins: "3", values: [0.1, 0.2, 0.3] } },+ { "choice:2": { bins: true, values: [0.1] } },+ { "choice:2": { values: [0.1] } },+ ];+ for (const bad of badBins) {+ expect(() => checkBinningMap(bad)).toThrowError(/must have an integer "bins" >= 1/);+ }+ });++ it("rejects mismatched values length", () => {+ const mismatches = [+ { "choice:2": { bins: 3, values: [0.1, 0.2] } },+ { "choice:2": { bins: 2, values: [0.1, 0.2, 0.3] } },+ { "choice:2": { bins: 2, values: "not-an-array" } },+ ];+ for (const bad of mismatches) {+ expect(() => checkBinningMap(bad)).toThrowError(/must have "values" of length "bins"/);+ }+ });++ it("rejects non-numbers and out of range [0, 1] numbers in values", () => {+ const badValues = [+ { "choice:2": { bins: 2, values: [0.1, "0.5"] } },+ { "choice:2": { bins: 2, values: [0.1, true] } },+ { "choice:2": { bins: 2, values: [0.1, false] } },+ { "choice:2": { bins: 2, values: [0.1, NaN] } },+ { "choice:2": { bins: 2, values: [0.1, Infinity] } },+ { "choice:2": { bins: 2, values: [-0.01, 0.5] } },+ { "choice:2": { bins: 2, values: [0.5, 1.01] } },+ ];+ for (const bad of badValues) {+ expect(() => checkBinningMap(bad)).toThrowError(/values must be numbers in \[0, 1\]/);+ }+ });+});++describe("applyBinningMap", () => {+ const map: BinningMap = {+ "choice:2": { bins: 4, values: [0.15, 0.35, 0.65, 0.95] },+ "score:2": { bins: 2, values: [0.2, 0.8] },+ };++ it("passes through confidence when map is null/undefined or bucket is missing", () => {+ expect(applyBinningMap(0.75, "choice:2", null)).toBe(0.75);+ expect(applyBinningMap(0.75, "choice:2", undefined)).toBe(0.75);+ expect(applyBinningMap(0.75, "choice:3-5", map)).toBe(0.75);+ expect(applyBinningMap(0.75, "unknown:bucket", map)).toBe(0.75);+ });++ it("passes through non-finite confidence", () => {+ expect(applyBinningMap(NaN, "choice:2", map)).toBeNaN();+ expect(applyBinningMap(Infinity, "choice:2", map)).toBe(Infinity);+ });++ it("correctly indexes bins for valid inputs", () => {+ // bins = 4: bin 0: [0, 0.25), bin 1: [0.25, 0.50), bin 2: [0.50, 0.75), bin 3: [0.75, 1.0]+ expect(applyBinningMap(0.0, "choice:2", map)).toBe(0.15);+ expect(applyBinningMap(0.1, "choice:2", map)).toBe(0.15);+ expect(applyBinningMap(0.24, "choice:2", map)).toBe(0.15);+ expect(applyBinningMap(0.25, "choice:2", map)).toBe(0.35);+ expect(applyBinningMap(0.49, "choice:2", map)).toBe(0.35);+ expect(applyBinningMap(0.50, "choice:2", map)).toBe(0.65);+ expect(applyBinningMap(0.74, "choice:2", map)).toBe(0.65);+ expect(applyBinningMap(0.75, "choice:2", map)).toBe(0.95);+ expect(applyBinningMap(0.99, "choice:2", map)).toBe(0.95);+ expect(applyBinningMap(1.0, "choice:2", map)).toBe(0.95);+ });++ it("clamps values outside [0, 1] to first and last bins", () => {+ expect(applyBinningMap(-0.5, "choice:2", map)).toBe(0.15);+ expect(applyBinningMap(1.5, "choice:2", map)).toBe(0.95);+ });+});++describe("Agent binning_map integration", () => {+ const binningMap: BinningMap = {+ "choice:2": { bins: 4, values: [0.1, 0.4, 0.7, 0.99] },+ "score:2": { bins: 2, values: [0.05, 0.95] },+ "noul:2": { bins: 2, values: [0.12, 0.88] },+ };++ it("accepts binning_map via cfg or options", () => {+ const a1 = new Agent({ provider: fakeProvider() as any, cfg: { binning_map: binningMap } });+ expect(a1.binningMap).toEqual(binningMap);++ const a2 = new Agent({ provider: fakeProvider() as any, binning_map: binningMap });+ expect(a2.binningMap).toEqual(binningMap);++ const a3 = new Agent({ provider: fakeProvider() as any, binningMap });+ expect(a3.binningMap).toEqual(binningMap);++ const a4 = new Agent({ provider: fakeProvider() as any });+ expect(a4.binningMap).toBeNull();+ });++ it("rejects invalid binning_map during Agent construction", () => {+ expect(+ () => new Agent({ provider: fakeProvider() as any, binning_map: "not-a-map" as any }),+ ).toThrowError(/binning_map must be an object/);+ });++ it("recalibrates answer_confidence on choice question", async () => {+ // logits: [2.0, 0.0] -> softmax([2, 0]) ~ [0.8808, 0.1192] -> ans_raw ~ 0.8808+ // With 4 bins: floor(0.8808 * 4) = floor(3.5232) = 3 -> values[3] = 0.99+ const agentWithBinning = new Agent({+ provider: fakeProvider([[2.0, 0.0]]) as any,+ binning_map: binningMap,+ });+ const agentWithout = new Agent({+ provider: fakeProvider([[2.0, 0.0]]) as any,+ });++ const resWith = await agentWithBinning.predict("dummy state", {+ q1: { type: "choice", instructions: "Pick one", criteria: ["A", "B"] },+ });+ const ansWith = resWith.answers.q1 as ChoiceAnswer;+ expect(ansWith.answer_confidence).toBe(0.99);++ const resWithout = await agentWithout.predict("dummy state", {+ q1: { type: "choice", instructions: "Pick one", criteria: ["A", "B"] },+ });+ const ansWithout = resWithout.answers.q1 as ChoiceAnswer;+ expect(ansWithout.answer_confidence).toBe(0.8808);+ // Uncalibrated entropy confidence remains untouched+ expect(ansWith.confidence).toBe(ansWithout.confidence);+ });++ it("recalibrates answer_confidence on score question", async () => {+ const agentWithBinning = new Agent({+ provider: fakeProvider([[2.0, 0.0]]) as any,+ binning_map: binningMap,+ });+ const res = await agentWithBinning.predict("dummy state", {+ q1: { type: "score", instructions: "Score this", criteria: ["Low", "High"] },+ });+ const ans = res.answers.q1 as ScoreAnswer;+ // max(p) ~ 0.8808 in bucket "score:2", bins=2 -> floor(0.8808 * 2) = 1 -> values[1] = 0.95+ expect(ans.answer_confidence).toBe(0.95);+ });++ it("recalibrates answer_confidence on noul question", async () => {+ const agentWithBinning = new Agent({+ provider: fakeProvider([[2.0, 0.0]]) as any,+ binning_map: binningMap,+ });+ const res = await agentWithBinning.predict("dummy state", {+ q1: { type: "noul", instructions: "Is this valid?" },+ });+ const ans = res.answers.q1 as NoulAnswer;+ // bucket "noul:2", bins=2 -> floor(0.8808 * 2) = 1 -> values[1] = 0.88+ expect(ans.answer_confidence).toBe(0.88);+ });++ it("bypasses binning recalibration when language override is active (Python parity)", async () => {+ const agent = new Agent({+ provider: fakeProvider([[2.0, 0.0]]) as any,+ binning_map: binningMap,+ lang_temperatures: {+ fr: { temperature: [1.0, 1.0, 1.0] },+ },+ });++ // Without lang override (or non-matching lang), binning recalibration applies+ const resEn = await agent.predict(+ "dummy state",+ { q1: { type: "choice", instructions: "Pick one", criteria: ["A", "B"] } },+ { lang: "en" },+ );+ expect((resEn.answers.q1 as ChoiceAnswer).answer_confidence).toBe(0.99);++ // With matching lang override "fr", binning recalibration is bypassed+ const resFr = await agent.predict(+ "dummy state",+ { q1: { type: "choice", instructions: "Pick one", criteria: ["A", "B"] } },+ { lang: "fr" },+ );+ expect((resFr.answers.q1 as ChoiceAnswer).answer_confidence).toBe(0.8808);+ });++ it("gates low confidence using recalibrated answer_confidence", async () => {+ // Without binning: ans_raw = 0.8808 >= 0.95 threshold -> low_confidence would be false.+ // With binning mapping into 0.70: 0.70 < 0.95 threshold -> low_confidence is flagged true!+ const mappingDown: BinningMap = {+ "choice:2": { bins: 1, values: [0.70] },+ };+ const agent = new Agent({+ provider: fakeProvider([[2.0, 0.0]]) as any,+ binning_map: mappingDown,+ });++ const res = await agent.predict(+ "dummy state",+ { q1: { type: "choice", instructions: "Pick one", criteria: ["A", "B"] } },+ { minConfidence: 0.85 },+ );+ const ans = res.answers.q1 as ChoiceAnswer;+ expect(ans.answer_confidence).toBe(0.70);+ expect(ans.low_confidence).toBe(true);+ });+});