Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
feat(ts): port per-bucket minConfidence map to laya-ts
Swatantra Yadav08a417a8a5f9
diff --git a/laya-ts/src/agent.ts b/laya-ts/src/agent.tsindex 83469fb8..5bbdad94 100644--- a/laya-ts/src/agent.ts+++ b/laya-ts/src/agent.ts@@ -16,7 +16,7 @@ import {softmax,tempBucket,} from "./common.js";-import type { OptionStats, SequenceStats } from "./common.js";+import type { MinConfidence, OptionStats, SequenceStats } from "./common.js";import type { Batch, SessionProvider } from "./providers.js";import { encodeWithData, parseTokenizerJson, type TokenizerLike } from "./tokenizer.js";import { decide, type DecideOptions, type DecisionResult } from "./structured.js";@@ -150,10 +150,10 @@ export interface PredictOptions {headMaxLen?: number | null;/** predictBatch: cap on states per shared forward pass; null sends them all in one pass. */batchSize?: number | null;- /** Minimum confidence threshold in [0.0, 1.0]. Low confidence answers get `low_confidence: true`. */- minConfidence?: number | null;+ /** Minimum confidence threshold in [0.0, 1.0] or per-bucket map. Low confidence answers get `low_confidence: true`. */+ minConfidence?: MinConfidence | null;/** Python parity alias for minConfidence. */- min_confidence?: number | null;+ min_confidence?: MinConfidence | null;}function qidStr(qid: string): string {diff --git a/laya-ts/src/common.ts b/laya-ts/src/common.tsindex 5047a405..89c52335 100644--- a/laya-ts/src/common.ts+++ b/laya-ts/src/common.ts@@ -229,33 +229,101 @@ function pyRepr(v: unknown): string {}if (typeof v === "string") return JSON.stringify(v);if (Array.isArray(v)) return `[${v.map(pyRepr).join(", ")}]`;+ if (typeof v === "object") {+ const entries = Object.entries(v as Record<string, unknown>).map(+ ([k, val]) => `${pyRepr(k)}: ${pyRepr(val)}`+ );+ return `{${entries.join(", ")}}`;+ }return String(v);}+export type MinConfidenceMap = Record<string, number>;+export type MinConfidence = number | MinConfidenceMap;++const BUCKET_KEY = /^(choice|score|noul):(2|3-5|6-10|11\+)$/;+/**- * Validate opt-in abstention threshold `min_confidence` (#361).+ * Validate a per-bucket abstention-threshold map (#394).*- * Must be a real number in [0.0, 1.0]. Booleans are rejected.+ * Keys are option-count bucket strings like "choice:2", "choice:3-5", "score:6-10", "noul:2",+ * plus an optional "default". Values are numbers in [0.0, 1.0].*/-export function checkMinConfidence(v: unknown): number {+export function checkMinConfidenceMap(m: unknown): MinConfidenceMap {+ if (!m || typeof m !== "object" || Array.isArray(m) || Object.keys(m).length === 0) {+ throw new Error(`a min_confidence map must be a non-empty dict of bucket -> float, got ${pyRepr(m)}`);+ }+ const out: MinConfidenceMap = {};+ for (const [key, val] of Object.entries(m as Record<string, unknown>)) {+ if (key !== "default" && !BUCKET_KEY.test(key)) {+ throw new Error(`min_confidence map keys must be strings like 'choice:3-5', got ${pyRepr(key)}`);+ }+ if (typeof val === "boolean" || typeof val !== "number" || !Number.isFinite(val) || val < 0.0 || val > 1.0) {+ throw new Error(`min_confidence must be a float in [0.0, 1.0], got ${pyRepr(val)}`);+ }+ out[key] = val;+ }+ return out;+}++/**+ * Validate opt-in abstention threshold `min_confidence` (#361, #394).+ *+ * Either a real number in [0.0, 1.0] or a per-bucket mapping. Booleans are rejected.+ */+export function checkMinConfidence(v: unknown): MinConfidence {+ if (v && typeof v === "object" && !Array.isArray(v)) {+ return checkMinConfidenceMap(v);+ }if (typeof v === "boolean" || typeof v !== "number" || !Number.isFinite(v) || v < 0.0 || v > 1.0) {throw new Error(`min_confidence must be a float in [0.0, 1.0], got ${pyRepr(v)}`);}return v;}+export function optionBucket(answer: Record<string, unknown>): string | null {+ const qt = answer.type;+ if (qt !== "choice" && qt !== "score" && qt !== "noul") return null;+ const probs = answer.probabilities;+ let k: number;+ if (probs && typeof probs === "object" && !Array.isArray(probs)) {+ k = Object.keys(probs).length;+ } else if (qt === "noul") {+ k = 2;+ } else {+ return null;+ }+ const size = k <= 2 ? "2" : k <= 5 ? "3-5" : k <= 10 ? "6-10" : "11+";+ return `${qt}:${size}`;+}++export function resolveMinConfidence(+ answer: Record<string, unknown>,+ thresholds: MinConfidenceMap,+ defaultVal: number = 0.0,+): number {+ const key = optionBucket(answer);+ if (key !== null && key in thresholds) {+ return thresholds[key];+ }+ return thresholds.default ?? defaultVal;+}+/**- * Opt-in abstention marker (#361): flag answers whose confidence falls below `min_confidence`.+ * Opt-in abstention marker (#361, #394): flag answers whose confidence falls below `min_confidence`.** Reads `answer_confidence` (the calibrated max(p) confidence, invariant to label count k),* falling back to `confidence` if `answer_confidence` is absent.* The raw answer and confidence stay intact; `low_confidence: true` is added.+ *+ * Supports both a scalar number in [0.0, 1.0] and a per-bucket mapping of thresholds.*/export function flagLowConfidence(results: Array<Record<string, unknown>> | Record<string, unknown>,- minConfidence: number,+ minConfidence: MinConfidence,): void {- if (minConfidence === 0.0) return;+ const isMap = typeof minConfidence === "object" && minConfidence !== null;+ if (!isMap && minConfidence === 0.0) return;const list = Array.isArray(results) ? results : [results];for (const res of list) {const answers = res && typeof res === "object" ? (res as Record<string, unknown>).answers : null;@@ -267,8 +335,11 @@ export function flagLowConfidence(if (conf === undefined || conf === null) {conf = ansObj.confidence;}- if (typeof conf === "number" && !Number.isNaN(conf) && conf < minConfidence) {- ansObj.low_confidence = true;+ if (typeof conf === "number" && !Number.isNaN(conf)) {+ const thr = isMap ? resolveMinConfidence(ansObj, minConfidence) : minConfidence;+ if (conf < thr) {+ ansObj.low_confidence = true;+ }}}}diff --git a/laya-ts/src/index.ts b/laya-ts/src/index.tsindex aa0fca66..c670a6ee 100644--- a/laya-ts/src/index.ts+++ b/laya-ts/src/index.ts@@ -60,14 +60,27 @@ export {confidenceFromProbs,answerConfidence,checkMinConfidence,+ checkMinConfidenceMap,flagLowConfidence,+ optionBucket,+ resolveMinConfidence,clampTemperature,tempBucket,collateItems,TEMP_MIN,TEMP_MAX,} from "./common.js";-export type { QType, InternalQ, CollateItem, CollatedBatch, QuestionPrefix, SequenceStats, OptionStats } from "./common.js";+export type {+ QType,+ InternalQ,+ CollateItem,+ CollatedBatch,+ QuestionPrefix,+ SequenceStats,+ OptionStats,+ MinConfidence,+ MinConfidenceMap,+} from "./common.js";export { q, best, topK, isConfident } from "./dx.js";export { bpeEncode, metaspaceEncode, encodeWithData, parseTokenizerJson, loadTokenizerJson, CHECKPOINT_IDS, SPECIAL_ALIASES, METASPACE_REPLACEMENT } from "./tokenizer.js";export type { TokenizerLike, TokenizerData, TokenizerIds, PreTokenizerKind } from "./tokenizer.js";diff --git a/laya-ts/src/structured.ts b/laya-ts/src/structured.tsindex 5cf7a6cf..bc5ef1b7 100644--- a/laya-ts/src/structured.ts+++ b/laya-ts/src/structured.ts@@ -20,6 +20,7 @@*/import type { QuestionDef } from "./agent.js";import { checkMinConfidence, flagLowConfidence } from "./common.js";+import type { MinConfidence } from "./common.js";export const MAX_PROPERTIES = 32;export const MAX_OPTIONS = 32;@@ -72,10 +73,10 @@ export interface DecideOptions {questions?: Record<string, QuestionDef>;/** Return a DecisionResult with confidence, probabilities and raw answers. */returnDetails?: boolean;- /** Minimum confidence threshold in [0.0, 1.0]. Low confidence answers project to null. */- minConfidence?: number | null;+ /** Minimum confidence threshold in [0.0, 1.0] or per-bucket map. Low confidence answers project to null. */+ minConfidence?: MinConfidence | null;/** Python parity alias for minConfidence. */- min_confidence?: number | null;+ min_confidence?: MinConfidence | null;/** Anything else is forwarded to runner.predict (hooks, model, ...). */[k: string]: unknown;}diff --git a/laya-ts/tests/confidence-gating.test.ts b/laya-ts/tests/confidence-gating.test.tsindex 02fb4470..846f8d30 100644--- a/laya-ts/tests/confidence-gating.test.ts+++ b/laya-ts/tests/confidence-gating.test.ts@@ -1,5 +1,11 @@import { describe, expect, it } from "vitest";-import { checkMinConfidence, flagLowConfidence } from "../src/common.js";+import {+ checkMinConfidence,+ checkMinConfidenceMap,+ flagLowConfidence,+ optionBucket,+ resolveMinConfidence,+} from "../src/common.js";import { Agent, type ChoiceAnswer } from "../src/agent.js";import { Router } from "../src/router.js";@@ -38,12 +44,91 @@ describe("checkMinConfidence", () => {null,undefined,[0.5],- {},];for (const invalid of invalids) {expect(() => checkMinConfidence(invalid)).toThrowError(/min_confidence must be a float in \[0\.0, 1\.0\]/);}});++ it("accepts and validates a per-bucket threshold map", () => {+ const map = { "choice:2": 0.9, default: 0.5 };+ expect(checkMinConfidence(map)).toEqual(map);+ });++ it("rejects empty min_confidence map", () => {+ expect(() => checkMinConfidence({})).toThrowError(/a min_confidence map must be a non-empty dict/);+ });+});++describe("checkMinConfidenceMap", () => {+ it("rejects non-object or non-dict inputs", () => {+ for (const invalid of [null, undefined, 42, "choice:2", [0.5], true]) {+ expect(() => checkMinConfidenceMap(invalid)).toThrowError(/a min_confidence map must be a non-empty dict/);+ }+ });++ it("rejects empty dict", () => {+ expect(() => checkMinConfidenceMap({})).toThrowError(/a min_confidence map must be a non-empty dict of bucket -> float, got \{\}/);+ });++ it("validates valid maps", () => {+ const map = { "choice:2": 0.9, "choice:11+": 0.5, "noul:2": 0.8, default: 0.3 };+ expect(checkMinConfidenceMap(map)).toEqual(map);+ });++ it("rejects invalid map values", () => {+ expect(() => checkMinConfidenceMap({ "choice:2": 1.5 })).toThrowError(/min_confidence must be a float in \[0\.0, 1\.0\]/);+ expect(() => checkMinConfidenceMap({ "choice:2": -0.1 })).toThrowError(/min_confidence must be a float in \[0\.0, 1\.0\]/);+ expect(() => checkMinConfidenceMap({ "choice:2": true })).toThrowError(/min_confidence must be a float in \[0\.0, 1\.0\]/);+ expect(() => checkMinConfidenceMap({ "choice:2": "0.9" as any })).toThrowError(/min_confidence must be a float in \[0\.0, 1\.0\]/);+ expect(() => checkMinConfidenceMap({ "choice:2": NaN })).toThrowError(/min_confidence must be a float in \[0\.0, 1\.0\]/);+ });++ it("rejects unknown bucket keys and typos", () => {+ for (const typo of ["choice:3_5", "Choice:2", "invalid", "score:1", "foo:2", "choice:12"]) {+ expect(() => checkMinConfidenceMap({ [typo]: 0.5 })).toThrowError(+ /min_confidence map keys must be strings like 'choice:3-5'/,+ );+ }+ });+});++describe("optionBucket & resolveMinConfidence", () => {+ it("computes optionBucket correctly", () => {+ expect(optionBucket({ type: "choice", probabilities: { a: 0.5, b: 0.5 } })).toBe("choice:2");+ expect(optionBucket({ type: "choice", probabilities: { a: 0.4, b: 0.3, c: 0.3 } })).toBe("choice:3-5");+ expect(optionBucket({ type: "choice", probabilities: { a: 0.2, b: 0.2, c: 0.2, d: 0.2, e: 0.2 } })).toBe("choice:3-5");+ const six: Record<string, number> = {};+ for (let i = 0; i < 6; i++) six[String(i)] = 1 / 6;+ expect(optionBucket({ type: "choice", probabilities: six })).toBe("choice:6-10");+ const ten: Record<string, number> = {};+ for (let i = 0; i < 10; i++) ten[String(i)] = 0.1;+ expect(optionBucket({ type: "choice", probabilities: ten })).toBe("choice:6-10");+ const eleven: Record<string, number> = {};+ for (let i = 0; i < 11; i++) eleven[String(i)] = 1 / 11;+ expect(optionBucket({ type: "choice", probabilities: eleven })).toBe("choice:11+");++ expect(optionBucket({ type: "score", probabilities: { "1": 0.3, "2": 0.3, "3": 0.4 } })).toBe("score:3-5");+ expect(optionBucket({ type: "noul" })).toBe("noul:2");+ expect(optionBucket({ type: "text" })).toBeNull();+ expect(optionBucket({ type: "choice" })).toBeNull();+ });++ it("resolves per-bucket thresholds with default and unconfigured fallback", () => {+ const MAP = { "choice:2": 0.9, "choice:11+": 0.5, "noul:2": 0.8, default: 0.3 };+ const ans = (type: string, k: number) => {+ const probs: Record<string, number> = {};+ for (let i = 0; i < k; i++) probs[String(i)] = 1 / k;+ return { type, probabilities: probs };+ };++ expect(resolveMinConfidence(ans("choice", 2), MAP)).toBe(0.9);+ expect(resolveMinConfidence(ans("choice", 12), MAP)).toBe(0.5);+ expect(resolveMinConfidence(ans("choice", 4), MAP)).toBe(0.3); // falls to default+ expect(resolveMinConfidence({ type: "noul" }, MAP)).toBe(0.8);+ // unconfigured bucket and no default in map -> returns defaultVal (0.0)+ expect(resolveMinConfidence(ans("score", 7), { "choice:2": 0.9 })).toBe(0.0);+ });});describe("flagLowConfidence", () => {@@ -93,6 +178,55 @@ describe("flagLowConfidence", () => {flagLowConfidence(single as any, 0.5);expect((single.answers.q as any).low_confidence).toBe(true);});++ it("gates with a per-bucket map", () => {+ const MAP = { "choice:2": 0.9, "choice:11+": 0.5, default: 0.3 };+ const sample = [+ {+ answers: {+ a: { type: "choice", probabilities: { "0": 0.5, "1": 0.5 }, answer_confidence: 0.85 },+ b: {+ type: "choice",+ probabilities: Object.fromEntries(Array.from({ length: 12 }, (_, i) => [String(i), 1 / 12])),+ answer_confidence: 0.6,+ },+ c: {+ type: "choice",+ probabilities: { "0": 0.25, "1": 0.25, "2": 0.25, "3": 0.25 },+ answer_confidence: 0.35,+ },+ d: {+ type: "score",+ probabilities: Object.fromEntries(Array.from({ length: 7 }, (_, i) => [String(i), 1 / 7])),+ answer_confidence: 0.05,+ },+ },+ },+ ];++ flagLowConfidence(sample as any, MAP);+ const ans = sample[0].answers as any;++ expect(ans.a.low_confidence).toBe(true); // 0.85 < 0.9 (choice:2)+ expect(ans.b.low_confidence).toBeUndefined(); // 0.60 >= 0.5 (choice:11+)+ expect(ans.c.low_confidence).toBeUndefined(); // 0.35 >= 0.3 (default)+ expect(ans.d.low_confidence).toBe(true); // 0.05 < 0.3 (falls back to default)++ // Unconfigured bucket with no default in map falls back to 0.0 (gate nothing)+ const unmapped = [+ {+ answers: {+ x: {+ type: "score",+ probabilities: Object.fromEntries(Array.from({ length: 7 }, (_, i) => [String(i), 1 / 7])),+ answer_confidence: 0.05,+ },+ },+ },+ ];+ flagLowConfidence(unmapped as any, { "choice:2": 0.9 });+ expect((unmapped[0].answers.x as any).low_confidence).toBeUndefined();+ });});describe("Agent confidence gating", () => {@@ -130,6 +264,27 @@ describe("Agent confidence gating", () => {expect(result.answers.choice_q.low_confidence).toBe(true);});+ it("supports per-bucket threshold maps in Agent.predict", async () => {+ const agent = new Agent({ provider: fakeProvider() } as any);+ const questions = {+ choice_q: {+ type: "choice" as const,+ instructions: "Pick one",+ criteria: { a: "option A", b: "option B" },+ },+ };++ // 2-option question matches choice:2 bucket+ const resultFlagged = await agent.predict("hello", questions, { minConfidence: { "choice:2": 0.99 } });+ expect(resultFlagged.answers.choice_q.low_confidence).toBe(true);++ const resultPassed = await agent.predict("hello", questions, { minConfidence: { "choice:2": 0.5 } });+ expect(resultPassed.answers.choice_q.low_confidence).toBeUndefined();++ const resultDefault = await agent.predict("hello", questions, { min_confidence: { default: 0.99 } });+ expect(resultDefault.answers.choice_q.low_confidence).toBe(true);+ });+it("validates minConfidence before running inference", async () => {const agent = new Agent({ provider: fakeProvider() } as any);await expect(@@ -163,6 +318,25 @@ describe("Router confidence gating", () => {expect(resPass.answers.q.low_confidence).toBeUndefined();});+ it("supports per-bucket threshold maps in Router.predict", async () => {+ const router = new Router();+ router.load = async () => new Agent({ provider: fakeProvider() } as any);++ const questions = {+ q: {+ type: "choice" as const,+ instructions: "Pick",+ criteria: { a: "A", b: "B" },+ },+ };++ const resFlagged = await router.predict("state text", questions, { minConfidence: { "choice:2": 0.99 } });+ expect(resFlagged.answers.q.low_confidence).toBe(true);++ const resPassed = await router.predict("state text", questions, { min_confidence: { "choice:2": 0.5 } });+ expect(resPassed.answers.q.low_confidence).toBeUndefined();+ });+it("validates minConfidence in Router.predict before routing or inference", async () => {const router = new Router();await expect(