Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(ts): port per-bucket minConfidence map to laya-ts

Swatantra Yadav08a417a8a5f9
diff --git a/laya-ts/src/agent.ts b/laya-ts/src/agent.ts
index 83469fb8..5bbdad94 100644
--- a/laya-ts/src/agent.ts
+++ b/laya-ts/src/agent.ts
@@ -16,7 +16,7 @@ import {
softmax,
tempBucket,
} from "./common.js";
-import type { OptionStats, SequenceStats } from "./common.js";
+import type { MinConfidence, OptionStats, SequenceStats } from "./common.js";
import type { Batch, SessionProvider } from "./providers.js";
import { encodeWithData, parseTokenizerJson, type TokenizerLike } from "./tokenizer.js";
import { decide, type DecideOptions, type DecisionResult } from "./structured.js";
@@ -150,10 +150,10 @@ export interface PredictOptions {
headMaxLen?: number | null;
/** predictBatch: cap on states per shared forward pass; null sends them all in one pass. */
batchSize?: number | null;
- /** Minimum confidence threshold in [0.0, 1.0]. Low confidence answers get `low_confidence: true`. */
- minConfidence?: number | null;
+ /** Minimum confidence threshold in [0.0, 1.0] or per-bucket map. Low confidence answers get `low_confidence: true`. */
+ minConfidence?: MinConfidence | null;
/** Python parity alias for minConfidence. */
- min_confidence?: number | null;
+ min_confidence?: MinConfidence | null;
}
function qidStr(qid: string): string {
diff --git a/laya-ts/src/common.ts b/laya-ts/src/common.ts
index 5047a405..89c52335 100644
--- a/laya-ts/src/common.ts
+++ b/laya-ts/src/common.ts
@@ -229,33 +229,101 @@ function pyRepr(v: unknown): string {
}
if (typeof v === "string") return JSON.stringify(v);
if (Array.isArray(v)) return `[${v.map(pyRepr).join(", ")}]`;
+ if (typeof v === "object") {
+ const entries = Object.entries(v as Record<string, unknown>).map(
+ ([k, val]) => `${pyRepr(k)}: ${pyRepr(val)}`
+ );
+ return `{${entries.join(", ")}}`;
+ }
return String(v);
}
+export type MinConfidenceMap = Record<string, number>;
+export type MinConfidence = number | MinConfidenceMap;
+
+const BUCKET_KEY = /^(choice|score|noul):(2|3-5|6-10|11\+)$/;
+
/**
- * Validate opt-in abstention threshold `min_confidence` (#361).
+ * Validate a per-bucket abstention-threshold map (#394).
*
- * Must be a real number in [0.0, 1.0]. Booleans are rejected.
+ * Keys are option-count bucket strings like "choice:2", "choice:3-5", "score:6-10", "noul:2",
+ * plus an optional "default". Values are numbers in [0.0, 1.0].
*/
-export function checkMinConfidence(v: unknown): number {
+export function checkMinConfidenceMap(m: unknown): MinConfidenceMap {
+ if (!m || typeof m !== "object" || Array.isArray(m) || Object.keys(m).length === 0) {
+ throw new Error(`a min_confidence map must be a non-empty dict of bucket -> float, got ${pyRepr(m)}`);
+ }
+ const out: MinConfidenceMap = {};
+ for (const [key, val] of Object.entries(m as Record<string, unknown>)) {
+ if (key !== "default" && !BUCKET_KEY.test(key)) {
+ throw new Error(`min_confidence map keys must be strings like 'choice:3-5', got ${pyRepr(key)}`);
+ }
+ if (typeof val === "boolean" || typeof val !== "number" || !Number.isFinite(val) || val < 0.0 || val > 1.0) {
+ throw new Error(`min_confidence must be a float in [0.0, 1.0], got ${pyRepr(val)}`);
+ }
+ out[key] = val;
+ }
+ return out;
+}
+
+/**
+ * Validate opt-in abstention threshold `min_confidence` (#361, #394).
+ *
+ * Either a real number in [0.0, 1.0] or a per-bucket mapping. Booleans are rejected.
+ */
+export function checkMinConfidence(v: unknown): MinConfidence {
+ if (v && typeof v === "object" && !Array.isArray(v)) {
+ return checkMinConfidenceMap(v);
+ }
if (typeof v === "boolean" || typeof v !== "number" || !Number.isFinite(v) || v < 0.0 || v > 1.0) {
throw new Error(`min_confidence must be a float in [0.0, 1.0], got ${pyRepr(v)}`);
}
return v;
}
+export function optionBucket(answer: Record<string, unknown>): string | null {
+ const qt = answer.type;
+ if (qt !== "choice" && qt !== "score" && qt !== "noul") return null;
+ const probs = answer.probabilities;
+ let k: number;
+ if (probs && typeof probs === "object" && !Array.isArray(probs)) {
+ k = Object.keys(probs).length;
+ } else if (qt === "noul") {
+ k = 2;
+ } else {
+ return null;
+ }
+ const size = k <= 2 ? "2" : k <= 5 ? "3-5" : k <= 10 ? "6-10" : "11+";
+ return `${qt}:${size}`;
+}
+
+export function resolveMinConfidence(
+ answer: Record<string, unknown>,
+ thresholds: MinConfidenceMap,
+ defaultVal: number = 0.0,
+): number {
+ const key = optionBucket(answer);
+ if (key !== null && key in thresholds) {
+ return thresholds[key];
+ }
+ return thresholds.default ?? defaultVal;
+}
+
/**
- * Opt-in abstention marker (#361): flag answers whose confidence falls below `min_confidence`.
+ * Opt-in abstention marker (#361, #394): flag answers whose confidence falls below `min_confidence`.
*
* Reads `answer_confidence` (the calibrated max(p) confidence, invariant to label count k),
* falling back to `confidence` if `answer_confidence` is absent.
* The raw answer and confidence stay intact; `low_confidence: true` is added.
+ *
+ * Supports both a scalar number in [0.0, 1.0] and a per-bucket mapping of thresholds.
*/
export function flagLowConfidence(
results: Array<Record<string, unknown>> | Record<string, unknown>,
- minConfidence: number,
+ minConfidence: MinConfidence,
): void {
- if (minConfidence === 0.0) return;
+ const isMap = typeof minConfidence === "object" && minConfidence !== null;
+ if (!isMap && minConfidence === 0.0) return;
const list = Array.isArray(results) ? results : [results];
for (const res of list) {
const answers = res && typeof res === "object" ? (res as Record<string, unknown>).answers : null;
@@ -267,8 +335,11 @@ export function flagLowConfidence(
if (conf === undefined || conf === null) {
conf = ansObj.confidence;
}
- if (typeof conf === "number" && !Number.isNaN(conf) && conf < minConfidence) {
- ansObj.low_confidence = true;
+ if (typeof conf === "number" && !Number.isNaN(conf)) {
+ const thr = isMap ? resolveMinConfidence(ansObj, minConfidence) : minConfidence;
+ if (conf < thr) {
+ ansObj.low_confidence = true;
+ }
}
}
}
diff --git a/laya-ts/src/index.ts b/laya-ts/src/index.ts
index aa0fca66..c670a6ee 100644
--- a/laya-ts/src/index.ts
+++ b/laya-ts/src/index.ts
@@ -60,14 +60,27 @@ export {
confidenceFromProbs,
answerConfidence,
checkMinConfidence,
+ checkMinConfidenceMap,
flagLowConfidence,
+ optionBucket,
+ resolveMinConfidence,
clampTemperature,
tempBucket,
collateItems,
TEMP_MIN,
TEMP_MAX,
} from "./common.js";
-export type { QType, InternalQ, CollateItem, CollatedBatch, QuestionPrefix, SequenceStats, OptionStats } from "./common.js";
+export type {
+ QType,
+ InternalQ,
+ CollateItem,
+ CollatedBatch,
+ QuestionPrefix,
+ SequenceStats,
+ OptionStats,
+ MinConfidence,
+ MinConfidenceMap,
+} from "./common.js";
export { q, best, topK, isConfident } from "./dx.js";
export { bpeEncode, metaspaceEncode, encodeWithData, parseTokenizerJson, loadTokenizerJson, CHECKPOINT_IDS, SPECIAL_ALIASES, METASPACE_REPLACEMENT } from "./tokenizer.js";
export type { TokenizerLike, TokenizerData, TokenizerIds, PreTokenizerKind } from "./tokenizer.js";
diff --git a/laya-ts/src/structured.ts b/laya-ts/src/structured.ts
index 5cf7a6cf..bc5ef1b7 100644
--- a/laya-ts/src/structured.ts
+++ b/laya-ts/src/structured.ts
@@ -20,6 +20,7 @@
*/
import type { QuestionDef } from "./agent.js";
import { checkMinConfidence, flagLowConfidence } from "./common.js";
+import type { MinConfidence } from "./common.js";
export const MAX_PROPERTIES = 32;
export const MAX_OPTIONS = 32;
@@ -72,10 +73,10 @@ export interface DecideOptions {
questions?: Record<string, QuestionDef>;
/** Return a DecisionResult with confidence, probabilities and raw answers. */
returnDetails?: boolean;
- /** Minimum confidence threshold in [0.0, 1.0]. Low confidence answers project to null. */
- minConfidence?: number | null;
+ /** Minimum confidence threshold in [0.0, 1.0] or per-bucket map. Low confidence answers project to null. */
+ minConfidence?: MinConfidence | null;
/** Python parity alias for minConfidence. */
- min_confidence?: number | null;
+ min_confidence?: MinConfidence | null;
/** Anything else is forwarded to runner.predict (hooks, model, ...). */
[k: string]: unknown;
}
diff --git a/laya-ts/tests/confidence-gating.test.ts b/laya-ts/tests/confidence-gating.test.ts
index 02fb4470..846f8d30 100644
--- a/laya-ts/tests/confidence-gating.test.ts
+++ b/laya-ts/tests/confidence-gating.test.ts
@@ -1,5 +1,11 @@
import { describe, expect, it } from "vitest";
-import { checkMinConfidence, flagLowConfidence } from "../src/common.js";
+import {
+ checkMinConfidence,
+ checkMinConfidenceMap,
+ flagLowConfidence,
+ optionBucket,
+ resolveMinConfidence,
+} from "../src/common.js";
import { Agent, type ChoiceAnswer } from "../src/agent.js";
import { Router } from "../src/router.js";
@@ -38,12 +44,91 @@ describe("checkMinConfidence", () => {
null,
undefined,
[0.5],
- {},
];
for (const invalid of invalids) {
expect(() => checkMinConfidence(invalid)).toThrowError(/min_confidence must be a float in \[0\.0, 1\.0\]/);
}
});
+
+ it("accepts and validates a per-bucket threshold map", () => {
+ const map = { "choice:2": 0.9, default: 0.5 };
+ expect(checkMinConfidence(map)).toEqual(map);
+ });
+
+ it("rejects empty min_confidence map", () => {
+ expect(() => checkMinConfidence({})).toThrowError(/a min_confidence map must be a non-empty dict/);
+ });
+});
+
+describe("checkMinConfidenceMap", () => {
+ it("rejects non-object or non-dict inputs", () => {
+ for (const invalid of [null, undefined, 42, "choice:2", [0.5], true]) {
+ expect(() => checkMinConfidenceMap(invalid)).toThrowError(/a min_confidence map must be a non-empty dict/);
+ }
+ });
+
+ it("rejects empty dict", () => {
+ expect(() => checkMinConfidenceMap({})).toThrowError(/a min_confidence map must be a non-empty dict of bucket -> float, got \{\}/);
+ });
+
+ it("validates valid maps", () => {
+ const map = { "choice:2": 0.9, "choice:11+": 0.5, "noul:2": 0.8, default: 0.3 };
+ expect(checkMinConfidenceMap(map)).toEqual(map);
+ });
+
+ it("rejects invalid map values", () => {
+ expect(() => checkMinConfidenceMap({ "choice:2": 1.5 })).toThrowError(/min_confidence must be a float in \[0\.0, 1\.0\]/);
+ expect(() => checkMinConfidenceMap({ "choice:2": -0.1 })).toThrowError(/min_confidence must be a float in \[0\.0, 1\.0\]/);
+ expect(() => checkMinConfidenceMap({ "choice:2": true })).toThrowError(/min_confidence must be a float in \[0\.0, 1\.0\]/);
+ expect(() => checkMinConfidenceMap({ "choice:2": "0.9" as any })).toThrowError(/min_confidence must be a float in \[0\.0, 1\.0\]/);
+ expect(() => checkMinConfidenceMap({ "choice:2": NaN })).toThrowError(/min_confidence must be a float in \[0\.0, 1\.0\]/);
+ });
+
+ it("rejects unknown bucket keys and typos", () => {
+ for (const typo of ["choice:3_5", "Choice:2", "invalid", "score:1", "foo:2", "choice:12"]) {
+ expect(() => checkMinConfidenceMap({ [typo]: 0.5 })).toThrowError(
+ /min_confidence map keys must be strings like 'choice:3-5'/,
+ );
+ }
+ });
+});
+
+describe("optionBucket & resolveMinConfidence", () => {
+ it("computes optionBucket correctly", () => {
+ expect(optionBucket({ type: "choice", probabilities: { a: 0.5, b: 0.5 } })).toBe("choice:2");
+ expect(optionBucket({ type: "choice", probabilities: { a: 0.4, b: 0.3, c: 0.3 } })).toBe("choice:3-5");
+ expect(optionBucket({ type: "choice", probabilities: { a: 0.2, b: 0.2, c: 0.2, d: 0.2, e: 0.2 } })).toBe("choice:3-5");
+ const six: Record<string, number> = {};
+ for (let i = 0; i < 6; i++) six[String(i)] = 1 / 6;
+ expect(optionBucket({ type: "choice", probabilities: six })).toBe("choice:6-10");
+ const ten: Record<string, number> = {};
+ for (let i = 0; i < 10; i++) ten[String(i)] = 0.1;
+ expect(optionBucket({ type: "choice", probabilities: ten })).toBe("choice:6-10");
+ const eleven: Record<string, number> = {};
+ for (let i = 0; i < 11; i++) eleven[String(i)] = 1 / 11;
+ expect(optionBucket({ type: "choice", probabilities: eleven })).toBe("choice:11+");
+
+ expect(optionBucket({ type: "score", probabilities: { "1": 0.3, "2": 0.3, "3": 0.4 } })).toBe("score:3-5");
+ expect(optionBucket({ type: "noul" })).toBe("noul:2");
+ expect(optionBucket({ type: "text" })).toBeNull();
+ expect(optionBucket({ type: "choice" })).toBeNull();
+ });
+
+ it("resolves per-bucket thresholds with default and unconfigured fallback", () => {
+ const MAP = { "choice:2": 0.9, "choice:11+": 0.5, "noul:2": 0.8, default: 0.3 };
+ const ans = (type: string, k: number) => {
+ const probs: Record<string, number> = {};
+ for (let i = 0; i < k; i++) probs[String(i)] = 1 / k;
+ return { type, probabilities: probs };
+ };
+
+ expect(resolveMinConfidence(ans("choice", 2), MAP)).toBe(0.9);
+ expect(resolveMinConfidence(ans("choice", 12), MAP)).toBe(0.5);
+ expect(resolveMinConfidence(ans("choice", 4), MAP)).toBe(0.3); // falls to default
+ expect(resolveMinConfidence({ type: "noul" }, MAP)).toBe(0.8);
+ // unconfigured bucket and no default in map -> returns defaultVal (0.0)
+ expect(resolveMinConfidence(ans("score", 7), { "choice:2": 0.9 })).toBe(0.0);
+ });
});
describe("flagLowConfidence", () => {
@@ -93,6 +178,55 @@ describe("flagLowConfidence", () => {
flagLowConfidence(single as any, 0.5);
expect((single.answers.q as any).low_confidence).toBe(true);
});
+
+ it("gates with a per-bucket map", () => {
+ const MAP = { "choice:2": 0.9, "choice:11+": 0.5, default: 0.3 };
+ const sample = [
+ {
+ answers: {
+ a: { type: "choice", probabilities: { "0": 0.5, "1": 0.5 }, answer_confidence: 0.85 },
+ b: {
+ type: "choice",
+ probabilities: Object.fromEntries(Array.from({ length: 12 }, (_, i) => [String(i), 1 / 12])),
+ answer_confidence: 0.6,
+ },
+ c: {
+ type: "choice",
+ probabilities: { "0": 0.25, "1": 0.25, "2": 0.25, "3": 0.25 },
+ answer_confidence: 0.35,
+ },
+ d: {
+ type: "score",
+ probabilities: Object.fromEntries(Array.from({ length: 7 }, (_, i) => [String(i), 1 / 7])),
+ answer_confidence: 0.05,
+ },
+ },
+ },
+ ];
+
+ flagLowConfidence(sample as any, MAP);
+ const ans = sample[0].answers as any;
+
+ expect(ans.a.low_confidence).toBe(true); // 0.85 < 0.9 (choice:2)
+ expect(ans.b.low_confidence).toBeUndefined(); // 0.60 >= 0.5 (choice:11+)
+ expect(ans.c.low_confidence).toBeUndefined(); // 0.35 >= 0.3 (default)
+ expect(ans.d.low_confidence).toBe(true); // 0.05 < 0.3 (falls back to default)
+
+ // Unconfigured bucket with no default in map falls back to 0.0 (gate nothing)
+ const unmapped = [
+ {
+ answers: {
+ x: {
+ type: "score",
+ probabilities: Object.fromEntries(Array.from({ length: 7 }, (_, i) => [String(i), 1 / 7])),
+ answer_confidence: 0.05,
+ },
+ },
+ },
+ ];
+ flagLowConfidence(unmapped as any, { "choice:2": 0.9 });
+ expect((unmapped[0].answers.x as any).low_confidence).toBeUndefined();
+ });
});
describe("Agent confidence gating", () => {
@@ -130,6 +264,27 @@ describe("Agent confidence gating", () => {
expect(result.answers.choice_q.low_confidence).toBe(true);
});
+ it("supports per-bucket threshold maps in Agent.predict", async () => {
+ const agent = new Agent({ provider: fakeProvider() } as any);
+ const questions = {
+ choice_q: {
+ type: "choice" as const,
+ instructions: "Pick one",
+ criteria: { a: "option A", b: "option B" },
+ },
+ };
+
+ // 2-option question matches choice:2 bucket
+ const resultFlagged = await agent.predict("hello", questions, { minConfidence: { "choice:2": 0.99 } });
+ expect(resultFlagged.answers.choice_q.low_confidence).toBe(true);
+
+ const resultPassed = await agent.predict("hello", questions, { minConfidence: { "choice:2": 0.5 } });
+ expect(resultPassed.answers.choice_q.low_confidence).toBeUndefined();
+
+ const resultDefault = await agent.predict("hello", questions, { min_confidence: { default: 0.99 } });
+ expect(resultDefault.answers.choice_q.low_confidence).toBe(true);
+ });
+
it("validates minConfidence before running inference", async () => {
const agent = new Agent({ provider: fakeProvider() } as any);
await expect(
@@ -163,6 +318,25 @@ describe("Router confidence gating", () => {
expect(resPass.answers.q.low_confidence).toBeUndefined();
});
+ it("supports per-bucket threshold maps in Router.predict", async () => {
+ const router = new Router();
+ router.load = async () => new Agent({ provider: fakeProvider() } as any);
+
+ const questions = {
+ q: {
+ type: "choice" as const,
+ instructions: "Pick",
+ criteria: { a: "A", b: "B" },
+ },
+ };
+
+ const resFlagged = await router.predict("state text", questions, { minConfidence: { "choice:2": 0.99 } });
+ expect(resFlagged.answers.q.low_confidence).toBe(true);
+
+ const resPassed = await router.predict("state text", questions, { min_confidence: { "choice:2": 0.5 } });
+ expect(resPassed.answers.q.low_confidence).toBeUndefined();
+ });
+
it("validates minConfidence in Router.predict before routing or inference", async () => {
const router = new Router();
await expect(