Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
feat(ts): port confidence-based abstention gating on answer_confidence (#361)
Swatantra Yadav3f0ff782a7f0
diff --git a/laya-ts/src/agent.ts b/laya-ts/src/agent.tsindex 6a8c8566..64c96022 100644--- a/laya-ts/src/agent.ts+++ b/laya-ts/src/agent.ts@@ -6,6 +6,8 @@ import {collateItems,confidenceFromProbs,answerConfidence,+ checkMinConfidence,+ flagLowConfidence,renderCriterion,renderOptions,sequenceWithState,@@ -48,6 +50,7 @@ export interface ChoiceAnswer {confidence: number;answer_confidence: number;action: ActionInfo;+ low_confidence?: boolean;}export interface ScoreAnswer {@@ -58,6 +61,7 @@ export interface ScoreAnswer {confidence: number;answer_confidence: number;action: ActionInfo;+ low_confidence?: boolean;}export interface NoulAnswer {@@ -66,6 +70,7 @@ export interface NoulAnswer {confidence: number;answer_confidence: number;action: ActionInfo;+ low_confidence?: boolean;}export type SystemAnswer = ChoiceAnswer | ScoreAnswer | NoulAnswer;@@ -128,6 +133,10 @@ export interface PredictOptions {onPredictStart?: PredictHook;onPredictEnd?: PredictHook;hooksRaise?: boolean;+ /** Minimum confidence threshold in [0.0, 1.0]. Low confidence answers get `low_confidence: true`. */+ minConfidence?: number | null;+ /** Python parity alias for minConfidence. */+ min_confidence?: number | null;}function qidStr(qid: string): string {@@ -372,6 +381,8 @@ export class Agent extends HookRegistry {questions: Record<string, QuestionDef>,opts: PredictOptions,): Promise<SystemOneResult[]> {+ const mcOpt = opts.minConfidence ?? opts.min_confidence;+ const mc = mcOpt !== undefined && mcOpt !== null ? checkMinConfidence(mcOpt) : null;const active = defaultsAlreadyRan(opts)? [...this.hooks, ...normaliseHooks(opts.hooks, opts.onPredictStart, opts.onPredictEnd)]: composeHooks(this.hooks, opts.hooks, opts.onPredictStart, opts.onPredictEnd);@@ -407,7 +418,12 @@ export class Agent extends HookRegistry {throw err;} finally {ctx.markElapsed();- if (ctx.results !== null) ctx.usage = aggregateUsage(ctx.results);+ if (ctx.results !== null) {+ ctx.usage = aggregateUsage(ctx.results);+ if (mc !== null) {+ flagLowConfidence(ctx.results as unknown as Record<string, unknown>[], mc);+ }+ }try {await dispatchAsync(active, "onPredictEnd", ctx, { raiseErrors });} catch (hookErr) {diff --git a/laya-ts/src/common.ts b/laya-ts/src/common.tsindex ae5b9da8..cc78c18b 100644--- a/laya-ts/src/common.ts+++ b/laya-ts/src/common.ts@@ -132,6 +132,62 @@ export function answerConfidence(p: number[]): number {if (p.length < 1) return 1.0;return Math.min(1, Math.max(0, Math.max(...p)));}++function pyRepr(v: unknown): string {+ if (typeof v === "boolean") return v ? "True" : "False";+ if (v === null || v === undefined) return "None";+ if (typeof v === "number") {+ if (Number.isNaN(v)) return "nan";+ if (v === Infinity) return "inf";+ if (v === -Infinity) return "-inf";+ return String(v);+ }+ if (typeof v === "string") return JSON.stringify(v);+ if (Array.isArray(v)) return `[${v.map(pyRepr).join(", ")}]`;+ return String(v);+}++/**+ * Validate opt-in abstention threshold `min_confidence` (#361).+ *+ * Must be a real number in [0.0, 1.0]. Booleans are rejected.+ */+export function checkMinConfidence(v: unknown): number {+ if (typeof v === "boolean" || typeof v !== "number" || !Number.isFinite(v) || v < 0.0 || v > 1.0) {+ throw new Error(`min_confidence must be a float in [0.0, 1.0], got ${pyRepr(v)}`);+ }+ return v;+}++/**+ * Opt-in abstention marker (#361): flag answers whose confidence falls below `min_confidence`.+ *+ * Reads `answer_confidence` (the calibrated max(p) confidence, invariant to label count k),+ * falling back to `confidence` if `answer_confidence` is absent.+ * The raw answer and confidence stay intact; `low_confidence: true` is added.+ */+export function flagLowConfidence(+ results: Array<Record<string, unknown>> | Record<string, unknown>,+ minConfidence: number,+): void {+ if (minConfidence === 0.0) return;+ const list = Array.isArray(results) ? results : [results];+ for (const res of list) {+ const answers = res && typeof res === "object" ? (res as Record<string, unknown>).answers : null;+ if (!answers || typeof answers !== "object") continue;+ for (const a of Object.values(answers as Record<string, unknown>)) {+ if (!a || typeof a !== "object") continue;+ const ansObj = a as Record<string, unknown>;+ let conf = ansObj.answer_confidence;+ if (conf === undefined || conf === null) {+ conf = ansObj.confidence;+ }+ if (typeof conf === "number" && !Number.isNaN(conf) && conf < minConfidence) {+ ansObj.low_confidence = true;+ }+ }+ }+}export const TEMP_MIN = 0.5, TEMP_MAX = 5.0;export function clampTemperature(t: unknown): number {if (t === null || t === undefined || t === "" || typeof t === "boolean") return 1.0;diff --git a/laya-ts/src/index.ts b/laya-ts/src/index.tsindex 38e105a3..7c0c8167 100644--- a/laya-ts/src/index.ts+++ b/laya-ts/src/index.ts@@ -58,6 +58,8 @@ export {softmax,confidenceFromProbs,answerConfidence,+ checkMinConfidence,+ flagLowConfidence,clampTemperature,tempBucket,collateItems,diff --git a/laya-ts/src/router.ts b/laya-ts/src/router.tsindex 33ded388..9f2dea5b 100644--- a/laya-ts/src/router.ts+++ b/laya-ts/src/router.ts@@ -1,5 +1,6 @@import { analyse, type AnalyseResult } from "./lang.js";import type { PredictOptions, QuestionDef, SystemOneResult } from "./agent.js";+import { checkMinConfidence, flagLowConfidence } from "./common.js";import { decide, type DecideOptions, type DecisionResult } from "./structured.js";import {HookRegistry,@@ -479,6 +480,8 @@ export class Router extends HookRegistry {questions: Record<string, QuestionDef>,opts: RouteOptions & PredictOptions = {},): Promise<RoutedResult> {+ const mcOpt = opts.minConfidence ?? opts.min_confidence;+ const mc = mcOpt !== undefined && mcOpt !== null ? checkMinConfidence(mcOpt) : null;const active = composeHooks(this.hooks, opts.hooks, opts.onPredictStart, opts.onPredictEnd);const raiseErrors = opts.hooksRaise ?? this.hooksRaise;@@ -488,7 +491,7 @@ export class Router extends HookRegistry {systemOne(s: unknown,q: Record<string, QuestionDef>,- opts?: { lang?: string | null },+ opts?: PredictOptions,): Promise<SystemOneResult>;};const ctx = new PredictContext({@@ -510,7 +513,8 @@ export class Router extends HookRegistry {// explicit lang="en" can select an "en" override.const detected = decision.detection?.language;const effectiveLang = opts.lang ?? (detected && detected !== "en" ? detected : null);- const agentOpts = { lang: effectiveLang };+ const agentOpts: PredictOptions = { lang: effectiveLang };+ if (mc !== null) agentOpts.minConfidence = mc;markDefaultsRan(agentOpts);const result = (await agent.systemOne(ctx.states[0],@@ -538,7 +542,12 @@ export class Router extends HookRegistry {throw err;} finally {ctx.markElapsed();- if (ctx.results !== null) ctx.usage = aggregateUsage(ctx.results);+ if (ctx.results !== null) {+ ctx.usage = aggregateUsage(ctx.results);+ if (mc !== null) {+ flagLowConfidence(ctx.results as unknown as Record<string, unknown>[], mc);+ }+ }try {await dispatchAsync(active, "onPredictEnd", ctx, { raiseErrors });} catch (hookErr) {diff --git a/laya-ts/src/structured.ts b/laya-ts/src/structured.tsindex c5326e04..c854c59a 100644--- a/laya-ts/src/structured.ts+++ b/laya-ts/src/structured.ts@@ -19,6 +19,7 @@* with a `toJSONSchema()` method is accepted.*/import type { QuestionDef } from "./agent.js";+import { checkMinConfidence, flagLowConfidence } from "./common.js";export const MAX_PROPERTIES = 32;export const MAX_OPTIONS = 32;@@ -71,6 +72,10 @@ export interface DecideOptions {questions?: Record<string, QuestionDef>;/** Return a DecisionResult with confidence, probabilities and raw answers. */returnDetails?: boolean;+ /** Minimum confidence threshold in [0.0, 1.0]. Low confidence answers project to null. */+ minConfidence?: number | null;+ /** Python parity alias for minConfidence. */+ min_confidence?: number | null;/** Anything else is forwarded to runner.predict (hooks, model, ...). */[k: string]: unknown;}@@ -236,6 +241,10 @@ function project(answers: Record<string, Record<string, any> | undefined>, fieldfor (const f of fields) {const answer = answers[f.name];if (answer == null) continue;+ if (answer.low_confidence) {+ values[f.name] = null;+ continue;+ }if (f.kind === "noul") {values[f.name] = Number(answer.noul ?? 0.0) >= 0.5;} else if (f.kind === "score") {@@ -316,10 +325,15 @@ export async function decide(schema?: unknown,opts: DecideOptions = {},): Promise<Record<string, unknown> | DecisionResult> {- const { questions, returnDetails = false, ...predictOpts } = opts;+ const { questions, returnDetails = false, minConfidence, min_confidence, ...predictOpts } = opts;if ((schema == null) === (questions == null)) {throw new Error("pass exactly one of schema= or questions=");}+ const mcOpt = minConfidence ?? min_confidence;+ const mc = mcOpt !== undefined && mcOpt !== null ? checkMinConfidence(mcOpt) : null;+ if (mc !== null) {+ predictOpts.minConfidence = mc;+ }let fields: PlannedField[] | null = null;let qs = questions;if (schema != null) {@@ -327,6 +341,9 @@ export async function decide(qs = Object.fromEntries(fields.map((f) => [f.name, f.question]));}const result = await runner.predict(state, qs as Record<string, QuestionDef>, predictOpts);+ if (mc !== null && result && typeof result === "object") {+ flagLowConfidence(result as Record<string, unknown>, mc);+ }const answers = result.answers ?? {};const values = fields ? project(answers, fields) : { ...answers };if (returnDetails) return detailsOf(values, answers, result);diff --git a/laya-ts/tests/confidence-gating.test.ts b/laya-ts/tests/confidence-gating.test.tsnew file mode 100644index 00000000..02fb4470--- /dev/null+++ b/laya-ts/tests/confidence-gating.test.ts@@ -0,0 +1,172 @@+import { describe, expect, it } from "vitest";+import { checkMinConfidence, flagLowConfidence } from "../src/common.js";+import { Agent, type ChoiceAnswer } from "../src/agent.js";+import { Router } from "../src/router.js";++const fakeProvider = () => ({+ async runEncoder(_b: any) {+ return { lastHidden: [[1, 0], [0, 1]] };+ },+ async runHead(_h: any) {+ // 2-option question with clear margin: argmax 0+ return {+ logits: [[2.0, 0.0]],+ act: [[3.0, 0.0]],+ };+ },+});++describe("checkMinConfidence", () => {+ it("validates real numbers in [0.0, 1.0]", () => {+ for (const valid of [0.0, 0.5, 1.0, 0, 1, 0.85]) {+ expect(checkMinConfidence(valid)).toBe(Number(valid));+ }+ });++ it("rejects booleans, out of range numbers, and non-finite values", () => {+ const invalids = [+ true,+ false,+ -0.01,+ 1.01,+ -1.0,+ 2.0,+ NaN,+ Infinity,+ -Infinity,+ "0.5",+ null,+ undefined,+ [0.5],+ {},+ ];+ for (const invalid of invalids) {+ expect(() => checkMinConfidence(invalid)).toThrowError(/min_confidence must be a float in \[0\.0, 1\.0\]/);+ }+ });+});++describe("flagLowConfidence", () => {+ it("is a no-op when minConfidence is 0.0", () => {+ const sample = [+ {+ answers: {+ q1: { type: "choice", choice: "a", answer_confidence: 0.1, confidence: 0.1 },+ },+ },+ ];+ flagLowConfidence(sample as any, 0.0);+ expect(sample[0].answers.q1).not.toHaveProperty("low_confidence");+ });++ it("flags answers where answer_confidence falls below threshold", () => {+ const sample = [+ {+ answers: {+ q_high: { type: "choice", choice: "a", answer_confidence: 0.92, confidence: 0.8 },+ q_low: { type: "score", score: 1, answer_confidence: 0.45, confidence: 0.4 },+ q_fallback: { type: "choice", choice: "b", confidence: 0.3 },+ q_exact: { type: "choice", choice: "c", answer_confidence: 0.7 },+ },+ },+ ];++ flagLowConfidence(sample as any, 0.7);+ const ans = sample[0].answers as any;++ expect(ans.q_low.low_confidence).toBe(true);+ expect(ans.q_fallback.low_confidence).toBe(true); // falls back to confidence 0.3 < 0.7+ expect(ans.q_high.low_confidence).toBeUndefined();+ expect(ans.q_exact.low_confidence).toBeUndefined(); // 0.70 >= 0.70++ // Raw answers and confidence fields remain intact+ expect(ans.q_low.answer_confidence).toBe(0.45);+ expect(ans.q_high.choice).toBe("a");+ });++ it("handles a single result object as well as an array", () => {+ const single = {+ answers: {+ q: { type: "choice", choice: "x", answer_confidence: 0.4 },+ },+ };+ flagLowConfidence(single as any, 0.5);+ expect((single.answers.q as any).low_confidence).toBe(true);+ });+});++describe("Agent confidence gating", () => {+ it("flags low-confidence answers when minConfidence is set", async () => {+ const agent = new Agent({ provider: fakeProvider() } as any);+ const questions = {+ choice_q: {+ type: "choice" as const,+ instructions: "Pick one",+ criteria: { a: "option A", b: "option B" },+ },+ };++ // Threshold 0.999 is above the predicted probability (softmax([2, 0]) ~ 0.88)+ const result = await agent.systemOne("hello", questions, { minConfidence: 0.99 });+ expect(result.answers.choice_q.low_confidence).toBe(true);+ expect((result.answers.choice_q as ChoiceAnswer).choice).toBe("a");++ // Threshold 0.5 is below the predicted probability+ const resultHigh = await agent.systemOne("hello", questions, { minConfidence: 0.5 });+ expect(resultHigh.answers.choice_q.low_confidence).toBeUndefined();+ });++ it("supports python parity alias min_confidence", async () => {+ const agent = new Agent({ provider: fakeProvider() } as any);+ const questions = {+ choice_q: {+ type: "choice" as const,+ instructions: "Pick one",+ criteria: { a: "option A", b: "option B" },+ },+ };++ const result = await agent.predict("hello", questions, { min_confidence: 0.99 });+ expect(result.answers.choice_q.low_confidence).toBe(true);+ });++ it("validates minConfidence before running inference", async () => {+ const agent = new Agent({ provider: fakeProvider() } as any);+ await expect(+ agent.predict("hello", {}, { minConfidence: 1.5 }),+ ).rejects.toThrowError(/min_confidence must be a float/);+ });+});++describe("Router confidence gating", () => {+ it("flags low-confidence answers in Router.predict and supports alias", async () => {+ const router = new Router();+ // Stub router.load to return an agent using our fakeProvider+ router.load = async () => new Agent({ provider: fakeProvider() } as any);++ const questions = {+ q: {+ type: "choice" as const,+ instructions: "Pick",+ criteria: { a: "A", b: "B" },+ },+ };++ const res = await router.predict("state text", questions, { minConfidence: 0.99 });+ expect(res.answers.q.low_confidence).toBe(true);+ expect(res.routing).toBeDefined();++ const resAlias = await router.predict("state text", questions, { min_confidence: 0.99 });+ expect(resAlias.answers.q.low_confidence).toBe(true);++ const resPass = await router.predict("state text", questions, { minConfidence: 0.5 });+ expect(resPass.answers.q.low_confidence).toBeUndefined();+ });++ it("validates minConfidence in Router.predict before routing or inference", async () => {+ const router = new Router();+ await expect(+ router.predict("state text", {}, { minConfidence: -0.5 }),+ ).rejects.toThrowError(/min_confidence must be a float/);+ });+});diff --git a/laya-ts/tests/structured.test.ts b/laya-ts/tests/structured.test.tsindex a20d0ba3..b7627460 100644--- a/laya-ts/tests/structured.test.ts+++ b/laya-ts/tests/structured.test.ts@@ -286,3 +286,73 @@ describe("structured/nullable-anyof", () => {});});+describe("structured/min_confidence", () => {+ const fakeRunner = (answers: Record<string, any>) => ({+ calls: [] as any[],+ async predict(state: unknown, questions: any, opts: any) {+ this.calls.push({ state, questions, opts });+ return { model: "fake", answers: JSON.parse(JSON.stringify(answers)) };+ },+ });++ it("default behavior unchanged when minConfidence is omitted", async () => {+ const runner = fakeRunner(ANSWERS);+ const out = await decide(runner, "s", SCHEMA);+ expect(out.department).toBe("billing");+ expect(out.urgency).toBe(2);+ expect(out.needs_human).toBe(true);+ expect(out.priority).toBe(2);+ expect(Object.values(ANSWERS).some((a: any) => a.low_confidence)).toBe(false);+ });++ it("with minConfidence=0.85, answers below threshold project as null", async () => {+ const runner = fakeRunner(ANSWERS);+ const out = await decide(runner, "s", SCHEMA, { minConfidence: 0.85 });+ expect(out.department).toBe("billing"); // 0.9 >= 0.85+ expect(out.urgency).toBeNull(); // 0.5 < 0.85+ expect(out.needs_human).toBeNull(); // 0.8 < 0.85+ expect(out.priority).toBeNull(); // 0.7 < 0.85+ });++ it("accepts python parity alias min_confidence", async () => {+ const runner = fakeRunner(ANSWERS);+ const out = await decide(runner, "s", SCHEMA, { min_confidence: 0.85 });+ expect(out.department).toBe("billing");+ expect(out.urgency).toBeNull();+ });++ it("with returnDetails: true, details keep raw confidence and answers dict", async () => {+ const runner = fakeRunner(ANSWERS);+ const det = await decide(runner, "s", SCHEMA, { minConfidence: 0.85, returnDetails: true });+ expect(det.values.urgency).toBeNull();+ expect(det.confidence.urgency).toBe(0.5);+ expect((det.answers.urgency as any).low_confidence).toBe(true);+ expect(det.values.department).toBe("billing");+ expect((det.answers.department as any).low_confidence).toBeUndefined();+ });++ it("direct answersToJson with low_confidence: true in answer projects to null", () => {+ const answersWithFlag = {+ department: { type: "choice", choice: "billing", confidence: 0.4, low_confidence: true },+ urgency: {+ type: "score",+ score: 2.0,+ confidence: 0.9,+ probabilities: { "0": 0.0, "1": 0.1, "2": 0.9 },+ legend: {},+ },+ };+ const proj = answersToJson(answersWithFlag as any, SCHEMA);+ expect(proj.department).toBeNull();+ expect(proj.urgency).toBe(2);+ });++ it("rejects invalid minConfidence values", async () => {+ const runner = fakeRunner(ANSWERS);+ await expect(decide(runner, "s", SCHEMA, { minConfidence: 1.2 })).rejects.toThrow(/min_confidence must be a float/);+ await expect(decide(runner, "s", SCHEMA, { minConfidence: -0.1 })).rejects.toThrow(/min_confidence must be a float/);+ await expect(decide(runner, "s", SCHEMA, { minConfidence: true as any })).rejects.toThrow(/min_confidence must be a float/);+ await expect(decide(runner, "s", SCHEMA, { minConfidence: "0.85" as any })).rejects.toThrow(/min_confidence must be a float/);+ });+});+