Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(ts): port confidence-based abstention gating on answer_confidence (#361)

Swatantra Yadav3f0ff782a7f0
diff --git a/laya-ts/src/agent.ts b/laya-ts/src/agent.ts
index 6a8c8566..64c96022 100644
--- a/laya-ts/src/agent.ts
+++ b/laya-ts/src/agent.ts
@@ -6,6 +6,8 @@ import {
collateItems,
confidenceFromProbs,
answerConfidence,
+ checkMinConfidence,
+ flagLowConfidence,
renderCriterion,
renderOptions,
sequenceWithState,
@@ -48,6 +50,7 @@ export interface ChoiceAnswer {
confidence: number;
answer_confidence: number;
action: ActionInfo;
+ low_confidence?: boolean;
}
export interface ScoreAnswer {
@@ -58,6 +61,7 @@ export interface ScoreAnswer {
confidence: number;
answer_confidence: number;
action: ActionInfo;
+ low_confidence?: boolean;
}
export interface NoulAnswer {
@@ -66,6 +70,7 @@ export interface NoulAnswer {
confidence: number;
answer_confidence: number;
action: ActionInfo;
+ low_confidence?: boolean;
}
export type SystemAnswer = ChoiceAnswer | ScoreAnswer | NoulAnswer;
@@ -128,6 +133,10 @@ export interface PredictOptions {
onPredictStart?: PredictHook;
onPredictEnd?: PredictHook;
hooksRaise?: boolean;
+ /** Minimum confidence threshold in [0.0, 1.0]. Low confidence answers get `low_confidence: true`. */
+ minConfidence?: number | null;
+ /** Python parity alias for minConfidence. */
+ min_confidence?: number | null;
}
function qidStr(qid: string): string {
@@ -372,6 +381,8 @@ export class Agent extends HookRegistry {
questions: Record<string, QuestionDef>,
opts: PredictOptions,
): Promise<SystemOneResult[]> {
+ const mcOpt = opts.minConfidence ?? opts.min_confidence;
+ const mc = mcOpt !== undefined && mcOpt !== null ? checkMinConfidence(mcOpt) : null;
const active = defaultsAlreadyRan(opts)
? [...this.hooks, ...normaliseHooks(opts.hooks, opts.onPredictStart, opts.onPredictEnd)]
: composeHooks(this.hooks, opts.hooks, opts.onPredictStart, opts.onPredictEnd);
@@ -407,7 +418,12 @@ export class Agent extends HookRegistry {
throw err;
} finally {
ctx.markElapsed();
- if (ctx.results !== null) ctx.usage = aggregateUsage(ctx.results);
+ if (ctx.results !== null) {
+ ctx.usage = aggregateUsage(ctx.results);
+ if (mc !== null) {
+ flagLowConfidence(ctx.results as unknown as Record<string, unknown>[], mc);
+ }
+ }
try {
await dispatchAsync(active, "onPredictEnd", ctx, { raiseErrors });
} catch (hookErr) {
diff --git a/laya-ts/src/common.ts b/laya-ts/src/common.ts
index ae5b9da8..cc78c18b 100644
--- a/laya-ts/src/common.ts
+++ b/laya-ts/src/common.ts
@@ -132,6 +132,62 @@ export function answerConfidence(p: number[]): number {
if (p.length < 1) return 1.0;
return Math.min(1, Math.max(0, Math.max(...p)));
}
+
+function pyRepr(v: unknown): string {
+ if (typeof v === "boolean") return v ? "True" : "False";
+ if (v === null || v === undefined) return "None";
+ if (typeof v === "number") {
+ if (Number.isNaN(v)) return "nan";
+ if (v === Infinity) return "inf";
+ if (v === -Infinity) return "-inf";
+ return String(v);
+ }
+ if (typeof v === "string") return JSON.stringify(v);
+ if (Array.isArray(v)) return `[${v.map(pyRepr).join(", ")}]`;
+ return String(v);
+}
+
+/**
+ * Validate opt-in abstention threshold `min_confidence` (#361).
+ *
+ * Must be a real number in [0.0, 1.0]. Booleans are rejected.
+ */
+export function checkMinConfidence(v: unknown): number {
+ if (typeof v === "boolean" || typeof v !== "number" || !Number.isFinite(v) || v < 0.0 || v > 1.0) {
+ throw new Error(`min_confidence must be a float in [0.0, 1.0], got ${pyRepr(v)}`);
+ }
+ return v;
+}
+
+/**
+ * Opt-in abstention marker (#361): flag answers whose confidence falls below `min_confidence`.
+ *
+ * Reads `answer_confidence` (the calibrated max(p) confidence, invariant to label count k),
+ * falling back to `confidence` if `answer_confidence` is absent.
+ * The raw answer and confidence stay intact; `low_confidence: true` is added.
+ */
+export function flagLowConfidence(
+ results: Array<Record<string, unknown>> | Record<string, unknown>,
+ minConfidence: number,
+): void {
+ if (minConfidence === 0.0) return;
+ const list = Array.isArray(results) ? results : [results];
+ for (const res of list) {
+ const answers = res && typeof res === "object" ? (res as Record<string, unknown>).answers : null;
+ if (!answers || typeof answers !== "object") continue;
+ for (const a of Object.values(answers as Record<string, unknown>)) {
+ if (!a || typeof a !== "object") continue;
+ const ansObj = a as Record<string, unknown>;
+ let conf = ansObj.answer_confidence;
+ if (conf === undefined || conf === null) {
+ conf = ansObj.confidence;
+ }
+ if (typeof conf === "number" && !Number.isNaN(conf) && conf < minConfidence) {
+ ansObj.low_confidence = true;
+ }
+ }
+ }
+}
export const TEMP_MIN = 0.5, TEMP_MAX = 5.0;
export function clampTemperature(t: unknown): number {
if (t === null || t === undefined || t === "" || typeof t === "boolean") return 1.0;
diff --git a/laya-ts/src/index.ts b/laya-ts/src/index.ts
index 38e105a3..7c0c8167 100644
--- a/laya-ts/src/index.ts
+++ b/laya-ts/src/index.ts
@@ -58,6 +58,8 @@ export {
softmax,
confidenceFromProbs,
answerConfidence,
+ checkMinConfidence,
+ flagLowConfidence,
clampTemperature,
tempBucket,
collateItems,
diff --git a/laya-ts/src/router.ts b/laya-ts/src/router.ts
index 33ded388..9f2dea5b 100644
--- a/laya-ts/src/router.ts
+++ b/laya-ts/src/router.ts
@@ -1,5 +1,6 @@
import { analyse, type AnalyseResult } from "./lang.js";
import type { PredictOptions, QuestionDef, SystemOneResult } from "./agent.js";
+import { checkMinConfidence, flagLowConfidence } from "./common.js";
import { decide, type DecideOptions, type DecisionResult } from "./structured.js";
import {
HookRegistry,
@@ -479,6 +480,8 @@ export class Router extends HookRegistry {
questions: Record<string, QuestionDef>,
opts: RouteOptions & PredictOptions = {},
): Promise<RoutedResult> {
+ const mcOpt = opts.minConfidence ?? opts.min_confidence;
+ const mc = mcOpt !== undefined && mcOpt !== null ? checkMinConfidence(mcOpt) : null;
const active = composeHooks(this.hooks, opts.hooks, opts.onPredictStart, opts.onPredictEnd);
const raiseErrors = opts.hooksRaise ?? this.hooksRaise;
@@ -488,7 +491,7 @@ export class Router extends HookRegistry {
systemOne(
s: unknown,
q: Record<string, QuestionDef>,
- opts?: { lang?: string | null },
+ opts?: PredictOptions,
): Promise<SystemOneResult>;
};
const ctx = new PredictContext({
@@ -510,7 +513,8 @@ export class Router extends HookRegistry {
// explicit lang="en" can select an "en" override.
const detected = decision.detection?.language;
const effectiveLang = opts.lang ?? (detected && detected !== "en" ? detected : null);
- const agentOpts = { lang: effectiveLang };
+ const agentOpts: PredictOptions = { lang: effectiveLang };
+ if (mc !== null) agentOpts.minConfidence = mc;
markDefaultsRan(agentOpts);
const result = (await agent.systemOne(
ctx.states[0],
@@ -538,7 +542,12 @@ export class Router extends HookRegistry {
throw err;
} finally {
ctx.markElapsed();
- if (ctx.results !== null) ctx.usage = aggregateUsage(ctx.results);
+ if (ctx.results !== null) {
+ ctx.usage = aggregateUsage(ctx.results);
+ if (mc !== null) {
+ flagLowConfidence(ctx.results as unknown as Record<string, unknown>[], mc);
+ }
+ }
try {
await dispatchAsync(active, "onPredictEnd", ctx, { raiseErrors });
} catch (hookErr) {
diff --git a/laya-ts/src/structured.ts b/laya-ts/src/structured.ts
index c5326e04..c854c59a 100644
--- a/laya-ts/src/structured.ts
+++ b/laya-ts/src/structured.ts
@@ -19,6 +19,7 @@
* with a `toJSONSchema()` method is accepted.
*/
import type { QuestionDef } from "./agent.js";
+import { checkMinConfidence, flagLowConfidence } from "./common.js";
export const MAX_PROPERTIES = 32;
export const MAX_OPTIONS = 32;
@@ -71,6 +72,10 @@ export interface DecideOptions {
questions?: Record<string, QuestionDef>;
/** Return a DecisionResult with confidence, probabilities and raw answers. */
returnDetails?: boolean;
+ /** Minimum confidence threshold in [0.0, 1.0]. Low confidence answers project to null. */
+ minConfidence?: number | null;
+ /** Python parity alias for minConfidence. */
+ min_confidence?: number | null;
/** Anything else is forwarded to runner.predict (hooks, model, ...). */
[k: string]: unknown;
}
@@ -236,6 +241,10 @@ function project(answers: Record<string, Record<string, any> | undefined>, field
for (const f of fields) {
const answer = answers[f.name];
if (answer == null) continue;
+ if (answer.low_confidence) {
+ values[f.name] = null;
+ continue;
+ }
if (f.kind === "noul") {
values[f.name] = Number(answer.noul ?? 0.0) >= 0.5;
} else if (f.kind === "score") {
@@ -316,10 +325,15 @@ export async function decide(
schema?: unknown,
opts: DecideOptions = {},
): Promise<Record<string, unknown> | DecisionResult> {
- const { questions, returnDetails = false, ...predictOpts } = opts;
+ const { questions, returnDetails = false, minConfidence, min_confidence, ...predictOpts } = opts;
if ((schema == null) === (questions == null)) {
throw new Error("pass exactly one of schema= or questions=");
}
+ const mcOpt = minConfidence ?? min_confidence;
+ const mc = mcOpt !== undefined && mcOpt !== null ? checkMinConfidence(mcOpt) : null;
+ if (mc !== null) {
+ predictOpts.minConfidence = mc;
+ }
let fields: PlannedField[] | null = null;
let qs = questions;
if (schema != null) {
@@ -327,6 +341,9 @@ export async function decide(
qs = Object.fromEntries(fields.map((f) => [f.name, f.question]));
}
const result = await runner.predict(state, qs as Record<string, QuestionDef>, predictOpts);
+ if (mc !== null && result && typeof result === "object") {
+ flagLowConfidence(result as Record<string, unknown>, mc);
+ }
const answers = result.answers ?? {};
const values = fields ? project(answers, fields) : { ...answers };
if (returnDetails) return detailsOf(values, answers, result);
diff --git a/laya-ts/tests/confidence-gating.test.ts b/laya-ts/tests/confidence-gating.test.ts
new file mode 100644
index 00000000..02fb4470
--- /dev/null
+++ b/laya-ts/tests/confidence-gating.test.ts
@@ -0,0 +1,172 @@
+import { describe, expect, it } from "vitest";
+import { checkMinConfidence, flagLowConfidence } from "../src/common.js";
+import { Agent, type ChoiceAnswer } from "../src/agent.js";
+import { Router } from "../src/router.js";
+
+const fakeProvider = () => ({
+ async runEncoder(_b: any) {
+ return { lastHidden: [[1, 0], [0, 1]] };
+ },
+ async runHead(_h: any) {
+ // 2-option question with clear margin: argmax 0
+ return {
+ logits: [[2.0, 0.0]],
+ act: [[3.0, 0.0]],
+ };
+ },
+});
+
+describe("checkMinConfidence", () => {
+ it("validates real numbers in [0.0, 1.0]", () => {
+ for (const valid of [0.0, 0.5, 1.0, 0, 1, 0.85]) {
+ expect(checkMinConfidence(valid)).toBe(Number(valid));
+ }
+ });
+
+ it("rejects booleans, out of range numbers, and non-finite values", () => {
+ const invalids = [
+ true,
+ false,
+ -0.01,
+ 1.01,
+ -1.0,
+ 2.0,
+ NaN,
+ Infinity,
+ -Infinity,
+ "0.5",
+ null,
+ undefined,
+ [0.5],
+ {},
+ ];
+ for (const invalid of invalids) {
+ expect(() => checkMinConfidence(invalid)).toThrowError(/min_confidence must be a float in \[0\.0, 1\.0\]/);
+ }
+ });
+});
+
+describe("flagLowConfidence", () => {
+ it("is a no-op when minConfidence is 0.0", () => {
+ const sample = [
+ {
+ answers: {
+ q1: { type: "choice", choice: "a", answer_confidence: 0.1, confidence: 0.1 },
+ },
+ },
+ ];
+ flagLowConfidence(sample as any, 0.0);
+ expect(sample[0].answers.q1).not.toHaveProperty("low_confidence");
+ });
+
+ it("flags answers where answer_confidence falls below threshold", () => {
+ const sample = [
+ {
+ answers: {
+ q_high: { type: "choice", choice: "a", answer_confidence: 0.92, confidence: 0.8 },
+ q_low: { type: "score", score: 1, answer_confidence: 0.45, confidence: 0.4 },
+ q_fallback: { type: "choice", choice: "b", confidence: 0.3 },
+ q_exact: { type: "choice", choice: "c", answer_confidence: 0.7 },
+ },
+ },
+ ];
+
+ flagLowConfidence(sample as any, 0.7);
+ const ans = sample[0].answers as any;
+
+ expect(ans.q_low.low_confidence).toBe(true);
+ expect(ans.q_fallback.low_confidence).toBe(true); // falls back to confidence 0.3 < 0.7
+ expect(ans.q_high.low_confidence).toBeUndefined();
+ expect(ans.q_exact.low_confidence).toBeUndefined(); // 0.70 >= 0.70
+
+ // Raw answers and confidence fields remain intact
+ expect(ans.q_low.answer_confidence).toBe(0.45);
+ expect(ans.q_high.choice).toBe("a");
+ });
+
+ it("handles a single result object as well as an array", () => {
+ const single = {
+ answers: {
+ q: { type: "choice", choice: "x", answer_confidence: 0.4 },
+ },
+ };
+ flagLowConfidence(single as any, 0.5);
+ expect((single.answers.q as any).low_confidence).toBe(true);
+ });
+});
+
+describe("Agent confidence gating", () => {
+ it("flags low-confidence answers when minConfidence is set", async () => {
+ const agent = new Agent({ provider: fakeProvider() } as any);
+ const questions = {
+ choice_q: {
+ type: "choice" as const,
+ instructions: "Pick one",
+ criteria: { a: "option A", b: "option B" },
+ },
+ };
+
+ // Threshold 0.999 is above the predicted probability (softmax([2, 0]) ~ 0.88)
+ const result = await agent.systemOne("hello", questions, { minConfidence: 0.99 });
+ expect(result.answers.choice_q.low_confidence).toBe(true);
+ expect((result.answers.choice_q as ChoiceAnswer).choice).toBe("a");
+
+ // Threshold 0.5 is below the predicted probability
+ const resultHigh = await agent.systemOne("hello", questions, { minConfidence: 0.5 });
+ expect(resultHigh.answers.choice_q.low_confidence).toBeUndefined();
+ });
+
+ it("supports python parity alias min_confidence", async () => {
+ const agent = new Agent({ provider: fakeProvider() } as any);
+ const questions = {
+ choice_q: {
+ type: "choice" as const,
+ instructions: "Pick one",
+ criteria: { a: "option A", b: "option B" },
+ },
+ };
+
+ const result = await agent.predict("hello", questions, { min_confidence: 0.99 });
+ expect(result.answers.choice_q.low_confidence).toBe(true);
+ });
+
+ it("validates minConfidence before running inference", async () => {
+ const agent = new Agent({ provider: fakeProvider() } as any);
+ await expect(
+ agent.predict("hello", {}, { minConfidence: 1.5 }),
+ ).rejects.toThrowError(/min_confidence must be a float/);
+ });
+});
+
+describe("Router confidence gating", () => {
+ it("flags low-confidence answers in Router.predict and supports alias", async () => {
+ const router = new Router();
+ // Stub router.load to return an agent using our fakeProvider
+ router.load = async () => new Agent({ provider: fakeProvider() } as any);
+
+ const questions = {
+ q: {
+ type: "choice" as const,
+ instructions: "Pick",
+ criteria: { a: "A", b: "B" },
+ },
+ };
+
+ const res = await router.predict("state text", questions, { minConfidence: 0.99 });
+ expect(res.answers.q.low_confidence).toBe(true);
+ expect(res.routing).toBeDefined();
+
+ const resAlias = await router.predict("state text", questions, { min_confidence: 0.99 });
+ expect(resAlias.answers.q.low_confidence).toBe(true);
+
+ const resPass = await router.predict("state text", questions, { minConfidence: 0.5 });
+ expect(resPass.answers.q.low_confidence).toBeUndefined();
+ });
+
+ it("validates minConfidence in Router.predict before routing or inference", async () => {
+ const router = new Router();
+ await expect(
+ router.predict("state text", {}, { minConfidence: -0.5 }),
+ ).rejects.toThrowError(/min_confidence must be a float/);
+ });
+});
diff --git a/laya-ts/tests/structured.test.ts b/laya-ts/tests/structured.test.ts
index a20d0ba3..b7627460 100644
--- a/laya-ts/tests/structured.test.ts
+++ b/laya-ts/tests/structured.test.ts
@@ -286,3 +286,73 @@ describe("structured/nullable-anyof", () => {
});
});
+describe("structured/min_confidence", () => {
+ const fakeRunner = (answers: Record<string, any>) => ({
+ calls: [] as any[],
+ async predict(state: unknown, questions: any, opts: any) {
+ this.calls.push({ state, questions, opts });
+ return { model: "fake", answers: JSON.parse(JSON.stringify(answers)) };
+ },
+ });
+
+ it("default behavior unchanged when minConfidence is omitted", async () => {
+ const runner = fakeRunner(ANSWERS);
+ const out = await decide(runner, "s", SCHEMA);
+ expect(out.department).toBe("billing");
+ expect(out.urgency).toBe(2);
+ expect(out.needs_human).toBe(true);
+ expect(out.priority).toBe(2);
+ expect(Object.values(ANSWERS).some((a: any) => a.low_confidence)).toBe(false);
+ });
+
+ it("with minConfidence=0.85, answers below threshold project as null", async () => {
+ const runner = fakeRunner(ANSWERS);
+ const out = await decide(runner, "s", SCHEMA, { minConfidence: 0.85 });
+ expect(out.department).toBe("billing"); // 0.9 >= 0.85
+ expect(out.urgency).toBeNull(); // 0.5 < 0.85
+ expect(out.needs_human).toBeNull(); // 0.8 < 0.85
+ expect(out.priority).toBeNull(); // 0.7 < 0.85
+ });
+
+ it("accepts python parity alias min_confidence", async () => {
+ const runner = fakeRunner(ANSWERS);
+ const out = await decide(runner, "s", SCHEMA, { min_confidence: 0.85 });
+ expect(out.department).toBe("billing");
+ expect(out.urgency).toBeNull();
+ });
+
+ it("with returnDetails: true, details keep raw confidence and answers dict", async () => {
+ const runner = fakeRunner(ANSWERS);
+ const det = await decide(runner, "s", SCHEMA, { minConfidence: 0.85, returnDetails: true });
+ expect(det.values.urgency).toBeNull();
+ expect(det.confidence.urgency).toBe(0.5);
+ expect((det.answers.urgency as any).low_confidence).toBe(true);
+ expect(det.values.department).toBe("billing");
+ expect((det.answers.department as any).low_confidence).toBeUndefined();
+ });
+
+ it("direct answersToJson with low_confidence: true in answer projects to null", () => {
+ const answersWithFlag = {
+ department: { type: "choice", choice: "billing", confidence: 0.4, low_confidence: true },
+ urgency: {
+ type: "score",
+ score: 2.0,
+ confidence: 0.9,
+ probabilities: { "0": 0.0, "1": 0.1, "2": 0.9 },
+ legend: {},
+ },
+ };
+ const proj = answersToJson(answersWithFlag as any, SCHEMA);
+ expect(proj.department).toBeNull();
+ expect(proj.urgency).toBe(2);
+ });
+
+ it("rejects invalid minConfidence values", async () => {
+ const runner = fakeRunner(ANSWERS);
+ await expect(decide(runner, "s", SCHEMA, { minConfidence: 1.2 })).rejects.toThrow(/min_confidence must be a float/);
+ await expect(decide(runner, "s", SCHEMA, { minConfidence: -0.1 })).rejects.toThrow(/min_confidence must be a float/);
+ await expect(decide(runner, "s", SCHEMA, { minConfidence: true as any })).rejects.toThrow(/min_confidence must be a float/);
+ await expect(decide(runner, "s", SCHEMA, { minConfidence: "0.85" as any })).rejects.toThrow(/min_confidence must be a float/);
+ });
+});
+