Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
feat(laya-ts): report answer_confidence (max(p)) on every answer type
Melvin3eb5039b8ff8
diff --git a/laya-ts/src/agent.ts b/laya-ts/src/agent.tsindex 294b7ad7..edc9149b 100644--- a/laya-ts/src/agent.ts+++ b/laya-ts/src/agent.ts@@ -5,6 +5,7 @@ import {clampTemperature,collateItems,confidenceFromProbs,+ answerConfidence,renderOptions,sequenceWithState,serializeState,@@ -42,6 +43,7 @@ export interface ChoiceAnswer {choice: string;probabilities: Record<string, number>;confidence: number;+ answer_confidence: number;action: ActionInfo;}@@ -51,6 +53,7 @@ export interface ScoreAnswer {legend: Record<string, unknown>;probabilities: Record<string, number>;confidence: number;+ answer_confidence: number;action: ActionInfo;}@@ -58,6 +61,7 @@ export interface NoulAnswer {type: "noul";noul: number;confidence: number;+ answer_confidence: number;action: ActionInfo;}@@ -399,6 +403,9 @@ export class Agent extends HookRegistry {const actRow = (act[r] as number[]) ?? [1, 0];const actP = softmax(actRow.slice(0, Math.max(2, actRow.length)));const ext = { act_probability: r4(actP[0]) };+ // Same quantity on every question type (max(p)), so callers can gate across types on+ // one number; `confidence` stays as-is for existing callers (entropy for choice/score).+ const ansConf = r4(answerConfidence(p));if (q.t === "choice") {const keys = Object.keys(q.crit as Record<string, unknown>);let best = 0;@@ -408,6 +415,7 @@ export class Agent extends HookRegistry {choice: keys[best],probabilities: Object.fromEntries(keys.map((kk, i) => [kk, r4(p[i] ?? 0)])),confidence: r4(confidenceFromProbs(p)),+ answer_confidence: ansConf,action: ext,};} else if (q.t === "score") {@@ -418,6 +426,7 @@ export class Agent extends HookRegistry {legend: Object.fromEntries((q.crit as unknown[]).map((c, i) => [String(i), c])),probabilities: Object.fromEntries(p.map((v, i) => [String(i), r4(v)])),confidence: r4(confidenceFromProbs(p)),+ answer_confidence: ansConf,action: ext,};} else {@@ -426,6 +435,8 @@ export class Agent extends HookRegistry {type: "noul",noul: r4(pt),confidence: r4(Math.max(pt, 1 - pt)),+ // over two options max(p_true, 1 - p_true) is max(p): identical to confidence here+ answer_confidence: ansConf,action: ext,};}diff --git a/laya-ts/src/common.ts b/laya-ts/src/common.tsindex f46f1b51..323bad34 100644--- a/laya-ts/src/common.ts+++ b/laya-ts/src/common.ts@@ -107,6 +107,14 @@ export function confidenceFromProbs(p: number[]): number {const ent = -p.reduce((a, v) => a + v * Math.log(Math.max(v, 1e-12)), 0);return Math.min(1, Math.max(0, 1 - ent / Math.log(k)));}+export function answerConfidence(p: number[]): number {+ // Probability mass on the reported answer: max(p). This is the quantity temperature+ // scaling fits and the one every calibration figure is computed on, so it is stable+ // across option counts and comparable across question types -- unlike+ // confidenceFromProbs, whose entropy scale moves with k.+ if (p.length < 1) return 1.0;+ return Math.min(1, Math.max(0, Math.max(...p)));+}export const TEMP_MIN = 0.5, TEMP_MAX = 5.0;export function clampTemperature(t: unknown): number {if (t === null || t === undefined || t === "" || typeof t === "boolean") return 1.0;diff --git a/laya-ts/src/index.ts b/laya-ts/src/index.tsindex 6f81120b..48e68cdd 100644--- a/laya-ts/src/index.ts+++ b/laya-ts/src/index.ts@@ -46,6 +46,7 @@ export {sequenceWithState,softmax,confidenceFromProbs,+ answerConfidence,clampTemperature,tempBucket,collateItems,diff --git a/laya-ts/tests/answer-confidence.test.ts b/laya-ts/tests/answer-confidence.test.tsnew file mode 100644index 00000000..6c7799da--- /dev/null+++ b/laya-ts/tests/answer-confidence.test.ts@@ -0,0 +1,56 @@+import { describe, expect, it } from "vitest";+import { answerConfidence, confidenceFromProbs } from "../src/common.js";+import { Agent } from "../src/agent.js";++const fakeProvider = (n: number) => ({+ // n logits with a known argmax at 0; softmax gives a fixed, checkable distribution+ async runEncoder(_b: any) { return { lastHidden: [[1, 0], [0, 1]] }; },+ async runHead(_h: any) {+ const row = Array.from({ length: n }, (_, i) => (i === 0 ? 2 : 0));+ return { logits: [row], act: [[3, 0]] };+ },+});++function probs(provider: any, a: Agent) { return a as any; }++describe("answerConfidence", () => {+ it("is max(p) on the reported answer", () => {+ expect(answerConfidence([0.25, 0.6, 0.15])).toBeCloseTo(0.6);+ expect(answerConfidence([1.0])).toBe(1.0);+ });++ it("returns 1.0 for an empty distribution (Python parity: k < 1)", () => {+ expect(answerConfidence([])).toBe(1.0);+ });++ it("stays at the same value across option counts, unlike the entropy confidence", () => {+ // The #361 table: same top probability 0.9, different label-set sizes.+ const fill = (k: number) => {+ const rest = Array.from({ length: k - 1 }, () => 0.1 / (k - 1));+ return [0.9, ...rest];+ };+ for (const k of [2, 5, 20, 77]) {+ expect(answerConfidence(fill(k))).toBeCloseTo(0.9, 5);+ }+ // entropy confidence rises with k for the same max(p)+ expect(confidenceFromProbs(fill(2))).toBeLessThan(confidenceFromProbs(fill(20)));+ });++ it("is attached to every answer type by the agent decode", async () => {+ const mk = (n: number) => new Agent({ provider: fakeProvider(n) } as any);+ const choice: any = await mk(3).systemOne("s", {+ c: { type: "choice", instructions: "q?", criteria: { a: "x", b: "y", c: "z" } },+ });+ const top = Math.max(...Object.values(choice.answers.c.probabilities).map(Number));+ expect(choice.answers.c.answer_confidence).toBeCloseTo(top, 4);++ const score: any = await mk(3).systemOne("s", {+ s: { type: "score", instructions: "q?", criteria: ["lo", "mid", "hi"] },+ });+ expect(score.answers.s.answer_confidence).toBeCloseTo(top, 4);++ const noul: any = await mk(2).systemOne("s", { n: { type: "noul", instructions: "q?" } });+ // over two options max(p) equals the existing noul confidence+ expect(noul.answers.n.answer_confidence).toBeCloseTo(noul.answers.n.confidence, 4);+ });+});