Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(laya-ts): report answer_confidence (max(p)) on every answer type

Melvin3eb5039b8ff8
diff --git a/laya-ts/src/agent.ts b/laya-ts/src/agent.ts
index 294b7ad7..edc9149b 100644
--- a/laya-ts/src/agent.ts
+++ b/laya-ts/src/agent.ts
@@ -5,6 +5,7 @@ import {
clampTemperature,
collateItems,
confidenceFromProbs,
+ answerConfidence,
renderOptions,
sequenceWithState,
serializeState,
@@ -42,6 +43,7 @@ export interface ChoiceAnswer {
choice: string;
probabilities: Record<string, number>;
confidence: number;
+ answer_confidence: number;
action: ActionInfo;
}
@@ -51,6 +53,7 @@ export interface ScoreAnswer {
legend: Record<string, unknown>;
probabilities: Record<string, number>;
confidence: number;
+ answer_confidence: number;
action: ActionInfo;
}
@@ -58,6 +61,7 @@ export interface NoulAnswer {
type: "noul";
noul: number;
confidence: number;
+ answer_confidence: number;
action: ActionInfo;
}
@@ -399,6 +403,9 @@ export class Agent extends HookRegistry {
const actRow = (act[r] as number[]) ?? [1, 0];
const actP = softmax(actRow.slice(0, Math.max(2, actRow.length)));
const ext = { act_probability: r4(actP[0]) };
+ // Same quantity on every question type (max(p)), so callers can gate across types on
+ // one number; `confidence` stays as-is for existing callers (entropy for choice/score).
+ const ansConf = r4(answerConfidence(p));
if (q.t === "choice") {
const keys = Object.keys(q.crit as Record<string, unknown>);
let best = 0;
@@ -408,6 +415,7 @@ export class Agent extends HookRegistry {
choice: keys[best],
probabilities: Object.fromEntries(keys.map((kk, i) => [kk, r4(p[i] ?? 0)])),
confidence: r4(confidenceFromProbs(p)),
+ answer_confidence: ansConf,
action: ext,
};
} else if (q.t === "score") {
@@ -418,6 +426,7 @@ export class Agent extends HookRegistry {
legend: Object.fromEntries((q.crit as unknown[]).map((c, i) => [String(i), c])),
probabilities: Object.fromEntries(p.map((v, i) => [String(i), r4(v)])),
confidence: r4(confidenceFromProbs(p)),
+ answer_confidence: ansConf,
action: ext,
};
} else {
@@ -426,6 +435,8 @@ export class Agent extends HookRegistry {
type: "noul",
noul: r4(pt),
confidence: r4(Math.max(pt, 1 - pt)),
+ // over two options max(p_true, 1 - p_true) is max(p): identical to confidence here
+ answer_confidence: ansConf,
action: ext,
};
}
diff --git a/laya-ts/src/common.ts b/laya-ts/src/common.ts
index f46f1b51..323bad34 100644
--- a/laya-ts/src/common.ts
+++ b/laya-ts/src/common.ts
@@ -107,6 +107,14 @@ export function confidenceFromProbs(p: number[]): number {
const ent = -p.reduce((a, v) => a + v * Math.log(Math.max(v, 1e-12)), 0);
return Math.min(1, Math.max(0, 1 - ent / Math.log(k)));
}
+export function answerConfidence(p: number[]): number {
+ // Probability mass on the reported answer: max(p). This is the quantity temperature
+ // scaling fits and the one every calibration figure is computed on, so it is stable
+ // across option counts and comparable across question types -- unlike
+ // confidenceFromProbs, whose entropy scale moves with k.
+ if (p.length < 1) return 1.0;
+ return Math.min(1, Math.max(0, Math.max(...p)));
+}
export const TEMP_MIN = 0.5, TEMP_MAX = 5.0;
export function clampTemperature(t: unknown): number {
if (t === null || t === undefined || t === "" || typeof t === "boolean") return 1.0;
diff --git a/laya-ts/src/index.ts b/laya-ts/src/index.ts
index 6f81120b..48e68cdd 100644
--- a/laya-ts/src/index.ts
+++ b/laya-ts/src/index.ts
@@ -46,6 +46,7 @@ export {
sequenceWithState,
softmax,
confidenceFromProbs,
+ answerConfidence,
clampTemperature,
tempBucket,
collateItems,
diff --git a/laya-ts/tests/answer-confidence.test.ts b/laya-ts/tests/answer-confidence.test.ts
new file mode 100644
index 00000000..6c7799da
--- /dev/null
+++ b/laya-ts/tests/answer-confidence.test.ts
@@ -0,0 +1,56 @@
+import { describe, expect, it } from "vitest";
+import { answerConfidence, confidenceFromProbs } from "../src/common.js";
+import { Agent } from "../src/agent.js";
+
+const fakeProvider = (n: number) => ({
+ // n logits with a known argmax at 0; softmax gives a fixed, checkable distribution
+ async runEncoder(_b: any) { return { lastHidden: [[1, 0], [0, 1]] }; },
+ async runHead(_h: any) {
+ const row = Array.from({ length: n }, (_, i) => (i === 0 ? 2 : 0));
+ return { logits: [row], act: [[3, 0]] };
+ },
+});
+
+function probs(provider: any, a: Agent) { return a as any; }
+
+describe("answerConfidence", () => {
+ it("is max(p) on the reported answer", () => {
+ expect(answerConfidence([0.25, 0.6, 0.15])).toBeCloseTo(0.6);
+ expect(answerConfidence([1.0])).toBe(1.0);
+ });
+
+ it("returns 1.0 for an empty distribution (Python parity: k < 1)", () => {
+ expect(answerConfidence([])).toBe(1.0);
+ });
+
+ it("stays at the same value across option counts, unlike the entropy confidence", () => {
+ // The #361 table: same top probability 0.9, different label-set sizes.
+ const fill = (k: number) => {
+ const rest = Array.from({ length: k - 1 }, () => 0.1 / (k - 1));
+ return [0.9, ...rest];
+ };
+ for (const k of [2, 5, 20, 77]) {
+ expect(answerConfidence(fill(k))).toBeCloseTo(0.9, 5);
+ }
+ // entropy confidence rises with k for the same max(p)
+ expect(confidenceFromProbs(fill(2))).toBeLessThan(confidenceFromProbs(fill(20)));
+ });
+
+ it("is attached to every answer type by the agent decode", async () => {
+ const mk = (n: number) => new Agent({ provider: fakeProvider(n) } as any);
+ const choice: any = await mk(3).systemOne("s", {
+ c: { type: "choice", instructions: "q?", criteria: { a: "x", b: "y", c: "z" } },
+ });
+ const top = Math.max(...Object.values(choice.answers.c.probabilities).map(Number));
+ expect(choice.answers.c.answer_confidence).toBeCloseTo(top, 4);
+
+ const score: any = await mk(3).systemOne("s", {
+ s: { type: "score", instructions: "q?", criteria: ["lo", "mid", "hi"] },
+ });
+ expect(score.answers.s.answer_confidence).toBeCloseTo(top, 4);
+
+ const noul: any = await mk(2).systemOne("s", { n: { type: "noul", instructions: "q?" } });
+ // over two options max(p) equals the existing noul confidence
+ expect(noul.answers.n.answer_confidence).toBeCloseTo(noul.answers.n.confidence, 4);
+ });
+});