Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(ts): gate on the number Python's _gate_confidence picks
Nandakishor3c645e903d43
diff --git a/laya-ts/src/common.ts b/laya-ts/src/common.tsindex 8ab30cbb..ec21ccbe 100644--- a/laya-ts/src/common.ts+++ b/laya-ts/src/common.ts@@ -328,11 +328,37 @@ export function resolveMinConfidence(return thresholds.default ?? defaultVal;}+/**+ * The number the abstention gate compares against the threshold, or null if there is none.+ *+ * Mirrors Python `laya.confidence._gate_confidence` exactly, including the two places it is+ * easy to get wrong:+ *+ * - the fallback is taken whenever `answer_confidence` is not a usable number, not only when+ * it is absent or null. An `answer_confidence` of `NaN` beside a usable `confidence` is+ * gated on the `confidence`, because "unusable" and "missing" are the same case here;+ * - a number outside [0, 1] is still a number. It is compared, not discarded: a negative+ * `answer_confidence` is below every threshold and so is flagged, and one above 1 clears+ * every threshold. Treating out-of-range as "nothing to gate on" would silently clear a+ * marker Python sets.+ *+ * null means no usable number, never an unusable one, which is what lets the caller tell+ * "nothing to gate on" from "a gate ran".+ */+function gateConfidence(answer: Record<string, unknown>): number | null {+ const primary = answer.answer_confidence;+ if (typeof primary === "number" && Number.isFinite(primary)) return primary;+ const fallback = answer.confidence;+ if (typeof fallback === "number" && Number.isFinite(fallback)) return fallback;+ return null;+}+/*** Opt-in abstention marker (#361, #394): flag answers whose confidence falls below `min_confidence`.** Reads `answer_confidence` (the calibrated max(p) confidence, invariant to label count k),- * falling back to `confidence` if `answer_confidence` is absent.+ * falling back to `confidence` whenever `answer_confidence` is not a usable number -- see+ * `gateConfidence`, which mirrors Python's `_gate_confidence`.* The raw answer and confidence stay intact; `low_confidence: true` is added when the answer* falls below the threshold, and removed if a previously-flagged answer now clears it (e.g.* when a result dict is reused or re-evaluated with a different threshold).@@ -355,11 +381,8 @@ export function flagLowConfidence(for (const a of Object.values(answers as Record<string, unknown>)) {if (!a || typeof a !== "object") continue;const ansObj = a as Record<string, unknown>;- let conf = ansObj.answer_confidence;- if (conf === undefined || conf === null) {- conf = ansObj.confidence;- }- if (typeof conf !== "number" || !Number.isFinite(conf) || conf < 0 || conf > 1) {+ const conf = gateConfidence(ansObj);+ if (conf === null) {delete ansObj.low_confidence;continue;}diff --git a/laya-ts/tests/confidence-gating.test.ts b/laya-ts/tests/confidence-gating.test.tsindex a56d7426..ce3203d5 100644--- a/laya-ts/tests/confidence-gating.test.ts+++ b/laya-ts/tests/confidence-gating.test.ts@@ -179,33 +179,50 @@ describe("flagLowConfidence", () => {expect(ans).not.toHaveProperty("low_confidence");});- it("clears stale low_confidence flag when confidence is missing, non-finite, or out of [0, 1] range", () => {+ it("clears stale low_confidence flag when there is no usable confidence to gate on", () => {const sample = [{answers: {q_missing: { choice: "a", low_confidence: true },q_nan: { choice: "b", answer_confidence: NaN, low_confidence: true },q_inf: { choice: "c", answer_confidence: Infinity, low_confidence: true },- q_neg: { choice: "d", answer_confidence: -0.1, low_confidence: true },- q_over: { choice: "e", answer_confidence: 1.5, low_confidence: true },+ q_bool: { choice: "d", answer_confidence: true, low_confidence: true },+ q_str: { choice: "e", answer_confidence: "0.9", low_confidence: true },},},];flagLowConfidence(sample as any, 0.5);- for (const key of ["q_missing", "q_nan", "q_inf", "q_neg", "q_over"]) {+ for (const key of ["q_missing", "q_nan", "q_inf", "q_bool", "q_str"]) {expect((sample[0].answers as any)[key]).not.toHaveProperty("low_confidence");}+ });- // Out-of-range confidence is also not flagged when minConfidence is 0.0- const sampleNeg = [- {- answers: {- q_neg: { choice: "d", answer_confidence: -0.1 },- },- },+ // Python's `_gate_confidence` is the reference. Two rules there are easy to lose, and both+ // were measured against `laya.confidence.flag_low_confidence` on the same inputs:+ //+ // 1. the fallback to `confidence` is taken whenever `answer_confidence` is not a usable+ // number, not only when it is absent or null. NaN, Infinity and a bool all take it.+ // 2. a number outside [0, 1] is still a number, so it is compared rather than discarded.+ // A negative `answer_confidence` is below every threshold, 0.0 included, and one above+ // 1 clears every threshold. Reading out-of-range as "nothing to gate on" would clear a+ // marker Python sets.+ it("gates on the same number Python's _gate_confidence picks", () => {+ const cases: Array<[string, Record<string, unknown>, number, boolean]> = [+ ["unusable answer_confidence falls back to confidence", { answer_confidence: NaN, confidence: 0.3 }, 0.5, true],+ ["and clears when that fallback passes", { answer_confidence: NaN, confidence: 0.9, low_confidence: true }, 0.5, false],+ ["Infinity takes the fallback too", { answer_confidence: Infinity, confidence: 0.3 }, 0.5, true],+ ["a bool is not a confidence, so it takes the fallback", { answer_confidence: true, confidence: 0.3 }, 0.5, true],+ ["null still takes the fallback", { answer_confidence: null, confidence: 0.3 }, 0.5, true],+ ["a negative confidence is below every threshold", { answer_confidence: -0.1 }, 0.5, true],+ ["including exactly 0.0", { answer_confidence: -0.1 }, 0.0, true],+ ["above 1 clears every threshold", { answer_confidence: 1.5, low_confidence: true }, 0.5, false],+ ["no usable number anywhere clears", { answer_confidence: NaN, confidence: NaN, low_confidence: true }, 0.5, false],];- flagLowConfidence(sampleNeg as any, 0.0);- expect((sampleNeg[0].answers as any).q_neg).not.toHaveProperty("low_confidence");+ for (const [label, answer, threshold, flagged] of cases) {+ const held = { ...answer };+ flagLowConfidence([{ answers: { q: held } }] as any, threshold);+ expect((held as any).low_confidence === true, label).toBe(flagged);+ }});it("flags answers where answer_confidence falls below threshold", () => {