Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(sdk): read the confidence an answer is gated on and the abstention report

Aashish7af12f470188
diff --git a/sdk/typescript/src/response.ts b/sdk/typescript/src/response.ts
index d6eb8468..8180c6f2 100644
--- a/sdk/typescript/src/response.ts
+++ b/sdk/typescript/src/response.ts
@@ -47,6 +47,25 @@ export function validatePrediction(value: unknown, questions: Questions): void {
if (answer.action !== undefined) {
expect(isRecord(answer.action) && probability(answer.action.act_probability), `answers.${id}.action`);
}
+ // Checked when the server sends them, like the optional fields of `health` and of `usage`:
+ // `answer_confidence` (#126) and the gate fields (#361) are recent additions to an answer that
+ // predates them, so requiring a key here would fail every prediction against a deployment that
+ // has not shipped it. A caller that asked for a gate (`minConfidence`) and got no `abstention`
+ // back reads it as `undefined` on the typed answer, which is the truth: the gate did not run.
+ if (answer.answer_confidence !== undefined) {
+ expect(probability(answer.answer_confidence), `answers.${id}.answer_confidence`);
+ }
+ if (answer.low_confidence !== undefined) {
+ expect(answer.low_confidence === true, `answers.${id}.low_confidence`);
+ }
+ if (answer.abstention !== undefined || answer.abstention_threshold !== undefined) {
+ // The gate writes both onto the same answer or onto neither, so one without the other is a
+ // half-report: an `abstention` with no echoed threshold cannot be re-split, and a threshold
+ // with no state says nothing about the answer it was measured against.
+ expect(answer.abstention === 'passed' || answer.abstention === 'abstained' ||
+ answer.abstention === 'unevaluated', `answers.${id}.abstention`);
+ expect(probability(answer.abstention_threshold), `answers.${id}.abstention_threshold`);
+ }
if (question.type === 'noul') {
expect(probability(answer.noul), `answers.${id}.noul`);
} else {
diff --git a/sdk/typescript/src/types.ts b/sdk/typescript/src/types.ts
index 5a0b62a3..ac3d7403 100644
--- a/sdk/typescript/src/types.ts
+++ b/sdk/typescript/src/types.ts
@@ -53,6 +53,20 @@ export interface PredictOptions extends RequestOptions {
interface AnswerBase {
/** Optional action metadata returned by Laya. */
action?: { act_probability: number };
+ /** The probability mass on the answer actually reported (`max(p)`). Unlike `confidence`, which
+ * means normalized entropy on `choice` and `score` and `max(p_yes, p_no)` on `noul`, this is
+ * the same quantity on all three types, so one threshold gates across them. */
+ answer_confidence: number;
+ /** Set by the abstention gate on the answers that fell below `minConfidence`, and only then:
+ * `flag_low_confidence` writes `True` and never writes the key otherwise. */
+ low_confidence?: true;
+ /** How the gate decided this answer, reported on every answer of a gated call. Absent means no
+ * gate ran (`minConfidence` unset); `unevaluated` means a gate ran and this answer carried no
+ * usable confidence, which the boolean above cannot tell apart from a pass. */
+ abstention?: 'passed' | 'abstained' | 'unevaluated';
+ /** The threshold that produced `abstention`, echoed on the same answers: the gate consumes
+ * `minConfidence`, so without this a batch run cannot be re-split by the threshold it used. */
+ abstention_threshold?: number;
}
export interface ChoiceAnswer<Label extends string = string> extends AnswerBase {
type: 'choice';