Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
test(sdk): pin the answer confidence and gate report at the type and on the wire
Aashish3ec02d78ce4a
diff --git a/sdk/typescript/test/client.test.mjs b/sdk/typescript/test/client.test.mjsindex c862f012..b2cec0bb 100644--- a/sdk/typescript/test/client.test.mjs+++ b/sdk/typescript/test/client.test.mjs@@ -224,6 +224,80 @@ test('prediction validation rejects malformed required fields and optional exten}});+// Captured from `tests/sdk_server_fixture.py` over HTTP with `min_confidence: 0.35`, so the three+// answer types each carry their real gate report. The counts are what that fixture's random weights+// give; `abstention: 'unevaluated'` is not reachable there -- every answer it builds carries a+// usable confidence -- so only the two states the gate really reported here are asserted verbatim.+const gateQuestions = {+ team: { type: 'choice', instructions: 'Team?', criteria: ['billing', 'technical'] },+ urgency: { type: 'score', instructions: 'Urgency?', criteria: ['low', 'medium', 'high'] },+ refund: questions.refund,+};+const gateReport = {+ model: 'laya-rl-agent', usage: { input_tokens: 10, output_tokens: 0 },+ answers: {+ team: { type: 'choice', choice: 'technical', probabilities: { billing: 0.4965, technical: 0.5035 },+ confidence: 0.0, answer_confidence: 0.5035, action: { act_probability: 0.5017 },+ abstention: 'passed', abstention_threshold: 0.35 },+ urgency: { type: 'score', score: 1.0105, probabilities: { '0': 0.3338, '1': 0.3218, '2': 0.3444 },+ legend: { '0': 'low', '1': 'medium', '2': 'high' }, confidence: 0.0003, answer_confidence: 0.3444,+ action: { act_probability: 0.5238 }, low_confidence: true, abstention: 'abstained',+ abstention_threshold: 0.35 },+ refund: { type: 'noul', noul: 0.499, confidence: 0.501, answer_confidence: 0.501,+ action: { act_probability: 0.3968 }, abstention: 'passed', abstention_threshold: 0.35 },+ },+};++test('the confidence an answer was gated on and the gate report reach the caller', async () => {+ const client = new Laya({ fetch: async () => json(gateReport) });+ const result = await client.predict('hello', gateQuestions);+ assert.deepEqual(result.answers, gateReport.answers);+ assert.equal(result.answers.urgency.low_confidence, true);+ assert.equal(result.answers.urgency.abstention, 'abstained');+ assert.equal(result.answers.team.abstention, 'passed');+ assert.equal(result.answers.team.abstention_threshold, 0.35);++ // 'unevaluated' is the third state core reports (laya/confidence.py); accepted like the others.+ const unevaluated = structuredClone(gateReport);+ unevaluated.answers.refund.abstention = 'unevaluated';+ assert.equal((await new Laya({ fetch: async () => json(unevaluated) })+ .predict('hello', gateQuestions)).answers.refund.abstention, 'unevaluated');++ // An ungated call answers exactly the payload it answered before the gate existed: no abstention,+ // no threshold, no flag. Absence is the report, so it must not read as a failure.+ const ungated = structuredClone(gateReport);+ for (const answer of Object.values(ungated.answers)) {+ delete answer.low_confidence;+ delete answer.abstention;+ delete answer.abstention_threshold;+ }+ const bare = await new Laya({ fetch: async () => json(ungated) }).predict('hello', gateQuestions);+ assert.equal(bare.answers.urgency.abstention, undefined);+ assert.equal(bare.answers.urgency.low_confidence, undefined);++ // A deployment predating `answer_confidence` (#126) still answers, exactly as it did before.+ const older = structuredClone(ungated);+ for (const answer of Object.values(older.answers)) delete answer.answer_confidence;+ assert.equal((await new Laya({ fetch: async () => json(older) })+ .predict('hello', gateQuestions)).answers.team.choice, 'technical');++ for (const mutate of [+ answer => { answer.abstention = 'skipped'; },+ answer => { delete answer.abstention_threshold; },+ answer => { answer.abstention_threshold = 1.5; },+ answer => { answer.abstention_threshold = '0.35'; },+ answer => { answer.low_confidence = false; },+ answer => { delete answer.abstention; },+ answer => { answer.answer_confidence = 'high'; },+ answer => { answer.answer_confidence = 2; },+ ]) {+ const payload = structuredClone(gateReport);+ mutate(payload.answers.urgency);+ await assert.rejects(new Laya({ fetch: async () => json(payload) })+ .predict('hello', gateQuestions), LayaResponseError);+ }+});+test('network errors preserve the cause', async () => {const cause = new TypeError('fetch failed');await assert.rejects(new Laya({ fetch: async () => { throw cause; } }).health(), error =>diff --git a/sdk/typescript/test/types.ts b/sdk/typescript/test/types.tsindex 7bbd0aa8..a8abc3ae 100644--- a/sdk/typescript/test/types.ts+++ b/sdk/typescript/test/types.ts@@ -1,4 +1,5 @@-import { Laya, defineQuestions, emailQuestions, triageQuestions, type Questions, type Answer } from 'laya-client';+import { Laya, defineQuestions, emailQuestions, triageQuestions,+ type Questions, type Answer, type ChoiceAnswer } from 'laya-client';const client = new Laya();const questions = defineQuestions({@@ -16,6 +17,21 @@ const noul: number = result.answers.refund.noul;const routingModel: string | undefined = result.routing?.model;const actionProbability: number | undefined = result.answers.team.action?.act_probability;const noulConfidence: number | undefined = result.answers.refund.confidence;+const answerConfidence: number = result.answers.team.answer_confidence;+const noulAnswerConfidence: number = result.answers.refund.answer_confidence;+const gateState: 'passed' | 'abstained' | 'unevaluated' | undefined = result.answers.priority.abstention;+const gateThreshold: number | undefined = result.answers.priority.abstention_threshold;+const flagged: true | undefined = result.answers.refund.low_confidence;+// @ts-expect-error `flag_low_confidence` writes `True` or writes nothing, so no answer reads false.+const flagAsFalse: false | undefined = result.answers.refund.low_confidence;+// @ts-expect-error A gate state is one of the three core reports, not any string.+const inventedGate: Answer['abstention'] = 'skipped';+// @ts-expect-error An answer must carry the calibrated confidence it was gated on.+const uncalibratedAnswer: ChoiceAnswer = { type: 'choice', confidence: 0.5, choice: 'a',+ probabilities: { a: 0.5 } };+const gatedAnswer: ChoiceAnswer = { type: 'choice', confidence: 0.5, answer_confidence: 0.42,+ choice: 'a', probabilities: { a: 0.42 }, low_confidence: true, abstention: 'abstained',+ abstention_threshold: 0.8 };// @ts-expect-error Routing metadata is optional.result.routing.model;// @ts-expect-error Action metadata is optional.@@ -58,4 +74,6 @@ if (answer?.type === 'choice') {const dynamicProbability: number | undefined = answer.probabilities.anyLabel;void [dynamicLabel, dynamicProbability];}-void [label, arrayLabel, probability, score, noul, inlineLabel, intent, category, routingModel, actionProbability, noulConfidence];+void [label, arrayLabel, probability, score, noul, inlineLabel, intent, category, routingModel,+ actionProbability, noulConfidence, answerConfidence, noulAnswerConfidence, gateState,+ gateThreshold, flagged, gatedAnswer];