Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(sdk): align TypeScript confidence contract with HTTP responses
debjyotikmf439434ff040
diff --git a/sdk/typescript/README.md b/sdk/typescript/README.mdindex 32ed6cba..efab44c6 100644--- a/sdk/typescript/README.md+++ b/sdk/typescript/README.md@@ -75,6 +75,12 @@ console.log(result.answers.refund_requested.noul); // P(true)console.log(result.routing?.model); // Laya-only metadata```+Choice and score answers may also contain `x_jev_confidence`, the HTTP server's+Jev-compatible confidence measure. It is distinct from Laya's `confidence` and+`answer_confidence`, and is absent on older servers, in strict Jev mode, and on+`noul` answers. The client preserves it and checks that it is a finite number in+`[0, 1]` when present; it does not compute or substitute it when absent.+CommonJS works too:```jsdiff --git a/sdk/typescript/src/response.ts b/sdk/typescript/src/response.tsindex ea1e1e4a..8c9db256 100644--- a/sdk/typescript/src/response.ts+++ b/sdk/typescript/src/response.ts@@ -92,6 +92,11 @@ export function validatePrediction(value: unknown, questions: Questions): void {if (answer.answer_confidence !== undefined) {expect(probability(answer.answer_confidence), `answers.${id}.answer_confidence`);}+ // HTTP-only metadata: older servers omit it, and Jev defines none for noul.+ if (answer.x_jev_confidence !== undefined) {+ expect(question.type !== 'noul' && probability(answer.x_jev_confidence),+ `answers.${id}.x_jev_confidence`);+ }if (answer.low_confidence !== undefined) {expect(answer.low_confidence === true, `answers.${id}.low_confidence`);}diff --git a/sdk/typescript/src/types.ts b/sdk/typescript/src/types.tsindex 4515e96f..51b6ef14 100644--- a/sdk/typescript/src/types.ts+++ b/sdk/typescript/src/types.ts@@ -88,12 +88,16 @@ interface AnswerBase {export interface ChoiceAnswer<Label extends string = string> extends AnswerBase {type: 'choice';confidence: number;+ /** Jev-compatible confidence added by HTTP servers; absent on older or strict-mode servers. */+ x_jev_confidence?: number;choice: Label;probabilities: Record<Label, number>;}export interface ScoreAnswer extends AnswerBase {type: 'score';confidence: number;+ /** Jev-compatible confidence added by HTTP servers; absent on older or strict-mode servers. */+ x_jev_confidence?: number;/** Expected zero-based rubric index, not a normalized probability. */score: number;legend: Record<string, JsonValue>;diff --git a/sdk/typescript/test/client.test.mjs b/sdk/typescript/test/client.test.mjsindex d67477f0..1706cbe6 100644--- a/sdk/typescript/test/client.test.mjs+++ b/sdk/typescript/test/client.test.mjs@@ -93,6 +93,38 @@ test('default request omits model and supports every Laya answer type', async ()assert.deepEqual(allAnswerQuestions.team.criteria, ['billing', 'support'], 'normalization must not mutate the schema');});+test('HTTP Jev confidence is preserved and remains optional for older servers', async () => {+ for (const value of [undefined, 0, 0.6, 1]) {+ const payload = structuredClone(allAnswerPrediction);+ if (value !== undefined) {+ payload.answers.team.x_jev_confidence = value;+ payload.answers.urgency.x_jev_confidence = value;+ }+ const client = new Laya({ fetch: async () => json(payload) });+ const result = await client.predict('hello', allAnswerQuestions);+ assert.deepEqual(result, payload);+ assert.equal(Object.hasOwn(result.answers.team, 'x_jev_confidence'), value !== undefined);+ assert.equal(Object.hasOwn(result.answers.urgency, 'x_jev_confidence'), value !== undefined);+ assert.equal(Object.hasOwn(result.answers.refund, 'x_jev_confidence'), false);+ }+});++test('malformed or noul Jev confidence is rejected', async () => {+ for (const id of ['team', 'urgency']) {+ for (const value of [null, false, true, '0.6', [], {}, -0.01, 1.01, NaN, Infinity, -Infinity]) {+ const payload = structuredClone(allAnswerPrediction);+ payload.answers[id].x_jev_confidence = value;+ const client = new Laya({ fetch: async () => json(payload) });+ await assert.rejects(client.predict('hello', allAnswerQuestions), error =>+ error instanceof LayaResponseError && error.message.includes(`answers.${id}.x_jev_confidence`));+ }+ }+ const payload = structuredClone(allAnswerPrediction);+ payload.answers.refund.x_jev_confidence = 0.9;+ await assert.rejects(new Laya({ fetch: async () => json(payload) }).predict('hello', allAnswerQuestions),+ LayaResponseError);+});+test('client model default and prediction overrides select local checkpoints', async () => {const models = [];const client = new Laya({ model: 'english', fetch: async (_url, init) => {diff --git a/sdk/typescript/test/integration.mjs b/sdk/typescript/test/integration.mjsindex 4233225a..2e5a1534 100644--- a/sdk/typescript/test/integration.mjs+++ b/sdk/typescript/test/integration.mjs@@ -7,6 +7,70 @@ import { setTimeout as delay } from 'node:timers/promises';import { test } from 'node:test';import { Laya, LayaAPIError, triageQuestions, emailQuestions, guardQuestions, moderationQuestions, routerQuestions } from 'laya-client';+function assertPredictionParity(result, expected) {+ const { routing, ...prediction } = structuredClone(result);+ for (const [id, answer] of Object.entries(expected.answers)) {+ const actual = prediction.answers[id];+ if (answer.type === 'noul') {+ assert.equal(Object.hasOwn(actual, 'x_jev_confidence'), false, `${id}: noul has no Jev confidence`);+ continue;+ }+ // Derive the HTTP extension independently from the direct model's probabilities,+ // not from the server's enrichment helper or from the received HTTP probabilities.+ const keys = Object.keys(answer.probabilities);+ if (answer.type === 'score') keys.sort((a, b) => Number(a) - Number(b));+ const values = keys.map(key => answer.probabilities[key]);+ const n = values.length;+ const total = values.reduce((a, b) => a + b, 0);+ const p = values.map(value => total ? value / total : 1 / n);+ let confidence = 1;+ if (n > 1 && answer.type === 'choice') {+ confidence = (Math.max(...p) - 1 / n) / (1 - 1 / n);+ } else if (n > 1) {+ const mode = p.indexOf(Math.max(...p));+ const spread = p.reduce((sum, value, i) => sum + value * Math.abs(i - mode), 0);+ const uniformSpread = p.reduce((sum, _, i) => sum + Math.abs(i - (n - 1) / 2) / n, 0);+ confidence = Math.max(0, 1 - spread / uniformSpread);+ }+ assert.equal(typeof actual.x_jev_confidence, 'number', `${id}: HTTP Jev confidence missing`);+ assert.ok(Math.abs(actual.x_jev_confidence - confidence) <= 0.0000500001,+ `${id}: HTTP Jev confidence must match the formula rounded to four decimals`);+ delete actual.x_jev_confidence;+ }+ assert.deepEqual(prediction, expected, 'SDK core answers must exactly match direct Python inference');+}++test('parity checks the HTTP extension without weakening core prediction equality', () => {+ const expected = { model: 'test', usage: { input_tokens: 1, output_tokens: 0 }, answers: {+ choice: { type: 'choice', probabilities: { a: 0.8, b: 0.2 }, choice: 'a' },+ score: { type: 'score', probabilities: { '0': 0.1, '1': 0.8, '2': 0.1 }, score: 1 },+ single: { type: 'choice', probabilities: { only: 1 }, choice: 'only' },+ singleScore: { type: 'score', probabilities: { '0': 1 }, score: 0 },+ noul: { type: 'noul', noul: 0.9 },+ } };+ const actual = structuredClone(expected);+ for (const [id, value] of Object.entries({ choice: 0.6, score: 0.7, single: 1, singleScore: 1 })) {+ actual.answers[id].x_jev_confidence = value;+ }+ const before = structuredClone(actual);+ assertPredictionParity(actual, expected);+ assert.deepEqual(actual, before, 'parity check must not mutate the response');+ for (const mutate of [+ r => { delete r.answers.choice.x_jev_confidence; },+ r => { r.answers.choice.x_jev_confidence = 0.61; },+ r => { r.answers.score.x_jev_confidence = 0.8; },+ r => { r.answers.noul.x_jev_confidence = 0.9; },+ r => { r.answers.choice.probabilities.a = 0.7; },+ r => { r.answers.choice.choice = 'b'; },+ r => { r.usage.input_tokens = 2; },+ r => { r.answers.score.unexpected = true; },+ ]) {+ const broken = structuredClone(actual);+ mutate(broken);+ assert.throws(() => assertPredictionParity(broken, expected), assert.AssertionError);+ }+});+test('JavaScript → HTTP → Python Router → real offline Agent inference', { timeout: 90_000 }, async t => {const script = fileURLToPath(new URL('../../../tests/sdk_server_fixture.py', import.meta.url));const server = spawn(process.env.PYTHON ?? 'python3', [script], { stdio: ['ignore', 'pipe', 'pipe'] });@@ -42,9 +106,8 @@ test('JavaScript → HTTP → Python Router → real offline Agent inference', {assert.ok(ready, `Server never became ready: ${stderr}`);const result = await client.predict(fixture.state, fixture.questions, { model: 'english' });- const { routing, ...prediction } = result;- assert.deepEqual(prediction, fixture.expected, 'SDK answers must exactly match direct Python inference');- assert.equal(routing.model, 'english');+ assertPredictionParity(result, fixture.expected);+ assert.equal(result.routing.model, 'english');assert.equal(result.answers.single.choice, 'only');assert.equal(result.answers.single.probabilities.only, 1);assert.ok(result.usage.input_tokens > 0);diff --git a/sdk/typescript/test/types.ts b/sdk/typescript/test/types.tsindex 078a0f2b..63c83c47 100644--- a/sdk/typescript/test/types.ts+++ b/sdk/typescript/test/types.ts@@ -17,6 +17,14 @@ const noul: number = result.answers.refund.noul;const routingModel: string | undefined = result.routing?.model;const actionProbability: number | undefined = result.answers.team.action?.act_probability;const noulConfidence: number | undefined = result.answers.refund.confidence;+const choiceJevConfidence: number | undefined = result.answers.team.x_jev_confidence;+const scoreJevConfidence: number | undefined = result.answers.priority.x_jev_confidence;+// @ts-expect-error Older and strict-mode servers omit the HTTP confidence extension.+const requiredJevConfidence: number = result.answers.team.x_jev_confidence;+// @ts-expect-error Jev defines no confidence extension for noul answers.+result.answers.refund.x_jev_confidence;+// @ts-expect-error Jev confidence is numeric when present.+const textJevConfidence: ChoiceAnswer['x_jev_confidence'] = '0.6';const stateTokens: number = result.usage.state_tokens;const stateTokensDropped: number = result.usage.state_tokens_dropped;const truncated: boolean = result.usage.truncated;diff --git a/tests/sdk_server_fixture.py b/tests/sdk_server_fixture.pyindex 15a02467..36ac2a98 100644--- a/tests/sdk_server_fixture.py+++ b/tests/sdk_server_fixture.py@@ -59,6 +59,8 @@ def main():for name in ["english", "multilingual", "typed-decisions"]:router.attach(name, agent)os.environ["LAYA_API_KEY"] = "integration-test"+ # This fixture exercises the default HTTP contract, including its confidence extension.+ os.environ["LAYA_JEV_STRICT"] = "0"app = create_app(router)with socket.socket() as listener:listener.bind(("127.0.0.1", 0))diff --git a/tests/test_hooks_api.py b/tests/test_hooks_api.pyindex 9b4b0c6d..4cf19f9a 100644--- a/tests/test_hooks_api.py+++ b/tests/test_hooks_api.py@@ -1522,6 +1522,14 @@ check("agreement provenance is required",inspect.signature(evaluate_agreement).parameters["student_provenance"].default,inspect.Parameter.empty)+# The TypeScript HTTP confidence extension follows this core contract, not entropy confidence.+from laya.confidence import jev_confidence # noqa: E402++check("jev_confidence signature", list(inspect.signature(jev_confidence).parameters), ["answer"])+check("Jev choice confidence", round(jev_confidence({"type": "choice", "probabilities": {"a": 0.8, "b": 0.2}}), 4), 0.6)+check("Jev score confidence", round(jev_confidence({"type": "score", "probabilities": {"0": 0.1, "1": 0.8, "2": 0.1}}), 4), 0.7)+check("Jev noul confidence absent", jev_confidence({"type": "noul", "noul": 0.9}), None)+print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))for f in FAIL:print(" FAIL", f)