Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(sdk): align TypeScript confidence contract with HTTP responses

debjyotikmf439434ff040
diff --git a/sdk/typescript/README.md b/sdk/typescript/README.md
index 32ed6cba..efab44c6 100644
--- a/sdk/typescript/README.md
+++ b/sdk/typescript/README.md
@@ -75,6 +75,12 @@ console.log(result.answers.refund_requested.noul); // P(true)
console.log(result.routing?.model); // Laya-only metadata
```
+Choice and score answers may also contain `x_jev_confidence`, the HTTP server's
+Jev-compatible confidence measure. It is distinct from Laya's `confidence` and
+`answer_confidence`, and is absent on older servers, in strict Jev mode, and on
+`noul` answers. The client preserves it and checks that it is a finite number in
+`[0, 1]` when present; it does not compute or substitute it when absent.
+
CommonJS works too:
```js
diff --git a/sdk/typescript/src/response.ts b/sdk/typescript/src/response.ts
index ea1e1e4a..8c9db256 100644
--- a/sdk/typescript/src/response.ts
+++ b/sdk/typescript/src/response.ts
@@ -92,6 +92,11 @@ export function validatePrediction(value: unknown, questions: Questions): void {
if (answer.answer_confidence !== undefined) {
expect(probability(answer.answer_confidence), `answers.${id}.answer_confidence`);
}
+ // HTTP-only metadata: older servers omit it, and Jev defines none for noul.
+ if (answer.x_jev_confidence !== undefined) {
+ expect(question.type !== 'noul' && probability(answer.x_jev_confidence),
+ `answers.${id}.x_jev_confidence`);
+ }
if (answer.low_confidence !== undefined) {
expect(answer.low_confidence === true, `answers.${id}.low_confidence`);
}
diff --git a/sdk/typescript/src/types.ts b/sdk/typescript/src/types.ts
index 4515e96f..51b6ef14 100644
--- a/sdk/typescript/src/types.ts
+++ b/sdk/typescript/src/types.ts
@@ -88,12 +88,16 @@ interface AnswerBase {
export interface ChoiceAnswer<Label extends string = string> extends AnswerBase {
type: 'choice';
confidence: number;
+ /** Jev-compatible confidence added by HTTP servers; absent on older or strict-mode servers. */
+ x_jev_confidence?: number;
choice: Label;
probabilities: Record<Label, number>;
}
export interface ScoreAnswer extends AnswerBase {
type: 'score';
confidence: number;
+ /** Jev-compatible confidence added by HTTP servers; absent on older or strict-mode servers. */
+ x_jev_confidence?: number;
/** Expected zero-based rubric index, not a normalized probability. */
score: number;
legend: Record<string, JsonValue>;
diff --git a/sdk/typescript/test/client.test.mjs b/sdk/typescript/test/client.test.mjs
index d67477f0..1706cbe6 100644
--- a/sdk/typescript/test/client.test.mjs
+++ b/sdk/typescript/test/client.test.mjs
@@ -93,6 +93,38 @@ test('default request omits model and supports every Laya answer type', async ()
assert.deepEqual(allAnswerQuestions.team.criteria, ['billing', 'support'], 'normalization must not mutate the schema');
});
+test('HTTP Jev confidence is preserved and remains optional for older servers', async () => {
+ for (const value of [undefined, 0, 0.6, 1]) {
+ const payload = structuredClone(allAnswerPrediction);
+ if (value !== undefined) {
+ payload.answers.team.x_jev_confidence = value;
+ payload.answers.urgency.x_jev_confidence = value;
+ }
+ const client = new Laya({ fetch: async () => json(payload) });
+ const result = await client.predict('hello', allAnswerQuestions);
+ assert.deepEqual(result, payload);
+ assert.equal(Object.hasOwn(result.answers.team, 'x_jev_confidence'), value !== undefined);
+ assert.equal(Object.hasOwn(result.answers.urgency, 'x_jev_confidence'), value !== undefined);
+ assert.equal(Object.hasOwn(result.answers.refund, 'x_jev_confidence'), false);
+ }
+});
+
+test('malformed or noul Jev confidence is rejected', async () => {
+ for (const id of ['team', 'urgency']) {
+ for (const value of [null, false, true, '0.6', [], {}, -0.01, 1.01, NaN, Infinity, -Infinity]) {
+ const payload = structuredClone(allAnswerPrediction);
+ payload.answers[id].x_jev_confidence = value;
+ const client = new Laya({ fetch: async () => json(payload) });
+ await assert.rejects(client.predict('hello', allAnswerQuestions), error =>
+ error instanceof LayaResponseError && error.message.includes(`answers.${id}.x_jev_confidence`));
+ }
+ }
+ const payload = structuredClone(allAnswerPrediction);
+ payload.answers.refund.x_jev_confidence = 0.9;
+ await assert.rejects(new Laya({ fetch: async () => json(payload) }).predict('hello', allAnswerQuestions),
+ LayaResponseError);
+});
+
test('client model default and prediction overrides select local checkpoints', async () => {
const models = [];
const client = new Laya({ model: 'english', fetch: async (_url, init) => {
diff --git a/sdk/typescript/test/integration.mjs b/sdk/typescript/test/integration.mjs
index 4233225a..2e5a1534 100644
--- a/sdk/typescript/test/integration.mjs
+++ b/sdk/typescript/test/integration.mjs
@@ -7,6 +7,70 @@ import { setTimeout as delay } from 'node:timers/promises';
import { test } from 'node:test';
import { Laya, LayaAPIError, triageQuestions, emailQuestions, guardQuestions, moderationQuestions, routerQuestions } from 'laya-client';
+function assertPredictionParity(result, expected) {
+ const { routing, ...prediction } = structuredClone(result);
+ for (const [id, answer] of Object.entries(expected.answers)) {
+ const actual = prediction.answers[id];
+ if (answer.type === 'noul') {
+ assert.equal(Object.hasOwn(actual, 'x_jev_confidence'), false, `${id}: noul has no Jev confidence`);
+ continue;
+ }
+ // Derive the HTTP extension independently from the direct model's probabilities,
+ // not from the server's enrichment helper or from the received HTTP probabilities.
+ const keys = Object.keys(answer.probabilities);
+ if (answer.type === 'score') keys.sort((a, b) => Number(a) - Number(b));
+ const values = keys.map(key => answer.probabilities[key]);
+ const n = values.length;
+ const total = values.reduce((a, b) => a + b, 0);
+ const p = values.map(value => total ? value / total : 1 / n);
+ let confidence = 1;
+ if (n > 1 && answer.type === 'choice') {
+ confidence = (Math.max(...p) - 1 / n) / (1 - 1 / n);
+ } else if (n > 1) {
+ const mode = p.indexOf(Math.max(...p));
+ const spread = p.reduce((sum, value, i) => sum + value * Math.abs(i - mode), 0);
+ const uniformSpread = p.reduce((sum, _, i) => sum + Math.abs(i - (n - 1) / 2) / n, 0);
+ confidence = Math.max(0, 1 - spread / uniformSpread);
+ }
+ assert.equal(typeof actual.x_jev_confidence, 'number', `${id}: HTTP Jev confidence missing`);
+ assert.ok(Math.abs(actual.x_jev_confidence - confidence) <= 0.0000500001,
+ `${id}: HTTP Jev confidence must match the formula rounded to four decimals`);
+ delete actual.x_jev_confidence;
+ }
+ assert.deepEqual(prediction, expected, 'SDK core answers must exactly match direct Python inference');
+}
+
+test('parity checks the HTTP extension without weakening core prediction equality', () => {
+ const expected = { model: 'test', usage: { input_tokens: 1, output_tokens: 0 }, answers: {
+ choice: { type: 'choice', probabilities: { a: 0.8, b: 0.2 }, choice: 'a' },
+ score: { type: 'score', probabilities: { '0': 0.1, '1': 0.8, '2': 0.1 }, score: 1 },
+ single: { type: 'choice', probabilities: { only: 1 }, choice: 'only' },
+ singleScore: { type: 'score', probabilities: { '0': 1 }, score: 0 },
+ noul: { type: 'noul', noul: 0.9 },
+ } };
+ const actual = structuredClone(expected);
+ for (const [id, value] of Object.entries({ choice: 0.6, score: 0.7, single: 1, singleScore: 1 })) {
+ actual.answers[id].x_jev_confidence = value;
+ }
+ const before = structuredClone(actual);
+ assertPredictionParity(actual, expected);
+ assert.deepEqual(actual, before, 'parity check must not mutate the response');
+ for (const mutate of [
+ r => { delete r.answers.choice.x_jev_confidence; },
+ r => { r.answers.choice.x_jev_confidence = 0.61; },
+ r => { r.answers.score.x_jev_confidence = 0.8; },
+ r => { r.answers.noul.x_jev_confidence = 0.9; },
+ r => { r.answers.choice.probabilities.a = 0.7; },
+ r => { r.answers.choice.choice = 'b'; },
+ r => { r.usage.input_tokens = 2; },
+ r => { r.answers.score.unexpected = true; },
+ ]) {
+ const broken = structuredClone(actual);
+ mutate(broken);
+ assert.throws(() => assertPredictionParity(broken, expected), assert.AssertionError);
+ }
+});
+
test('JavaScript → HTTP → Python Router → real offline Agent inference', { timeout: 90_000 }, async t => {
const script = fileURLToPath(new URL('../../../tests/sdk_server_fixture.py', import.meta.url));
const server = spawn(process.env.PYTHON ?? 'python3', [script], { stdio: ['ignore', 'pipe', 'pipe'] });
@@ -42,9 +106,8 @@ test('JavaScript → HTTP → Python Router → real offline Agent inference', {
assert.ok(ready, `Server never became ready: ${stderr}`);
const result = await client.predict(fixture.state, fixture.questions, { model: 'english' });
- const { routing, ...prediction } = result;
- assert.deepEqual(prediction, fixture.expected, 'SDK answers must exactly match direct Python inference');
- assert.equal(routing.model, 'english');
+ assertPredictionParity(result, fixture.expected);
+ assert.equal(result.routing.model, 'english');
assert.equal(result.answers.single.choice, 'only');
assert.equal(result.answers.single.probabilities.only, 1);
assert.ok(result.usage.input_tokens > 0);
diff --git a/sdk/typescript/test/types.ts b/sdk/typescript/test/types.ts
index 078a0f2b..63c83c47 100644
--- a/sdk/typescript/test/types.ts
+++ b/sdk/typescript/test/types.ts
@@ -17,6 +17,14 @@ const noul: number = result.answers.refund.noul;
const routingModel: string | undefined = result.routing?.model;
const actionProbability: number | undefined = result.answers.team.action?.act_probability;
const noulConfidence: number | undefined = result.answers.refund.confidence;
+const choiceJevConfidence: number | undefined = result.answers.team.x_jev_confidence;
+const scoreJevConfidence: number | undefined = result.answers.priority.x_jev_confidence;
+// @ts-expect-error Older and strict-mode servers omit the HTTP confidence extension.
+const requiredJevConfidence: number = result.answers.team.x_jev_confidence;
+// @ts-expect-error Jev defines no confidence extension for noul answers.
+result.answers.refund.x_jev_confidence;
+// @ts-expect-error Jev confidence is numeric when present.
+const textJevConfidence: ChoiceAnswer['x_jev_confidence'] = '0.6';
const stateTokens: number = result.usage.state_tokens;
const stateTokensDropped: number = result.usage.state_tokens_dropped;
const truncated: boolean = result.usage.truncated;
diff --git a/tests/sdk_server_fixture.py b/tests/sdk_server_fixture.py
index 15a02467..36ac2a98 100644
--- a/tests/sdk_server_fixture.py
+++ b/tests/sdk_server_fixture.py
@@ -59,6 +59,8 @@ def main():
for name in ["english", "multilingual", "typed-decisions"]:
router.attach(name, agent)
os.environ["LAYA_API_KEY"] = "integration-test"
+ # This fixture exercises the default HTTP contract, including its confidence extension.
+ os.environ["LAYA_JEV_STRICT"] = "0"
app = create_app(router)
with socket.socket() as listener:
listener.bind(("127.0.0.1", 0))
diff --git a/tests/test_hooks_api.py b/tests/test_hooks_api.py
index 9b4b0c6d..4cf19f9a 100644
--- a/tests/test_hooks_api.py
+++ b/tests/test_hooks_api.py
@@ -1522,6 +1522,14 @@ check("agreement provenance is required",
inspect.signature(evaluate_agreement).parameters["student_provenance"].default,
inspect.Parameter.empty)
+# The TypeScript HTTP confidence extension follows this core contract, not entropy confidence.
+from laya.confidence import jev_confidence # noqa: E402
+
+check("jev_confidence signature", list(inspect.signature(jev_confidence).parameters), ["answer"])
+check("Jev choice confidence", round(jev_confidence({"type": "choice", "probabilities": {"a": 0.8, "b": 0.2}}), 4), 0.6)
+check("Jev score confidence", round(jev_confidence({"type": "score", "probabilities": {"0": 0.1, "1": 0.8, "2": 0.1}}), 4), 0.7)
+check("Jev noul confidence absent", jev_confidence({"type": "noul", "noul": 0.9}), None)
+
print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))
for f in FAIL:
print(" FAIL", f)