Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(ts): honour a question's `option_order` across encoding and decoding

Aashishe894d167a2e1
diff --git a/laya-ts/src/agent.ts b/laya-ts/src/agent.ts
index dfa8077e..40013c11 100644
--- a/laya-ts/src/agent.ts
+++ b/laya-ts/src/agent.ts
@@ -17,8 +17,9 @@ import {
serializeState,
softmax,
tempBucket,
+ unpermuteProbs,
} from "./common.js";
-import type { BinningMap, MinConfidence, OptionStats, SequenceStats } from "./common.js";
+import type { BinningMap, InternalQ, MinConfidence, OptionStats, SequenceStats } from "./common.js";
import type { Batch, SessionProvider } from "./providers.js";
import { encodeWithData, parseTokenizerJson, type TokenizerLike } from "./tokenizer.js";
import { decide, type DecideOptions, type DecisionResult } from "./structured.js";
@@ -170,6 +171,15 @@ function qidStr(qid: string): string {
return JSON.stringify(qid);
}
+/** How many options a validated question definition renders to -- `laya.agent._option_count`
+ * (`laya/agent.py:321-330`): a choice has one per criterion, a score one per level, a noul is always
+ * the pair [false, true]. */
+function optionCount(qdef: Record<string, unknown>): number {
+ if (qdef["type"] === "noul") return 2;
+ const crit = qdef["criteria"];
+ return Array.isArray(crit) || (typeof crit === "object" && crit !== null) ? Object.keys(crit).length : 0;
+}
+
export function checkQuestion(qid: string, qdef: unknown): void {
if (typeof qid !== "string" || !qid.trim()) {
throw new Error(`question id must be a non-empty string, got ${qidStr(qid)}`);
@@ -290,6 +300,29 @@ export function checkQuestion(qid: string, qdef: unknown): void {
);
}
}
+ // Slot `s` of the head's row shows option `order[s]`, so an order that is not a permutation of the
+ // option indices would either drop an option or show one twice. Python refuses it here
+ // (`laya/agent.py:1088-1099`) rather than let it reach the encoder, and the same numbers say why:
+ // the decoder un-permutes by the same list, so a short order leaves slots unmapped.
+ if (Object.hasOwn(q, "option_order")) {
+ const order = q["option_order"];
+ const n = optionCount(q);
+ const ints = Array.isArray(order)
+ // Python keeps `isinstance(i, int) and not isinstance(i, bool)`, which rejects a float option
+ // index; JavaScript has one number type, so an integral value is the closest match, and
+ // `[1.0, 0.0]` is not a value a caller can even write here.
+ ? order.filter((i) => typeof i === "number" && Number.isInteger(i))
+ : null;
+ const isPermutation =
+ ints !== null && ints.length === n &&
+ [...ints].sort((a, b) => a - b).join(",") === Array.from({ length: n }, (_, i) => i).join(",");
+ if (!isPermutation) {
+ throw new Error(
+ `question ${qidStr(qid)}: 'option_order' must be a permutation of range(${n}) -- one slot per ` +
+ `option, each option once -- got ${JSON.stringify(order)}`,
+ );
+ }
+ }
if ("labels" in q && t !== "noul") {
throw new Error(`question ${qidStr(qid)}: 'labels' is only supported for noul questions`);
}
@@ -332,7 +365,7 @@ function isEmptyOverride(v: unknown): boolean {
return Array.isArray(v) && v.length === 0; // [] -- truthy in JavaScript
}
-export function toInternal(qdef: QuestionDef): { t: "choice" | "score" | "noul"; ins: string; crit: unknown; labels?: { false: string; true: string } } {
+export function toInternal(qdef: QuestionDef): InternalQ {
const t = qdef["type"] as "choice" | "score" | "noul";
let crit: unknown = qdef["criteria"];
if (t === "choice" && Array.isArray(crit)) {
@@ -342,13 +375,19 @@ export function toInternal(qdef: QuestionDef): { t: "choice" | "score" | "noul";
}
let ins: unknown = qdef["instructions"];
if (typeof ins !== "string") ins = serializeState(ins);
- const out: { t: "choice" | "score" | "noul"; ins: string; crit: unknown; labels?: { false: string; true: string } } = {
+ const out: InternalQ = {
t, ins: ins as string, crit,
};
if (t === "noul" && qdef["labels"] && typeof qdef["labels"] === "object" && !Array.isArray(qdef["labels"])) {
const labels = qdef["labels"] as Record<string, unknown>;
out.labels = { false: String(labels["false"]).trim(), true: String(labels["true"]).trim() };
}
+ // `Agent._to_internal` carries the validated order onto the internal question (`laya/agent.py:1128-1129`),
+ // which is what lets one object drive both halves: `_encodeItems` presents options in it and
+ // `_decodeRow` inverts the row back out of it.
+ if (Object.hasOwn(qdef, "option_order")) {
+ out.optionOrder = (qdef["option_order"] as number[]).map((i) => Math.trunc(Number(i)));
+ }
return out;
}
@@ -590,7 +629,7 @@ export class Agent extends HookRegistry {
private _encodeItems(
state: unknown,
ids: string[],
- internals: { t: "choice" | "score" | "noul"; ins: string; crit: unknown }[],
+ internals: InternalQ[],
maxLen: number,
headMaxLen: number,
): { items: { ids: number[]; markers: number[]; qtype: number }[]; nTokens: number; stats: SequenceStats[] } {
@@ -602,7 +641,7 @@ export class Agent extends HookRegistry {
for (let qi = 0; qi < ids.length; qi++) {
const qid = ids[qi];
const q = internals[qi];
- const prefix = buildQuestionPrefix(this.tok, q, maxLen, headMaxLen);
+ const prefix = buildQuestionPrefix(this.tok, q, maxLen, headMaxLen, q.optionOrder);
const seq = sequenceWithState(
prefix,
stAll,
@@ -686,7 +725,13 @@ export class Agent extends HookRegistry {
for (let qi = 0; qi < ids.length; qi++) {
answers[ids[qi]] = this._decodeRow(
logits[row] as number[],
- (act[row] as number[]) ?? [1, 0],
+ // The action head is normalised here, at the model boundary, as `laya/agent.py`'s
+ // `_forward` (agent.py:1329) and `laya/onnx_agent.py` (onnx_agent.py:805) do: the decode
+ // layer's contract -- the Java decoder's `actionProbabilities`, and the `act` rows in
+ // `laya-java/fixtures/decode.json` -- is a probability row. Softmaxing it a second time
+ // inside `_decodeRow` reported 0.6899 for the row `[0.9, 0.1]`, where the reference
+ // reports 0.9.
+ softmax((act[row] as number[]) ?? [1, 0]),
internals[qi],
built[s].items[qi].markers.length,
opts.lang ?? null,
@@ -720,11 +765,15 @@ export class Agent extends HookRegistry {
return out;
}
- /** Decode one head row into the answer for its question. */
+ /**
+ * Decode one head row into the answer for its question. `actRow` is the model boundary's
+ * normalised action-probability row, so its first entry is the reported `act_probability` --
+ * exactly what `Agent._decode_answers` reads as `act[r, 0]`.
+ */
private _decodeRow(
logitsRow: number[],
actRow: number[],
- q: { t: "choice" | "score" | "noul"; ins: string; crit: unknown },
+ q: InternalQ,
k: number,
lang: string | null = null,
): SystemAnswer {
@@ -738,9 +787,13 @@ export class Agent extends HookRegistry {
? (langCfg.temperatureByOptions[bucket] ?? langCfg.temperature[qt] ?? 1.0)
: (this.temperatureByOptions[bucket] ?? this.temperature[qt] ?? 1.0);
const z = logitsRow.slice(0, k).map((v) => v / scale);
- const p = softmax(z);
- const actP = softmax(actRow.slice(0, Math.max(2, actRow.length)));
- const ext = { act_probability: r4(actP[0]) };
+ // The row comes back in slot order and everything below indexes by option, so invert it
+ // here -- `p.argmax()`/`keys[best]`, `i * p[i]` and `p[1]` are all option-indexed. Same
+ // position as `laya/agent.py:1350`, immediately after the softmax.
+ const p = unpermuteProbs(softmax(z), q.optionOrder);
+ // Read, not re-normalised: the row arrives as probabilities. The `slice(0, Math.max(2, ...))`
+ // this replaces could only pad an Act row that validation had already refused for being empty.
+ const ext = { act_probability: r4(actRow[0]) };
// Same quantity on every question type (max(p)), so callers can gate across types on
// one number; `confidence` stays as-is for existing callers (entropy for choice/score).
// Histogram-binning recalibration (issue #871, Python parity) recalibrates ans_raw
diff --git a/laya-ts/src/common.ts b/laya-ts/src/common.ts
index ec21ccbe..bd697073 100644
--- a/laya-ts/src/common.ts
+++ b/laya-ts/src/common.ts
@@ -1,6 +1,14 @@
import type { TokenizerLike } from "./tokenizer.js";
export type QType = "choice" | "score" | "noul";
-export interface InternalQ { t: QType; ins: string; crit: unknown; labels?: { false: string; true: string } }
+export interface InternalQ {
+ t: QType;
+ ins: string;
+ crit: unknown;
+ labels?: { false: string; true: string };
+ /** The caller's option indices in the order the head should see them: slot `s` shows option
+ `optionOrder[s]`. Set by `toInternal` from a question's `option_order`. */
+ optionOrder?: number[];
+}
/** Python `json.dumps` for a finite number. JavaScript prints the same shortest round-trip
digits, but Python switches to exponent notation below 1e-4 (JavaScript: below 1e-6) and pads the
exponent to two digits (`5e-05`, not `5e-5`). An integer-valued number prints as before, which
@@ -197,6 +205,20 @@ export function buildSequence(tok: TokenizerLike, state: unknown, q: InternalQ,
const stAll = tok.encode(serializeState(state).split(tok.maskToken).join(" "));
return sequenceWithState(buildQuestionPrefix(tok, q, maxLen, headMaxLen, optionOrder), stAll, tok.sepId, maxLen, truncateLeft);
}
+/** Put a slot-ordered probability row back into the caller's option order.
+ *
+ * `buildQuestionPrefix` puts option `order[s]` in slot `s`, so a head row comes back indexed by
+ * slot, while everything downstream indexes by option -- `keys[best]` for a choice, `i * p[i]` for a
+ * score level, `p[1]` for noul-true. Without the inverse the probabilities stay attached to the
+ * wrong options, which is silent. Mirrors `laya.common.unpermute_probs` (`laya/common.py:557`): a
+ * missing order, or one whose length is not the row's, returns the row untouched, so the canonical
+ * path is unaffected. */
+export function unpermuteProbs(p: number[], order?: number[] | null): number[] {
+ if (!order || order.length !== p.length) return p;
+ const canonical = new Array<number>(p.length);
+ for (let s = 0; s < order.length; s++) canonical[order[s]] = p[s];
+ return canonical;
+}
export function softmax(z: number[]): number[] {
const m = Math.max(...z);
const e = z.map((v) => Math.exp(v - m));
diff --git a/laya-ts/src/shortlist.ts b/laya-ts/src/shortlist.ts
index 1d1fdca7..83cb7eb4 100644
--- a/laya-ts/src/shortlist.ts
+++ b/laya-ts/src/shortlist.ts
@@ -136,6 +136,34 @@ function subsetCriteria(criteria: unknown, labels: string[]): unknown {
return Object.fromEntries(labels.map((l) => [l, (criteria as Record<string, unknown>)[l]]));
}
+/** Return `question` without an `option_order` that no longer fits it.
+ *
+ * `option_order` holds one slot per option, so an order that fitted the caller's label set does not
+ * fit a narrowed one -- and once the agent reads the field at all, that mismatch is a refused
+ * question, not a silently ignored one. Remapping the order onto the kept labels would move options
+ * between slots, so it is dropped instead, once per question per call. Mirrors
+ * `laya.shortlist._drop_stale_order` (`laya/shortlist.py:428`), which both `predict_shortlist` and
+ * `predict_tournament` run on every narrowed choice; a passthrough keeps its order. */
+function dropStaleOrder(qid: string, question: Record<string, unknown>, nOptions: number,
+ warned: Set<string>): Record<string, unknown> {
+ if (!Object.hasOwn(question, "option_order")) return question;
+ const out = { ...question };
+ const order = out["option_order"];
+ delete out["option_order"];
+ if (!warned.has(qid)) {
+ warned.add(qid);
+ // The same shape as the reference's warning: the slot count the order carried, so a caller can
+ // see how much of the label set the narrowing dropped.
+ const had = Array.isArray(order) ? `${order.length} slots` : JSON.stringify(order);
+ console.warn(
+ `laya.shortlist: question ${JSON.stringify(qid)}: 'option_order' (${had}) was dropped because the ` +
+ `label set was narrowed to ${nOptions}; the order no longer fits the kept options and remapping ` +
+ `it onto them would move options between slots.`,
+ );
+ }
+ return out;
+}
+
/** Return the top-k choice labels for state. Skips embed_fn when k >= n. */
export async function shortlistChoice(
state: unknown,
@@ -172,6 +200,7 @@ export async function predictShortlist(
const embed = embedFn ?? (needsEmbed ? embedFnFromAgent(agent) : null);
const reduced: Record<string, unknown> = {};
const meta: Record<string, ShortlistMeta> = {};
+ const warned = new Set<string>();
for (const [qid, qdef] of Object.entries(questions)) {
if (typeof qdef !== "object" || qdef === null || Array.isArray(qdef) || (qdef as Record<string, unknown>)["type"] !== "choice") {
reduced[qid] = qdef;
@@ -187,7 +216,8 @@ export async function predictShortlist(
reduced[qid] = qdef;
continue;
}
- reduced[qid] = { ...q, criteria: subsetCriteria(q["criteria"], labels) };
+ reduced[qid] = dropStaleOrder(
+ qid, { ...q, criteria: subsetCriteria(q["criteria"], labels) }, labels.length, warned);
}
const a = agent as Record<string, unknown>;
const fn = a?.["predict"] ?? a?.["systemOne"] ?? a?.["system_one"];
@@ -248,20 +278,12 @@ export async function predictTournament(
return result as SystemOneResult;
};
const warned = new Set<string>();
- const cut = (qid: string, labels: string[]): QuestionDef => {
- const original = questions[qid];
- const narrowed: QuestionDef = { ...original, criteria: subsetCriteria(original["criteria"], labels) };
- if (Object.hasOwn(narrowed, "option_order")) {
- delete narrowed["option_order"];
- if (!warned.has(qid)) {
- warned.add(qid);
- console.warn(
- `laya.shortlist: question ${JSON.stringify(qid)}: 'option_order' was dropped because the label set was narrowed to ${labels.length}; the order no longer fits the kept options.`,
- );
- }
- }
- return narrowed;
- };
+ const cut = (qid: string, labels: string[]): QuestionDef => dropStaleOrder(
+ qid,
+ { ...questions[qid], criteria: subsetCriteria(questions[qid]["criteria"], labels) },
+ labels.length,
+ warned,
+ ) as QuestionDef;
while (true) {
const groups: { qid: string; labels: string[] }[] = [];
diff --git a/laya-ts/tests/option-order.test.ts b/laya-ts/tests/option-order.test.ts
new file mode 100644
index 00000000..3c41a396
--- /dev/null
+++ b/laya-ts/tests/option-order.test.ts
@@ -0,0 +1,391 @@
+// laya-ts/tests/option-order.test.ts — a question's `option_order` reaches both halves of the
+// prediction: the prompt presents option `order[s]` in slot `s`, and the decoded row is put back
+// into the caller's option order before an answer is chosen.
+//
+// The expected answers are not hand-computed. They are printed by `/tmp/probe_option_order.py`,
+// which calls laya's own `Agent._decode_answers` through the same `Agent.__new__` shim
+// `laya-java/scripts/gen_fixtures.py:1838-1860` uses to build `decode.json` -- clamped temperature
+// tables, an `[0.25, -0.5]` action row, a logits block three columns wider than the option count --
+// so the numbers below are the reference decoder's output for these rows, not a restatement of this
+// port's arithmetic.
+import { describe, expect, it, vi } from "vitest";
+import { Agent, checkQuestion, toInternal } from "../src/agent.js";
+import {
+ buildQuestionPrefix,
+ renderOptions,
+ unpermuteProbs,
+ type InternalQ,
+} from "../src/common.js";
+import { predictShortlist } from "../src/shortlist.js";
+import type { TokenizerLike } from "../src/tokenizer.js";
+
+type QuestionDef = Record<string, unknown>;
+
+const padded = (logits: number[]): number[] => [...logits, 99.0, 99.0, 99.0];
+
+/** `Agent._decodeRow` at the same seam `decode-parity.test.ts` uses. */
+function decodeRow(agent: Agent, q: InternalQ, logits: number[]): Record<string, unknown> {
+ const row = (agent as unknown as {
+ _decodeRow(
+ logits: number[], act: number[], q: InternalQ, k: number, lang: string | null,
+ ): Record<string, unknown>;
+ })._decodeRow;
+ return row.call(agent, padded(logits), [0.25, -0.5], q, logits.length, null);
+}
+
+const stubAgent = () => new Agent({
+ provider: {
+ async runEncoder() { return { lastHidden: [[1, 0]] }; },
+ async runHead() { return { logits: [[2, 0]], act: [[1, 0]] }; },
+ },
+} as never);
+
+describe("option_order validation", () => {
+ const choice = (order: unknown): QuestionDef => ({
+ type: "choice", instructions: "pick", criteria: { a: "1", b: "2", c: "3" }, option_order: order,
+ });
+
+ it("accepts every permutation of the option indices", () => {
+ for (const order of [[0, 1, 2], [2, 0, 1], [1, 2, 0]]) {
+ expect(() => checkQuestion("q", choice(order))).not.toThrow();
+ }
+ });
+
+ it("refuses an order that is not one option per slot", () => {
+ // Each arm is a different way an order loses or duplicates an option. All of them would reach
+ // the encoder silently if the field were not validated here; Python refuses them at the same
+ // point (`laya/agent.py:1088-1099`).
+ const bad: unknown[] = [
+ [0, 1], [0, 1, 2, 3], [0, 0, 1], [2, 0, 3], [1, 2], [], [0, 1, "2"], [0, 1, 1.5],
+ ["a", "b", "c"], [true, false, true], 2, null, { 0: 1 },
+ ];
+ for (const order of bad) {
+ expect(() => checkQuestion("q", choice(order))).toThrow(/permutation of range\(3\)/);
+ }
+ });
+
+ it("names the question, the required range and the value it refused", () => {
+ expect(() => checkQuestion("q", choice([3, 0, 0]))).toThrow(
+ /question "q": 'option_order' must be a permutation of range\(3\) -- one slot per option, each option once -- got \[3,0,0\]/,
+ );
+ });
+
+ it("sizes a score by its levels and a noul by its pair", () => {
+ expect(() => checkQuestion("q", {
+ type: "score", instructions: "rate", criteria: ["bad", "ok", "good"], option_order: [2, 0, 1],
+ })).not.toThrow();
+ expect(() => checkQuestion("q", {
+ type: "score", instructions: "rate", criteria: ["bad", "ok", "good"], option_order: [1, 0],
+ })).toThrow(/permutation of range\(3\)/);
+ expect(() => checkQuestion("q", {
+ type: "noul", instructions: "holds", option_order: [1, 0],
+ })).not.toThrow();
+ expect(() => checkQuestion("q", {
+ type: "noul", instructions: "holds", option_order: [0, 1, 2],
+ })).toThrow(/permutation of range\(2\)/);
+ });
+
+ it("counts a criteria list as one option per entry", () => {
+ expect(() => checkQuestion("q", {
+ type: "choice", instructions: "pick", criteria: ["a", "b"], option_order: [1, 0],
+ })).not.toThrow();
+ expect(() => checkQuestion("q", {
+ type: "choice", instructions: "pick", criteria: ["a", "b"], option_order: [1, 0, 0],
+ })).toThrow(/permutation of range\(2\)/);
+ });
+});
+
+describe("toInternal carries the order", () => {
+ it("sets optionOrder from the definition", () => {
+ expect(toInternal({
+ type: "choice", instructions: "pick", criteria: { a: "1", b: "2" }, option_order: [1, 0],
+ } as never).optionOrder).toEqual([1, 0]);
+ });
+
+ it("leaves it unset when the question has none", () => {
+ const q = toInternal({ type: "choice", instructions: "pick", criteria: { a: "1" } } as never);
+ expect(Object.hasOwn(q, "optionOrder")).toBe(false);
+ });
+
+ it("truncates a numeric index the way int() does", () => {
+ // Validation is what normally guarantees integral entries; the reference still writes `int(i)`
+ // (`laya/agent.py:1129`) onto the internal question, so the port cannot hold a fractional index.
+ expect(toInternal({
+ type: "choice", instructions: "pick", criteria: { a: "1", b: "2", c: "3" },
+ option_order: [2.7, 0.1, 1.9],
+ } as never).optionOrder).toEqual([2, 0, 1]);
+ });
+});
+
+describe("unpermuteProbs", () => {
+ it("maps slot s back onto option order[s]", () => {
+ expect(unpermuteProbs([0.1, 0.6, 0.3], [1, 2, 0])).toEqual([0.3, 0.1, 0.6]);
+ });
+
+ it("is the inverse of presenting the options in that order", () => {
+ const opts = renderOptions({ t: "choice", ins: "i", crit: { a: "1", b: "2", c: "3" } });
+ const order = [2, 0, 1];
+ // Slot s carries option `order[s]`, so inverting the slot row has to land every value on its own
+ // option -- asserted against the rendered option texts, not against another formula.
+ const back = unpermuteProbs([0.7, 0.2, 0.1], order);
+ expect(back.map((v, i) => [opts[i], v])).toEqual([["a: 1", 0.2], ["b: 2", 0.1], ["c: 3", 0.7]]);
+ });
+
+ it("returns the row untouched without an order or when the lengths disagree", () => {
+ // `laya/common.py:567` leaves the row alone for None and for a length mismatch. The mismatch is
+ // the arm that matters: a head row can be shorter than the order once options are lost to the
+ // token budget, and re-indexing it would move probabilities onto options never shown.
+ expect(unpermuteProbs([0.6, 0.4])).toEqual([0.6, 0.4]);
+ expect(unpermuteProbs([0.6, 0.4], null)).toEqual([0.6, 0.4]);
+ expect(unpermuteProbs([0.6, 0.4], [0, 1, 2])).toEqual([0.6, 0.4]);
+ });
+});
+
+/** One id per encoded string, remembered so a slot's token traces back to the option it shows. */
+type IdentityTok = TokenizerLike & { idOf(text: string): number };
+
+function identityTokenizer(): IdentityTok {
+ const ids = new Map<string, number>();
+ const tok: IdentityTok = {
+ clsId: 101,
+ sepId: 102,
+ maskId: 103,
+ padId: 0,
+ maskToken: "[MASK]",
+ encode(text: string): number[] {
+ const key = text.trim();
+ if (!ids.has(key)) ids.set(key, 1000 + ids.size);
+ return [ids.get(key)!];
+ },
+ idOf(text: string): number {
+ return ids.get(text.trim()) ?? -1;
+ },
+ };
+ return tok;
+}
+
+// Criteria written as bare labels: `renderOptions` drops an empty description, so the text the
+// model sees for option `i` is exactly the key, which is what makes a slot traceable.
+const THREE = { billing: "", refund: "", technical: "" };
+
+describe("the prompt presents options in the caller's order", () => {
+ const q: InternalQ = { t: "choice", ins: "i", crit: THREE };
+ const opts = renderOptions(q);
+
+ it("puts option order[s] in slot s", () => {
+ for (const order of [[0, 1, 2], [2, 0, 1], [1, 2, 0]]) {
+ const tok = identityTokenizer();
+ const built = buildQuestionPrefix(tok, q, 512, 192, order);
+ expect(built.markers).toHaveLength(3);
+ // A slot's chunk is [maskId, ...option tokens], so the id right after a marker is the option.
+ order.forEach((optIdx, s) => {
+ expect(built.ids[built.markers[s] + 1]).toBe(tok.idOf(opts[optIdx]));
+ });
+ }
+ });
+
+ it("shows the same options the answer keys will name", () => {
+ const tok = identityTokenizer();
+ const built = buildQuestionPrefix(tok, q, 512, 192, [2, 0, 1]);
+ expect(built.markers.map((m) => built.ids[m + 1])).toEqual(
+ ["technical", "billing", "refund"].map((o) => tok.idOf(o)),
+ );
+ // Without the prompt half -- order ignored -- the same row would be decoded against the wrong
+ // options, which is the silent failure this field exists to prevent.
+ const canonical = buildQuestionPrefix(tok, q, 512, 192);
+ expect(canonical.ids[canonical.markers[0] + 1]).toBe(tok.idOf("billing"));
+ });
+});
+
+describe("the decoded row answers in the caller's option order", () => {
+ const a = stubAgent();
+
+ it("matches the reference on a rotated choice", () => {
+ expect(decodeRow(a, {
+ t: "choice", ins: "i", crit: { a: "1", b: "2", c: "3" }, optionOrder: [1, 2, 0],
+ }, [0.4, 2.2, 0.9])).toEqual({
+ type: "choice", choice: "c",
+ probabilities: { a: 0.1895, b: 0.115, c: 0.6955 },
+ confidence: 0.2568, answer_confidence: 0.6955,
+ action: { act_probability: 0.25 },
+ });
+ });
+
+ it("matches the reference on the same row read in canonical order", () => {
+ // The pair is the point: the logits are unchanged, only the slots differ, so the answer moves
+ // from `c` to `b` because the order says which option each slot is showing.
+ expect(decodeRow(a, {
+ t: "choice", ins: "i", crit: { a: "1", b: "2", c: "3" }, optionOrder: [0, 1, 2],
+ }, [0.4, 2.2, 0.9])).toEqual({
+ type: "choice", choice: "b",
+ probabilities: { a: 0.115, b: 0.6955, c: 0.1895 },
+ confidence: 0.2568, answer_confidence: 0.6955,
+ action: { act_probability: 0.25 },
+ });
+ });
+
+ it("matches the reference on a reversed four-option row", () => {
+ expect(decodeRow(a, {
+ t: "choice", ins: "i", crit: { a: "1", b: "2", c: "3", d: "4" }, optionOrder: [3, 2, 1, 0],
+ }, [0.5, 2.5, -1.0, 2.5])).toEqual({
+ type: "choice", choice: "a",
+ probabilities: { a: 0.4618, b: 0.0139, c: 0.4618, d: 0.0625 },
+ confidence: 0.3173, answer_confidence: 0.4618,
+ action: { act_probability: 0.25 },
+ });
+ });
+
+ it("matches the reference on a permuted score, where the level index is the value", () => {
+ expect(decodeRow(a, {
+ t: "score", ins: "i", crit: ["l0", "l1", "l2", "l3"], optionOrder: [2, 0, 3, 1],
+ }, [0.2, 1.4, 0.9, -0.3])).toEqual({
+ type: "score", score: 1.246,
+ legend: { "0": "l0", "1": "l1", "2": "l2", "3": "l3" },
+ probabilities: { "0": 0.4784, "1": 0.0874, "2": 0.1441, "3": 0.2901 },
+ confidence: 0.1316, answer_confidence: 0.4784,
+ action: { act_probability: 0.25 },
+ });
+ });
+
+ it("matches the reference on a flipped noul, where slot 0 is showing the true option", () => {
+ // Without the inversion `p[1]` is whatever the second slot happened to show, so a flipped noul
+ // reports the complement of the reference: 0.832 where laya reports 0.168.
+ expect(decodeRow(a, { t: "noul", ins: "i", crit: null, optionOrder: [1, 0] }, [0.3, 1.9])).toEqual({
+ type: "noul", noul: 0.168, confidence: 0.832, answer_confidence: 0.832,
+ action: { act_probability: 0.25 },
+ });
+ expect(decodeRow(a, { t: "noul", ins: "i", crit: null, optionOrder: [0, 1] }, [0.3, 1.9])).toEqual({
+ type: "noul", noul: 0.832, confidence: 0.832, answer_confidence: 0.832,
+ action: { act_probability: 0.25 },
+ });
+ });
+
+ it("leaves a row shorter than the order alone, as the reference does", () => {
+ // Options lost to the head budget make the row shorter than `option_order`; re-indexing by the
+ // order would name an option that has no span on screen.
+ expect(decodeRow(a, {
+ t: "choice", ins: "i", crit: { a: "1", b: "2" }, optionOrder: [0, 1, 2],
+ }, [2.0, 0.1])).toEqual({
+ type: "choice", choice: "a", probabilities: { a: 0.8699, b: 0.1301 },
+ confidence: 0.4423, answer_confidence: 0.8699,
+ action: { act_probability: 0.25 },
+ });
+ });
+
+ it("does not move an answer for a question with no order", () => {
+ const plain = { t: "choice", ins: "i", crit: { a: "1", b: "2", c: "3" } } as InternalQ;
+ expect(decodeRow(a, plain, [0.4, 2.2, 0.9]))
+ .toEqual(decodeRow(a, { ...plain, optionOrder: undefined }, [0.4, 2.2, 0.9]));
+ });
+});
+
+describe("predict() honours option_order end to end", () => {
+ it("shows the permuted options and answers in the caller's order", async () => {
+ let seen: { inputIds: number[][]; markerPos: number[][] } | null = null;
+ const tok = identityTokenizer();
+ const provider = {
+ async runEncoder(batch: { inputIds: number[][]; markerPos: number[][] }) {
+ seen = { inputIds: batch.inputIds, markerPos: batch.markerPos };
+ return { lastHidden: batch.inputIds.map((row: number[]) => row.map(() => [1, 0])) };
+ },
+ // The reference arm's row: under this order slot 0 shows `b`, slot 1 shows `c`, slot 2 `a`.
+ async runHead() { return { logits: [[0.4, 2.2, 0.9, 99.0, 99.0, 99.0]], act: [[1.0, 0.0]] }; },
+ };
+ const a = new Agent({ provider, tok } as never);
+ const r = await a.systemOne("s", {
+ q: {
+ type: "choice", instructions: "i",
+ criteria: { a: "", b: "", c: "" },
+ option_order: [1, 2, 0],
+ },
+ } as never);
+
+ // The prompt half, read back off the batch the model actually received.
+ const { inputIds, markerPos } = seen as unknown as { inputIds: number[][]; markerPos: number[][] };
+ expect(markerPos[0].map((m) => inputIds[0][m + 1])).toEqual(
+ ["b", "c", "a"].map((o) => tok.idOf(o)),
+ );
+ // The answer half: the winning slot is reported against the option it was showing, and the
+ // probabilities are keyed the way the reference keys them for this row and order.
+ expect(r.answers.q.type).toBe("choice");
+ const c = r.answers.q as typeof r.answers.q & { choice: string; probabilities: Record<string, number> };
+ expect(c.choice).toBe("c");
+ expect(c.probabilities).toEqual({ a: 0.1895, b: 0.115, c: 0.6955 });
+ });
+
+ it("refuses a bad order before the model is consulted", async () => {
+ let called = false;
+ const provider = {
+ async runEncoder(_b: unknown) { called = true; return { lastHidden: [[1, 0]] }; },
+ async runHead() { return { logits: [[1, 0]], act: [[1, 0]] }; },
+ };
+ const a = new Agent({ provider } as never);
+ await expect(a.systemOne("s", {
+ q: { type: "choice", instructions: "i", criteria: { a: "1", b: "2" }, option_order: [0, 2] },
+ } as never)).rejects.toThrow(/permutation of range\(2\)/);
+ expect(called).toBe(false);
+ });
+
+ it("changes nothing for a question that does not carry the field", async () => {
+ const provider = {
+ async runEncoder(b: { inputIds: number[][] }) {
+ return { lastHidden: b.inputIds.map((row: number[]) => row.map(() => [1, 0])) };
+ },
+ async runHead() { return { logits: [[0.4, 2.2, 0.9, 99.0, 99.0, 99.0]], act: [[1.0, 0.0]] }; },
+ };
+ const questions = (order?: number[]): Record<string, QuestionDef> => ({
+ q: {
+ type: "choice", instructions: "i", criteria: { a: "", b: "", c: "" },
+ ...(order === undefined ? {} : { option_order: order }),
+ },
+ });
+ const identity = await new Agent({ provider, tok: identityTokenizer() } as never)
+ .systemOne("s", questions([0, 1, 2]) as never);
+ const none = await new Agent({ provider, tok: identityTokenizer() } as never)
+ .systemOne("s", questions() as never);
+ expect(identity.answers).toEqual(none.answers);
+ expect(identity.usage).toEqual(none.usage);
+ // And the rotated order does move it, so the equality above is not two copies of one behaviour.
+ const rotated = await new Agent({ provider, tok: identityTokenizer() } as never)
+ .systemOne("s", questions([1, 2, 0]) as never);
+ expect(rotated.answers).not.toEqual(none.answers);
+ });
+});
+
+describe("a narrowed question loses an order that no longer fits it", () => {
+ // The drop already lived in `shortlist.ts`; the validation added here is what makes it
+ // load-bearing, because a one-option shortlist still carrying a three-slot order would now be
+ // refused outright. Python's remedy for the same case is the drop, not the refusal
+ // (`laya/shortlist.py:426`).
+ it("removes option_order from the narrowed definition and keeps the question valid", async () => {
+ const warn = vi.spyOn(console, "warn").mockImplementation(() => {});
+ let reduced: Record<string, QuestionDef> | null = null;
+ const stub = {
+ async predict(_s: unknown, q: Record<string, QuestionDef>) {
+ reduced = q;
+ return { model: "m", answers: {}, usage: { input_tokens: 0, output_tokens: 0 } };
+ },
+ } as never;
+ const embed = async (texts: string[]) =>
+ texts.map((t) => (t.includes("refund") ? [1, 0] : [0, 1]));
+ await predictShortlist(stub, "billed twice refund", {
+ d: {
+ type: "choice", instructions: "pick",
+ criteria: { card_arrival: "where is my card", transfer_fee: "fee on transfer", refund: "refund money" },
+ option_order: [2, 0, 1],
+ },
+ }, embed as never, 1);
+
+ const kept = reduced as unknown as Record<string, QuestionDef>;
+ expect(Object.keys(kept.d["criteria"] as object)).toEqual(["refund"]);
+ expect(Object.hasOwn(kept.d, "option_order")).toBe(false);
+ expect(warn).toHaveBeenCalledTimes(1);
+ expect(() => checkQuestion("d", kept.d)).not.toThrow();
+ // The counterfactual the drop exists to prevent: the same narrowed criteria still carrying the
+ // original three-slot order is refused by the validation above.
+ expect(() => checkQuestion("d", {
+ type: "choice", instructions: "pick", criteria: { refund: "refund money" }, option_order: [2, 0, 1],
+ })).toThrow(/permutation of range\(1\)/);
+ warn.mockRestore();
+ });
+});