Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
import { serializeState } from "./common.js";
import { BaseHook, type PredictContext } from "./hooks.js";
import { StabilityRandom } from "./_option-stability-random.js";
export type StabilityProbe = "reorder" | "rename";
export type ReliabilityDecision = "ACCEPT" | "VERIFY" | "ESCALATE";
export interface StabilityVariant {
name: string;
order: string[];
relabel: boolean;
}
export interface VariantOptions {
nShuffles?: number;
relabel?: boolean;
include?: readonly string[] | null;
reorder?: boolean;
}
export interface CalibrationOptions {
targetAccuracy?: number;
maxEscalateAccuracy?: number;
}
export interface OptionStabilityOptions {
/** Domain-calibrated thresholds; pass null for uncalibrated metrics only. */
accept: number | null;
escalate: number | null;
probes?: "auto" | "all" | StabilityProbe | readonly StabilityProbe[] | null;
nShuffles?: number;
relabel?: boolean;
renameMaxOptions?: number;
variants?: "full" | "fast" | readonly string[] | null;
/** Use an explicit seed for cross-language parity with structured states. */
seed?: string | null;
}
export interface OptionStabilityReport {
probes: StabilityProbe[];
option_layout: "parallel" | "sequential";
decision: ReliabilityDecision | null;
/** The unrounded mean used by decideReliability and calibrateReliability. */
soft_stability: number | null;
stability: number | null;
rename_stability: number | null;
reorder_stability: number | null;
n_variants: number;
distinct_choices: string[];
variant_choices: Record<string, string | null>;
variant_support: Record<string, number>;
}
const SEP = "::rel";
const ALL_VARIANTS = ["reversed", "shuffle1", "shuffle2", "shuffle3", "letters", "letters_reversed", "letters_shuffle"];
/** Deterministic option permutations, including the original, with duplicates removed.
* The CRC32 / Python MT19937 sequence matches laya.hooks.make_variants. Filtering
* happens after shuffling so a named subset retains the full preset's permutations. */
export function makeVariants(labels: readonly string[], seedKey: string, opts: VariantOptions = {}): StabilityVariant[] {
const { nShuffles = 3, relabel = true, include = null, reorder = true } = opts;
checkShuffles(nShuffles);
const rng = new StabilityRandom(seedKey);
const candidates: StabilityVariant[] = [{ name: "original", order: [...labels], relabel: false }];
if (reorder) {
candidates.push({ name: "reversed", order: [...labels].reverse(), relabel: false });
for (let i = 0; i < nShuffles; i++) {
const order = [...labels];
rng.shuffle(order);
candidates.push({ name: `shuffle${i + 1}`, order, relabel: false });
}
}
if (relabel && labels.length <= 26) {
candidates.push({ name: "letters", order: [...labels], relabel: true });
candidates.push({ name: "letters_reversed", order: [...labels].reverse(), relabel: true });
const order = [...labels];
rng.shuffle(order);
candidates.push({ name: "letters_shuffle", order, relabel: true });
}
const wanted = include === null ? null : new Set(include);
const seen = new Set<string>();
return candidates.filter((v) => {
if (wanted && v.name !== "original" && !wanted.has(v.name)) return false;
const key = JSON.stringify([v.order, v.relabel]);
if (seen.has(key)) return false;
seen.add(key);
return true;
});
}
/** Uncalibrated thresholds fail closed; equality with accept accepts, while equality
* with escalate verifies. A fitted accept of 1.01 accepts no probability score. */
export function decideReliability(softStability: number, accept: number | null,
escalate: number | null = null): ReliabilityDecision | null {
checkProbability(softStability, "softStability");
checkThresholds(accept, escalate);
if (accept === null) return null;
if (softStability >= accept) return "ACCEPT";
if (escalate !== null && softStability < escalate) return "ESCALATE";
return "VERIFY";
}
/** Fit [accept, escalate] on labelled examples. Distinct raw scores are candidate
* cutoffs; equal scores stay together and report JSON can be replayed without rounding.
* Inputs must be equally sized, non-empty score/label arrays. */
export function calibrateReliability(scores: readonly number[], correct: readonly boolean[],
opts: CalibrationOptions = {}): [number, number] {
const { targetAccuracy = 0.90, maxEscalateAccuracy = 1 / 3 } = opts;
if (scores.length !== correct.length) throw new Error("scores and correct must have the same length");
if (!scores.length) throw new Error("cannot calibrate reliability on empty scores/correct pairs");
checkProbability(targetAccuracy, "targetAccuracy");
checkProbability(maxEscalateAccuracy, "maxEscalateAccuracy");
const pairs = Array.from(scores, (score, i) => {
checkProbability(score, `scores[${i}]`);
if (typeof correct[i] !== "boolean") throw new TypeError(`correct[${i}] must be a bool`);
return [score, correct[i]] as const;
});
const cuts = [...new Set(pairs.map(([score]) => score))].sort((a, b) => a - b);
let accept = 1.01;
for (const t of cuts) {
const kept = pairs.filter(([s]) => s >= t);
if (kept.length && kept.filter(([, c]) => c).length / kept.length >= targetAccuracy) {
accept = t;
break;
}
}
let escalate = 0;
for (const t of cuts) {
if (t >= accept) break;
const below = pairs.filter(([s]) => s < t);
if (below.length && below.filter(([, c]) => c).length / below.length <= maxEscalateAccuracy) escalate = t;
}
return [accept, escalate];
}
function isRecord(value: unknown): value is Record<string, unknown> {
return value !== null && typeof value === "object" && !Array.isArray(value);
}
function checkShuffles(value: number): void {
if (typeof value !== "number" || !Number.isInteger(value) || value < 0) {
throw new Error("nShuffles must be a non-negative integer");
}
}
function checkThreshold(value: number | null, name: string, max: number): void {
if (value !== null && (typeof value !== "number" || !Number.isFinite(value) || value < 0 || value > max)) {
throw new Error(`${name} must be null or a number in [0, ${max}]`);
}
}
function checkProbability(value: number, name: string): void {
if (typeof value !== "number" || !Number.isFinite(value) || value < 0 || value > 1) {
throw new Error(`${name} must be a finite number in [0, 1]`);
}
}
function checkThresholds(accept: number | null, escalate: number | null): void {
checkThreshold(accept, "accept", 1.01);
checkThreshold(escalate, "escalate", 1);
if (accept !== null && escalate !== null && escalate > accept) {
throw new Error("escalate must not be greater than accept");
}
}
function resolveProbes(probes: OptionStabilityOptions["probes"]): StabilityProbe[] | null {
if (probes === undefined || probes === null || probes === "auto") return null;
if (probes === "all") return ["reorder", "rename"];
if (probes === "rename" || probes === "reorder") return [probes];
if (!Array.isArray(probes) || !probes.length || probes.some((p) => p !== "reorder" && p !== "rename")) {
throw new Error("probes must be 'auto', 'all', 'reorder', 'rename', or a non-empty sequence of probe names");
}
return [...new Set(probes)];
}
function resolveVariants(variants: OptionStabilityOptions["variants"]): string[] | null {
if (variants === undefined || variants === null || variants === "full") return null;
if (variants === "fast") return ["reversed", "letters_reversed"];
if (!Array.isArray(variants) || !variants.length || variants.some((v) => !ALL_VARIANTS.includes(v))) {
throw new Error(`variants must be 'full', 'fast', or a non-empty sequence from ${ALL_VARIANTS.join(", ")}`);
}
return [...variants];
}
function detectLayout(ctx: PredictContext): "parallel" | "sequential" {
const cfg = isRecord(ctx.agent) && isRecord(ctx.agent.cfg) ? ctx.agent.cfg : null;
const raw = cfg?.option_layout;
if (typeof raw === "string") {
const layout = raw.trim().toLowerCase();
if (layout === "parallel" || layout === "sequential") return layout;
console.warn(`laya: unknown option_layout ${JSON.stringify(raw)} in agent config; defaulting to 'sequential'`);
}
return "sequential";
}
function variantQuestion(q: Record<string, unknown>, order: string[], relabel: boolean):
{ question: Record<string, unknown>; back: Map<string, string> } {
const shown = relabel ? order.map((_, i) => String.fromCharCode(65 + i)) : [...order];
const criteria = q.criteria;
const entries = shown.map((label, i) => {
const original = order[i];
let description = Array.isArray(criteria) ? null : (criteria as Record<string, unknown>)[original];
if (relabel && (description === null || description === undefined ||
(typeof description === "string" && description.trim() === ""))) description = original;
return [label, description] as const;
});
const nextCriteria = Object.fromEntries(entries);
const question = { ...q, criteria: Array.isArray(criteria) && !relabel ? [...order] : nextCriteria };
// Integer-like keys enumerate numerically in JS regardless of insertion order. The
// encoder already supports a slot permutation, so preserve the requested physical
// order there, including the caller's permutation of each generated variant.
const actual = Object.keys(nextCriteria);
const callerOrder = Array.isArray(q.option_order) ? q.option_order as number[] : order.map((_, i) => i);
const effectiveOrder = callerOrder.map((i) => actual.indexOf(shown[i]));
if (Object.hasOwn(q, "option_order") || effectiveOrder.some((i, slot) => i !== slot)) {
(question as Record<string, unknown>).option_order = effectiveOrder;
}
return { question, back: new Map(shown.map((label, i) => [label, order[i]])) };
}
/** Python round(value, 4): round the binary value itself, with ties to even.
* Multiplying a double by 1e4 first can manufacture a tie, and toFixed ties away. */
function round4(value: number): number {
if (!Number.isFinite(value) || value === 0) return value;
const data = new DataView(new ArrayBuffer(8));
data.setFloat64(0, value);
const bits = data.getBigUint64(0);
const exponent = Number((bits >> 52n) & 0x7ffn);
const mantissa = (bits & ((1n << 52n) - 1n)) | (exponent ? 1n << 52n : 0n);
const power = exponent ? exponent - 1075 : -1074;
let numerator = mantissa * 10000n;
const denominator = power < 0 ? 1n << BigInt(-power) : 1n;
if (power > 0) numerator <<= BigInt(power);
let rounded = numerator / denominator;
const remainder = numerator % denominator;
if (remainder * 2n > denominator || (remainder * 2n === denominator && (rounded & 1n))) rounded++;
return (bits >> 63n ? -Number(rounded) : Number(rounded)) / 10000;
}
function compareCodePoints(a: string, b: string): number {
const x = Array.from(a, (c) => c.codePointAt(0)!);
const y = Array.from(b, (c) => c.codePointAt(0)!);
for (let i = 0; i < Math.min(x.length, y.length); i++) if (x[i] !== y[i]) return x[i] - y[i];
return x.length - y.length;
}
interface PlannedVariant {
name: string;
key: string;
back: Map<string, string>;
relabel: boolean;
}
interface QuestionPlan {
layout: "parallel" | "sequential";
variants: PlannedVariant[];
probes: StabilityProbe[];
}
/** Opt-in stability probes, batched with the original choice questions. Metrics are
* signals for calibration, not correctness probabilities. Layout detection selects
* probes; it does not add parallel-layout support to a provider. */
export class OptionStabilityHook extends BaseHook {
private readonly accept: number | null;
private readonly escalate: number | null;
private readonly probes: StabilityProbe[] | null;
private readonly nShuffles: number;
private readonly relabel: boolean;
private readonly renameMaxOptions: number;
private readonly include: string[] | null;
private readonly seed: string | null;
private readonly plans = new WeakMap<PredictContext, Map<string, QuestionPlan>>();
constructor(opts: OptionStabilityOptions) {
super();
if (!opts || !Object.hasOwn(opts, "accept") || !Object.hasOwn(opts, "escalate")) {
throw new Error("accept and escalate are required; pass null for uncalibrated metrics");
}
checkThresholds(opts.accept, opts.escalate);
this.accept = opts.accept;
this.escalate = opts.escalate;
this.probes = resolveProbes(opts.probes);
this.nShuffles = opts.nShuffles === undefined ? 3 : opts.nShuffles;
checkShuffles(this.nShuffles);
this.relabel = opts.relabel === undefined ? true : opts.relabel;
if (typeof this.relabel !== "boolean") throw new TypeError("relabel must be a bool");
this.renameMaxOptions = opts.renameMaxOptions === undefined ? 20 : opts.renameMaxOptions;
if (!Number.isInteger(this.renameMaxOptions) || this.renameMaxOptions < 2) {
throw new Error("renameMaxOptions must be an integer >= 2");
}
this.include = resolveVariants(opts.variants);
this.seed = opts.seed === undefined || opts.seed === null ? null : String(opts.seed);
}
onPredictStart(ctx: PredictContext): void {
this.plans.delete(ctx);
for (const qid of Object.keys(ctx.questions)) {
if (qid.includes(SEP)) throw new Error(`question id ${JSON.stringify(qid)} must not contain ${JSON.stringify(SEP)}`);
}
const layout = detectLayout(ctx);
const configured = new Set(this.probes ?? (layout === "parallel"
? (this.relabel ? ["rename"] : []) : (this.relabel ? ["reorder", "rename"] : ["reorder"])));
// Python uses repr for structured states; TS keeps its existing serializer.
// Explicit seed, string states, and multi-state "batch" seeds are shared.
const baseSeed = this.seed ?? (ctx.states.length === 1 ? serializeState(ctx.states[0]) : "batch");
const plan = new Map<string, QuestionPlan>();
const questions = { ...ctx.questions };
for (const [qid, q] of Object.entries(ctx.questions)) {
if (!isRecord(q) || q.type !== "choice") continue;
const criteria = q.criteria;
if (!Array.isArray(criteria) && !isRecord(criteria)) continue;
// Match the engine's list-to-dict answer keys, including numeric-like labels.
const labels = Object.keys(Array.isArray(criteria)
? Object.fromEntries(criteria.map((label) => [String(label), null])) : criteria);
const variants: PlannedVariant[] = [{ name: "original", key: qid,
back: new Map(labels.map((label) => [label, label])), relabel: false }];
const probes: StabilityProbe[] = [];
if (labels.length >= 2) {
const generated = makeVariants(labels, `${baseSeed}|${qid}`, {
nShuffles: this.nShuffles,
relabel: configured.has("rename") && labels.length <= this.renameMaxOptions,
reorder: configured.has("reorder"), include: this.include,
}).slice(1);
if (generated.some((v) => !v.relabel)) probes.push("reorder");
if (generated.some((v) => v.relabel)) probes.push("rename");
generated.forEach((variant, i) => {
const key = `${qid}${SEP}${i + 1}`;
const { question, back } = variantQuestion(q, variant.order, variant.relabel);
questions[key] = question;
variants.push({ name: variant.name, key, back, relabel: variant.relabel });
});
}
plan.set(qid, { layout, variants, probes });
}
if (plan.size) {
ctx.questions = questions;
this.plans.set(ctx, plan);
}
}
onPredictEnd(ctx: PredictContext): void {
const plan = this.plans.get(ctx);
this.plans.delete(ctx);
if (!plan || !ctx.results) return;
const internalKeys = new Set([...plan.values()].flatMap((p) => p.variants.slice(1).map((v) => v.key)));
for (const result of ctx.results) {
if (!isRecord(result)) continue;
const answers = result.answers;
if (isRecord(answers)) {
for (const [qid, p] of plan) {
if (!Object.hasOwn(answers, qid)) continue;
const answer = answers[qid];
if (!isRecord(answer) || answer.type !== "choice" || typeof answer.choice !== "string") continue;
const seen: { name: string; choice: string | null; support: number; relabel: boolean }[] = [];
for (const variant of p.variants) {
if (!Object.hasOwn(answers, variant.key)) break;
const value = answers[variant.key];
if (!isRecord(value) || value.type !== "choice" || typeof value.choice !== "string" ||
!isRecord(value.probabilities)) break;
const probabilities = Object.entries(value.probabilities);
if (probabilities.some(([, v]) => typeof v !== "number" || !Number.isFinite(v) || v < 0 || v > 1)) break;
const probs = new Map(probabilities.map(([label, probability]) => [variant.back.get(label) ?? label, probability as number]));
const choice = variant.back.get(value.choice) ?? value.choice;
seen.push({ name: variant.name, choice, support: probs.get(answer.choice) ?? 0, relabel: variant.relabel });
}
// A partial or malformed answer set cannot establish stability.
if (seen.length !== p.variants.length) continue;
const others = seen.slice(1);
const probed = others.length > 0 && p.probes.length > 0;
const rate = (items: typeof seen): number | null => items.length
? round4(items.filter((v) => v.choice === answer.choice).length / items.length) : null;
const soft = probed ? seen.reduce((sum, v) => sum + v.support, 0) / seen.length : null;
const report: OptionStabilityReport = {
probes: probed ? [...p.probes] : [], option_layout: p.layout,
decision: soft === null ? null : decideReliability(soft, this.accept, this.escalate),
soft_stability: soft, stability: probed ? rate(others) : null,
rename_stability: probed ? rate(others.filter((v) => v.relabel)) : null,
reorder_stability: probed ? rate(others.filter((v) => !v.relabel)) : null,
n_variants: seen.length,
distinct_choices: [...new Set(seen.flatMap((v) => v.choice === null ? [] : [v.choice]))].sort(compareCodePoints),
variant_choices: Object.fromEntries(seen.map((v) => [v.name, v.choice])),
variant_support: Object.fromEntries(seen.map((v) => [v.name, round4(v.support)])),
};
answer.reliability = report;
}
for (const key of internalKeys) delete answers[key];
}
const usage = result.usage;
if (isRecord(usage)) {
if (Array.isArray(usage.truncated_questions)) {
usage.truncated_questions = usage.truncated_questions.filter((key) => !internalKeys.has(key));
}
if (isRecord(usage.options)) {
usage.options = Object.fromEntries(Object.entries(usage.options).filter(([key]) => !internalKeys.has(key)));
}
}
}
}
onError(ctx: PredictContext): void {
this.plans.delete(ctx);
}
}