Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
import { maxOf, renderOptions, serializeState } from "./common.js";
import type { Batch } from "./providers.js";
import type { QuestionDef, SystemOneResult } from "./agent.js";
export const DEFAULT_SHORTLIST_K = 20;
export const DEFAULT_TOURNAMENT_GROUP = 16;
export const DEFAULT_SHORTLIST_MAXSIZE = 4096;
export type EmbedFn = (texts: string[]) => Promise<number[][]> | number[][];
export interface ShortlistMeta {
labels: string[];
scores: number[] | null;
k: number;
n: number;
passthrough: boolean;
}
export interface TournamentMeta {
labels: string[];
n: number;
rounds: number;
}
export interface CacheInfo {
size: number;
maxsize: number;
hits: number;
misses: number;
}
export interface CachedEmbedFn {
(texts: string[]): Promise<number[][]>;
cacheInfo(): CacheInfo;
cache_info(): CacheInfo;
cacheClear(): void;
cache_clear(): void;
}
function checkK(k: unknown): number {
if (typeof k !== "number" || !Number.isInteger(k) || k < 1) {
throw new Error(`k must be a positive integer, got ${JSON.stringify(k)}`);
}
return k;
}
function criteriaItems(criteria: unknown): [string, unknown][] {
let items: [string, unknown][];
if (Array.isArray(criteria)) {
items = (criteria as unknown[]).map((item) => [String(item), null] as [string, unknown]);
} else if (typeof criteria === "object" && criteria !== null) {
items = Object.entries(criteria as Record<string, unknown>);
} else {
throw new TypeError(
`choice criteria must be a dict or list, got ${Array.isArray(criteria) ? "list" : criteria === null ? "NoneType" : typeof criteria}`,
);
}
if (items.length === 0) throw new Error("choice criteria must contain at least one option");
const seen = new Set<string>();
for (const [key] of items) {
if (seen.has(key)) throw new Error(`choice criteria label ${JSON.stringify(key)} is duplicated`);
seen.add(key);
}
return items;
}
function optionTexts(items: [string, unknown][]): string[] {
const crit: Record<string, unknown> = Object.fromEntries(items);
const rendered = renderOptions({ t: "choice", ins: "", crit });
if (rendered.length !== items.length) throw new Error("could not render every choice option");
return rendered.map((p) => (typeof p === "string" ? p : String(p)));
}
function queryText(state: unknown, instructions?: unknown): string {
const body = serializeState(state);
if (instructions === null || instructions === undefined || instructions === "") return body;
if (typeof instructions !== "string") return `${JSON.stringify(instructions)}\n${body}`;
return `${instructions}\n${body}`;
}
function clean(v: number): number {
return typeof v !== "number" || !Number.isFinite(v) ? 0 : v;
}
async function embeddings(embedFn: unknown, texts: string[]): Promise<number[][]> {
if (typeof embedFn !== "function") throw new TypeError("embed_fn must be callable");
const raw = await (embedFn as EmbedFn)([...texts]);
const arr = raw as unknown;
if (!Array.isArray(arr) || (arr as unknown[]).length !== texts.length) {
throw new Error(`embed_fn must return an array of shape (${texts.length}, dim), got ${Array.isArray(arr) ? `(${(arr as unknown[]).length}, ?)` : typeof arr}`);
}
const rows = (arr as unknown[]).map((row) => {
if (!Array.isArray(row) || row.length < 1) {
throw new Error(`embed_fn must return an array of shape (${texts.length}, dim), got row ${JSON.stringify(row)?.slice(0, 60)}`);
}
return (row as unknown[]).map((v) => clean(Number(v)));
});
return rows;
}
function cosine(query: number[], docs: number[][]): number[] {
const qn = Math.sqrt(query.reduce((a, v) => a + v * v, 0));
if (qn === 0 || docs.length === 0) return docs.map(() => 0);
return docs.map((d) => {
const dn = Math.sqrt(d.reduce((a, v) => a + v * v, 0));
if (dn === 0) return 0;
const dot = d.reduce((a, v, i) => a + v * (query[i] ?? 0), 0);
return Math.min(1, Math.max(-1, dot / (dn * qn)));
});
}
async function rank(
state: unknown,
criteria: unknown,
embedFn: EmbedFn,
k: number,
instructions?: unknown,
): Promise<{ labels: string[]; scores: number[] | null; passthrough: boolean; n: number }> {
const checked = checkK(k);
const items = criteriaItems(criteria);
const n = items.length;
const keys = items.map(([key]) => key);
if (checked >= n) return { labels: [...keys], scores: null, passthrough: true, n };
const q = queryText(state, instructions);
const matrix = await embeddings(embedFn, [q, ...optionTexts(items)]);
const sims = cosine(matrix[0], matrix.slice(1));
const order = sims
.map((s, i) => i)
.sort((a, b) => sims[b] - sims[a]);
const top = order.slice(0, checked);
return { labels: top.map((i) => keys[i]), scores: top.map((i) => sims[i]), passthrough: false, n };
}
function subsetCriteria(criteria: unknown, labels: string[]): unknown {
if (Array.isArray(criteria)) return [...labels];
return Object.fromEntries(labels.map((l) => [l, (criteria as Record<string, unknown>)[l]]));
}
/** Return `question` without an `option_order` that no longer fits it.
*
* `option_order` holds one slot per option, so an order that fitted the caller's label set does not
* fit a narrowed one -- and once the agent reads the field at all, that mismatch is a refused
* question, not a silently ignored one. Remapping the order onto the kept labels would move options
* between slots, so it is dropped instead, once per question per call. Mirrors
* `laya.shortlist._drop_stale_order` (`laya/shortlist.py:428`), which both `predict_shortlist` and
* `predict_tournament` run on every narrowed choice; a passthrough keeps its order. */
function dropStaleOrder(qid: string, question: Record<string, unknown>, nOptions: number,
warned: Set<string>): Record<string, unknown> {
if (!Object.hasOwn(question, "option_order")) return question;
const out = { ...question };
const order = out["option_order"];
delete out["option_order"];
if (!warned.has(qid)) {
warned.add(qid);
// The same shape as the reference's warning: the slot count the order carried, so a caller can
// see how much of the label set the narrowing dropped.
const had = Array.isArray(order) ? `${order.length} slots` : JSON.stringify(order);
console.warn(
`laya.shortlist: question ${JSON.stringify(qid)}: 'option_order' (${had}) was dropped because the ` +
`label set was narrowed to ${nOptions}; the order no longer fits the kept options and remapping ` +
`it onto them would move options between slots.`,
);
}
return out;
}
/** Return the top-k choice labels for state. Skips embed_fn when k >= n. */
export async function shortlistChoice(
state: unknown,
criteria: unknown,
embedFn: EmbedFn,
k: number = DEFAULT_SHORTLIST_K,
instructions?: unknown,
): Promise<string[]> {
const { labels } = await rank(state, criteria, embedFn, k, instructions);
return labels;
}
/** Shortlist each choice question, then call predict/systemOne once. */
export async function predictShortlist(
agent: unknown,
state: unknown,
questions: Record<string, QuestionDef>,
embedFn?: EmbedFn | null,
k: number = DEFAULT_SHORTLIST_K,
predictKwargs: Record<string, unknown> = {},
): Promise<SystemOneResult & { shortlist: Record<string, ShortlistMeta> }> {
if (typeof questions !== "object" || questions === null || Array.isArray(questions)) {
throw new TypeError("questions must be a dict of question id -> definition");
}
const checked = checkK(k);
const needsEmbed = Object.values(questions).some(
(qdef) =>
typeof qdef === "object" && qdef !== null && !Array.isArray(qdef) &&
(qdef as Record<string, unknown>)["type"] === "choice" &&
"criteria" in (qdef as Record<string, unknown>) &&
Object.keys((qdef as Record<string, unknown>)["criteria"] as object ?? {}).length > checked,
);
// Default bi-encoder is the agent itself; pass a dedicated embedFn only when it shortlists better.
const embed = embedFn ?? (needsEmbed ? embedFnFromAgent(agent) : null);
const reduced: Record<string, unknown> = {};
const meta: Record<string, ShortlistMeta> = {};
const warned = new Set<string>();
for (const [qid, qdef] of Object.entries(questions)) {
if (typeof qdef !== "object" || qdef === null || Array.isArray(qdef) || (qdef as Record<string, unknown>)["type"] !== "choice") {
reduced[qid] = qdef;
continue;
}
if (!("criteria" in (qdef as Record<string, unknown>))) {
throw new Error(`question ${JSON.stringify(qid)} is a choice but has no criteria`);
}
const q = qdef as Record<string, unknown>;
const { labels, scores, passthrough, n } = await rank(state, q["criteria"], embed as EmbedFn, checked, q["instructions"]);
meta[qid] = { labels: [...labels], scores, k: checked, n, passthrough };
if (passthrough) {
reduced[qid] = qdef;
continue;
}
reduced[qid] = dropStaleOrder(
qid, { ...q, criteria: subsetCriteria(q["criteria"], labels) }, labels.length, warned);
}
const a = agent as Record<string, unknown>;
const fn = a?.["predict"] ?? a?.["systemOne"] ?? a?.["system_one"];
if (typeof fn !== "function") throw new TypeError("agent must provide predict or system_one");
const extra = Object.keys(predictKwargs ?? {}).length ? [predictKwargs] : [];
const result = await (fn as (...args: unknown[]) => Promise<unknown>).call(agent, state, reduced, ...extra);
if (typeof result !== "object" || result === null || Array.isArray(result)) {
throw new TypeError("predict/system_one must return a dict");
}
return { ...(result as SystemOneResult), shortlist: meta };
}
/**
* Reduce large choices through balanced elimination rounds, then predict all questions.
* Each round batches every group's question in one call. Metadata lists the finalists;
* probabilities, confidence, usage and other result fields come from the final call only.
*/
export async function predictTournament(
agent: unknown,
state: unknown,
questions: Record<string, QuestionDef>,
groupSize: number = DEFAULT_TOURNAMENT_GROUP,
predictKwargs: Record<string, unknown> = {},
): Promise<SystemOneResult & { tournament: Record<string, TournamentMeta> }> {
if (typeof questions !== "object" || questions === null || Array.isArray(questions) ||
(Object.getPrototypeOf(questions) !== null && Object.getPrototypeOf(questions) !== Object.prototype)) {
throw new TypeError("questions must be a dict of question id -> definition");
}
if (!Number.isSafeInteger(groupSize) || groupSize < 2) {
throw new Error(`groupSize must be a safe integer >= 2, got ${JSON.stringify(groupSize)}`);
}
const meta = new Map<string, TournamentMeta>();
for (const [qid, qdef] of Object.entries(questions)) {
if (typeof qdef !== "object" || qdef === null || Array.isArray(qdef) || qdef["type"] !== "choice") continue;
if (!("criteria" in qdef)) {
throw new Error(`question ${JSON.stringify(qid)} is a choice but has no criteria`);
}
const criteria = qdef["criteria"];
if (Array.isArray(criteria)) {
for (const label of criteria) {
if (label === null || label === undefined || typeof label === "object") {
throw new Error(`question ${JSON.stringify(qid)}: choice labels must not be null, lists or dicts`);
}
}
}
const labels = criteriaItems(criteria).map(([label]) => label);
meta.set(qid, { labels, n: labels.length, rounds: 0 });
}
const a = agent as Record<string, unknown>;
const fn = a?.["predict"] ?? a?.["systemOne"] ?? a?.["system_one"];
if (typeof fn !== "function") throw new TypeError("agent must provide predict or system_one");
const extra = Object.keys(predictKwargs ?? {}).length ? [predictKwargs] : [];
const predict = async (current: Record<string, QuestionDef>): Promise<SystemOneResult> => {
const result = await (fn as (...args: unknown[]) => unknown).call(agent, state, current, ...extra);
if (typeof result !== "object" || result === null || Array.isArray(result)) {
throw new TypeError("predict/system_one must return a dict");
}
return result as SystemOneResult;
};
const warned = new Set<string>();
const cut = (qid: string, labels: string[]): QuestionDef => dropStaleOrder(
qid,
{ ...questions[qid], criteria: subsetCriteria(questions[qid]["criteria"], labels) },
labels.length,
warned,
) as QuestionDef;
while (true) {
const groups: { qid: string; labels: string[] }[] = [];
for (const [qid, entry] of meta) {
const n = entry.labels.length;
const parts = Math.ceil(n / groupSize);
if (parts <= 1) continue;
for (let i = 0; i < parts; i++) {
groups.push({ qid, labels: entry.labels.slice(Math.floor(i * n / parts), Math.floor((i + 1) * n / parts)) });
}
}
if (groups.length === 0) break;
const roundQuestions = Object.fromEntries(groups.map(({ qid, labels }, i) => [String(i), cut(qid, labels)]));
const result = await predict(roundQuestions);
const answers = result["answers"];
if (typeof answers !== "object" || answers === null || Array.isArray(answers)) {
throw new TypeError("tournament round must return an answers dict");
}
const winners = new Map<string, string[]>();
for (const [i, { qid, labels }] of groups.entries()) {
const answer = Object.hasOwn(answers, String(i)) ? (answers as Record<string, unknown>)[String(i)] : undefined;
const choice = typeof answer === "object" && answer !== null && !Array.isArray(answer) && Object.hasOwn(answer, "choice")
? (answer as Record<string, unknown>)["choice"] : undefined;
if (typeof choice !== "string" || !labels.includes(choice)) {
throw new Error(`tournament group ${i} for question ${JSON.stringify(qid)} must return a choice from its criteria`);
}
const kept = winners.get(qid) ?? [];
kept.push(choice);
winners.set(qid, kept);
}
for (const [qid, labels] of winners) {
const entry = meta.get(qid)!;
entry.labels = labels;
entry.rounds++;
}
}
const finalQuestions = Object.fromEntries(Object.entries(questions).map(([qid, qdef]) => {
const entry = meta.get(qid);
return [qid, entry?.rounds ? cut(qid, entry.labels) : qdef];
}));
const result = await predict(finalQuestions);
return { ...result, tournament: Object.fromEntries(meta) };
}
/** Mean-pool the agent's provider encoder. Throws honestly without encoder access. */
export function embedFnFromAgent(agent: unknown, maxLength = 512, batchSize = 32): EmbedFn {
if (!Number.isInteger(maxLength) || (maxLength as number) < 1) {
throw new Error(`max_length must be a positive integer, got ${JSON.stringify(maxLength)}`);
}
if (!Number.isInteger(batchSize) || (batchSize as number) < 1) {
throw new Error(`batch_size must be a positive integer, got ${JSON.stringify(batchSize)}`);
}
const a = agent as { tok?: { encode(t: string): number[]; padId: number }; provider?: { runEncoder(b: Batch): Promise<{ lastHidden: number[][][] }> } };
if (!a?.provider || typeof a.provider.runEncoder !== "function" || !a?.tok || typeof a.tok.encode !== "function") {
throw new Error("embedFnFromAgent needs encoder access: agent.provider.runEncoder + agent.tok.encode required");
}
const tok = a.tok;
const provider = a.provider;
return async (texts: string[]): Promise<number[][]> => {
const rows = (texts ?? []).map((t) => (t === null || t === undefined ? "" : String(t)));
if (rows.length === 0) return [];
const out: number[][] = [];
for (let s = 0; s < rows.length; s += batchSize) {
const chunk = rows.slice(s, s + batchSize);
const ids = chunk.map((t) => tok.encode(t).slice(0, maxLength));
const L = maxOf(ids.map((r) => r.length), 1);
const pad = tok.padId ?? 0;
const batch: Batch = {
inputIds: ids.map((r) => [...r, ...Array(L - r.length).fill(pad)]),
attentionMask: ids.map((r) => [...Array(r.length).fill(1), ...Array(L - r.length).fill(0)]),
markerPos: ids.map(() => [0]),
markerMask: ids.map(() => [true]),
qtype: ids.map(() => 0),
};
const { lastHidden } = await provider.runEncoder(batch);
for (let i = 0; i < chunk.length; i++) {
const mask = batch.attentionMask[i];
const denom = Math.max(1, mask.reduce((x, y) => x + y, 0));
const dim = lastHidden[i]?.[0] ? (lastHidden[i][0] as number[]).length : 0;
const pooled = new Array(dim).fill(0);
for (let j = 0; j < mask.length; j++) {
if (!mask[j]) continue;
const hv = lastHidden[i]?.[j] as number[] | undefined;
if (!hv) continue;
for (let h = 0; h < dim; h++) pooled[h] += Number(hv[h] ?? 0);
}
out.push(pooled.map((v) => v / denom));
}
}
return out;
};
}
/**
* Wrap `embedFn` with an LRU cache keyed by text.
*
* Shortlisting embeds the query plus every option on every call. When the option
* set is fixed -- a label list, an intent taxonomy -- those rows never change
* between requests and are re-embedded every time. Wrapping the embedder once
* leaves the first call unchanged and reduces each repeat to embedding the new
* query alone.
*
* A cold cache costs the same number of embedder calls as the unwrapped function:
* the misses are deduplicated and embedded in one call. Nothing is cached when
* the embedder throws or returns a bad shape, so a transient failure cannot
* poison the cache.
*
* @param embedFn The underlying embedding function
* @param maxsize Maximum number of distinct text rows to cache (default: 4096)
*/
export function cachedEmbedFn(embedFn: EmbedFn, maxsize: number = DEFAULT_SHORTLIST_MAXSIZE): CachedEmbedFn {
if (typeof embedFn !== "function") {
throw new TypeError("embed_fn must be callable");
}
if (typeof maxsize !== "number" || !Number.isInteger(maxsize) || maxsize < 1) {
throw new Error(`maxsize must be a positive integer, got ${JSON.stringify(maxsize)}`);
}
const rowsByText = new Map<string, number[]>();
let hitsTotal = 0;
let missesTotal = 0;
let generation = 0;
const fn = async (texts: string[]): Promise<number[][]> => {
const keys = (texts ?? []).map((t) => (t === null || t === undefined ? "" : String(t)));
if (keys.length === 0) return [];
const found = new Map<string, number[]>();
let hitsThisCall = 0;
for (const key of keys) {
const row = rowsByText.get(key);
if (row !== undefined) {
found.set(key, row);
hitsThisCall++;
}
}
hitsTotal += hitsThisCall;
missesTotal += keys.length - hitsThisCall;
const missing: string[] = [];
const seenMissing = new Set<string>();
for (const key of keys) {
if (!found.has(key) && !seenMissing.has(key)) {
seenMissing.add(key);
missing.push(key);
}
}
const genAtStart = generation;
if (missing.length > 0) {
const fresh = await embeddings(embedFn, missing);
// Validate dimensions of all returned rows before caching any
const heldDim = rowsByText.size > 0 ? rowsByText.values().next().value?.length : undefined;
const expectedDim = heldDim ?? (fresh.length > 0 ? fresh[0]?.length : undefined);
for (let i = 0; i < fresh.length; i++) {
const rowDim = fresh[i]?.length ?? 0;
if (expectedDim !== undefined && rowDim !== expectedDim) {
if (heldDim !== undefined) {
throw new Error(
`the embedder returned dim ${rowDim}, but the cache holds dim ${heldDim}; call cacheClear() if the model behind it changed`,
);
} else {
throw new Error(
`the embedder returned inconsistent embedding dimensions (${expectedDim} vs ${rowDim})`,
);
}
}
}
for (let i = 0; i < missing.length; i++) {
found.set(missing[i], fresh[i]);
}
// If cacheClear() ran while embeddings() was pending, do not restore entries into the cleared cache
if (generation !== genAtStart) {
return keys.map((key) => [...(found.get(key) ?? [])]);
}
}
// Apply recency updates and evictions in input key order
for (const key of keys) {
const row = found.get(key);
if (row !== undefined) {
rowsByText.delete(key);
rowsByText.set(key, row);
while (rowsByText.size > maxsize) {
const victim = rowsByText.keys().next().value!;
rowsByText.delete(victim);
}
}
}
return keys.map((key) => [...(found.get(key) ?? [])]);
};
const cacheInfo = (): CacheInfo => ({
size: rowsByText.size,
maxsize,
hits: hitsTotal,
misses: missesTotal,
});
const cacheClear = (): void => {
rowsByText.clear();
hitsTotal = 0;
missesTotal = 0;
generation++;
};
const cachedResult = fn as CachedEmbedFn;
cachedResult.cacheInfo = cacheInfo;
cachedResult.cache_info = cacheInfo;
cachedResult.cacheClear = cacheClear;
cachedResult.cache_clear = cacheClear;
return cachedResult;
}
export const cached = cachedEmbedFn;