Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
feat(ts): port cached embedder to laya-ts
Swatantra Yadav4739b93c77b3
diff --git a/laya-ts/README.md b/laya-ts/README.mdindex 713f7e56..82e6738f 100644--- a/laya-ts/README.md+++ b/laya-ts/README.md@@ -170,12 +170,14 @@ Python's `sort_by_length` grouping is not ported yet.## Shortlist (many labels)```ts-import { shortlistChoice, predictShortlist, embedFnFromAgent } from "laya-ts";+import { shortlistChoice, predictShortlist, embedFnFromAgent, cached } from "laya-ts";-const keep = await shortlistChoice(state, bigCriteriaDict, embedFn, 20);-const out = await predictShortlist(agent, state, questions, embedFn, 20);+const cachedEmbed = cached(embedFn);+const keep = await shortlistChoice(state, bigCriteriaDict, cachedEmbed, 20);+const out = await predictShortlist(agent, state, questions, cachedEmbed, 20);// out.shortlist[qid] = { labels, scores, k, n, passthrough }// embedFnFromAgent(agent) mean-pools the loaded encoder; a dedicated bi-encoder usually shortlists better.+// cached(embedFn) keeps an LRU cache across calls; cachedEmbed.cacheInfo() reports size/hits/misses.```## Per-language calibration (`lang_temperatures`)diff --git a/laya-ts/src/index.ts b/laya-ts/src/index.tsindex a9dd789d..f08738db 100644--- a/laya-ts/src/index.ts+++ b/laya-ts/src/index.ts@@ -17,8 +17,8 @@ export { createNodeProvider, createWebProvider, feed, feedHead, loadNodeBundle,export type { Batch, SessionProvider, ProviderOptions, NodeBundle, WebBundle } from "./providers.js";export { Router, normaliseName, DEFAULT_MODELS, digestsFromEnv, _digestsFromEnv } from "./router.js";export type { RoutedResult, RouteDecision, ModelName, ModelSpec, BatchRequest, PredictBatchOptions, RouterOptions, RouteOptions } from "./router.js";-export { shortlistChoice, predictShortlist, embedFnFromAgent, DEFAULT_SHORTLIST_K } from "./shortlist.js";-export type { EmbedFn, ShortlistMeta } from "./shortlist.js";+export { shortlistChoice, predictShortlist, embedFnFromAgent, DEFAULT_SHORTLIST_K, cached, cachedEmbedFn, DEFAULT_SHORTLIST_MAXSIZE } from "./shortlist.js";+export type { EmbedFn, ShortlistMeta, CacheInfo, CachedEmbedFn } from "./shortlist.js";export { analyse, isEnglish, guessLatinLanguage, detectScript } from "./lang.js";export type { AnalyseResult, LatinProfile } from "./lang.js";export { cleanEmailBody, emailState } from "./email.js";diff --git a/laya-ts/src/shortlist.ts b/laya-ts/src/shortlist.tsindex f856ae96..5e3f0815 100644--- a/laya-ts/src/shortlist.ts+++ b/laya-ts/src/shortlist.ts@@ -3,6 +3,7 @@ import type { Batch } from "./providers.js";import type { QuestionDef, SystemOneResult } from "./agent.js";export const DEFAULT_SHORTLIST_K = 20;+export const DEFAULT_SHORTLIST_MAXSIZE = 4096;export type EmbedFn = (texts: string[]) => Promise<number[][]> | number[][];@@ -14,6 +15,21 @@ export interface ShortlistMeta {passthrough: boolean;}+export interface CacheInfo {+ size: number;+ maxsize: number;+ hits: number;+ misses: number;+}++export interface CachedEmbedFn {+ (texts: string[]): Promise<number[][]>;+ cacheInfo(): CacheInfo;+ cache_info(): CacheInfo;+ cacheClear(): void;+ cache_clear(): void;+}+function checkK(k: unknown): number {if (typeof k !== "number" || !Number.isInteger(k) || k < 1) {throw new Error(`k must be a positive integer, got ${JSON.stringify(k)}`);@@ -225,3 +241,136 @@ export function embedFnFromAgent(agent: unknown, maxLength = 512, batchSize = 32return out;};}++/**+ * Wrap `embedFn` with an LRU cache keyed by text.+ *+ * Shortlisting embeds the query plus every option on every call. When the option+ * set is fixed -- a label list, an intent taxonomy -- those rows never change+ * between requests and are re-embedded every time. Wrapping the embedder once+ * leaves the first call unchanged and reduces each repeat to embedding the new+ * query alone.+ *+ * A cold cache costs the same number of embedder calls as the unwrapped function:+ * the misses are deduplicated and embedded in one call. Nothing is cached when+ * the embedder throws or returns a bad shape, so a transient failure cannot+ * poison the cache.+ *+ * @param embedFn The underlying embedding function+ * @param maxsize Maximum number of distinct text rows to cache (default: 4096)+ */+export function cachedEmbedFn(embedFn: EmbedFn, maxsize: number = DEFAULT_SHORTLIST_MAXSIZE): CachedEmbedFn {+ if (typeof embedFn !== "function") {+ throw new TypeError("embed_fn must be callable");+ }+ if (typeof maxsize !== "number" || !Number.isInteger(maxsize) || maxsize < 1) {+ throw new Error(`maxsize must be a positive integer, got ${JSON.stringify(maxsize)}`);+ }++ const rowsByText = new Map<string, number[]>();+ let hitsTotal = 0;+ let missesTotal = 0;+ let generation = 0;++ const fn = async (texts: string[]): Promise<number[][]> => {+ const keys = (texts ?? []).map((t) => (t === null || t === undefined ? "" : String(t)));+ if (keys.length === 0) return [];++ const found = new Map<string, number[]>();+ let hitsThisCall = 0;++ for (const key of keys) {+ const row = rowsByText.get(key);+ if (row !== undefined) {+ found.set(key, row);+ hitsThisCall++;+ }+ }++ hitsTotal += hitsThisCall;+ missesTotal += keys.length - hitsThisCall;++ const missing: string[] = [];+ const seenMissing = new Set<string>();+ for (const key of keys) {+ if (!found.has(key) && !seenMissing.has(key)) {+ seenMissing.add(key);+ missing.push(key);+ }+ }++ const genAtStart = generation;++ if (missing.length > 0) {+ const fresh = await embeddings(embedFn, missing);++ // Validate dimensions of all returned rows before caching any+ const heldDim = rowsByText.size > 0 ? rowsByText.values().next().value?.length : undefined;+ const expectedDim = heldDim ?? (fresh.length > 0 ? fresh[0]?.length : undefined);++ for (let i = 0; i < fresh.length; i++) {+ const rowDim = fresh[i]?.length ?? 0;+ if (expectedDim !== undefined && rowDim !== expectedDim) {+ if (heldDim !== undefined) {+ throw new Error(+ `the embedder returned dim ${rowDim}, but the cache holds dim ${heldDim}; call cacheClear() if the model behind it changed`,+ );+ } else {+ throw new Error(+ `the embedder returned inconsistent embedding dimensions (${expectedDim} vs ${rowDim})`,+ );+ }+ }+ }++ for (let i = 0; i < missing.length; i++) {+ found.set(missing[i], fresh[i]);+ }++ // If cacheClear() ran while embeddings() was pending, do not restore entries into the cleared cache+ if (generation !== genAtStart) {+ return keys.map((key) => [...(found.get(key) ?? [])]);+ }+ }++ // Apply recency updates and evictions in input key order+ for (const key of keys) {+ const row = found.get(key);+ if (row !== undefined) {+ rowsByText.delete(key);+ rowsByText.set(key, row);+ while (rowsByText.size > maxsize) {+ const victim = rowsByText.keys().next().value!;+ rowsByText.delete(victim);+ }+ }+ }++ return keys.map((key) => [...(found.get(key) ?? [])]);+ };++ const cacheInfo = (): CacheInfo => ({+ size: rowsByText.size,+ maxsize,+ hits: hitsTotal,+ misses: missesTotal,+ });++ const cacheClear = (): void => {+ rowsByText.clear();+ hitsTotal = 0;+ missesTotal = 0;+ generation++;+ };++ const cachedResult = fn as CachedEmbedFn;+ cachedResult.cacheInfo = cacheInfo;+ cachedResult.cache_info = cacheInfo;+ cachedResult.cacheClear = cacheClear;+ cachedResult.cache_clear = cacheClear;++ return cachedResult;+}++export const cached = cachedEmbedFn;+diff --git a/laya-ts/tests/shortlist.test.ts b/laya-ts/tests/shortlist.test.tsindex 945fb460..665e765d 100644--- a/laya-ts/tests/shortlist.test.ts+++ b/laya-ts/tests/shortlist.test.ts@@ -1,7 +1,9 @@import { describe, expect, it } from "vitest";-import { shortlistChoice } from "../src/shortlist.js";+import { cached, cachedEmbedFn, shortlistChoice } from "../src/shortlist.js";+const emb = async (texts: string[]) =>texts.map((t) => (t.includes("refund") || t.includes("billed") ? [1, 0] : [0, 1]));+describe("shortlist", () => {it("keeps top-k by cosine", async () => {const labels = await shortlistChoice({ text: "billed twice refund" },@@ -9,9 +11,255 @@ describe("shortlist", () => {emb as any, 1);expect(labels).toEqual(["refund"]);});+it("passthrough when k >= n skips embed", async () => {let called = false;await shortlistChoice("x", { a: "1" }, (async (t: string[]) => { called = true; return t.map(() => [1]); }) as any, 5);expect(called).toBe(false);});});++describe("cached embedder", () => {+ it("refuses non-callable embed_fn with TypeError", () => {+ expect(() => cached(null as any)).toThrow(TypeError);+ expect(() => cached(undefined as any)).toThrow(TypeError);+ expect(() => cached("not a function" as any)).toThrow(TypeError);+ });++ it("refuses non-positive or non-integer maxsize", () => {+ expect(() => cached(emb, 0)).toThrow(/maxsize must be a positive integer/);+ expect(() => cached(emb, -2)).toThrow(/maxsize must be a positive integer/);+ expect(() => cached(emb, 2.5)).toThrow(/maxsize must be a positive integer/);+ expect(() => cached(emb, "10" as any)).toThrow(/maxsize must be a positive integer/);+ expect(() => cached(emb, true as any)).toThrow(/maxsize must be a positive integer/);+ });++ it("records hits and misses and deduplicates cold lookups", async () => {+ const calls: string[][] = [];+ const countingEmbed = async (texts: string[]) => {+ calls.push([...texts]);+ return texts.map((t) => [t.length, 1]);+ };++ const c = cached(countingEmbed);+ expect(c.cacheInfo()).toEqual({ size: 0, maxsize: 4096, hits: 0, misses: 0 });++ // Cold call with duplicates+ const res1 = await c(["apple", "banana", "apple"]);+ expect(res1).toHaveLength(3);+ expect(res1[0]).toEqual([5, 1]);+ expect(res1[1]).toEqual([6, 1]);+ expect(res1[2]).toEqual([5, 1]);+ expect(calls).toEqual([["apple", "banana"]]); // deduplicated miss query+ expect(c.cacheInfo()).toEqual({ size: 2, maxsize: 4096, hits: 0, misses: 3 });++ // Second call: pure hits+ const res2 = await c(["banana", "apple"]);+ expect(res2).toHaveLength(2);+ expect(calls).toHaveLength(1); // no new embed calls+ expect(c.cacheInfo()).toEqual({ size: 2, maxsize: 4096, hits: 2, misses: 3 });++ // Mixed call: 1 hit, 1 miss+ const res3 = await c(["apple", "cherry"]);+ expect(res3).toHaveLength(2);+ expect(calls).toEqual([["apple", "banana"], ["cherry"]]);+ expect(c.cacheInfo()).toEqual({ size: 3, maxsize: 4096, hits: 3, misses: 4 });+ });++ it("evicts least-recently-used entries when size exceeds maxsize", async () => {+ const calls: string[][] = [];+ const tracking = async (texts: string[]) => {+ calls.push([...texts]);+ return texts.map((t) => [t.length]);+ };++ const c = cachedEmbedFn(tracking, 2);++ await c(["a", "b"]); // cache: [a, b]+ expect(c.cacheInfo().size).toBe(2);++ await c(["a"]); // access a -> LRU order: [b, a]+ expect(calls.length).toBe(1);+ expect(c.cacheInfo().hits).toBe(1);++ await c(["c"]); // adds c -> evicts b! cache: [a, c]+ expect(calls[calls.length - 1]).toEqual(["c"]);+ expect(c.cacheInfo().size).toBe(2);++ // both survivors are in cache: hits+ await c(["a", "c"]);+ expect(calls.length).toBe(2);++ // b was evicted, so it re-embeds+ await c(["b"]);+ expect(calls[calls.length - 1]).toEqual(["b"]);+ expect(c.cacheInfo().size).toBe(2);++ // most recent pair [c, b] survives+ await c(["c", "b"]);+ expect(calls.length).toBe(3);++ // a was now the oldest of the three and was evicted when b was inserted+ await c(["a"]);+ expect(calls[calls.length - 1]).toEqual(["a"]);+ expect(calls.length).toBe(4);+ });++ it("resets cache and counters on cacheClear() and cache_clear()", async () => {+ const c = cached(async (texts) => texts.map(() => [1, 2]));+ await c(["hello", "world"]);+ expect(c.cacheInfo().size).toBe(2);+ expect(c.cacheInfo().misses).toBe(2);++ c.cacheClear();+ expect(c.cacheInfo()).toEqual({ size: 0, maxsize: 4096, hits: 0, misses: 0 });+ expect(c.cache_info()).toEqual({ size: 0, maxsize: 4096, hits: 0, misses: 0 });++ c.cache_clear();+ expect(c.cacheInfo().size).toBe(0);+ });++ it("handles empty arrays and normalizes null/undefined keys", async () => {+ let called = false;+ const c = cached(async (t) => { called = true; return t.map(() => [1]); });++ const empty = await c([]);+ expect(empty).toEqual([]);+ expect(called).toBe(false);++ const normed = await c([null as any, undefined as any, ""]);+ expect(normed).toHaveLength(3);+ expect(c.cacheInfo().size).toBe(1); // all normalized to ""+ expect(c.cacheInfo().misses).toBe(3);+ });++ it("defensively copies cached vectors against caller mutations", async () => {+ const c = cached(async (texts) => texts.map(() => [10, 20]));+ const first = await c(["immutable"]);+ first[0][0] = 999;++ const second = await c(["immutable"]);+ expect(second[0][0]).toBe(10);+ });++ it("refuses changed embedding dimensions with descriptive error", async () => {+ let dim = 2;+ const changing = async (texts: string[]) => texts.map(() => Array(dim).fill(1));+ const c = cached(changing);++ await c(["first"]); // caches dim 2++ dim = 4;+ await expect(c(["second"])).rejects.toThrow(+ /embedder returned dim 4, but the cache holds dim 2; call cacheClear\(\)/,+ );+ });++ it("does not poison the cache on transient embedder failure", async () => {+ let fail = true;+ const flaky = async (texts: string[]) => {+ if (fail) throw new Error("network down");+ return texts.map(() => [1, 2]);+ };++ const c = cached(flaky);+ await expect(c(["retry_me"])).rejects.toThrow("network down");+ expect(c.cacheInfo().size).toBe(0);++ fail = false;+ const ok = await c(["retry_me"]);+ expect(ok).toEqual([[1, 2]]);+ expect(c.cacheInfo().size).toBe(1);+ });++ it("integrates seamlessly with shortlistChoice across repeated calls", async () => {+ let embedCalls = 0;+ const embedFn = async (texts: string[]) => {+ embedCalls++;+ return texts.map((t) => (t.includes("refund") ? [1, 0] : [0, 1]));+ };++ const cachedEmbed = cached(embedFn);+ const criteria = { card: "card arrival", refund: "refund money", fee: "transfer fee" };++ // First call: embeds query + 3 criteria (4 texts)+ const first = await shortlistChoice("first query refund", criteria, cachedEmbed, 1);+ expect(first).toEqual(["refund"]);+ const firstCalls = embedCalls;+ const firstHits = cachedEmbed.cacheInfo().hits;++ // Second call: criteria is cached! Only embeds the new query (1 text)+ const second = await shortlistChoice("second query refund", criteria, cachedEmbed, 1);+ expect(second).toEqual(["refund"]);+ expect(embedCalls).toBe(firstCalls + 1);+ expect(cachedEmbed.cacheInfo().hits - firstHits).toBe(3);+ });++ it("applies LRU recency in input order so missing keys before cached keys do not displace later inputs", async () => {+ let reembedded = false;+ const tracking = cachedEmbedFn(async (texts) => {+ reembedded = true;+ return texts.map((t) => [t.length]);+ }, 2);++ await tracking(["a", "b"]); // cache has [a, b]+ expect(tracking.cacheInfo().size).toBe(2);++ // "q" is missing, "a" and "b" are cached.+ // In input order: "q" is evaluated first (evicts a -> [b, q]),+ // then "a" is accessed (evicts b -> [q, a]),+ // then "b" is accessed (evicts q -> [a, b]).+ await tracking(["q", "a", "b"]);+ expect(tracking.cacheInfo().size).toBe(2);++ // Calling [a, b] should be complete hits without re-embedding+ reembedded = false;+ await tracking(["a", "b"]);+ expect(reembedded).toBe(false);+ });++ it("validates all fresh rows for dimension consistency before caching", async () => {+ // Case 1: Inconsistent dimensions from fresh call on empty cache+ const inconsistent = cached(async (texts) => texts.map((t, i) => Array(i + 1).fill(1)));+ await expect(inconsistent(["one", "two"])).rejects.toThrow(+ /inconsistent embedding dimensions/,+ );+ expect(inconsistent.cacheInfo().size).toBe(0);++ // Case 2: fresh[0] has matching dim but fresh[1] has differing dim on populated cache+ let secondCalls = 0;+ const mixed = cached(async (texts) => {+ secondCalls++;+ if (secondCalls === 1) return texts.map(() => [1, 2]);+ return [[1, 2], [1, 2, 3]]; // fresh[0] is dim 2, fresh[1] is dim 3+ });+ await mixed(["first"]);+ await expect(mixed(["good", "bad"])).rejects.toThrow(+ /embedder returned dim 3, but the cache holds dim 2; call cacheClear\(\)/,+ );+ });++ it("prevents in-flight embeddings from restoring entries into a cleared cache", async () => {+ let resolveEmbed: ((val: number[][]) => void) | null = null;+ const delayed = cached(+ () =>+ new Promise<number[][]>((res) => {+ resolveEmbed = res;+ }),+ );++ const pending = delayed(["slow"]);+ expect(delayed.cacheInfo().size).toBe(0);++ // Clear cache while request is in-flight+ delayed.cacheClear();++ // Resolve the delayed embeddings+ resolveEmbed!([[42]]);+ const res = await pending;+ expect(res).toEqual([[42]]);++ // Cache should NOT be populated with the stale in-flight result+ expect(delayed.cacheInfo().size).toBe(0);+ });+});