Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(ts): port cached embedder to laya-ts

Swatantra Yadav4739b93c77b3
diff --git a/laya-ts/README.md b/laya-ts/README.md
index 713f7e56..82e6738f 100644
--- a/laya-ts/README.md
+++ b/laya-ts/README.md
@@ -170,12 +170,14 @@ Python's `sort_by_length` grouping is not ported yet.
## Shortlist (many labels)
```ts
-import { shortlistChoice, predictShortlist, embedFnFromAgent } from "laya-ts";
+import { shortlistChoice, predictShortlist, embedFnFromAgent, cached } from "laya-ts";
-const keep = await shortlistChoice(state, bigCriteriaDict, embedFn, 20);
-const out = await predictShortlist(agent, state, questions, embedFn, 20);
+const cachedEmbed = cached(embedFn);
+const keep = await shortlistChoice(state, bigCriteriaDict, cachedEmbed, 20);
+const out = await predictShortlist(agent, state, questions, cachedEmbed, 20);
// out.shortlist[qid] = { labels, scores, k, n, passthrough }
// embedFnFromAgent(agent) mean-pools the loaded encoder; a dedicated bi-encoder usually shortlists better.
+// cached(embedFn) keeps an LRU cache across calls; cachedEmbed.cacheInfo() reports size/hits/misses.
```
## Per-language calibration (`lang_temperatures`)
diff --git a/laya-ts/src/index.ts b/laya-ts/src/index.ts
index a9dd789d..f08738db 100644
--- a/laya-ts/src/index.ts
+++ b/laya-ts/src/index.ts
@@ -17,8 +17,8 @@ export { createNodeProvider, createWebProvider, feed, feedHead, loadNodeBundle,
export type { Batch, SessionProvider, ProviderOptions, NodeBundle, WebBundle } from "./providers.js";
export { Router, normaliseName, DEFAULT_MODELS, digestsFromEnv, _digestsFromEnv } from "./router.js";
export type { RoutedResult, RouteDecision, ModelName, ModelSpec, BatchRequest, PredictBatchOptions, RouterOptions, RouteOptions } from "./router.js";
-export { shortlistChoice, predictShortlist, embedFnFromAgent, DEFAULT_SHORTLIST_K } from "./shortlist.js";
-export type { EmbedFn, ShortlistMeta } from "./shortlist.js";
+export { shortlistChoice, predictShortlist, embedFnFromAgent, DEFAULT_SHORTLIST_K, cached, cachedEmbedFn, DEFAULT_SHORTLIST_MAXSIZE } from "./shortlist.js";
+export type { EmbedFn, ShortlistMeta, CacheInfo, CachedEmbedFn } from "./shortlist.js";
export { analyse, isEnglish, guessLatinLanguage, detectScript } from "./lang.js";
export type { AnalyseResult, LatinProfile } from "./lang.js";
export { cleanEmailBody, emailState } from "./email.js";
diff --git a/laya-ts/src/shortlist.ts b/laya-ts/src/shortlist.ts
index f856ae96..5e3f0815 100644
--- a/laya-ts/src/shortlist.ts
+++ b/laya-ts/src/shortlist.ts
@@ -3,6 +3,7 @@ import type { Batch } from "./providers.js";
import type { QuestionDef, SystemOneResult } from "./agent.js";
export const DEFAULT_SHORTLIST_K = 20;
+export const DEFAULT_SHORTLIST_MAXSIZE = 4096;
export type EmbedFn = (texts: string[]) => Promise<number[][]> | number[][];
@@ -14,6 +15,21 @@ export interface ShortlistMeta {
passthrough: boolean;
}
+export interface CacheInfo {
+ size: number;
+ maxsize: number;
+ hits: number;
+ misses: number;
+}
+
+export interface CachedEmbedFn {
+ (texts: string[]): Promise<number[][]>;
+ cacheInfo(): CacheInfo;
+ cache_info(): CacheInfo;
+ cacheClear(): void;
+ cache_clear(): void;
+}
+
function checkK(k: unknown): number {
if (typeof k !== "number" || !Number.isInteger(k) || k < 1) {
throw new Error(`k must be a positive integer, got ${JSON.stringify(k)}`);
@@ -225,3 +241,136 @@ export function embedFnFromAgent(agent: unknown, maxLength = 512, batchSize = 32
return out;
};
}
+
+/**
+ * Wrap `embedFn` with an LRU cache keyed by text.
+ *
+ * Shortlisting embeds the query plus every option on every call. When the option
+ * set is fixed -- a label list, an intent taxonomy -- those rows never change
+ * between requests and are re-embedded every time. Wrapping the embedder once
+ * leaves the first call unchanged and reduces each repeat to embedding the new
+ * query alone.
+ *
+ * A cold cache costs the same number of embedder calls as the unwrapped function:
+ * the misses are deduplicated and embedded in one call. Nothing is cached when
+ * the embedder throws or returns a bad shape, so a transient failure cannot
+ * poison the cache.
+ *
+ * @param embedFn The underlying embedding function
+ * @param maxsize Maximum number of distinct text rows to cache (default: 4096)
+ */
+export function cachedEmbedFn(embedFn: EmbedFn, maxsize: number = DEFAULT_SHORTLIST_MAXSIZE): CachedEmbedFn {
+ if (typeof embedFn !== "function") {
+ throw new TypeError("embed_fn must be callable");
+ }
+ if (typeof maxsize !== "number" || !Number.isInteger(maxsize) || maxsize < 1) {
+ throw new Error(`maxsize must be a positive integer, got ${JSON.stringify(maxsize)}`);
+ }
+
+ const rowsByText = new Map<string, number[]>();
+ let hitsTotal = 0;
+ let missesTotal = 0;
+ let generation = 0;
+
+ const fn = async (texts: string[]): Promise<number[][]> => {
+ const keys = (texts ?? []).map((t) => (t === null || t === undefined ? "" : String(t)));
+ if (keys.length === 0) return [];
+
+ const found = new Map<string, number[]>();
+ let hitsThisCall = 0;
+
+ for (const key of keys) {
+ const row = rowsByText.get(key);
+ if (row !== undefined) {
+ found.set(key, row);
+ hitsThisCall++;
+ }
+ }
+
+ hitsTotal += hitsThisCall;
+ missesTotal += keys.length - hitsThisCall;
+
+ const missing: string[] = [];
+ const seenMissing = new Set<string>();
+ for (const key of keys) {
+ if (!found.has(key) && !seenMissing.has(key)) {
+ seenMissing.add(key);
+ missing.push(key);
+ }
+ }
+
+ const genAtStart = generation;
+
+ if (missing.length > 0) {
+ const fresh = await embeddings(embedFn, missing);
+
+ // Validate dimensions of all returned rows before caching any
+ const heldDim = rowsByText.size > 0 ? rowsByText.values().next().value?.length : undefined;
+ const expectedDim = heldDim ?? (fresh.length > 0 ? fresh[0]?.length : undefined);
+
+ for (let i = 0; i < fresh.length; i++) {
+ const rowDim = fresh[i]?.length ?? 0;
+ if (expectedDim !== undefined && rowDim !== expectedDim) {
+ if (heldDim !== undefined) {
+ throw new Error(
+ `the embedder returned dim ${rowDim}, but the cache holds dim ${heldDim}; call cacheClear() if the model behind it changed`,
+ );
+ } else {
+ throw new Error(
+ `the embedder returned inconsistent embedding dimensions (${expectedDim} vs ${rowDim})`,
+ );
+ }
+ }
+ }
+
+ for (let i = 0; i < missing.length; i++) {
+ found.set(missing[i], fresh[i]);
+ }
+
+ // If cacheClear() ran while embeddings() was pending, do not restore entries into the cleared cache
+ if (generation !== genAtStart) {
+ return keys.map((key) => [...(found.get(key) ?? [])]);
+ }
+ }
+
+ // Apply recency updates and evictions in input key order
+ for (const key of keys) {
+ const row = found.get(key);
+ if (row !== undefined) {
+ rowsByText.delete(key);
+ rowsByText.set(key, row);
+ while (rowsByText.size > maxsize) {
+ const victim = rowsByText.keys().next().value!;
+ rowsByText.delete(victim);
+ }
+ }
+ }
+
+ return keys.map((key) => [...(found.get(key) ?? [])]);
+ };
+
+ const cacheInfo = (): CacheInfo => ({
+ size: rowsByText.size,
+ maxsize,
+ hits: hitsTotal,
+ misses: missesTotal,
+ });
+
+ const cacheClear = (): void => {
+ rowsByText.clear();
+ hitsTotal = 0;
+ missesTotal = 0;
+ generation++;
+ };
+
+ const cachedResult = fn as CachedEmbedFn;
+ cachedResult.cacheInfo = cacheInfo;
+ cachedResult.cache_info = cacheInfo;
+ cachedResult.cacheClear = cacheClear;
+ cachedResult.cache_clear = cacheClear;
+
+ return cachedResult;
+}
+
+export const cached = cachedEmbedFn;
+
diff --git a/laya-ts/tests/shortlist.test.ts b/laya-ts/tests/shortlist.test.ts
index 945fb460..665e765d 100644
--- a/laya-ts/tests/shortlist.test.ts
+++ b/laya-ts/tests/shortlist.test.ts
@@ -1,7 +1,9 @@
import { describe, expect, it } from "vitest";
-import { shortlistChoice } from "../src/shortlist.js";
+import { cached, cachedEmbedFn, shortlistChoice } from "../src/shortlist.js";
+
const emb = async (texts: string[]) =>
texts.map((t) => (t.includes("refund") || t.includes("billed") ? [1, 0] : [0, 1]));
+
describe("shortlist", () => {
it("keeps top-k by cosine", async () => {
const labels = await shortlistChoice({ text: "billed twice refund" },
@@ -9,9 +11,255 @@ describe("shortlist", () => {
emb as any, 1);
expect(labels).toEqual(["refund"]);
});
+
it("passthrough when k >= n skips embed", async () => {
let called = false;
await shortlistChoice("x", { a: "1" }, (async (t: string[]) => { called = true; return t.map(() => [1]); }) as any, 5);
expect(called).toBe(false);
});
});
+
+describe("cached embedder", () => {
+ it("refuses non-callable embed_fn with TypeError", () => {
+ expect(() => cached(null as any)).toThrow(TypeError);
+ expect(() => cached(undefined as any)).toThrow(TypeError);
+ expect(() => cached("not a function" as any)).toThrow(TypeError);
+ });
+
+ it("refuses non-positive or non-integer maxsize", () => {
+ expect(() => cached(emb, 0)).toThrow(/maxsize must be a positive integer/);
+ expect(() => cached(emb, -2)).toThrow(/maxsize must be a positive integer/);
+ expect(() => cached(emb, 2.5)).toThrow(/maxsize must be a positive integer/);
+ expect(() => cached(emb, "10" as any)).toThrow(/maxsize must be a positive integer/);
+ expect(() => cached(emb, true as any)).toThrow(/maxsize must be a positive integer/);
+ });
+
+ it("records hits and misses and deduplicates cold lookups", async () => {
+ const calls: string[][] = [];
+ const countingEmbed = async (texts: string[]) => {
+ calls.push([...texts]);
+ return texts.map((t) => [t.length, 1]);
+ };
+
+ const c = cached(countingEmbed);
+ expect(c.cacheInfo()).toEqual({ size: 0, maxsize: 4096, hits: 0, misses: 0 });
+
+ // Cold call with duplicates
+ const res1 = await c(["apple", "banana", "apple"]);
+ expect(res1).toHaveLength(3);
+ expect(res1[0]).toEqual([5, 1]);
+ expect(res1[1]).toEqual([6, 1]);
+ expect(res1[2]).toEqual([5, 1]);
+ expect(calls).toEqual([["apple", "banana"]]); // deduplicated miss query
+ expect(c.cacheInfo()).toEqual({ size: 2, maxsize: 4096, hits: 0, misses: 3 });
+
+ // Second call: pure hits
+ const res2 = await c(["banana", "apple"]);
+ expect(res2).toHaveLength(2);
+ expect(calls).toHaveLength(1); // no new embed calls
+ expect(c.cacheInfo()).toEqual({ size: 2, maxsize: 4096, hits: 2, misses: 3 });
+
+ // Mixed call: 1 hit, 1 miss
+ const res3 = await c(["apple", "cherry"]);
+ expect(res3).toHaveLength(2);
+ expect(calls).toEqual([["apple", "banana"], ["cherry"]]);
+ expect(c.cacheInfo()).toEqual({ size: 3, maxsize: 4096, hits: 3, misses: 4 });
+ });
+
+ it("evicts least-recently-used entries when size exceeds maxsize", async () => {
+ const calls: string[][] = [];
+ const tracking = async (texts: string[]) => {
+ calls.push([...texts]);
+ return texts.map((t) => [t.length]);
+ };
+
+ const c = cachedEmbedFn(tracking, 2);
+
+ await c(["a", "b"]); // cache: [a, b]
+ expect(c.cacheInfo().size).toBe(2);
+
+ await c(["a"]); // access a -> LRU order: [b, a]
+ expect(calls.length).toBe(1);
+ expect(c.cacheInfo().hits).toBe(1);
+
+ await c(["c"]); // adds c -> evicts b! cache: [a, c]
+ expect(calls[calls.length - 1]).toEqual(["c"]);
+ expect(c.cacheInfo().size).toBe(2);
+
+ // both survivors are in cache: hits
+ await c(["a", "c"]);
+ expect(calls.length).toBe(2);
+
+ // b was evicted, so it re-embeds
+ await c(["b"]);
+ expect(calls[calls.length - 1]).toEqual(["b"]);
+ expect(c.cacheInfo().size).toBe(2);
+
+ // most recent pair [c, b] survives
+ await c(["c", "b"]);
+ expect(calls.length).toBe(3);
+
+ // a was now the oldest of the three and was evicted when b was inserted
+ await c(["a"]);
+ expect(calls[calls.length - 1]).toEqual(["a"]);
+ expect(calls.length).toBe(4);
+ });
+
+ it("resets cache and counters on cacheClear() and cache_clear()", async () => {
+ const c = cached(async (texts) => texts.map(() => [1, 2]));
+ await c(["hello", "world"]);
+ expect(c.cacheInfo().size).toBe(2);
+ expect(c.cacheInfo().misses).toBe(2);
+
+ c.cacheClear();
+ expect(c.cacheInfo()).toEqual({ size: 0, maxsize: 4096, hits: 0, misses: 0 });
+ expect(c.cache_info()).toEqual({ size: 0, maxsize: 4096, hits: 0, misses: 0 });
+
+ c.cache_clear();
+ expect(c.cacheInfo().size).toBe(0);
+ });
+
+ it("handles empty arrays and normalizes null/undefined keys", async () => {
+ let called = false;
+ const c = cached(async (t) => { called = true; return t.map(() => [1]); });
+
+ const empty = await c([]);
+ expect(empty).toEqual([]);
+ expect(called).toBe(false);
+
+ const normed = await c([null as any, undefined as any, ""]);
+ expect(normed).toHaveLength(3);
+ expect(c.cacheInfo().size).toBe(1); // all normalized to ""
+ expect(c.cacheInfo().misses).toBe(3);
+ });
+
+ it("defensively copies cached vectors against caller mutations", async () => {
+ const c = cached(async (texts) => texts.map(() => [10, 20]));
+ const first = await c(["immutable"]);
+ first[0][0] = 999;
+
+ const second = await c(["immutable"]);
+ expect(second[0][0]).toBe(10);
+ });
+
+ it("refuses changed embedding dimensions with descriptive error", async () => {
+ let dim = 2;
+ const changing = async (texts: string[]) => texts.map(() => Array(dim).fill(1));
+ const c = cached(changing);
+
+ await c(["first"]); // caches dim 2
+
+ dim = 4;
+ await expect(c(["second"])).rejects.toThrow(
+ /embedder returned dim 4, but the cache holds dim 2; call cacheClear\(\)/,
+ );
+ });
+
+ it("does not poison the cache on transient embedder failure", async () => {
+ let fail = true;
+ const flaky = async (texts: string[]) => {
+ if (fail) throw new Error("network down");
+ return texts.map(() => [1, 2]);
+ };
+
+ const c = cached(flaky);
+ await expect(c(["retry_me"])).rejects.toThrow("network down");
+ expect(c.cacheInfo().size).toBe(0);
+
+ fail = false;
+ const ok = await c(["retry_me"]);
+ expect(ok).toEqual([[1, 2]]);
+ expect(c.cacheInfo().size).toBe(1);
+ });
+
+ it("integrates seamlessly with shortlistChoice across repeated calls", async () => {
+ let embedCalls = 0;
+ const embedFn = async (texts: string[]) => {
+ embedCalls++;
+ return texts.map((t) => (t.includes("refund") ? [1, 0] : [0, 1]));
+ };
+
+ const cachedEmbed = cached(embedFn);
+ const criteria = { card: "card arrival", refund: "refund money", fee: "transfer fee" };
+
+ // First call: embeds query + 3 criteria (4 texts)
+ const first = await shortlistChoice("first query refund", criteria, cachedEmbed, 1);
+ expect(first).toEqual(["refund"]);
+ const firstCalls = embedCalls;
+ const firstHits = cachedEmbed.cacheInfo().hits;
+
+ // Second call: criteria is cached! Only embeds the new query (1 text)
+ const second = await shortlistChoice("second query refund", criteria, cachedEmbed, 1);
+ expect(second).toEqual(["refund"]);
+ expect(embedCalls).toBe(firstCalls + 1);
+ expect(cachedEmbed.cacheInfo().hits - firstHits).toBe(3);
+ });
+
+ it("applies LRU recency in input order so missing keys before cached keys do not displace later inputs", async () => {
+ let reembedded = false;
+ const tracking = cachedEmbedFn(async (texts) => {
+ reembedded = true;
+ return texts.map((t) => [t.length]);
+ }, 2);
+
+ await tracking(["a", "b"]); // cache has [a, b]
+ expect(tracking.cacheInfo().size).toBe(2);
+
+ // "q" is missing, "a" and "b" are cached.
+ // In input order: "q" is evaluated first (evicts a -> [b, q]),
+ // then "a" is accessed (evicts b -> [q, a]),
+ // then "b" is accessed (evicts q -> [a, b]).
+ await tracking(["q", "a", "b"]);
+ expect(tracking.cacheInfo().size).toBe(2);
+
+ // Calling [a, b] should be complete hits without re-embedding
+ reembedded = false;
+ await tracking(["a", "b"]);
+ expect(reembedded).toBe(false);
+ });
+
+ it("validates all fresh rows for dimension consistency before caching", async () => {
+ // Case 1: Inconsistent dimensions from fresh call on empty cache
+ const inconsistent = cached(async (texts) => texts.map((t, i) => Array(i + 1).fill(1)));
+ await expect(inconsistent(["one", "two"])).rejects.toThrow(
+ /inconsistent embedding dimensions/,
+ );
+ expect(inconsistent.cacheInfo().size).toBe(0);
+
+ // Case 2: fresh[0] has matching dim but fresh[1] has differing dim on populated cache
+ let secondCalls = 0;
+ const mixed = cached(async (texts) => {
+ secondCalls++;
+ if (secondCalls === 1) return texts.map(() => [1, 2]);
+ return [[1, 2], [1, 2, 3]]; // fresh[0] is dim 2, fresh[1] is dim 3
+ });
+ await mixed(["first"]);
+ await expect(mixed(["good", "bad"])).rejects.toThrow(
+ /embedder returned dim 3, but the cache holds dim 2; call cacheClear\(\)/,
+ );
+ });
+
+ it("prevents in-flight embeddings from restoring entries into a cleared cache", async () => {
+ let resolveEmbed: ((val: number[][]) => void) | null = null;
+ const delayed = cached(
+ () =>
+ new Promise<number[][]>((res) => {
+ resolveEmbed = res;
+ }),
+ );
+
+ const pending = delayed(["slow"]);
+ expect(delayed.cacheInfo().size).toBe(0);
+
+ // Clear cache while request is in-flight
+ delayed.cacheClear();
+
+ // Resolve the delayed embeddings
+ resolveEmbed!([[42]]);
+ const res = await pending;
+ expect(res).toEqual([[42]]);
+
+ // Cache should NOT be populated with the stale in-flight result
+ expect(delayed.cacheInfo().size).toBe(0);
+ });
+});