#1 Persistent memory for AI coding agents based on real-world benchmarks
TypeScript agentmemoryagentsaiclaudeclaudecodecodexcopilotcursorgenaiharnesshermesmemoryompopenclawopencodepi
chore(deps): migrate @xenova/transformers to @huggingface/transformers v4 (#1096)
Abdullah Alaqeel8c90741c633c
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex ed53c6da..8b60a787 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -52,7 +52,7 @@ jobs:# src/functions/obsidian-export.ts needs os.tmpdir() + path.join# rework before Windows can be added back. Tracked as follow-up.os: [ubuntu-latest, macos-latest]- node-version: [20, 22]+ node-version: [20, 22, 24, 26]steps:- uses: actions/checkout@v6with:diff --git a/README.md b/README.mdindex 24751192..48f4045c 100644--- a/README.md+++ b/README.md@@ -940,7 +940,7 @@ BM25 tokenizes Greek, Cyrillic, Hebrew, Arabic, and accented Latin out of the boagentmemory auto-detects your provider. For best results, install local embeddings (free):```bash-npm install @xenova/transformers+npm install @huggingface/transformers```| Provider | Model | Cost | Notes |@@ -1271,7 +1271,7 @@ OPENAI_MODEL=qwen2.5-coder-7b-instruct # match the model name from LM StReasoning-class models (`o1`-style with `<think>` blocks) can return empty `content` with a `reasoning` field your local server may not surface. If extractions come back blank, switch to a non-reasoning model first. The `OPENAI_REASONING_EFFORT=none` env can also disable thinking on Ollama Cloud thinking models that mirror the OpenAI reasoning schema.-Local embeddings ship out of the box via `@xenova/transformers` — `EMBEDDING_PROVIDER=local` (default) gives you BGE-small entirely on-device. No extra config needed.+Local embeddings ship out of the box via `@huggingface/transformers` — `EMBEDDING_PROVIDER=local` (default) gives you `Xenova/all-MiniLM-L6-v2` (384-dim) entirely on-device. No extra config needed.### Cost-aware model selectiondiff --git a/READMEs/README.de-DE.md b/READMEs/README.de-DE.mdindex 512e81d8..332519b4 100644--- a/READMEs/README.de-DE.md+++ b/READMEs/README.de-DE.md@@ -809,7 +809,7 @@ BM25 tokenisiert Griechisch, Kyrillisch, Hebräisch, Arabisch und akzentuiertesagentmemory erkennt Ihren Provider automatisch. Für die besten Ergebnisse installieren Sie lokale Embeddings (kostenlos):```bash-npm install @xenova/transformers+npm install @huggingface/transformers```| Provider | Modell | Kosten | Hinweise |diff --git a/READMEs/README.es-ES.md b/READMEs/README.es-ES.mdindex 8b5654ea..af7e7b35 100644--- a/READMEs/README.es-ES.md+++ b/READMEs/README.es-ES.md@@ -802,7 +802,7 @@ BM25 tokeniza griego, cirílico, hebreo, árabe y latín con tildes de serie. Paagentmemory autodetecta tu proveedor. Para mejores resultados, instala embeddings locales (gratis):```bash-npm install @xenova/transformers+npm install @huggingface/transformers```| Proveedor | Modelo | Coste | Notas |diff --git a/READMEs/README.fr-FR.md b/READMEs/README.fr-FR.mdindex d00206cb..f3952260 100644--- a/READMEs/README.fr-FR.md+++ b/READMEs/README.fr-FR.md@@ -809,7 +809,7 @@ BM25 tokenise nativement le grec, le cyrillique, l'hébreu, l'arabe et le latinagentmemory détecte automatiquement votre fournisseur. Pour de meilleurs résultats, installez les embeddings locaux (gratuits) :```bash-npm install @xenova/transformers+npm install @huggingface/transformers```| Fournisseur | Modèle | Coût | Notes |diff --git a/READMEs/README.hi-IN.md b/READMEs/README.hi-IN.mdindex 8733236a..dc6f8d9f 100644--- a/READMEs/README.hi-IN.md+++ b/READMEs/README.hi-IN.md@@ -812,7 +812,7 @@ BM25 box से बाहर ही Greek, Cyrillic, Hebrew, Arabic, और acceagentmemory आपके provider को auto-detect करता है। सर्वोत्तम परिणामों के लिए, local embeddings install करें (free):```bash-npm install @xenova/transformers+npm install @huggingface/transformers```| Provider | Model | Cost | नोट्स |diff --git a/READMEs/README.ja-JP.md b/READMEs/README.ja-JP.mdindex 30d535d9..bca18961 100644--- a/READMEs/README.ja-JP.md+++ b/READMEs/README.ja-JP.md@@ -812,7 +812,7 @@ BM25 は箱から出してすぐにギリシャ文字、キリル文字、ヘブagentmemory はプロバイダーを自動検出します。最良の結果を得るには、ローカル埋め込み(無料)をインストール:```bash-npm install @xenova/transformers+npm install @huggingface/transformers```| プロバイダー | モデル | コスト | 備考 |diff --git a/READMEs/README.ko-KR.md b/READMEs/README.ko-KR.mdindex 395c91c2..7bd90050 100644--- a/READMEs/README.ko-KR.md+++ b/READMEs/README.ko-KR.md@@ -793,7 +793,7 @@ BM25는 기본적으로 그리스어, 키릴 문자, 히브리어, 아랍어,agentmemory는 프로바이더를 자동 감지합니다. 최상의 결과를 위해 로컬 임베딩을 설치하십시오 (무료):```bash-npm install @xenova/transformers+npm install @huggingface/transformers```| 프로바이더 | 모델 | 비용 | 비고 |diff --git a/READMEs/README.pt-BR.md b/READMEs/README.pt-BR.mdindex 0e80ff8f..e56fd799 100644--- a/READMEs/README.pt-BR.md+++ b/READMEs/README.pt-BR.md@@ -802,7 +802,7 @@ BM25 tokeniza grego, cirílico, hebraico, árabe e latim acentuado de fábrica.agentmemory autodetecta seu provider. Para melhores resultados, instale embeddings locais (gratuito):```bash-npm install @xenova/transformers+npm install @huggingface/transformers```| Provider | Modelo | Custo | Notas |diff --git a/READMEs/README.ru-RU.md b/READMEs/README.ru-RU.mdindex f6c1e554..0e112f70 100644--- a/READMEs/README.ru-RU.md+++ b/READMEs/README.ru-RU.md@@ -809,7 +809,7 @@ BM25 «из коробки» токенизирует греческий, кирagentmemory автоматически определяет вашего провайдера. Для лучших результатов поставьте локальные эмбеддинги (бесплатно):```bash-npm install @xenova/transformers+npm install @huggingface/transformers```| Провайдер | Модель | Стоимость | Заметки |diff --git a/READMEs/README.tr-TR.md b/READMEs/README.tr-TR.mdindex fefcf6cc..4b68acc7 100644--- a/READMEs/README.tr-TR.md+++ b/READMEs/README.tr-TR.md@@ -813,7 +813,7 @@ BM25, Yunanca, Kiril, İbranice, Arapça ve aksanlı Latin'i kutudan çıkar çagentmemory sağlayıcınızı otomatik algılar. En iyi sonuçlar için yerel embedding'leri kurun (ücretsiz):```bash-npm install @xenova/transformers+npm install @huggingface/transformers```| Sağlayıcı | Model | Maliyet | Notlar |diff --git a/READMEs/README.zh-CN.md b/READMEs/README.zh-CN.mdindex a11ab29a..e7b50d52 100644--- a/READMEs/README.zh-CN.md+++ b/READMEs/README.zh-CN.md@@ -810,7 +810,7 @@ BM25 开箱即用支持希腊语、西里尔语、希伯来语、阿拉伯语和agentmemory 自动检测你的提供者。为获得最佳效果,安装本地嵌入(免费):```bash-npm install @xenova/transformers+npm install @huggingface/transformers```| 提供者 | 模型 | 成本 | 备注 |diff --git a/READMEs/README.zh-TW.md b/READMEs/README.zh-TW.mdindex c7a20fb5..ce87e241 100644--- a/READMEs/README.zh-TW.md+++ b/READMEs/README.zh-TW.md@@ -810,7 +810,7 @@ BM25 開箱即用支援希臘文、西里爾文、希伯來文、阿拉伯文和agentmemory 自動偵測你的提供者。為獲得最佳效果,安裝本地嵌入(免費):```bash-npm install @xenova/transformers+npm install @huggingface/transformers```| 提供者 | 模型 | 成本 | 備註 |diff --git a/SECURITY.md b/SECURITY.mdindex a24b6e96..ff51d01e 100644--- a/SECURITY.md+++ b/SECURITY.md@@ -72,7 +72,7 @@ Supply-chain monitoring we already do:- Dependabot opens PRs for every minor/patch bump on the production dep list (visible in the open PRs).- Every PR runs the full test suite on ubuntu-latest + macos-latest, Node 20 + 22, before any merge.-- `optionalDependencies` (`@xenova/transformers`, `onnxruntime-node`, etc.) are guarded by `try { await import("...") } catch` so a missing or compromised optional dep cannot break the core runtime path.+- `optionalDependencies` (`@huggingface/transformers`, transitively `onnxruntime-node`, etc.) are guarded by `try { await import("...") } catch` so a missing or compromised optional dep cannot break the core runtime path.If you find a malicious package in our dep tree, file via the GHSA flow at the top of this document — that's the fastest path to a fixed release on npm.diff --git a/__mocks__/@huggingface/transformers.ts b/__mocks__/@huggingface/transformers.tsnew file mode 100644index 00000000..debd57ad--- /dev/null+++ b/__mocks__/@huggingface/transformers.ts@@ -0,0 +1,4 @@+throw Object.assign(+ new Error("Cannot find package '@huggingface/transformers'"),+ { code: "ERR_MODULE_NOT_FOUND" },+);diff --git a/benchmark/LONGMEMEVAL.md b/benchmark/LONGMEMEVAL.mdindex 00de938e..9b1afa41 100644--- a/benchmark/LONGMEMEVAL.md+++ b/benchmark/LONGMEMEVAL.md@@ -74,6 +74,6 @@ hf_hub_download(repo_id='xiaowu0162/longmemeval-cleaned', filename='longmemeval_# Run BM25-onlynpx tsx benchmark/longmemeval-bench.ts bm25-# Run BM25+Vector hybrid (requires @xenova/transformers)+# Run BM25+Vector hybrid (requires @huggingface/transformers)npx tsx benchmark/longmemeval-bench.ts hybrid```diff --git a/benchmark/REAL-EMBEDDINGS.md b/benchmark/REAL-EMBEDDINGS.mdindex 95c58636..f7db8f03 100644--- a/benchmark/REAL-EMBEDDINGS.md+++ b/benchmark/REAL-EMBEDDINGS.md@@ -59,7 +59,7 @@ Embedding is a one-time cost at ingestion. Search is sub-millisecond after index## Recommendation-Enable local embeddings by default (`EMBEDDING_PROVIDER=local` or install `@xenova/transformers`).+Enable local embeddings by default (`EMBEDDING_PROVIDER=local` or install `@huggingface/transformers`).This gives agentmemory genuine semantic search that built-in agent memories cannot match —understanding that "database performance optimization" relates to "N+1 query fix" and "eager loading".diff --git a/benchmark/real-embeddings-eval.ts b/benchmark/real-embeddings-eval.tsindex 1e4628e5..7804649d 100644--- a/benchmark/real-embeddings-eval.ts+++ b/benchmark/real-embeddings-eval.ts@@ -335,7 +335,7 @@ function generateReport(systems: SystemResult[], obsCount: number): string {w("");w("## Recommendation");w("");- w("Enable local embeddings by default (`EMBEDDING_PROVIDER=local` or install `@xenova/transformers`).");+ w("Enable local embeddings by default (`EMBEDDING_PROVIDER=local` or install `@huggingface/transformers`).");w("This gives agentmemory genuine semantic search that built-in agent memories cannot match —");w("understanding that \"database performance optimization\" relates to \"N+1 query fix\" and \"eager loading\".");w("");@@ -357,7 +357,7 @@ async function main() {console.log(`Model loaded. Dimensions: ${testEmbed.length}\n`);} catch (err) {console.error("Failed to load Xenova model:", err);- console.error("Install with: npm install @xenova/transformers");+ console.error("Install with: npm install @huggingface/transformers");process.exit(1);}diff --git a/package.json b/package.jsonindex 588e94da..77185ad5 100644--- a/package.json+++ b/package.json@@ -69,10 +69,8 @@"zod": "^4.0.0"},"optionalDependencies": {+ "@huggingface/transformers": "^4.2.0","@node-rs/jieba": "^2.0.1",- "@xenova/transformers": "^2.17.2",- "onnxruntime-node": "^1.14.0",- "onnxruntime-web": "^1.14.0","tiny-segmenter": "^0.2.0"},"devDependencies": {diff --git a/src/providers/embedding/clip.ts b/src/providers/embedding/clip.tsindex 2a0143ad..4ee102bc 100644--- a/src/providers/embedding/clip.ts+++ b/src/providers/embedding/clip.ts@@ -1,32 +1,20 @@import { readFile } from "node:fs/promises";+import type { RawImage } from "@huggingface/transformers";import type { EmbeddingProvider } from "../../types.js";-type TransformersModule = {- pipeline: (- task: string,- model: string,- ) => Promise<ClipPipeline>;- RawImage: {- fromBlob: (blob: Blob) => Promise<RawImageInstance>;- };-};--type RawImageInstance = unknown;-+type TransformersModule = typeof import("@huggingface/transformers");type ClipPipeline = (- input: string[] | RawImageInstance | RawImageInstance[],+ input: string[] | RawImage | RawImage[],options?: { pooling?: string; normalize?: boolean },) => Promise<{ tolist: () => number[][]; data: Float32Array }>;const DEFAULT_MODEL = "Xenova/clip-vit-base-patch32";-const DIMENSIONS = 512;export class ClipEmbeddingProvider implements EmbeddingProvider {readonly name = "clip";- readonly dimensions = DIMENSIONS;+ readonly dimensions = 512;private textExtractor: ClipPipeline | null = null;private imageExtractor: ClipPipeline | null = null;- private transformers: TransformersModule | null = null;private readonly modelId: string;constructor(modelId: string = DEFAULT_MODEL) {@@ -45,7 +33,7 @@ export class ClipEmbeddingProvider implements EmbeddingProvider {}async embedImage(src: string): Promise<Float32Array> {- const t = await this.getTransformers();+ const t = await loadTransformers();const image = await loadImage(t, src);const extractor = await this.getImageExtractor();const output = await extractor(image);@@ -53,47 +41,46 @@ export class ClipEmbeddingProvider implements EmbeddingProvider {return normalize(vec);}- private async getTransformers(): Promise<TransformersModule> {- if (this.transformers) return this.transformers;- try {- this.transformers = (await import("@xenova/transformers")) as unknown as TransformersModule;- } catch {- throw new Error(- "Install @xenova/transformers for CLIP image embeddings: npm install @xenova/transformers",- );- }- return this.transformers;- }-private async getTextExtractor(): Promise<ClipPipeline> {if (this.textExtractor) return this.textExtractor;- const t = await this.getTransformers();- this.textExtractor = await t.pipeline("feature-extraction", this.modelId);+ const t = await loadTransformers();+ this.textExtractor = (await t.pipeline("feature-extraction", this.modelId, { dtype: "q8" })) as ClipPipeline;return this.textExtractor;}private async getImageExtractor(): Promise<ClipPipeline> {if (this.imageExtractor) return this.imageExtractor;- const t = await this.getTransformers();- this.imageExtractor = await t.pipeline("image-feature-extraction", this.modelId);+ const t = await loadTransformers();+ this.imageExtractor = (await t.pipeline("image-feature-extraction", this.modelId, { dtype: "q8" })) as ClipPipeline;return this.imageExtractor;}}+async function loadTransformers(): Promise<TransformersModule> {+ try {+ return await import("@huggingface/transformers");+ } catch (err) {+ if ((err as NodeJS.ErrnoException).code === "ERR_MODULE_NOT_FOUND") {+ throw new Error(+ "Install @huggingface/transformers for CLIP embeddings: npm install @huggingface/transformers",+ );+ }+ throw err;+ }+}+async function loadImage(t: TransformersModule,src: string,-): Promise<RawImageInstance> {+): Promise<RawImage> {if (src.startsWith("data:")) {const comma = src.indexOf(",");const b64 = comma >= 0 ? src.slice(comma + 1) : src;const buf = Buffer.from(b64, "base64");- const blob = new Blob([buf]);- return t.RawImage.fromBlob(blob);+ return t.RawImage.fromBlob(new Blob([buf]));}const data = await readFile(src);- const blob = new Blob([data]);- return t.RawImage.fromBlob(blob);+ return t.RawImage.fromBlob(new Blob([data]));}function normalize(vec: Float32Array): Float32Array {diff --git a/src/providers/embedding/local.ts b/src/providers/embedding/local.tsindex ad6c2d21..7646815d 100644--- a/src/providers/embedding/local.ts+++ b/src/providers/embedding/local.ts@@ -1,19 +1,14 @@import type { EmbeddingProvider } from "../../types.js";-type Pipeline = (- task: string,- model: string,-) => Promise<- (- texts: string[],- options: { pooling: string; normalize: boolean },- ) => Promise<{ tolist: () => number[][] }>->;+type FeatureExtractor = (+ texts: string[],+ options: { pooling: string; normalize: boolean },+) => Promise<{ tolist: () => number[][] }>;export class LocalEmbeddingProvider implements EmbeddingProvider {readonly name = "local";readonly dimensions = 384;- private extractor: Awaited<ReturnType<Pipeline>> | null = null;+ private extractor: FeatureExtractor | null = null;async embed(text: string): Promise<Float32Array> {const [result] = await this.embedBatch([text]);@@ -26,27 +21,27 @@ export class LocalEmbeddingProvider implements EmbeddingProvider {pooling: "mean",normalize: true,});- const vectors = output.tolist();- return vectors.map((v: number[]) => new Float32Array(v));+ return output.tolist().map((v) => new Float32Array(v));}private async getExtractor() {if (this.extractor) return this.extractor;-- let transformers: { pipeline: Pipeline };+ let transformers: typeof import("@huggingface/transformers");try {- // @ts-ignore - optional peer dependency- transformers = await import("@xenova/transformers");- } catch {- throw new Error(- "Install @xenova/transformers for local embeddings: npm install @xenova/transformers",- );+ transformers = await import("@huggingface/transformers");+ } catch (err) {+ if ((err as NodeJS.ErrnoException).code === "ERR_MODULE_NOT_FOUND") {+ throw new Error(+ "Install @huggingface/transformers for local embeddings: npm install @huggingface/transformers",+ );+ }+ throw err;}-- this.extractor = await transformers.pipeline(+ this.extractor = (await transformers.pipeline("feature-extraction","Xenova/all-MiniLM-L6-v2",- );+ { dtype: "q8" },+ )) as FeatureExtractor;return this.extractor;}}diff --git a/src/state/reranker.ts b/src/state/reranker.tsindex d0aae68c..ae92f8e4 100644--- a/src/state/reranker.ts+++ b/src/state/reranker.ts@@ -12,12 +12,12 @@ async function loadPipeline(): Promise<any> {pipelineLoading = (async () => {try {const { pipeline: createPipeline } = await import(- "@xenova/transformers"+ "@huggingface/transformers");pipeline = await createPipeline("text-classification","Xenova/ms-marco-MiniLM-L-6-v2",- { quantized: true },+ { dtype: "q8" },);return pipeline;} catch {diff --git a/src/xenova.d.ts b/src/xenova.d.tsdeleted file mode 100644index 7abd4b09..00000000--- a/src/xenova.d.ts+++ /dev/null@@ -1,3 +0,0 @@-declare module "@xenova/transformers" {- export function pipeline(task: string, model: string): Promise<any>;-}diff --git a/test/clip-embedding-provider.test.ts b/test/clip-embedding-provider.test.tsnew file mode 100644index 00000000..b605c1ef--- /dev/null+++ b/test/clip-embedding-provider.test.ts@@ -0,0 +1,100 @@+import { describe, it, expect, vi, afterEach } from "vitest";++afterEach(() => {+ vi.doUnmock("@huggingface/transformers");+ vi.resetModules();+});++describe("ClipEmbeddingProvider (package unavailable)", () => {+ it("throws clean install hint when @huggingface/transformers is missing", async () => {+ vi.doMock("@huggingface/transformers");+ vi.resetModules();+ const { ClipEmbeddingProvider: Fresh } = await import(+ "../src/providers/embedding/clip.js"+ );+ await expect(new Fresh().embed("hello")).rejects.toThrow(+ "Install @huggingface/transformers for CLIP embeddings",+ );+ });+});++describe("ClipEmbeddingProvider (with loaded pipeline)", () => {+ function mockSuccessModule() {+ const textExtractor = vi.fn(async (texts: string[]) => ({+ tolist: () => texts.map(() => [0.1, 0.2]),+ }));+ const imageExtractor = vi.fn(async () => ({+ tolist: () => [[0.3, 0.4]],+ data: new Float32Array([0.3, 0.4]),+ }));+ const fromBlob = vi.fn(async () => ({}));+ const pipeline = vi.fn((task: string) => {+ if (task === "feature-extraction") return Promise.resolve(textExtractor);+ if (task === "image-feature-extraction") return Promise.resolve(imageExtractor);+ return Promise.reject(new Error(`unmocked task: ${task}`));+ });+ vi.doMock("@huggingface/transformers", () => ({+ pipeline,+ RawImage: { fromBlob },+ }));+ vi.resetModules();+ return { pipeline, textExtractor, imageExtractor, fromBlob };+ }++ it("loads text pipeline with dtype: q8 and returns mapped Float32Array", async () => {+ const { pipeline } = mockSuccessModule();+ const { ClipEmbeddingProvider: Fresh } = await import(+ "../src/providers/embedding/clip.js"+ );+ const vec = await new Fresh().embed("hello");++ expect(pipeline).toHaveBeenCalledWith(+ "feature-extraction",+ "Xenova/clip-vit-base-patch32",+ { dtype: "q8" },+ );+ expect(vec).toBeInstanceOf(Float32Array);+ expect(vec).toEqual(new Float32Array([0.1, 0.2]));+ });++ it("embedBatch returns one Float32Array per input", async () => {+ mockSuccessModule();+ const { ClipEmbeddingProvider: Fresh } = await import(+ "../src/providers/embedding/clip.js"+ );+ const vecs = await new Fresh().embedBatch(["a", "b"]);++ expect(vecs).toHaveLength(2);+ for (const v of vecs) expect(v).toBeInstanceOf(Float32Array);+ });++ it("embedImage loads image pipeline with dtype: q8 and decodes data: URL", async () => {+ const { pipeline, fromBlob } = mockSuccessModule();+ const { ClipEmbeddingProvider: Fresh } = await import(+ "../src/providers/embedding/clip.js"+ );+ const vec = await new Fresh().embedImage("data:image/png;base64,AAAA");++ expect(pipeline).toHaveBeenCalledWith(+ "image-feature-extraction",+ "Xenova/clip-vit-base-patch32",+ { dtype: "q8" },+ );+ expect(fromBlob).toHaveBeenCalled();+ expect(vec).toBeInstanceOf(Float32Array);+ });++ it("accepts custom model ID via constructor", async () => {+ const { pipeline } = mockSuccessModule();+ const { ClipEmbeddingProvider: Fresh } = await import(+ "../src/providers/embedding/clip.js"+ );+ await new Fresh("Xenova/clip-vit-large-patch14").embed("hello");++ expect(pipeline).toHaveBeenCalledWith(+ "feature-extraction",+ "Xenova/clip-vit-large-patch14",+ { dtype: "q8" },+ );+ });+});diff --git a/test/local-embedding-provider.test.ts b/test/local-embedding-provider.test.tsnew file mode 100644index 00000000..69ce049d--- /dev/null+++ b/test/local-embedding-provider.test.ts@@ -0,0 +1,62 @@+import { describe, it, expect, vi, afterEach } from "vitest";++afterEach(() => {+ vi.doUnmock("@huggingface/transformers");+ vi.resetModules();+});++describe("LocalEmbeddingProvider (package unavailable)", () => {+ it("throws clean install hint when @huggingface/transformers is missing", async () => {+ vi.doMock("@huggingface/transformers");+ vi.resetModules();+ const { LocalEmbeddingProvider: Fresh } = await import(+ "../src/providers/embedding/local.js"+ );+ await expect(new Fresh().embed("hello")).rejects.toThrow(+ "Install @huggingface/transformers for local embeddings",+ );+ });+});++describe("LocalEmbeddingProvider (with loaded pipeline)", () => {+ function mockSuccessModule() {+ const extractor = vi.fn(async (texts: string[]) => ({+ tolist: () => texts.map(() => [0.1, 0.2, 0.3]),+ }));+ const pipeline = vi.fn(() => Promise.resolve(extractor));+ vi.doMock("@huggingface/transformers", () => ({ pipeline }));+ vi.resetModules();+ return { pipeline, extractor };+ }++ it("calls pipeline with dtype: q8, passes extractor opts, returns mapped Float32Array", async () => {+ const { pipeline, extractor } = mockSuccessModule();+ const { LocalEmbeddingProvider: Fresh } = await import(+ "../src/providers/embedding/local.js"+ );+ const vec = await new Fresh().embed("hello");++ expect(pipeline).toHaveBeenCalledWith(+ "feature-extraction",+ "Xenova/all-MiniLM-L6-v2",+ { dtype: "q8" },+ );+ expect(extractor).toHaveBeenCalledWith(["hello"], {+ pooling: "mean",+ normalize: true,+ });+ expect(vec).toBeInstanceOf(Float32Array);+ expect(vec).toEqual(new Float32Array([0.1, 0.2, 0.3]));+ });++ it("embedBatch returns one Float32Array per input text", async () => {+ mockSuccessModule();+ const { LocalEmbeddingProvider: Fresh } = await import(+ "../src/providers/embedding/local.js"+ );+ const vecs = await new Fresh().embedBatch(["a", "b", "c"]);++ expect(vecs).toHaveLength(3);+ for (const v of vecs) expect(v).toBeInstanceOf(Float32Array);+ });+});diff --git a/test/local-embedding-smoke.test.ts b/test/local-embedding-smoke.test.tsnew file mode 100644index 00000000..9dc17e0d--- /dev/null+++ b/test/local-embedding-smoke.test.ts@@ -0,0 +1,12 @@+import { describe, it, expect } from "vitest";+import { LocalEmbeddingProvider } from "../src/providers/embedding/local.js";++const run = process.env.RUN_HF_SMOKE === "1";++describe.runIf(run)("LocalEmbeddingProvider v4 smoke (real model)", () => {+ it("embeds via Xenova/all-MiniLM-L6-v2 -> 384 finite dims", async () => {+ const vec = await new LocalEmbeddingProvider().embed("hello world");+ expect(vec.length).toBe(384);+ for (const v of vec) expect(Number.isFinite(v)).toBe(true);+ }, 60_000);+});diff --git a/test/reranker.test.ts b/test/reranker.test.tsindex 0694e6d5..904a1980 100644--- a/test/reranker.test.ts+++ b/test/reranker.test.ts@@ -1,13 +1,13 @@-import { describe, it, expect, vi } from "vitest";+import { describe, it, expect, vi, afterEach } from "vitest";-vi.mock("@xenova/transformers", () => {+vi.mock("@huggingface/transformers", () => {throw new Error("not installed");});import { rerank, isRerankerAvailable } from "../src/state/reranker.js";describe("reranker", () => {- it("returns results unchanged when @xenova/transformers is unavailable", async () => {+ it("returns results unchanged when @huggingface/transformers is unavailable", async () => {const results = [{observation: {@@ -60,3 +60,32 @@ describe("reranker", () => {expect(reranked).toHaveLength(0);});});++describe("reranker with loaded pipeline", () => {+ afterEach(() => {+ vi.doUnmock("@huggingface/transformers");+ vi.resetModules();+ });++ it("invokes the @huggingface/transformers pipeline and reorders by score", async () => {+ const mockPipeline = vi.fn(async (text: string) => [+ { score: text.includes("First") ? 0.9 : 0.1 },+ ]);+ vi.doMock("@huggingface/transformers", () => ({+ pipeline: () => Promise.resolve(mockPipeline),+ }));+ vi.resetModules();++ const { rerank } = await import("../src/state/reranker.js");++ const results = [+ { observation: { id: "o2", title: "Second", narrative: "" }, combinedScore: 0.9 },+ { observation: { id: "o1", title: "First", narrative: "" }, combinedScore: 0.5 },+ ] as any;++ const reranked = await rerank("query", results);++ expect(mockPipeline).toHaveBeenCalled();+ expect(reranked[0].observation.id).toBe("o1");+ });+});diff --git a/tsdown.config.ts b/tsdown.config.tsindex 5cf10846..390a4df6 100644--- a/tsdown.config.ts+++ b/tsdown.config.ts@@ -20,19 +20,17 @@ const shared = {format: ["esm"] as const,target: "node20" as const,// Keep these as node_modules imports (deps.neverBundle). We never import- // onnxruntime-{node,web} directly; they come in transitively through- // @xenova/transformers, which is lazy-loaded from+ // onnxruntime-{node,web} or sharp directly; they come in transitively+ // through @huggingface/transformers, which is lazy-loaded from// src/providers/embedding/{clip,local}.ts and src/state/reranker.ts.// Bundling inlines relative paths like// `../bin/napi-v3/darwin/arm64/onnxruntime_binding.node` that no longer- // resolve from dist/. All are declared as optionalDependencies in- // package.json so users can install them only when they enable local- // embeddings / CLIP / reranker.+ // resolve from dist/. @huggingface/transformers is declared as an+ // optionalDependency in package.json so users can install it only when+ // they enable local embeddings / CLIP / reranker.deps: {neverBundle: [- "@xenova/transformers",- "onnxruntime-node",- "onnxruntime-web",+ "@huggingface/transformers","@anthropic-ai/claude-agent-sdk","@anthropic-ai/sdk",],