Skip to content

armgit / ruflo

Public Imported from GitHub
0

🌊 The original agent harness. Deploy intelligent multi-player swarms, coordinate autonomous workflows, and build conversational AI systems. Features adaptive memory, self-learning intelligence, federation, vector RAG integration, and native Claude Code / Codex / Hermes and many more Integrated

De-Lattice the WASM embedder tier + enforceable no-stub mode (3.25.1) (#2582)

rUv4b55fb0fc746
diff --git a/.gitignore b/.gitignore
index 29b0ec55e3..feb7d8b7dd 100644
--- a/.gitignore
+++ b/.gitignore
@@ -177,3 +177,7 @@ v3/@claude-flow/cli/plugins/
# in v3/@claude-flow/cli/.claude/helpers/helpers.manifest.json.
.claude/helpers/.helpers-version
.claude/helpers/helpers.manifest.json
+
+# Runtime proven-config adoption state (per-install, not source)
+.claude/proven-config.json
+.claude/.proven-config-version
diff --git a/v3/@claude-flow/cli/__tests__/embedding-policy.test.ts b/v3/@claude-flow/cli/__tests__/embedding-policy.test.ts
new file mode 100644
index 0000000000..3d36b08d66
--- /dev/null
+++ b/v3/@claude-flow/cli/__tests__/embedding-policy.test.ts
@@ -0,0 +1,24 @@
+/**
+ * Embedding policy β€” "no more stubs" enforcement (RUFLO_REQUIRE_REAL_EMBEDDINGS).
+ */
+import { describe, it, expect, afterEach } from 'vitest';
+import { requireRealEmbeddings, enforceNoStub } from '../src/memory/embedding-policy.js';
+
+const KEY = 'RUFLO_REQUIRE_REAL_EMBEDDINGS';
+afterEach(() => { delete process.env[KEY]; });
+
+describe('embedding policy (no-stub strict mode)', () => {
+ it('is off by default β€” enforceNoStub is a no-op (degrade allowed)', () => {
+ delete process.env[KEY];
+ expect(requireRealEmbeddings()).toBe(false);
+ expect(() => enforceNoStub('x')).not.toThrow();
+ });
+
+ it('when strict, a hash last-resort THROWS loudly instead of returning a stub', () => {
+ for (const v of ['1', 'true', 'strict', 'on']) {
+ process.env[KEY] = v;
+ expect(requireRealEmbeddings()).toBe(true);
+ expect(() => enforceNoStub('neural-tools.generateEmbedding')).toThrow(/no-stub.*real embeddings required/i);
+ }
+ });
+});
diff --git a/v3/@claude-flow/cli/__tests__/lattice-wasm.test.ts b/v3/@claude-flow/cli/__tests__/lattice-wasm.test.ts
deleted file mode 100644
index 8864ac5c06..0000000000
--- a/v3/@claude-flow/cli/__tests__/lattice-wasm.test.ts
+++ /dev/null
@@ -1,23 +0,0 @@
-/**
- * Lattice WASM embedder adapter β€” fail-closed degradation (embedding substrate).
- * The package is optional + not installed in CI, so these assert the ZERO-
- * REGRESSION path: absent Lattice β‡’ unavailable + null, callers fall through.
- */
-import { describe, it, expect } from 'vitest';
-import {
- latticeAvailable, latticeEmbed, latticeModels, LATTICE_WASM_PKG, DEFAULT_LATTICE_MODEL,
-} from '../src/ruvector/lattice-wasm.js';
-
-describe('lattice-wasm adapter (fail-closed)', () => {
- it('reports unavailable and returns null when the package is absent', async () => {
- expect(await latticeAvailable()).toBe(false); // optional dep not installed
- expect(await latticeEmbed('hello world')).toBeNull(); // β†’ caller falls through to next tier
- expect(await latticeEmbed('x', 'qwen3-0.6b')).toBeNull(); // GPU model path also degrades
- expect(latticeModels()).toEqual([]); // no models advertised until available
- });
-
- it('exposes a configurable package specifier and a default model', () => {
- expect(LATTICE_WASM_PKG).toContain('lattice'); // overridable via RUFLO_LATTICE_WASM_PKG
- expect(DEFAULT_LATTICE_MODEL).toBeTruthy(); // selectable via RUFLO_EMBED_MODEL
- });
-});
diff --git a/v3/@claude-flow/cli/__tests__/wasm-embedder.test.ts b/v3/@claude-flow/cli/__tests__/wasm-embedder.test.ts
new file mode 100644
index 0000000000..f621f86fa9
--- /dev/null
+++ b/v3/@claude-flow/cli/__tests__/wasm-embedder.test.ts
@@ -0,0 +1,22 @@
+/**
+ * Optional WASM embedder tier β€” opt-in + fail-closed degradation.
+ * Not configured (no RUFLO_EMBED_WASM_PKG) β‡’ inert; callers fall through to
+ * ruvector ONNX β†’ hash with zero regression.
+ */
+import { describe, it, expect } from 'vitest';
+import {
+ wasmEmbedderAvailable, wasmEmbed, wasmEmbedderModels, EMBED_WASM_PKG, DEFAULT_EMBED_MODEL,
+} from '../src/ruvector/wasm-embedder.js';
+
+describe('optional wasm embedder (opt-in, fail-closed)', () => {
+ it('is INERT when no package is configured (no 404-by-default)', async () => {
+ expect(EMBED_WASM_PKG).toBe(''); // no default package β€” opt-in only
+ expect(await wasmEmbedderAvailable()).toBe(false);
+ expect(await wasmEmbed('hello world')).toBeNull(); // β†’ caller falls through to next tier
+ expect(wasmEmbedderModels()).toEqual([]);
+ });
+
+ it('exposes a default model + is env-configurable', () => {
+ expect(DEFAULT_EMBED_MODEL).toBeTruthy(); // via RUFLO_EMBED_MODEL
+ });
+});
diff --git a/v3/@claude-flow/cli/src/mcp-tools/neural-tools.ts b/v3/@claude-flow/cli/src/mcp-tools/neural-tools.ts
index 3fa2ec2fcc..73843fe111 100644
--- a/v3/@claude-flow/cli/src/mcp-tools/neural-tools.ts
+++ b/v3/@claude-flow/cli/src/mcp-tools/neural-tools.ts
@@ -97,24 +97,24 @@ function ensureEmbeddings(): Promise<void> {
if (embeddingsPromise) return embeddingsPromise;
embeddingsPromise = (async () => {
try {
- // Tier -1 (PRIMARY): Lattice WASM β€” real multi-model semantic embeddings
- // (miniLM / bge / paraphrase-miniLM / GPU qwen3-0.6b) ahead of everything.
- // Optional + FAIL-CLOSED: absent/init-fail β‡’ fall straight through to the
- // existing ruvector-ONNX β†’ hash tiers with zero regression.
+ // Tier -1 (PRIMARY): optional WASM embedder β€” a pluggable real embedder
+ // ahead of everything. OPT-IN (RUFLO_EMBED_WASM_PKG) + FAIL-CLOSED: unset
+ // or absent/init-fail β‡’ fall straight through to the existing
+ // ruvector-ONNX β†’ hash tiers with zero regression.
try {
- const lat = await import('../ruvector/lattice-wasm.js').catch(() => null);
- if (lat && await lat.latticeAvailable()) {
- const model = lat.DEFAULT_LATTICE_MODEL;
+ const we = await import('../ruvector/wasm-embedder.js').catch(() => null);
+ if (we && await we.wasmEmbedderAvailable()) {
+ const model = we.DEFAULT_EMBED_MODEL;
realEmbeddings = {
embed: async (text: string) => {
- const v = await lat.latticeEmbed(text, model);
- if (!v || !v.length) throw new Error('lattice embed failed'); // β†’ generateEmbedding falls to next tier
+ const v = await we.wasmEmbed(text, model);
+ if (!v || !v.length) throw new Error('wasm embed failed'); // β†’ generateEmbedding falls to next tier
return v;
},
};
- embeddingServiceName = `lattice-wasm/${model} (${lat.latticeModels().length} model${lat.latticeModels().length === 1 ? '' : 's'})`;
+ embeddingServiceName = `wasm-embedder/${model} (${we.wasmEmbedderModels().length} model${we.wasmEmbedderModels().length === 1 ? '' : 's'})`;
}
- } catch { /* lattice absent β€” fall through */ }
+ } catch { /* not configured β€” fall through */ }
// Tier 0: [email protected] β€” bundled all-MiniLM-L6-v2 + parallel worker pool.
// Probe with isOnnxAvailable() and verify an actual embed succeeds (avoids
@@ -370,6 +370,7 @@ async function generateEmbedding(text?: string, dims: number = 384): Promise<num
// Hash-based deterministic embedding (better than pure random for consistency)
// NOTE: No semantic meaning β€” only useful for consistent deduplication, not similarity search
if (text) {
+ (await import('../memory/embedding-policy.js')).enforceNoStub('neural-tools.generateEmbedding'); // "no stubs" strict mode
if (embeddingServiceName === 'none') {
embeddingServiceName = 'hash-fallback';
}
diff --git a/v3/@claude-flow/cli/src/memory/embedding-policy.ts b/v3/@claude-flow/cli/src/memory/embedding-policy.ts
new file mode 100644
index 0000000000..8a2b5f581c
--- /dev/null
+++ b/v3/@claude-flow/cli/src/memory/embedding-policy.ts
@@ -0,0 +1,32 @@
+/**
+ * Embedding policy β€” "no more stubs" enforcement.
+ *
+ * Every embedder in the codebase tries REAL models first (Lattice WASM β†’
+ * ruvector ONNX β†’ AgentDB bridge β†’ transformers.js) and only falls back to a
+ * deterministic HASH embedding when none loads. A hash embedding has no semantic
+ * meaning β€” it silently degrades similarity search into noise.
+ *
+ * This module makes that failure mode a POLICY choice instead of a silent
+ * default. With RUFLO_REQUIRE_REAL_EMBEDDINGS truthy, any hash last-resort throws
+ * loudly ("no stubs") rather than returning a meaningless vector β€” fail-closed,
+ * so a broken embedding substrate is a hard error, not corrupt retrieval.
+ *
+ * Default (unset): unchanged behavior (warn + degrade), so nothing breaks for
+ * installs where a real embedder genuinely can't load. Set the flag in any
+ * environment that must never serve pseudo-embeddings.
+ */
+export function requireRealEmbeddings(): boolean {
+ return /^(1|true|yes|on|strict)$/i.test(process.env.RUFLO_REQUIRE_REAL_EMBEDDINGS ?? '');
+}
+
+/** Throw the canonical "no stubs" error if strict mode is on; else no-op. */
+export function enforceNoStub(where: string): void {
+ if (requireRealEmbeddings()) {
+ throw new Error(
+ `[no-stub] real embeddings required but only a hash fallback was available at ${where}. ` +
+ `RUFLO_REQUIRE_REAL_EMBEDDINGS is set β€” install a real embedder ` +
+ `(ruvector, or @claude-flow/embeddings with "embeddings init --download") ` +
+ `or unset the flag to allow degraded hash embeddings.`,
+ );
+ }
+}
diff --git a/v3/@claude-flow/cli/src/memory/memory-initializer.ts b/v3/@claude-flow/cli/src/memory/memory-initializer.ts
index 4d35a9d9ad..c6ef105cc2 100644
--- a/v3/@claude-flow/cli/src/memory/memory-initializer.ts
+++ b/v3/@claude-flow/cli/src/memory/memory-initializer.ts
@@ -2291,6 +2291,7 @@ export async function generateLocalEmbedding(text: string): Promise<{
// Deterministic hash-based fallback (for testing/demo without ONNX).
// AUDIT #3: backend='mock' β€” these vectors do NOT carry real semantics.
+ (await import('./embedding-policy.js')).enforceNoStub('memory-initializer.generateLocalEmbedding'); // "no stubs" strict mode
const embedding = generateHashEmbedding(text, state.dimensions);
return {
embedding,
diff --git a/v3/@claude-flow/cli/src/ruvector/lattice-wasm.ts b/v3/@claude-flow/cli/src/ruvector/lattice-wasm.ts
deleted file mode 100644
index a2cda46b8c..0000000000
--- a/v3/@claude-flow/cli/src/ruvector/lattice-wasm.ts
+++ /dev/null
@@ -1,126 +0,0 @@
-/**
- * Lattice WASM embedder adapter β€” the primary embedding tier (ADR: embedding
- * substrate upgrade). Replaces hash placeholders with real semantic embeddings
- * and adds multiple models (miniLM, bge, multi-paraphrase-miniLM, GPU qwen3-0.6B).
- *
- * Built on the proven `@ruvector/ruvllm-wasm` optional-dependency convention:
- * - dynamically imported (never a hard dependency),
- * - WASM initialized from bundled bytes,
- * - fully FAIL-CLOSED: if the package is absent, the WASM fails to init, or the
- * embed API differs, `latticeAvailable()` returns false and callers fall
- * through to the existing ruvector-ONNX β†’ hash tiers with ZERO regression.
- *
- * The package specifier is configurable (`RUFLO_LATTICE_WASM_PKG`) so the exact
- * published name can be set without a code change; the API is probed tolerantly.
- * Opt-in for the GPU model: qwen3 is only selected when explicitly requested.
- */
-import { createRequire } from 'module';
-import { readFileSync } from 'fs';
-
-const require_ = createRequire(import.meta.url);
-
-/** Default package name (ruvnet WASM convention); override with RUFLO_LATTICE_WASM_PKG. */
-export const LATTICE_WASM_PKG = process.env.RUFLO_LATTICE_WASM_PKG || '@ruvector/lattice-wasm';
-
-export type LatticeModel = 'minilm' | 'bge' | 'paraphrase-minilm' | 'qwen3-0.6b' | string;
-export const DEFAULT_LATTICE_MODEL: LatticeModel = (process.env.RUFLO_EMBED_MODEL as LatticeModel) || 'minilm';
-
-/* eslint-disable @typescript-eslint/no-explicit-any */
-let _mod: any = null;
-let _ready = false;
-let _probed = false;
-let _available = false;
-let _models: LatticeModel[] = [];
-
-async function loadModule(): Promise<any> {
- if (_mod) return _mod;
- const spec: string = LATTICE_WASM_PKG;
- _mod = await import(spec).catch(() => null); // optional β€” absent β‡’ null β‡’ unavailable
- return _mod;
-}
-
-async function ensureInit(): Promise<boolean> {
- if (_ready) return true;
- const mod = await loadModule();
- if (!mod) return false;
- try {
- // ruvnet WASM convention: initSync({ module: <wasm bytes> }); tolerate variants.
- if (!_ready && typeof mod.initSync === 'function') {
- let wasmBytes: Buffer | undefined;
- for (const cand of ['lattice_wasm_bg.wasm', 'lattice_bg.wasm', 'index_bg.wasm']) {
- try { wasmBytes = readFileSync(require_.resolve(`${LATTICE_WASM_PKG}/${cand}`)); break; } catch { /* try next */ }
- }
- mod.initSync(wasmBytes ? { module: wasmBytes } : undefined);
- } else if (typeof mod.default === 'function') {
- await mod.default(); // some wasm-bindgen builds export an async default init
- }
- _ready = true;
- return true;
- } catch {
- return false; // init failed β‡’ fail closed
- }
-}
-
-/** Extract a plain number[] from whatever shape the module's embed returns. */
-function toVec(r: unknown): number[] | null {
- const v = (r && typeof r === 'object' && 'embedding' in (r as any)) ? (r as any).embedding : r;
- if (!v) return null;
- if (Array.isArray(v)) return v as number[];
- if ((v as ArrayLike<number>).length !== undefined) return Array.from(v as ArrayLike<number>);
- return null;
-}
-
-/** Is the Lattice WASM embedder installed + initializable? Cached; never throws. */
-export async function latticeAvailable(): Promise<boolean> {
- if (_probed) return _available;
- _probed = true;
- try {
- if (!(await ensureInit())) { _available = false; return false; }
- const mod = _mod;
- // discover models (tolerant): listModels() / models / MODELS
- try {
- const list = typeof mod.listModels === 'function' ? mod.listModels() : (mod.models ?? mod.MODELS);
- if (Array.isArray(list) && list.length) _models = list as LatticeModel[];
- } catch { /* leave default */ }
- if (!_models.length) _models = [DEFAULT_LATTICE_MODEL];
- // verify an actual embed succeeds (the ADR-086 "loads but runtime-fails" trap).
- const probe = await latticeEmbedRaw('probe', DEFAULT_LATTICE_MODEL);
- _available = !!probe && probe.length > 0;
- return _available;
- } catch {
- _available = false;
- return false;
- }
-}
-
-/** The models Lattice reports (empty until availability is probed). */
-export function latticeModels(): LatticeModel[] { return [..._models]; }
-
-async function latticeEmbedRaw(text: string, model: LatticeModel): Promise<number[] | null> {
- const mod = _mod;
- if (!mod) return null;
- // tolerant API probing across plausible wasm-bindgen surfaces.
- const attempts: Array<() => Promise<unknown> | unknown> = [
- () => typeof mod.embed === 'function' ? mod.embed(text, model) : undefined,
- () => typeof mod.embed === 'function' ? mod.embed(text) : undefined,
- () => typeof mod.embedText === 'function' ? mod.embedText(text, model) : undefined,
- () => typeof mod.Embedder === 'function' ? new mod.Embedder(model).embed(text) : undefined,
- ];
- for (const a of attempts) {
- try { const r = await a(); const v = toVec(r); if (v) return v; } catch { /* try next surface */ }
- }
- return null;
-}
-
-/**
- * Embed `text` with `model` via Lattice WASM. Returns null on any failure so the
- * caller can fall through to the next tier. Never throws.
- */
-export async function latticeEmbed(text: string, model: LatticeModel = DEFAULT_LATTICE_MODEL): Promise<number[] | null> {
- try {
- if (!(await latticeAvailable())) return null;
- return await latticeEmbedRaw(text, model);
- } catch {
- return null;
- }
-}
diff --git a/v3/@claude-flow/cli/src/ruvector/vector-db.ts b/v3/@claude-flow/cli/src/ruvector/vector-db.ts
index f83f43c582..249b614578 100644
--- a/v3/@claude-flow/cli/src/ruvector/vector-db.ts
+++ b/v3/@claude-flow/cli/src/ruvector/vector-db.ts
@@ -14,6 +14,7 @@
import os from 'node:os';
import path from 'node:path';
import { randomUUID } from 'node:crypto';
+import { enforceNoStub } from '../memory/embedding-policy.js';
// ============================================================================
// Types
@@ -304,6 +305,7 @@ export function generateEmbedding(text: string, dimensions: number = 768): Float
}
}
+ enforceNoStub('vector-db.generateEmbedding'); // "no stubs" strict mode β†’ throw instead of hash
const embedding = generateHashEmbedding(text, dimensions);
// Tag the result so consumers can detect it came from hash fallback
Object.defineProperty(embedding, '_warning', {
diff --git a/v3/@claude-flow/cli/src/ruvector/wasm-embedder.ts b/v3/@claude-flow/cli/src/ruvector/wasm-embedder.ts
new file mode 100644
index 0000000000..dd5f8fec1a
--- /dev/null
+++ b/v3/@claude-flow/cli/src/ruvector/wasm-embedder.ts
@@ -0,0 +1,119 @@
+/**
+ * Optional WASM embedder tier β€” a pluggable PRIMARY embedding tier ahead of
+ * ruvector-ONNX β†’ hash.
+ *
+ * OPT-IN by design: there is NO default package (an earlier iteration defaulted
+ * to `@ruvector/lattice-wasm`, which does not exist β€” npm 404). Point
+ * `RUFLO_EMBED_WASM_PKG` at any real WASM embedder that follows the ruvnet
+ * wasm-bindgen convention (a text→vector embed export). With the env unset, this
+ * tier is INERT and everything resolves exactly as before (ruvector ONNX). Fully
+ * fail-closed; never throws.
+ *
+ * The adapter probes the module's API tolerantly and verifies a real embed
+ * succeeds before accepting the tier (guards the ADR-086 loads-but-runtime-fails
+ * trap), so an incompatible package simply reports unavailable.
+ */
+import { createRequire } from 'module';
+import { readFileSync } from 'fs';
+
+const require_ = createRequire(import.meta.url);
+
+/**
+ * Package specifier for the optional WASM embedder. EMPTY by default (opt-in).
+ * `RUFLO_LATTICE_WASM_PKG` is accepted as a back-compat alias.
+ */
+export const EMBED_WASM_PKG = process.env.RUFLO_EMBED_WASM_PKG || process.env.RUFLO_LATTICE_WASM_PKG || '';
+export const DEFAULT_EMBED_MODEL = process.env.RUFLO_EMBED_MODEL || 'default';
+
+/* eslint-disable @typescript-eslint/no-explicit-any */
+let _mod: any = null;
+let _ready = false;
+let _probed = false;
+let _available = false;
+let _models: string[] = [];
+
+async function loadModule(): Promise<any> {
+ if (!EMBED_WASM_PKG) return null; // opt-in: no package configured β‡’ inert
+ if (_mod) return _mod;
+ _mod = await import(EMBED_WASM_PKG).catch(() => null); // optional β€” absent β‡’ null β‡’ unavailable
+ return _mod;
+}
+
+async function ensureInit(): Promise<boolean> {
+ if (_ready) return true;
+ const mod = await loadModule();
+ if (!mod) return false;
+ try {
+ if (typeof mod.initSync === 'function') {
+ let wasmBytes: Buffer | undefined;
+ for (const cand of ['index_bg.wasm', 'embedder_bg.wasm', 'wasm_bg.wasm']) {
+ try { wasmBytes = readFileSync(require_.resolve(`${EMBED_WASM_PKG}/${cand}`)); break; } catch { /* try next */ }
+ }
+ mod.initSync(wasmBytes ? { module: wasmBytes } : undefined);
+ } else if (typeof mod.default === 'function') {
+ await mod.default();
+ }
+ _ready = true;
+ return true;
+ } catch {
+ return false;
+ }
+}
+
+function toVec(r: unknown): number[] | null {
+ const v = (r && typeof r === 'object' && 'embedding' in (r as any)) ? (r as any).embedding : r;
+ if (!v) return null;
+ if (Array.isArray(v)) return v as number[];
+ if ((v as ArrayLike<number>).length !== undefined) return Array.from(v as ArrayLike<number>);
+ return null;
+}
+
+async function embedRaw(text: string, model: string): Promise<number[] | null> {
+ const mod = _mod;
+ if (!mod) return null;
+ const attempts: Array<() => Promise<unknown> | unknown> = [
+ () => typeof mod.embed === 'function' ? mod.embed(text, model) : undefined,
+ () => typeof mod.embed === 'function' ? mod.embed(text) : undefined,
+ () => typeof mod.embedText === 'function' ? mod.embedText(text, model) : undefined,
+ () => typeof mod.Embedder === 'function' ? new mod.Embedder(model).embed(text) : undefined,
+ ];
+ for (const a of attempts) {
+ try { const r = await a(); const v = toVec(r); if (v) return v; } catch { /* try next surface */ }
+ }
+ return null;
+}
+
+/** Is an optional WASM embedder configured + installed + initializable? Cached; never throws. */
+export async function wasmEmbedderAvailable(): Promise<boolean> {
+ if (_probed) return _available;
+ _probed = true;
+ try {
+ if (!EMBED_WASM_PKG) { _available = false; return false; } // opt-in; not configured
+ if (!(await ensureInit())) { _available = false; return false; }
+ const mod = _mod;
+ try {
+ const list = typeof mod.listModels === 'function' ? mod.listModels() : (mod.models ?? mod.MODELS);
+ if (Array.isArray(list) && list.length) _models = list as string[];
+ } catch { /* leave default */ }
+ if (!_models.length) _models = [DEFAULT_EMBED_MODEL];
+ const probe = await embedRaw('probe', DEFAULT_EMBED_MODEL);
+ _available = !!probe && probe.length > 0;
+ return _available;
+ } catch {
+ _available = false;
+ return false;
+ }
+}
+
+/** Models the configured WASM embedder reports (empty until available). */
+export function wasmEmbedderModels(): string[] { return [..._models]; }
+
+/** Embed via the optional WASM embedder, or null (caller falls through). Never throws. */
+export async function wasmEmbed(text: string, model: string = DEFAULT_EMBED_MODEL): Promise<number[] | null> {
+ try {
+ if (!(await wasmEmbedderAvailable())) return null;
+ return await embedRaw(text, model);
+ } catch {
+ return null;
+ }
+}