Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

fix(embeddings): walk every configured embedder at ingest, and rank lexically when a search cannot embed its query

nirholas9b60b6fda9b7
diff --git a/api/_lib/embeddings.js b/api/_lib/embeddings.js
index a4bb06120..98de6e942 100644
--- a/api/_lib/embeddings.js
+++ b/api/_lib/embeddings.js
@@ -31,6 +31,13 @@ const OPENAI_EMBED_URL = 'https://api.openai.com/v1/embeddings';
// it; "global" does not for this model family, unlike the Gemini chat/image
// models elsewhere in the codebase) — default to the region every other
// GCP-hosted worker in this platform already runs in.
+import {
+ AUTH_COOLDOWN_SECONDS,
+ clearProviderCooldown,
+ markProviderCooldown,
+ providersInCooldown,
+} from './provider-health.js';
+
function vertexEmbedUrl() {
const project = process.env.GOOGLE_CLOUD_PROJECT;
const location = process.env.GOOGLE_CLOUD_LOCATION_EMBED || 'us-central1';
@@ -163,6 +170,78 @@ export function embedPassages(tag, texts) {
return embedWith(tag, texts, 'passage');
}
+// ── Ingest-time provider walk ───────────────────────────────────────────────
+//
+// embedWith is deliberately strict: a stored vector space has ONE embedder, and
+// the three lanes have three different dimensions (NIM 1024, Vertex 768, OpenAI
+// 256), so answering a query against an existing space with another lane's
+// vectors would compare points in unrelated geometries. Nothing here weakens
+// that, and this must never be used to query an existing space.
+//
+// What was missing is the case where no space is fixed yet. At INGEST the caller
+// is free to choose any lane, but defaultIngestEmbedderTag() picked the first
+// CONFIGURED one and embedWith threw when that provider was down, so a single
+// NIM 429 failed the whole embed with Vertex and OpenAI configured and idle
+// (the /api/watsonx/embed 502s of 2026-08-27). This walks the preference order
+// instead and returns the tag that actually answered, so the caller records the
+// space it really got rather than the one it asked for.
+const EMBED_LANE_COOLDOWN_SECONDS = 45;
+const embedLaneKey = (tag) => `embed:${tag}`;
+
+/**
+ * Embed `texts` as passages with the first lane that answers, preferring
+ * `preferredTag` and then the standard free-first order.
+ *
+ * @param {string|null} preferredTag tried first when configured
+ * @param {string[]} texts
+ * @returns {Promise<{ tag: string, info: object, vectors: Float64Array[] }>}
+ * @throws the last lane's error, or { code: 'no_embedder' } when none is configured
+ */
+export async function embedPassagesAny(preferredTag, texts) {
+ const resolved = resolveEmbedderTag(preferredTag);
+ const order = [
+ ...(resolved && EMBEDDERS[resolved].configured() ? [resolved] : []),
+ ...INGEST_PREFERENCE.filter((t) => t !== resolved && EMBEDDERS[t].configured()),
+ ];
+ if (!order.length) {
+ throw Object.assign(new Error('no embedding provider is configured'), { code: 'no_embedder' });
+ }
+ if (!texts.length) return { tag: order[0], info: EMBEDDERS[order[0]], vectors: [] };
+
+ // A lane a recent request found throttled goes to the back rather than being
+ // dropped, so a chain whose lanes are all cooling still answers.
+ let lanes = order;
+ if (order.length > 1) {
+ const cooling = await providersInCooldown(order.map(embedLaneKey));
+ if (cooling.size) {
+ const hot = order.filter((t) => !cooling.has(embedLaneKey(t)));
+ if (hot.length) lanes = [...hot, ...order.filter((t) => cooling.has(embedLaneKey(t)))];
+ }
+ }
+
+ let lastErr;
+ for (const tag of lanes) {
+ try {
+ const vectors = await embedWith(tag, texts, 'passage');
+ void clearProviderCooldown(embedLaneKey(tag));
+ return { tag, info: EMBEDDERS[tag], vectors };
+ } catch (err) {
+ // A misconfigured or unknown lane is not a health signal, it simply
+ // cannot serve; only a real upstream failure earns a cooldown.
+ if (err?.code !== 'no_embedder' && err?.code !== 'unknown_embedder') {
+ const authFault = err?.status === 401 || err?.status === 403 || err?.status === 402;
+ void markProviderCooldown(
+ embedLaneKey(tag),
+ authFault ? AUTH_COOLDOWN_SECONDS : EMBED_LANE_COOLDOWN_SECONDS,
+ authFault ? 'auth' : 'health',
+ );
+ }
+ lastErr = err;
+ }
+ }
+ throw lastErr || Object.assign(new Error('every embedding lane failed'), { code: 'embedder_error' });
+}
+
/** Convenience: embed one search string; resolves to a single Float64Array. */
export async function embedQuery(tag, text) {
const [vec] = await embedWith(tag, [text], 'query');
diff --git a/api/_lib/lexical-rank.js b/api/_lib/lexical-rank.js
new file mode 100644
index 000000000..13af625f9
--- /dev/null
+++ b/api/_lib/lexical-rank.js
@@ -0,0 +1,101 @@
+// Lexical fallback ranking.
+//
+// Semantic search is the platform's primary ranking everywhere it is available,
+// but a vector query has a hard dependency the rest of the page does not: the
+// query has to be embedded in the SAME space the corpus lives in. When that one
+// upstream is down there is no cross-provider failover to reach for, because a
+// 1024-dim NIM vector compared against a 768-dim Granite corpus is not a worse
+// answer, it is a meaningless one.
+//
+// So the fallback is a different METHOD rather than a different provider: rank
+// the same corpus by term overlap and hand it back explicitly labelled
+// `match: 'lexical'` with a null semantic score. That is the pattern
+// api/_lib/memory-store.js already established for the same reason (its
+// "substring + salience fill" keeps recall working before embeddings backfill),
+// generalised here so every semantic surface can degrade the same way instead of
+// answering a search page with a 502.
+//
+// It is deliberately simple and deterministic: no index to build, no dependency
+// to add, and nothing about it can fabricate a relevance number that reads like
+// a cosine similarity.
+
+// Words carrying no discriminating power. Kept short on purpose: an aggressive
+// list would drop meaningful query terms in a domain full of short names.
+const STOP_WORDS = new Set([
+ 'a', 'an', 'and', 'are', 'as', 'at', 'be', 'but', 'by', 'for', 'from', 'has',
+ 'have', 'how', 'in', 'is', 'it', 'its', 'of', 'on', 'or', 'that', 'the',
+ 'their', 'them', 'they', 'this', 'to', 'was', 'were', 'what', 'when', 'which',
+ 'who', 'why', 'will', 'with', 'you', 'your',
+]);
+
+/**
+ * Split text into lowercase word tokens, dropping stop words and single
+ * characters. Unicode-aware so non-latin queries tokenize rather than vanish.
+ *
+ * @param {string} text
+ * @returns {string[]}
+ */
+export function tokenize(text) {
+ if (typeof text !== 'string' || !text) return [];
+ return text
+ .toLowerCase()
+ .split(/[^\p{L}\p{N}]+/u)
+ .filter((t) => t.length > 1 && !STOP_WORDS.has(t));
+}
+
+/**
+ * Overlap score for one document, in [0, 1].
+ *
+ * The base is the share of DISTINCT query terms the document contains, which
+ * keeps a long document from outranking a precise one merely by repeating a
+ * word. A document containing the query as a contiguous phrase gets a bounded
+ * bonus, because a phrase hit is the strongest lexical evidence available
+ * without an index.
+ *
+ * @param {string[]} queryTokens pre-tokenized query (tokenize once per search)
+ * @param {string} queryRaw the original query, for the phrase check
+ * @param {string} text document text
+ * @returns {number} 0 when nothing matches
+ */
+export function lexicalScore(queryTokens, queryRaw, text) {
+ if (!queryTokens.length || typeof text !== 'string' || !text) return 0;
+ const haystack = text.toLowerCase();
+ const docTokens = new Set(tokenize(text));
+ let hits = 0;
+ for (const t of queryTokens) {
+ // Prefix match so "avatar" finds "avatars" without a stemmer, which would
+ // be a dependency and a source of surprising misses in a name-heavy corpus.
+ if (docTokens.has(t) || haystack.includes(t)) hits++;
+ }
+ if (!hits) return 0;
+ const base = hits / queryTokens.length;
+ const phrase = queryRaw.trim().length > 2 && haystack.includes(queryRaw.trim().toLowerCase()) ? 0.25 : 0;
+ return Math.min(1, base * 0.75 + phrase + (base === 1 ? 0.1 : 0));
+}
+
+/**
+ * Rank documents by lexical overlap with `query`.
+ *
+ * Every returned row carries `match: 'lexical'` and `score: null` for the
+ * semantic score it does NOT have, alongside the overlap in `lexicalScore`, so
+ * no caller can mistake this ordering for a vector one.
+ *
+ * @param {string} query
+ * @param {Array<{id: any, text: string}>} docs
+ * @param {object} [opts]
+ * @param {number} [opts.limit=20]
+ * @param {number} [opts.minScore=0.15] drop weak coincidental overlaps
+ * @returns {Array<{id: any, score: null, lexicalScore: number, match: 'lexical'}>}
+ */
+export function rankLexically(query, docs, { limit = 20, minScore = 0.15 } = {}) {
+ const q = String(query || '').trim();
+ const tokens = tokenize(q);
+ if (!tokens.length || !Array.isArray(docs)) return [];
+ const scored = [];
+ for (const d of docs) {
+ const s = lexicalScore(tokens, q, d?.text || '');
+ if (s >= minScore) scored.push({ id: d.id, score: null, lexicalScore: Number(s.toFixed(4)), match: 'lexical' });
+ }
+ scored.sort((a, b) => b.lexicalScore - a.lexicalScore);
+ return scored.slice(0, limit);
+}
diff --git a/api/galaxy.js b/api/galaxy.js
index 581f28c2a..0ab2f8dcb 100644
--- a/api/galaxy.js
+++ b/api/galaxy.js
@@ -25,7 +25,8 @@ import { clampInt } from './_lib/http-params.js';
import { limits, clientIp } from './_lib/rate-limit.js';
import { thumbnailUrl } from './_lib/r2.js';
import { watsonxConfig, watsonxEmbed, watsonxChatComplete } from './_lib/watsonx.js';
-import { ensureAgentEmbeddings, readAgentVectors } from './_lib/agent-embeddings.js';
+import { agentEmbedText, ensureAgentEmbeddings, readAgentVectors } from './_lib/agent-embeddings.js';
+import { rankLexically } from './_lib/lexical-rank.js';
import {
assembleGalaxy,
rankBySimilarity,
@@ -184,14 +185,49 @@ async function handleSearch(req, res, cfg) {
// Embed the query with the same Granite model the agents were embedded with, so
// query and agent vectors share a space.
- const { vectors } = await watsonxEmbed(cfg, { inputs: [query], model: cfg.embedModel });
- const queryVec = vectors[0];
- if (!queryVec?.length) {
- return error(res, 502, 'embed_failed', 'watsonx returned no embedding for the query');
+ //
+ // There is no provider failover to reach for here, and that is not an
+ // oversight: the stored agent vectors live in Granite's space, so a vector
+ // from another lane would be compared against them in a different geometry
+ // and different dimension. The fallback is a different METHOD instead. When
+ // the embedder cannot answer, the same corpus is ranked lexically and the
+ // response says so, because a search page that returns keyword matches is
+ // useful and a search page that returns 502 is not.
+ let queryVec = null;
+ let embedError = null;
+ try {
+ const { vectors } = await watsonxEmbed(cfg, { inputs: [query], model: cfg.embedModel });
+ queryVec = vectors[0]?.length ? vectors[0] : null;
+ if (!queryVec) embedError = 'the embedder returned no vector for the query';
+ } catch (err) {
+ embedError = err?.message || 'the embedder is unreachable';
}
const agents = await loadPublishedAgents(1200);
const byId = new Map(agents.map((a) => [a.id, a]));
+
+ if (!queryVec) {
+ const ranked = rankLexically(
+ query,
+ agents.map((a) => ({ id: a.id, text: agentEmbedText(a) || `${a.name || ''} ${a.description || ''}` })),
+ { limit: topK },
+ );
+ const results = ranked
+ .map(({ id, score, lexicalScore: lex }) => {
+ const a = byId.get(id);
+ return a ? { id: a.id, name: a.name, description: a.description || '', thumbnail: a.thumbnail || null, score, lexicalScore: lex } : null;
+ })
+ .filter(Boolean);
+ return json(res, 200, {
+ query,
+ model: cfg.embedModel,
+ count: results.length,
+ results,
+ ranking: 'lexical',
+ degraded: { reason: 'embedder_unavailable', detail: embedError, retryable: true },
+ });
+ }
+
const vectorsById = await readAgentVectors([...byId.keys()], { model: cfg.embedModel });
const ranked = rankBySimilarity(queryVec, vectorsById, { topK });
@@ -209,7 +245,7 @@ async function handleSearch(req, res, cfg) {
})
.filter(Boolean);
- return json(res, 200, { query, model: cfg.embedModel, count: results.length, results });
+ return json(res, 200, { query, model: cfg.embedModel, count: results.length, results, ranking: 'semantic' });
}
// ── Shared: published-agent population ──────────────────────────────────────
diff --git a/api/ibm/galaxy.js b/api/ibm/galaxy.js
index c8dd9932d..0a0ba1f15 100644
--- a/api/ibm/galaxy.js
+++ b/api/ibm/galaxy.js
@@ -33,6 +33,7 @@ import {
unit,
dot,
} from '../_lib/embedding-math.js';
+import { rankLexically } from '../_lib/lexical-rank.js';
import { createHash } from 'node:crypto';
// Hard cap on agents in one galaxy — keeps embedding cost bounded and the scene
@@ -431,10 +432,39 @@ async function handleSearch(req, res) {
if (!ids.length) return json(res, 200, { results: [], query, model: cfg.embedModel });
const vectorMap = await readAgentVectors(ids, { model: cfg.embedModel });
- const { vectors, model } = await watsonxEmbed(cfg, { inputs: [query] });
- const qvec = vectors[0];
- if (!qvec?.length)
- return error(res, 502, 'embed_failed', 'watsonx returned no query embedding');
+ // No provider failover exists for this read and none would be correct: the
+ // stored agent vectors live in Granite's space, so a vector minted by another
+ // lane would be a different dimension in a different geometry. Degrade the
+ // METHOD instead, ranking the same corpus lexically and saying so, rather
+ // than answering a search with a 502.
+ let qvec = null;
+ let model = cfg.embedModel;
+ let embedError = null;
+ try {
+ const out = await watsonxEmbed(cfg, { inputs: [query] });
+ qvec = out.vectors?.[0]?.length ? out.vectors[0] : null;
+ model = out.model || model;
+ if (!qvec) embedError = 'the embedder returned no vector for the query';
+ } catch (err) {
+ embedError = err?.message || 'the embedder is unreachable';
+ }
+ if (!qvec) {
+ const byId = new Map(rows.map((a) => [a.id, a]));
+ const lexical = rankLexically(
+ query,
+ rows.map((a) => ({ id: a.id, text: agentEmbedText(a) || '' })),
+ { limit: 16 },
+ );
+ return json(res, 200, {
+ query,
+ model,
+ count: lexical.length,
+ best: lexical[0] || null,
+ results: lexical.map((r) => ({ ...r, name: byId.get(r.id)?.name || null })),
+ ranking: 'lexical',
+ degraded: { reason: 'embedder_unavailable', detail: embedError, retryable: true },
+ });
+ }
const ranked = [];
for (const [id, vec] of vectorMap) {
@@ -450,6 +480,7 @@ async function handleSearch(req, res) {
count: ranked.length,
best: top[0] || null,
results: top,
+ ranking: 'semantic',
});
}
diff --git a/api/watsonx/embed.js b/api/watsonx/embed.js
index f44111bb6..f80b5cfe0 100644
--- a/api/watsonx/embed.js
+++ b/api/watsonx/embed.js
@@ -30,7 +30,7 @@ import { createHash } from 'node:crypto';
import { cors, method, readJson, error, json, wrap, rateLimited } from '../_lib/http.js';
import { watsonxConfig, watsonxEmbed } from '../_lib/watsonx.js';
import {
- embedPassages,
+ embedPassagesAny,
embeddingsConfigured,
defaultIngestEmbedderTag,
embedderInfo,
@@ -161,24 +161,38 @@ export default wrap(async function handler(req, res) {
if (!result && fallbackReady) {
// These texts are peers laid out against each other (not query-vs-corpus),
// so they all embed as 'passage': one consistent space per response.
- const fallbackTag = defaultIngestEmbedderTag();
- const fallback = embedderInfo(fallbackTag);
+ //
+ // The lane is chosen by embedPassagesAny, which walks the whole free-first
+ // order rather than failing on whichever lane happened to be preferred: a
+ // single NIM throttle used to 502 this endpoint with Vertex and OpenAI
+ // configured and idle. Every text in one response embeds through one lane,
+ // so the space stays consistent, and the response reports the model and
+ // dimension that actually answered.
+ const preferredTag = defaultIngestEmbedderTag();
try {
- result = await embedBatch(texts, fallback.model, async (inputs) => {
- const vecs = await embedPassages(fallbackTag, inputs);
+ let servedModel = embedderInfo(preferredTag)?.model || preferredTag;
+ result = await embedBatch(texts, servedModel, async (inputs) => {
+ const out = await embedPassagesAny(preferredTag, inputs);
+ servedModel = out.info.model;
return {
- vectors: vecs.map((v) => Array.from(v)),
- dimensions: vecs[0]?.length ?? 0,
+ vectors: out.vectors.map((v) => Array.from(v)),
+ dimensions: out.vectors[0]?.length ?? out.info.dim,
};
});
+ // embedBatch keys its cache by the model it was handed, so re-stamp the
+ // result with the lane that answered rather than the one we hoped for.
+ if (result) result.model = servedModel;
} catch (e) {
lastError = e;
}
}
if (!result) {
- // Both tiers failed at the network level. Surface the real upstream cause.
- return error(res, 502, 'embed_error', lastError?.message || 'embeddings failed');
+ // Every tier failed at the network level. This is upstream weather, not a
+ // bad request, so it is a retryable 503 with the real cause rather than a
+ // 502 the caller reads as permanent.
+ res.setHeader('retry-after', '15');
+ return error(res, 503, 'embed_unavailable', lastError?.message || 'embeddings failed');
}
// json() defaults to no-store, which is correct here: this is a POST whose
diff --git a/tests/lexical-rank.test.js b/tests/lexical-rank.test.js
new file mode 100644
index 000000000..4032084cd
--- /dev/null
+++ b/tests/lexical-rank.test.js
@@ -0,0 +1,75 @@
+// The lexical fallback ranking used when a semantic surface cannot embed its
+// query (a down embedder has no cross-provider failover: a vector from another
+// lane lives in another space entirely).
+import { describe, it, expect } from 'vitest';
+import { tokenize, lexicalScore, rankLexically } from '../api/_lib/lexical-rank.js';
+
+describe('tokenize', () => {
+ it('lowercases, splits on punctuation and drops stop words and single characters', () => {
+ expect(tokenize('The 3D Avatar, and a Rig!')).toEqual(['3d', 'avatar', 'rig']);
+ });
+ it('keeps non-latin words rather than dropping them', () => {
+ expect(tokenize('三次元 avatar')).toEqual(['三次元', 'avatar']);
+ });
+ it('is shape safe', () => {
+ expect(tokenize('')).toEqual([]);
+ expect(tokenize(null)).toEqual([]);
+ expect(tokenize(undefined)).toEqual([]);
+ });
+});
+
+describe('lexicalScore', () => {
+ const q = 'rigged knight avatar';
+ const tokens = tokenize(q);
+
+ it('scores by the share of distinct query terms present', () => {
+ const all = lexicalScore(tokens, q, 'a rigged knight avatar you can animate');
+ const some = lexicalScore(tokens, q, 'a knight, unrigged and static');
+ expect(all).toBeGreaterThan(some);
+ expect(some).toBeGreaterThan(0);
+ });
+ it('gives a contiguous phrase match the strongest evidence available', () => {
+ const phrase = lexicalScore(tokens, q, 'this is a rigged knight avatar');
+ const scattered = lexicalScore(tokens, q, 'avatar. later: knight. later still: rigged');
+ expect(phrase).toBeGreaterThan(scattered);
+ });
+ it('does not let repetition outrank precision', () => {
+ const precise = lexicalScore(tokens, q, 'rigged knight avatar');
+ const padded = lexicalScore(tokens, q, `knight ${'knight '.repeat(50)}`);
+ expect(precise).toBeGreaterThan(padded);
+ });
+ it('returns 0 when nothing matches, and stays in range', () => {
+ expect(lexicalScore(tokens, q, 'completely unrelated text')).toBe(0);
+ expect(lexicalScore(tokens, q, '')).toBe(0);
+ expect(lexicalScore([], '', 'anything')).toBe(0);
+ expect(lexicalScore(tokens, q, 'rigged knight avatar rigged knight avatar')).toBeLessThanOrEqual(1);
+ });
+});
+
+describe('rankLexically', () => {
+ const docs = [
+ { id: 'a', text: 'A rigged knight avatar ready to animate' },
+ { id: 'b', text: 'A static rock prop' },
+ { id: 'c', text: 'knight helmet, no rig' },
+ ];
+
+ it('orders by overlap and drops coincidental matches', () => {
+ const out = rankLexically('rigged knight avatar', docs);
+ expect(out[0].id).toBe('a');
+ expect(out.map((r) => r.id)).not.toContain('b');
+ });
+
+ it('labels every row as lexical and never invents a semantic score', () => {
+ const [top] = rankLexically('rigged knight avatar', docs);
+ expect(top.match).toBe('lexical');
+ expect(top.score).toBeNull();
+ expect(top.lexicalScore).toBeGreaterThan(0);
+ });
+
+ it('honours the limit and is shape safe', () => {
+ expect(rankLexically('knight', docs, { limit: 1 })).toHaveLength(1);
+ expect(rankLexically('', docs)).toEqual([]);
+ expect(rankLexically('knight', null)).toEqual([]);
+ expect(rankLexically('knight', [{ id: 'x' }])).toEqual([]);
+ });
+});