Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

fix(watsonx): treat a short embedding batch as a failure rather than a partial success

nirholas461ac29d9218
diff --git a/api/watsonx/embed.js b/api/watsonx/embed.js
index 6682c8c23..f44111bb6 100644
--- a/api/watsonx/embed.js
+++ b/api/watsonx/embed.js
@@ -1,4 +1,4 @@
-// POST /api/watsonx/embed — IBM Granite embeddings on watsonx.ai.
+// POST /api/watsonx/embed: IBM Granite embeddings on watsonx.ai.
//
// Body: { texts: string[], model?: string }
// Response: { model, dimensions, count, cachedHits, vectors: number[][] }
@@ -9,15 +9,22 @@
// lay them out in semantic space (e.g. the watsonx Constellation at
// /constellation). Inference runs on watsonx.ai with the server's IBM Cloud key.
//
-// There is no mock path — every vector is a real embedding call. The provider
+// There is no mock path: every vector is a real embedding call. The provider
// chain degrades gracefully so a watsonx outage never blanks the page:
// 1. IBM Granite on watsonx.ai (primary, when WATSONX_* is configured)
-// 2. The platform's free-first embedding chain (NVIDIA NIM when keyed,
-// OpenAI text-embedding-3-small as the paid backstop)
+// 2. The platform's free-first embedding chain (NVIDIA NIM when keyed, then
+// Vertex text-embedding-005 on the GCP credit pool, then OpenAI
+// text-embedding-3-small as the paid backstop). See api/_lib/embeddings.js
+// for the authoritative preference order.
// 3. 503 `embed_unconfigured` only when NO provider is available, so the
// client can show an honest "not configured" state instead of inventing
// vectors. Within a single response all vectors come from one provider, so
// `dimensions` is uniform regardless of which tier served the request.
+//
+// A provider that answers with a short or empty batch is treated as a failure,
+// not as a partial success: the response never carries a null in place of a
+// vector, because a caller laying these out in semantic space cannot tell a
+// missing embedding from a real one at the origin.
import { createHash } from 'node:crypto';
import { cors, method, readJson, error, json, wrap, rateLimited } from '../_lib/http.js';
@@ -66,7 +73,7 @@ function validateTexts(input) {
throw Object.assign(new Error('texts must be an array'), { status: 400 });
}
if (input.length === 0 || input.length > MAX_TEXTS) {
- throw Object.assign(new Error(`texts must hold 1–${MAX_TEXTS} items`), { status: 400 });
+ throw Object.assign(new Error(`texts must hold 1 to ${MAX_TEXTS} items`), { status: 400 });
}
const out = [];
for (const t of input) {
@@ -89,11 +96,11 @@ export default wrap(async function handler(req, res) {
const ip = clientIp(req);
const perIp = await limits.watsonxEmbedIp(ip);
if (!perIp.success) {
- return rateLimited(res, perIp, 'too many embedding requests — slow down');
+ return rateLimited(res, perIp, 'too many embedding requests, slow down');
}
const global = await limits.watsonxEmbedGlobal();
if (!global.success) {
- return rateLimited(res, global, 'embedding capacity reached — try again shortly');
+ return rateLimited(res, global, 'embedding capacity reached, try again shortly');
}
let body;
@@ -105,7 +112,10 @@ export default wrap(async function handler(req, res) {
let texts;
try {
- texts = validateTexts(body.texts);
+ // `body` is whatever parsed: a literal `null` or a bare scalar is valid
+ // JSON, so reach for `.texts` defensively and let validateTexts reject it
+ // with the same 400 as any other malformed payload.
+ texts = validateTexts(body?.texts);
} catch (e) {
return error(res, e.status || 400, 'bad_request', e.message);
}
@@ -113,20 +123,20 @@ export default wrap(async function handler(req, res) {
const cfg = watsonxConfig();
const fallbackReady = embeddingsConfigured();
if (!cfg.configured && !fallbackReady) {
- // No provider at all. No fabricated vectors — tell the client exactly
+ // No provider at all. No fabricated vectors. Tell the client exactly
// what's missing so the UI can render an honest "not configured" state.
return error(
res,
503,
'embed_unconfigured',
- 'No embedding provider is configured. Set WATSONX_API_KEY + WATSONX_PROJECT_ID (IBM Granite), NVIDIA_API_KEY (free fallback), or OPENAI_API_KEY (paid fallback) to enable embeddings.',
+ 'No embedding provider is configured. Set WATSONX_API_KEY + WATSONX_PROJECT_ID (IBM Granite), NVIDIA_API_KEY (free fallback), GOOGLE_CLOUD_PROJECT (Vertex fallback), or OPENAI_API_KEY (paid fallback) to enable embeddings.',
);
}
- // Provider chain: Granite (primary) → the platform free-first embedding
- // chain (NIM, then OpenAI). The first provider that returns a full batch
- // wins; a watsonx outage transparently falls through so /constellation
- // keeps rendering.
+ // Provider chain: Granite (primary) then the platform free-first embedding
+ // chain (NIM, Vertex, OpenAI). The first provider that returns a FULL batch
+ // wins; a watsonx outage, or a watsonx response that covers only part of the
+ // batch, transparently falls through so /constellation keeps rendering.
let result = null;
let lastError = null;
@@ -143,14 +153,14 @@ export default wrap(async function handler(req, res) {
})),
);
} catch (e) {
- // Hold the cause; if OpenAI can cover we still serve a 200.
+ // Hold the cause; if the fallback chain can cover we still serve a 200.
lastError = e;
}
}
if (!result && fallbackReady) {
// These texts are peers laid out against each other (not query-vs-corpus),
- // so they all embed as 'passage' — one consistent space per response.
+ // so they all embed as 'passage': one consistent space per response.
const fallbackTag = defaultIngestEmbedderTag();
const fallback = embedderInfo(fallbackTag);
try {
@@ -167,7 +177,7 @@ export default wrap(async function handler(req, res) {
}
if (!result) {
- // Both tiers failed at the network level — surface the real upstream cause.
+ // Both tiers failed at the network level. Surface the real upstream cause.
return error(res, 502, 'embed_error', lastError?.message || 'embeddings failed');
}
@@ -187,8 +197,13 @@ export default wrap(async function handler(req, res) {
* Embed `texts` with a single provider, reusing the process-local cache for
* already-seen (model, text) pairs so repeat lookups cost nothing. `fetcher`
* receives the genuinely-uncached inputs and returns { vectors, dimensions }.
- * Caching is keyed by model, so Granite and OpenAI vectors never mix — every
+ * Caching is keyed by model, so Granite and OpenAI vectors never mix: every
* returned batch is uniform in dimensionality.
+ *
+ * Throws when the provider covers only part of the batch. That is what makes
+ * the caller's fallback chain correct: a half-answered batch has to look like a
+ * failure, or the next provider is never tried and the client is handed nulls
+ * dressed up as embeddings.
*/
async function embedBatch(texts, model, fetcher) {
const vectors = new Array(texts.length).fill(null);
@@ -208,9 +223,14 @@ async function embedBatch(texts, model, fetcher) {
if (missText.length) {
const fetched = await fetcher(missText);
dimensions = fetched.dimensions || dimensions;
+ const returned = Array.isArray(fetched.vectors) ? fetched.vectors : [];
for (let k = 0; k < missIdx.length; k++) {
- const vec = fetched.vectors[k];
- if (!vec?.length) continue;
+ const vec = returned[k];
+ if (!vec?.length) {
+ throw new Error(
+ `${model} returned ${returned.filter((v) => v?.length).length} of ${missText.length} embeddings`,
+ );
+ }
vectors[missIdx[k]] = vec;
cacheSet(cacheKey(model, missText[k]), vec);
}
diff --git a/docs/ibm.md b/docs/ibm.md
index 0a7d7c4ac..872d52d40 100644
--- a/docs/ibm.md
+++ b/docs/ibm.md
@@ -211,7 +211,9 @@ curl -s https://three.ws/api/watsonx/embed \
-d '{"texts":["a witty Solana trading assistant","a calm meditation guide"]}'
```
-It returns one vector per input (the response reports the model and its native `dimensions`), plus a `cachedHits` count — a warm process-local LRU and CDN cache-control headers keep repeat embeddings free, and per-IP + global rate limits cap watsonx spend.
+It returns one vector per input (the response reports the model and its native `dimensions`), plus a `cachedHits` count: a warm process-local LRU keeps repeat embeddings free, and per-IP + global rate limits cap watsonx spend. The response itself is sent `cache-control: no-store`, because a POST body varies per request and must never be shared-cached.
+
+A batch is 1 to 96 texts, each truncated to 512 characters. When watsonx is not configured on a deployment (or is failing), the endpoint falls through to the platform's free-first embedding chain (NVIDIA NIM, then Vertex `text-embedding-005`, then OpenAI `text-embedding-3-small`) and reports whichever `model` actually served, so `dimensions` is uniform within a response but can differ between them. A provider that covers only part of a batch counts as a failure and the next provider takes over: the response never carries a null where a vector belongs. With no provider configured at all you get a `503 embed_unconfigured` naming the env vars to set, never invented vectors.
---
diff --git a/tests/api/watsonx-embed.test.js b/tests/api/watsonx-embed.test.js
new file mode 100644
index 000000000..f274d7602
--- /dev/null
+++ b/tests/api/watsonx-embed.test.js
@@ -0,0 +1,279 @@
+// Unit tests for POST /api/watsonx/embed, the standalone Granite embedding
+// endpoint behind the watsonx Constellation.
+//
+// No network: the watsonx client and the platform embedding chain are both
+// stubbed, so these lock down the handler's own contract:
+// - input validation returns JSON 4xx, never a stack trace
+// - the Granite lane serves the batch when watsonx is configured
+// - the process-local cache is reused on a repeat call (cachedHits)
+// - a watsonx failure falls through to the platform chain and still 200s
+// - the regression this file exists for: a provider that answers only PART
+// of the batch used to be treated as success, so the response shipped
+// `null` where a vector belonged and the fallback tier was never tried.
+// Now a short batch is a failure, the next provider covers it, and a
+// response never contains a null vector.
+
+import { describe, it, expect, beforeEach, vi } from 'vitest';
+import { Readable } from 'node:stream';
+
+const state = {
+ wxConfigured: true,
+ // How many leading vectors the watsonx lane answers with; null means "all".
+ wxCoverage: null,
+ wxError: null,
+ fallbackConfigured: true,
+ fallbackError: null,
+};
+
+vi.mock('../../api/_lib/rate-limit.js', () => ({
+ limits: {
+ watsonxEmbedIp: vi.fn(async () => ({ success: true })),
+ watsonxEmbedGlobal: vi.fn(async () => ({ success: true })),
+ },
+ clientIp: vi.fn(() => '127.0.0.1'),
+}));
+
+// Deterministic 4-dim "embeddings": vector i leans on axis i % 4, so a test can
+// tell Granite output (leading 1) from fallback output (leading 2) by value.
+const graniteVec = (i) => [1, i, 0, 0];
+const fallbackVec = (i) => [2, i, 0, 0];
+
+vi.mock('../../api/_lib/watsonx.js', () => ({
+ watsonxConfig: vi.fn(() =>
+ state.wxConfigured
+ ? {
+ configured: true,
+ url: 'https://wx',
+ projectId: 'proj',
+ apiVersion: '2024-05-31',
+ embedModel: 'ibm/granite-embedding-278m-multilingual',
+ }
+ : { configured: false },
+ ),
+ watsonxEmbed: vi.fn(async (_cfg, { inputs, model }) => {
+ if (state.wxError) throw new Error(state.wxError);
+ const cover = state.wxCoverage === null ? inputs.length : state.wxCoverage;
+ const vectors = inputs.map((_, i) => (i < cover ? graniteVec(i) : []));
+ return { model, vectors, dimensions: 4, inputCount: inputs.length };
+ }),
+}));
+
+vi.mock('../../api/_lib/embeddings.js', () => ({
+ embeddingsConfigured: vi.fn(() => state.fallbackConfigured),
+ defaultIngestEmbedderTag: vi.fn(() => 'nvidia/nv-embedqa-e5-v5@1024'),
+ embedderInfo: vi.fn(() => ({
+ tag: 'nvidia/nv-embedqa-e5-v5@1024',
+ provider: 'nim',
+ model: 'nvidia/nv-embedqa-e5-v5',
+ dim: 4,
+ })),
+ embedPassages: vi.fn(async (_tag, texts) => {
+ if (state.fallbackError) throw new Error(state.fallbackError);
+ return texts.map((_, i) => Float64Array.from(fallbackVec(i)));
+ }),
+}));
+
+import { limits } from '../../api/_lib/rate-limit.js';
+import { watsonxEmbed } from '../../api/_lib/watsonx.js';
+import { embedPassages } from '../../api/_lib/embeddings.js';
+
+const handler = (await import('../../api/watsonx/embed.js')).default;
+
+// `rawBody` sends the exact bytes instead of JSON.stringify(body), so a test can
+// post literal `null`: valid JSON that is not an object.
+function makeReq({ method = 'POST', body = null, rawBody = null } = {}) {
+ const payload = rawBody ?? (body === null ? null : JSON.stringify(body));
+ const req = payload === null ? Readable.from([]) : Readable.from([Buffer.from(payload)]);
+ req.method = method;
+ req.url = '/api/watsonx/embed';
+ req.headers = {
+ host: 'localhost',
+ 'content-type': 'application/json',
+ origin: 'http://localhost',
+ };
+ return req;
+}
+
+function makeRes() {
+ return {
+ statusCode: 200,
+ headers: {},
+ body: '',
+ writableEnded: false,
+ setHeader(k, v) {
+ this.headers[k.toLowerCase()] = v;
+ },
+ getHeader(k) {
+ return this.headers[k.toLowerCase()];
+ },
+ end(chunk) {
+ if (chunk !== undefined) this.body += chunk;
+ this.writableEnded = true;
+ },
+ };
+}
+
+async function invoke(opts) {
+ const res = makeRes();
+ await handler(makeReq(opts), res);
+ return { status: res.statusCode, body: res.body ? JSON.parse(res.body) : null, res };
+}
+
+// Fresh text per test: the handler's vector cache is process-local and lives for
+// the whole module, so reusing a string across tests would leak cache hits.
+let seq = 0;
+const uniqueTexts = (n) => {
+ seq += 1;
+ return Array.from({ length: n }, (_, i) => `probe ${seq} item ${i}`);
+};
+
+beforeEach(() => {
+ vi.clearAllMocks();
+ limits.watsonxEmbedIp.mockResolvedValue({ success: true });
+ limits.watsonxEmbedGlobal.mockResolvedValue({ success: true });
+ state.wxConfigured = true;
+ state.wxCoverage = null;
+ state.wxError = null;
+ state.fallbackConfigured = true;
+ state.fallbackError = null;
+});
+
+describe('POST /api/watsonx/embed validation', () => {
+ it('rejects a non-POST method', async () => {
+ const { status } = await invoke({ method: 'GET' });
+ expect(status).toBe(405);
+ });
+
+ it('rejects a missing texts field with a JSON 400', async () => {
+ const { status, body } = await invoke({ body: {} });
+ expect(status).toBe(400);
+ expect(body.error).toBe('bad_request');
+ expect(body.error_description).toContain('texts must be an array');
+ });
+
+ it('rejects a literal null body without throwing', async () => {
+ const { status, body } = await invoke({ rawBody: 'null' });
+ expect(status).toBe(400);
+ expect(body.error).toBe('bad_request');
+ });
+
+ it('rejects an empty batch and an oversized batch', async () => {
+ const empty = await invoke({ body: { texts: [] } });
+ expect(empty.status).toBe(400);
+ const tooMany = await invoke({ body: { texts: uniqueTexts(97) } });
+ expect(tooMany.status).toBe(400);
+ expect(tooMany.body.error_description).toContain('1 to 96');
+ });
+
+ it('rejects non-string and blank entries', async () => {
+ expect((await invoke({ body: { texts: ['ok', 7] } })).status).toBe(400);
+ expect((await invoke({ body: { texts: [' '] } })).status).toBe(400);
+ });
+
+ it('503s with an actionable message when no provider is configured', async () => {
+ state.wxConfigured = false;
+ state.fallbackConfigured = false;
+ const { status, body } = await invoke({ body: { texts: uniqueTexts(1) } });
+ expect(status).toBe(503);
+ expect(body.error).toBe('embed_unconfigured');
+ expect(body.error_description).toContain('WATSONX_API_KEY');
+ expect(body.vectors).toBeUndefined();
+ });
+});
+
+describe('POST /api/watsonx/embed provider chain', () => {
+ it('serves the Granite lane and preserves input order', async () => {
+ const texts = uniqueTexts(3);
+ const { status, body } = await invoke({ body: { texts } });
+ expect(status).toBe(200);
+ expect(body.model).toBe('ibm/granite-embedding-278m-multilingual');
+ expect(body.dimensions).toBe(4);
+ expect(body.count).toBe(3);
+ expect(body.cachedHits).toBe(0);
+ expect(body.vectors).toEqual([graniteVec(0), graniteVec(1), graniteVec(2)]);
+ expect(embedPassages).not.toHaveBeenCalled();
+ });
+
+ it('honours a caller-supplied model id', async () => {
+ await invoke({ body: { texts: uniqueTexts(1), model: 'ibm/granite-embedding-107m-multilingual' } });
+ expect(watsonxEmbed).toHaveBeenCalledWith(
+ expect.anything(),
+ expect.objectContaining({ model: 'ibm/granite-embedding-107m-multilingual' }),
+ );
+ });
+
+ it('reuses the process-local cache on a repeat call', async () => {
+ const texts = uniqueTexts(2);
+ await invoke({ body: { texts } });
+ const second = await invoke({ body: { texts } });
+ expect(second.status).toBe(200);
+ expect(second.body.cachedHits).toBe(2);
+ expect(second.body.vectors).toEqual([graniteVec(0), graniteVec(1)]);
+ // One upstream call total: the repeat was served entirely from cache.
+ expect(watsonxEmbed).toHaveBeenCalledTimes(1);
+ });
+
+ it('falls through to the platform chain when watsonx fails', async () => {
+ state.wxError = 'watsonx.ai error (503): service unavailable';
+ const texts = uniqueTexts(2);
+ const { status, body } = await invoke({ body: { texts } });
+ expect(status).toBe(200);
+ expect(body.model).toBe('nvidia/nv-embedqa-e5-v5');
+ expect(body.vectors).toEqual([fallbackVec(0), fallbackVec(1)]);
+ });
+
+ it('skips watsonx entirely when it is not configured', async () => {
+ state.wxConfigured = false;
+ const { status, body } = await invoke({ body: { texts: uniqueTexts(1) } });
+ expect(status).toBe(200);
+ expect(watsonxEmbed).not.toHaveBeenCalled();
+ expect(body.model).toBe('nvidia/nv-embedqa-e5-v5');
+ });
+
+ it('treats a partially-covered batch as a failure and lets the fallback cover it', async () => {
+ // watsonx answers 1 of 3. The old code returned 200 with two nulls.
+ state.wxCoverage = 1;
+ const texts = uniqueTexts(3);
+ const { status, body } = await invoke({ body: { texts } });
+ expect(status).toBe(200);
+ expect(body.model).toBe('nvidia/nv-embedqa-e5-v5');
+ expect(body.count).toBe(3);
+ expect(body.vectors).toHaveLength(3);
+ expect(body.vectors.every((v) => Array.isArray(v) && v.length === 4)).toBe(true);
+ expect(embedPassages).toHaveBeenCalledTimes(1);
+ });
+
+ it('502s with the upstream cause when every provider fails', async () => {
+ state.wxError = 'watsonx.ai error (401): unauthorized';
+ state.fallbackError = 'nim embedder unreachable';
+ const { status, body } = await invoke({ body: { texts: uniqueTexts(2) } });
+ expect(status).toBe(502);
+ expect(body.error).toBe('embed_error');
+ expect(body.error_description).toContain('nim embedder unreachable');
+ expect(body.vectors).toBeUndefined();
+ });
+
+ it('never serves a null vector when both tiers answer short', async () => {
+ state.wxCoverage = 0;
+ state.fallbackError = 'nim embedder unreachable';
+ const { status, body } = await invoke({ body: { texts: uniqueTexts(2) } });
+ expect(status).toBe(502);
+ expect(body.vectors).toBeUndefined();
+ });
+});
+
+describe('POST /api/watsonx/embed rate limits', () => {
+ it('429s on the per-IP limit before touching a provider', async () => {
+ limits.watsonxEmbedIp.mockResolvedValue({ success: false, limit: 20, remaining: 0, reset: Date.now() + 1000 });
+ const { status } = await invoke({ body: { texts: uniqueTexts(1) } });
+ expect(status).toBe(429);
+ expect(watsonxEmbed).not.toHaveBeenCalled();
+ });
+
+ it('429s on the global ceiling', async () => {
+ limits.watsonxEmbedGlobal.mockResolvedValue({ success: false, limit: 500, remaining: 0, reset: Date.now() + 1000 });
+ const { status } = await invoke({ body: { texts: uniqueTexts(1) } });
+ expect(status).toBe(429);
+ expect(watsonxEmbed).not.toHaveBeenCalled();
+ });
+});