Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
// Knowledge ingest + list + delete for the talking-agent widget. Accepts:
// { source_type: 'url', source_url: 'https://…', title?: '…' }
// { source_type: 'text', content: '…', title: '…' }
// { source_type: 'pdf', content: '<extracted>', title: '…', source_url?: '…', byte_size?: n }
//
// PDFs are extracted client-side (pdfjs in the studio bundle) and posted as
// text — the server never decodes binary PDFs, which keeps this route in the
// edge-friendly fetch-only path.
import { z } from 'zod';
import { sql } from '../../_lib/db.js';
import {
embedPassages,
embeddingsConfigured,
embedderConfigured,
defaultIngestEmbedderTag,
resolveEmbedderTag,
scoreRowsBySpace,
} from '../../_lib/embeddings.js';
import { rerankConfigured, rerankPassages } from '../../_lib/rerank.js';
import { chunk, estimateTokens } from '../../_lib/chunker.js';
import { fetchAndExtract } from '../../_lib/text-extract.js';
import { shortId } from '../../_lib/ids.js';
import { publishJob, qstashEnabled } from '../../_lib/qstash.js';
import { env } from '../../_lib/env.js';
const MAX_DOCS_PER_WIDGET = 25;
const MAX_TEXT_BYTES = 4 * 1024 * 1024;
const MAX_CHUNKS_PER_DOC = 400;
const MAX_PREVIEW_QS = 3;
// Embed text in batches of N. text-embedding-3-small accepts up to 2048
// inputs per call, but 64 keeps memory predictable and lets failures retry
// at a small unit.
const EMBED_BATCH = 64;
// Docs bigger than this run async via QStash to avoid Vercel function timeouts
// on large PDFs. Small docs stay inline so the studio UI doesn't have to poll
// for the common case.
const INLINE_CHUNK_THRESHOLD = 32;
const baseSchema = z.object({
title: z.string().trim().min(1).max(160).optional(),
source_type: z.enum(['url', 'text', 'pdf', 'markdown']),
});
const urlSchema = baseSchema.extend({
source_type: z.literal('url'),
source_url: z.string().url(),
});
const textSchema = baseSchema.extend({
source_type: z.enum(['text', 'markdown', 'pdf']),
content: z.string().min(1).max(MAX_TEXT_BYTES),
source_url: z.string().url().optional(),
byte_size: z.number().int().nonnegative().optional(),
});
const docInput = z.discriminatedUnion('source_type', [
urlSchema,
textSchema.extend({ source_type: z.literal('text') }),
textSchema.extend({ source_type: z.literal('markdown') }),
textSchema.extend({ source_type: z.literal('pdf') }),
]);
export async function listKnowledge(widgetId) {
const docs = await sql`
select id, title, source_type, source_url, byte_size, chunk_count,
token_count, status, error, created_at, updated_at
from widget_knowledge_docs
where widget_id = ${widgetId}
order by created_at desc
`;
return {
docs: docs.map((d) => ({
id: d.id,
title: d.title,
source_type: d.source_type,
source_url: d.source_url,
byte_size: Number(d.byte_size || 0),
chunk_count: Number(d.chunk_count || 0),
token_count: Number(d.token_count || 0),
status: d.status,
error: d.error,
created_at: d.created_at,
updated_at: d.updated_at,
})),
};
}
// Inkeep-style retrieval debugger: takes a probe query and returns the top-K
// chunks with cosine scores + the source doc. Lets the creator verify their
// docs are actually retrievable without having to chat. No LLM call — pure
// vector lookup, mirrors what handleChat would inject as grounding.
export async function testRetrieval({ widgetId, query, topK = 5 }) {
if (!embeddingsConfigured()) {
throw httpError(
503,
'embedder_unavailable',
'Knowledge test needs an embedding provider on the server: set NVIDIA_API_KEY (free) or OPENAI_API_KEY.',
);
}
const q = String(query || '').trim();
if (q.length < 2) throw httpError(400, 'invalid_request', 'query must be at least 2 chars');
const rows = await sql`
select c.id, c.doc_id, c.chunk_index, c.content, c.embedding, c.token_count,
c.embedder, d.title, d.source_url, d.source_type
from widget_knowledge_chunks c
join widget_knowledge_docs d on d.id = c.doc_id
where c.widget_id = ${widgetId}
`;
if (!rows.length) return { query: q, results: [], chunks_searched: 0 };
// Same-space rule: the query must be embedded with the SAME model each
// stored chunk was embedded with. Group chunks by their embedder tag,
// embed the query once per servable space, and refuse (never guess) for
// spaces whose provider isn't configured.
const { scored, needsReembed } = await scoreRowsBySpace(rows, q);
if (!scored.length && needsReembed.length) {
const spaces = needsReembed.map((n) => n.embedder).join(', ');
throw httpError(
503,
'needs_reembed',
`Stored knowledge was embedded with ${spaces}, which no configured provider can query. ` +
'Re-embed it with scripts/reembed-widget-knowledge.mjs, or configure the matching provider key.',
);
}
let top = scored
.sort((a, b) => b.score - a.score)
.slice(0, Math.max(1, Math.min(20, topK)));
let reranked = false;
if (rerankConfigured() && top.length > 1) {
const order = await rerankPassages(q, top.map((r) => r.content));
if (order) {
top = order.map((i) => top[i]);
reranked = true;
}
}
return {
query: q,
results: top.map((r) => ({
id: Number(r.id),
doc_id: r.doc_id,
chunk_index: Number(r.chunk_index),
score: r.score,
embedder: r.embedder,
token_count: Number(r.token_count || 0),
excerpt: truncate(String(r.content), 280),
title: r.title,
source_url: r.source_url,
source_type: r.source_type,
})),
chunks_searched: rows.length,
reranked,
...(needsReembed.length ? { needs_reembed: needsReembed } : {}),
};
}
function truncate(s, n) {
if (s.length <= n) return s;
return s.slice(0, n).trim() + '…';
}
export async function deleteKnowledge({ widgetId, userId, docId }) {
const rows = await sql`
delete from widget_knowledge_docs
where id = ${docId} and widget_id = ${widgetId} and user_id = ${userId}
returning id
`;
return !!rows[0];
}
export async function ingestKnowledge({ widgetId, userId, input }) {
// Free-first: new document sets embed via NVIDIA NIM when its key is
// present, OpenAI otherwise. The chosen embedder tag is stamped on the doc
// and every chunk so query time can stay in the same vector space.
const ingestTag = defaultIngestEmbedderTag();
if (!ingestTag) {
throw httpError(
503,
'embedder_unavailable',
'Knowledge upload needs an embedding provider on the server: set NVIDIA_API_KEY (free) or OPENAI_API_KEY.',
);
}
const parsed = docInput.safeParse(input);
if (!parsed.success) {
const msg = parsed.error.issues
.map((i) => `${i.path.join('.') || 'body'}: ${i.message}`)
.join('; ');
throw httpError(400, 'validation_error', msg);
}
const body = parsed.data;
const [{ n }] = await sql`
select count(*)::int as n from widget_knowledge_docs where widget_id = ${widgetId}
`;
if (n >= MAX_DOCS_PER_WIDGET) {
throw httpError(
400,
'too_many_docs',
`Each widget can hold up to ${MAX_DOCS_PER_WIDGET} knowledge docs. Delete one to add another.`,
);
}
let title,
text,
sourceUrl = null,
byteSize = 0;
if (body.source_type === 'url') {
const ext = await fetchAndExtract(body.source_url);
title = body.title || ext.title;
text = ext.text;
sourceUrl = body.source_url;
byteSize = ext.byteSize;
} else {
text = String(body.content || '').trim();
title = body.title || autoTitle(text);
sourceUrl = body.source_url || null;
byteSize = body.byte_size || Buffer.byteLength(text, 'utf8');
}
if (!text || text.length < 12) {
throw httpError(400, 'empty_source', 'No readable text was extracted from this source.');
}
const chunks = chunk(text).slice(0, MAX_CHUNKS_PER_DOC);
if (!chunks.length) {
throw httpError(400, 'empty_source', 'No chunks produced from this source.');
}
const docId = shortId('wkd');
const tokenSum = chunks.reduce((s, c) => s + (c.token_count || 0), 0);
// Route to QStash worker when the doc is large enough to risk a function
// timeout AND a publish endpoint is configured. The worker reads source_text
// straight off the doc row — no chunk payload over the wire.
const shouldQueue = qstashEnabled() && chunks.length > INLINE_CHUNK_THRESHOLD;
await sql`
insert into widget_knowledge_docs
(id, widget_id, user_id, title, source_type, source_url, byte_size,
chunk_count, token_count, status, source_text, embedder)
values
(${docId}, ${widgetId}, ${userId}, ${title.slice(0, 160)}, ${body.source_type},
${sourceUrl}, ${byteSize}, ${chunks.length}, ${tokenSum},
${shouldQueue ? 'queued' : 'processing'},
${shouldQueue ? text : null}, ${ingestTag})
`;
if (shouldQueue) {
try {
await publishJob({
url: `${env.APP_ORIGIN}/api/widgets/${widgetId}/knowledge-process`,
body: { doc_id: docId, widget_id: widgetId },
deduplicationId: `wkd:${docId}`,
retries: 3,
});
} catch (err) {
// QStash itself failed — mark failed so the UI doesn't spin forever.
await sql`
update widget_knowledge_docs
set status = 'failed', error = ${String(err?.message || 'queue failed').slice(0, 500)},
source_text = null
where id = ${docId}
`.catch(() => {});
throw httpError(502, 'queue_failed', err.message || 'queueing failed');
}
return {
id: docId,
title: title.slice(0, 160),
source_type: body.source_type,
source_url: sourceUrl,
byte_size: byteSize,
chunk_count: chunks.length,
token_count: tokenSum,
status: 'queued',
preview_questions: synthesizePreviewQs(chunks),
};
}
// Inline path for small docs: embed + insert here, return when ready.
try {
await embedAndInsertChunks({ docId, widgetId, chunks, embedderTag: ingestTag });
await sql`update widget_knowledge_docs set status = 'ready' where id = ${docId}`;
} catch (err) {
await sql`
update widget_knowledge_docs
set status = 'failed', error = ${String(err?.message || 'embed failed').slice(0, 500)}
where id = ${docId}
`.catch(() => {});
throw httpError(err.status || 502, 'embed_failed', err.message || 'embedding failed');
}
return {
id: docId,
title: title.slice(0, 160),
source_type: body.source_type,
source_url: sourceUrl,
byte_size: byteSize,
chunk_count: chunks.length,
token_count: tokenSum,
status: 'ready',
preview_questions: synthesizePreviewQs(chunks),
};
}
// Shared embed+insert loop used by both the inline path and the QStash worker.
// Chunks are corpus text, so they embed as 'passage' (NIM's asymmetric
// retrieval models require the ingest/search distinction); every row stores
// the embedder tag it was produced with.
async function embedAndInsertChunks({ docId, widgetId, chunks, embedderTag }) {
for (let i = 0; i < chunks.length; i += EMBED_BATCH) {
const slice = chunks.slice(i, i + EMBED_BATCH);
const embedded = await embedPassages(embedderTag, slice.map((c) => c.content));
const inserts = slice.map((c, j) => {
const vec = Array.from(embedded[j]);
return sql`
insert into widget_knowledge_chunks
(doc_id, widget_id, chunk_index, content, embedding, token_count, embedder)
values
(${docId}, ${widgetId}, ${i + j}, ${c.content},
${JSON.stringify(vec)}::jsonb, ${c.token_count}, ${embedderTag})
`;
});
await sql.transaction(inserts);
}
}
// Worker entrypoint — called by QStash after publishJob in ingestKnowledge.
// Reads source_text off the doc row, re-chunks (chunker is deterministic so
// indices match what we returned to the client), embeds + inserts chunks,
// clears source_text, marks ready. Idempotent: re-running after partial
// failure picks up where chunks left off.
export async function processQueuedDoc({ docId, widgetId }) {
const [doc] = await sql`
select id, widget_id, source_text, status, chunk_count, embedder
from widget_knowledge_docs
where id = ${docId} and widget_id = ${widgetId}
limit 1
`;
if (!doc) throw httpError(404, 'not_found', 'doc not found');
if (doc.status === 'ready') return { ok: true, status: 'ready' };
// Re-resolve the embedder at processing time: the worker may run on a
// deployment whose keys differ from the one that queued the doc. The doc
// has no chunks yet (any partials are wiped below), so switching to the
// current default embedder is safe — the tag we actually use is written
// back to the doc and to every chunk.
const queuedTag = resolveEmbedderTag(doc.embedder);
const embedderTag =
queuedTag && embedderConfigured(queuedTag) ? queuedTag : defaultIngestEmbedderTag();
if (!embedderTag) {
await sql`
update widget_knowledge_docs set status = 'failed',
error = 'no embedding provider configured'
where id = ${docId}
`.catch(() => {});
throw httpError(
503,
'embedder_unavailable',
'No embedding provider configured: set NVIDIA_API_KEY (free) or OPENAI_API_KEY.',
);
}
if (!doc.source_text || doc.source_text.length < 12) {
await sql`
update widget_knowledge_docs set status = 'failed',
error = 'source_text missing on queued doc'
where id = ${docId}
`.catch(() => {});
throw httpError(500, 'empty_source', 'no source_text on queued doc');
}
await sql`update widget_knowledge_docs set status = 'processing' where id = ${docId}`;
try {
const chunks = chunk(doc.source_text).slice(0, MAX_CHUNKS_PER_DOC);
// Idempotency: drop any partial chunks from a previous failed run so we
// don't end up with duplicates at the same chunk_index.
await sql`delete from widget_knowledge_chunks where doc_id = ${docId}`;
await embedAndInsertChunks({ docId, widgetId: doc.widget_id, chunks, embedderTag });
await sql`
update widget_knowledge_docs
set status = 'ready', source_text = null, error = null,
chunk_count = ${chunks.length}, embedder = ${embedderTag}
where id = ${docId}
`;
return { ok: true, status: 'ready', chunk_count: chunks.length };
} catch (err) {
await sql`
update widget_knowledge_docs
set status = 'failed', error = ${String(err?.message || 'embed failed').slice(0, 500)}
where id = ${docId}
`.catch(() => {});
throw err;
}
}
// "Your bot now knows X" preview — pulls salient sentences from the first few
// chunks and reshapes them into starter questions so the creator can sanity-
// check that the doc landed in retrieval. Heuristic, not LLM-generated.
function synthesizePreviewQs(chunks) {
const sentences = [];
for (const c of chunks.slice(0, 3)) {
for (const s of String(c.content).split(/(?<=[.!?])\s+/)) {
const trimmed = s.trim();
if (trimmed.length >= 32 && trimmed.length <= 180) sentences.push(trimmed);
}
}
if (!sentences.length) return [];
// Pick the longest distinct-prefix sentences first.
const seen = new Set();
const picked = [];
for (const s of sentences.sort((a, b) => b.length - a.length)) {
const key = s.slice(0, 40).toLowerCase();
if (seen.has(key)) continue;
seen.add(key);
picked.push(s);
if (picked.length >= MAX_PREVIEW_QS) break;
}
return picked.map(toQuestion);
}
function toQuestion(sentence) {
const s = sentence.replace(/[.!?]+$/, '').trim();
if (/^(what|when|where|why|how|who|which|can|do|does|is|are)\b/i.test(s)) return s + '?';
const noun = s.split(/[.,;:—-]/)[0].trim();
return noun
? `What does the doc say about ${lower(noun.slice(0, 80))}?`
: `Can you tell me more about this?`;
}
function lower(s) {
return s.charAt(0).toLowerCase() + s.slice(1);
}
function autoTitle(text) {
const firstLine =
String(text)
.split('\n')
.find((l) => l.trim().length > 0) || '';
return firstLine.trim().slice(0, 120) || `Pasted text (${estimateTokens(text)} tokens)`;
}
function httpError(status, code, message) {
return Object.assign(new Error(message), { status, code });
}