Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

15.3 KB
Raw Blame
// Knowledge ingest + list + delete for the talking-agent widget. Accepts:
//   { source_type: 'url',  source_url: 'https://…', title?: '…' }
//   { source_type: 'text', content:    '…',         title:  '…' }
//   { source_type: 'pdf',  content:    '<extracted>', title: '…', source_url?: '…', byte_size?: n }
//
// PDFs are extracted client-side (pdfjs in the studio bundle) and posted as
// text — the server never decodes binary PDFs, which keeps this route in the
// edge-friendly fetch-only path.

import { z } from 'zod';

import { sql } from '../../_lib/db.js';
import {
	embedPassages,
	embeddingsConfigured,
	embedderConfigured,
	defaultIngestEmbedderTag,
	resolveEmbedderTag,
	scoreRowsBySpace,
} from '../../_lib/embeddings.js';
import { rerankConfigured, rerankPassages } from '../../_lib/rerank.js';
import { chunk, estimateTokens } from '../../_lib/chunker.js';
import { fetchAndExtract } from '../../_lib/text-extract.js';
import { shortId } from '../../_lib/ids.js';
import { publishJob, qstashEnabled } from '../../_lib/qstash.js';
import { env } from '../../_lib/env.js';

const MAX_DOCS_PER_WIDGET = 25;
const MAX_TEXT_BYTES = 4 * 1024 * 1024;
const MAX_CHUNKS_PER_DOC = 400;
const MAX_PREVIEW_QS = 3;

// Embed text in batches of N. text-embedding-3-small accepts up to 2048
// inputs per call, but 64 keeps memory predictable and lets failures retry
// at a small unit.
const EMBED_BATCH = 64;

// Docs bigger than this run async via QStash to avoid Vercel function timeouts
// on large PDFs. Small docs stay inline so the studio UI doesn't have to poll
// for the common case.
const INLINE_CHUNK_THRESHOLD = 32;

const baseSchema = z.object({
	title: z.string().trim().min(1).max(160).optional(),
	source_type: z.enum(['url', 'text', 'pdf', 'markdown']),
});

const urlSchema = baseSchema.extend({
	source_type: z.literal('url'),
	source_url: z.string().url(),
});

const textSchema = baseSchema.extend({
	source_type: z.enum(['text', 'markdown', 'pdf']),
	content: z.string().min(1).max(MAX_TEXT_BYTES),
	source_url: z.string().url().optional(),
	byte_size: z.number().int().nonnegative().optional(),
});

const docInput = z.discriminatedUnion('source_type', [
	urlSchema,
	textSchema.extend({ source_type: z.literal('text') }),
	textSchema.extend({ source_type: z.literal('markdown') }),
	textSchema.extend({ source_type: z.literal('pdf') }),
]);

export async function listKnowledge(widgetId) {
	const docs = await sql`
		select id, title, source_type, source_url, byte_size, chunk_count,
		       token_count, status, error, created_at, updated_at
		from widget_knowledge_docs
		where widget_id = ${widgetId}
		order by created_at desc
	`;
	return {
		docs: docs.map((d) => ({
			id: d.id,
			title: d.title,
			source_type: d.source_type,
			source_url: d.source_url,
			byte_size: Number(d.byte_size || 0),
			chunk_count: Number(d.chunk_count || 0),
			token_count: Number(d.token_count || 0),
			status: d.status,
			error: d.error,
			created_at: d.created_at,
			updated_at: d.updated_at,
		})),
	};
}

// Inkeep-style retrieval debugger: takes a probe query and returns the top-K
// chunks with cosine scores + the source doc. Lets the creator verify their
// docs are actually retrievable without having to chat. No LLM call — pure
// vector lookup, mirrors what handleChat would inject as grounding.
export async function testRetrieval({ widgetId, query, topK = 5 }) {
	if (!embeddingsConfigured()) {
		throw httpError(
			503,
			'embedder_unavailable',
			'Knowledge test needs an embedding provider on the server: set NVIDIA_API_KEY (free) or OPENAI_API_KEY.',
		);
	}
	const q = String(query || '').trim();
	if (q.length < 2) throw httpError(400, 'invalid_request', 'query must be at least 2 chars');

	const rows = await sql`
		select c.id, c.doc_id, c.chunk_index, c.content, c.embedding, c.token_count,
		       c.embedder, d.title, d.source_url, d.source_type
		from widget_knowledge_chunks c
		join widget_knowledge_docs   d on d.id = c.doc_id
		where c.widget_id = ${widgetId}
	`;
	if (!rows.length) return { query: q, results: [], chunks_searched: 0 };

	// Same-space rule: the query must be embedded with the SAME model each
	// stored chunk was embedded with. Group chunks by their embedder tag,
	// embed the query once per servable space, and refuse (never guess) for
	// spaces whose provider isn't configured.
	const { scored, needsReembed } = await scoreRowsBySpace(rows, q);

	if (!scored.length && needsReembed.length) {
		const spaces = needsReembed.map((n) => n.embedder).join(', ');
		throw httpError(
			503,
			'needs_reembed',
			`Stored knowledge was embedded with ${spaces}, which no configured provider can query. ` +
				'Re-embed it with scripts/reembed-widget-knowledge.mjs, or configure the matching provider key.',
		);
	}

	let top = scored
		.sort((a, b) => b.score - a.score)
		.slice(0, Math.max(1, Math.min(20, topK)));
	let reranked = false;
	if (rerankConfigured() && top.length > 1) {
		const order = await rerankPassages(q, top.map((r) => r.content));
		if (order) {
			top = order.map((i) => top[i]);
			reranked = true;
		}
	}

	return {
		query: q,
		results: top.map((r) => ({
			id: Number(r.id),
			doc_id: r.doc_id,
			chunk_index: Number(r.chunk_index),
			score: r.score,
			embedder: r.embedder,
			token_count: Number(r.token_count || 0),
			excerpt: truncate(String(r.content), 280),
			title: r.title,
			source_url: r.source_url,
			source_type: r.source_type,
		})),
		chunks_searched: rows.length,
		reranked,
		...(needsReembed.length ? { needs_reembed: needsReembed } : {}),
	};
}


function truncate(s, n) {
	if (s.length <= n) return s;
	return s.slice(0, n).trim() + '…';
}

export async function deleteKnowledge({ widgetId, userId, docId }) {
	const rows = await sql`
		delete from widget_knowledge_docs
		where id = ${docId} and widget_id = ${widgetId} and user_id = ${userId}
		returning id
	`;
	return !!rows[0];
}

export async function ingestKnowledge({ widgetId, userId, input }) {
	// Free-first: new document sets embed via NVIDIA NIM when its key is
	// present, OpenAI otherwise. The chosen embedder tag is stamped on the doc
	// and every chunk so query time can stay in the same vector space.
	const ingestTag = defaultIngestEmbedderTag();
	if (!ingestTag) {
		throw httpError(
			503,
			'embedder_unavailable',
			'Knowledge upload needs an embedding provider on the server: set NVIDIA_API_KEY (free) or OPENAI_API_KEY.',
		);
	}

	const parsed = docInput.safeParse(input);
	if (!parsed.success) {
		const msg = parsed.error.issues
			.map((i) => `${i.path.join('.') || 'body'}: ${i.message}`)
			.join('; ');
		throw httpError(400, 'validation_error', msg);
	}
	const body = parsed.data;

	const [{ n }] = await sql`
		select count(*)::int as n from widget_knowledge_docs where widget_id = ${widgetId}
	`;
	if (n >= MAX_DOCS_PER_WIDGET) {
		throw httpError(
			400,
			'too_many_docs',
			`Each widget can hold up to ${MAX_DOCS_PER_WIDGET} knowledge docs. Delete one to add another.`,
		);
	}

	let title,
		text,
		sourceUrl = null,
		byteSize = 0;
	if (body.source_type === 'url') {
		const ext = await fetchAndExtract(body.source_url);
		title = body.title || ext.title;
		text = ext.text;
		sourceUrl = body.source_url;
		byteSize = ext.byteSize;
	} else {
		text = String(body.content || '').trim();
		title = body.title || autoTitle(text);
		sourceUrl = body.source_url || null;
		byteSize = body.byte_size || Buffer.byteLength(text, 'utf8');
	}

	if (!text || text.length < 12) {
		throw httpError(400, 'empty_source', 'No readable text was extracted from this source.');
	}

	const chunks = chunk(text).slice(0, MAX_CHUNKS_PER_DOC);
	if (!chunks.length) {
		throw httpError(400, 'empty_source', 'No chunks produced from this source.');
	}

	const docId = shortId('wkd');
	const tokenSum = chunks.reduce((s, c) => s + (c.token_count || 0), 0);

	// Route to QStash worker when the doc is large enough to risk a function
	// timeout AND a publish endpoint is configured. The worker reads source_text
	// straight off the doc row — no chunk payload over the wire.
	const shouldQueue = qstashEnabled() && chunks.length > INLINE_CHUNK_THRESHOLD;

	await sql`
		insert into widget_knowledge_docs
			(id, widget_id, user_id, title, source_type, source_url, byte_size,
			 chunk_count, token_count, status, source_text, embedder)
		values
			(${docId}, ${widgetId}, ${userId}, ${title.slice(0, 160)}, ${body.source_type},
			 ${sourceUrl}, ${byteSize}, ${chunks.length}, ${tokenSum},
			 ${shouldQueue ? 'queued' : 'processing'},
			 ${shouldQueue ? text : null}, ${ingestTag})
	`;

	if (shouldQueue) {
		try {
			await publishJob({
				url: `${env.APP_ORIGIN}/api/widgets/${widgetId}/knowledge-process`,
				body: { doc_id: docId, widget_id: widgetId },
				deduplicationId: `wkd:${docId}`,
				retries: 3,
			});
		} catch (err) {
			// QStash itself failed — mark failed so the UI doesn't spin forever.
			await sql`
				update widget_knowledge_docs
				set status = 'failed', error = ${String(err?.message || 'queue failed').slice(0, 500)},
				    source_text = null
				where id = ${docId}
			`.catch(() => {});
			throw httpError(502, 'queue_failed', err.message || 'queueing failed');
		}

		return {
			id: docId,
			title: title.slice(0, 160),
			source_type: body.source_type,
			source_url: sourceUrl,
			byte_size: byteSize,
			chunk_count: chunks.length,
			token_count: tokenSum,
			status: 'queued',
			preview_questions: synthesizePreviewQs(chunks),
		};
	}

	// Inline path for small docs: embed + insert here, return when ready.
	try {
		await embedAndInsertChunks({ docId, widgetId, chunks, embedderTag: ingestTag });
		await sql`update widget_knowledge_docs set status = 'ready' where id = ${docId}`;
	} catch (err) {
		await sql`
			update widget_knowledge_docs
			set status = 'failed', error = ${String(err?.message || 'embed failed').slice(0, 500)}
			where id = ${docId}
		`.catch(() => {});
		throw httpError(err.status || 502, 'embed_failed', err.message || 'embedding failed');
	}

	return {
		id: docId,
		title: title.slice(0, 160),
		source_type: body.source_type,
		source_url: sourceUrl,
		byte_size: byteSize,
		chunk_count: chunks.length,
		token_count: tokenSum,
		status: 'ready',
		preview_questions: synthesizePreviewQs(chunks),
	};
}

// Shared embed+insert loop used by both the inline path and the QStash worker.
// Chunks are corpus text, so they embed as 'passage' (NIM's asymmetric
// retrieval models require the ingest/search distinction); every row stores
// the embedder tag it was produced with.
async function embedAndInsertChunks({ docId, widgetId, chunks, embedderTag }) {
	for (let i = 0; i < chunks.length; i += EMBED_BATCH) {
		const slice = chunks.slice(i, i + EMBED_BATCH);
		const embedded = await embedPassages(embedderTag, slice.map((c) => c.content));
		const inserts = slice.map((c, j) => {
			const vec = Array.from(embedded[j]);
			return sql`
				insert into widget_knowledge_chunks
					(doc_id, widget_id, chunk_index, content, embedding, token_count, embedder)
				values
					(${docId}, ${widgetId}, ${i + j}, ${c.content},
					 ${JSON.stringify(vec)}::jsonb, ${c.token_count}, ${embedderTag})
			`;
		});
		await sql.transaction(inserts);
	}
}

// Worker entrypoint — called by QStash after publishJob in ingestKnowledge.
// Reads source_text off the doc row, re-chunks (chunker is deterministic so
// indices match what we returned to the client), embeds + inserts chunks,
// clears source_text, marks ready. Idempotent: re-running after partial
// failure picks up where chunks left off.
export async function processQueuedDoc({ docId, widgetId }) {
	const [doc] = await sql`
		select id, widget_id, source_text, status, chunk_count, embedder
		from widget_knowledge_docs
		where id = ${docId} and widget_id = ${widgetId}
		limit 1
	`;
	if (!doc) throw httpError(404, 'not_found', 'doc not found');
	if (doc.status === 'ready') return { ok: true, status: 'ready' };

	// Re-resolve the embedder at processing time: the worker may run on a
	// deployment whose keys differ from the one that queued the doc. The doc
	// has no chunks yet (any partials are wiped below), so switching to the
	// current default embedder is safe — the tag we actually use is written
	// back to the doc and to every chunk.
	const queuedTag = resolveEmbedderTag(doc.embedder);
	const embedderTag =
		queuedTag && embedderConfigured(queuedTag) ? queuedTag : defaultIngestEmbedderTag();
	if (!embedderTag) {
		await sql`
			update widget_knowledge_docs set status = 'failed',
				error = 'no embedding provider configured'
			where id = ${docId}
		`.catch(() => {});
		throw httpError(
			503,
			'embedder_unavailable',
			'No embedding provider configured: set NVIDIA_API_KEY (free) or OPENAI_API_KEY.',
		);
	}

	if (!doc.source_text || doc.source_text.length < 12) {
		await sql`
			update widget_knowledge_docs set status = 'failed',
				error = 'source_text missing on queued doc'
			where id = ${docId}
		`.catch(() => {});
		throw httpError(500, 'empty_source', 'no source_text on queued doc');
	}

	await sql`update widget_knowledge_docs set status = 'processing' where id = ${docId}`;

	try {
		const chunks = chunk(doc.source_text).slice(0, MAX_CHUNKS_PER_DOC);

		// Idempotency: drop any partial chunks from a previous failed run so we
		// don't end up with duplicates at the same chunk_index.
		await sql`delete from widget_knowledge_chunks where doc_id = ${docId}`;

		await embedAndInsertChunks({ docId, widgetId: doc.widget_id, chunks, embedderTag });

		await sql`
			update widget_knowledge_docs
			set status = 'ready', source_text = null, error = null,
			    chunk_count = ${chunks.length}, embedder = ${embedderTag}
			where id = ${docId}
		`;
		return { ok: true, status: 'ready', chunk_count: chunks.length };
	} catch (err) {
		await sql`
			update widget_knowledge_docs
			set status = 'failed', error = ${String(err?.message || 'embed failed').slice(0, 500)}
			where id = ${docId}
		`.catch(() => {});
		throw err;
	}
}

// "Your bot now knows X" preview — pulls salient sentences from the first few
// chunks and reshapes them into starter questions so the creator can sanity-
// check that the doc landed in retrieval. Heuristic, not LLM-generated.
function synthesizePreviewQs(chunks) {
	const sentences = [];
	for (const c of chunks.slice(0, 3)) {
		for (const s of String(c.content).split(/(?<=[.!?])\s+/)) {
			const trimmed = s.trim();
			if (trimmed.length >= 32 && trimmed.length <= 180) sentences.push(trimmed);
		}
	}
	if (!sentences.length) return [];

	// Pick the longest distinct-prefix sentences first.
	const seen = new Set();
	const picked = [];
	for (const s of sentences.sort((a, b) => b.length - a.length)) {
		const key = s.slice(0, 40).toLowerCase();
		if (seen.has(key)) continue;
		seen.add(key);
		picked.push(s);
		if (picked.length >= MAX_PREVIEW_QS) break;
	}

	return picked.map(toQuestion);
}

function toQuestion(sentence) {
	const s = sentence.replace(/[.!?]+$/, '').trim();
	if (/^(what|when|where|why|how|who|which|can|do|does|is|are)\b/i.test(s)) return s + '?';
	const noun = s.split(/[.,;:—-]/)[0].trim();
	return noun
		? `What does the doc say about ${lower(noun.slice(0, 80))}?`
		: `Can you tell me more about this?`;
}

function lower(s) {
	return s.charAt(0).toLowerCase() + s.slice(1);
}

function autoTitle(text) {
	const firstLine =
		String(text)
			.split('\n')
			.find((l) => l.trim().length > 0) || '';
	return firstLine.trim().slice(0, 120) || `Pasted text (${estimateTokens(text)} tokens)`;
}

function httpError(status, code, message) {
	return Object.assign(new Error(message), { status, code });
}