Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
// Documentation search: builder + runtime.
//
// The two halves of this feature live in different files and different
// languages of the stack (a node ESM build script, a browser IIFE), but they
// share one encoding and one tokenizer. Every test here that looks pedantic is
// guarding a way those two can silently drift apart, because the failure mode
// is never an exception: it is a search that quietly stops finding things.
import { describe, it, expect, beforeAll } from 'vitest';
import { readFileSync, cpSync, mkdtempSync, rmSync } from 'node:fs';
import { resolve, join } from 'node:path';
import { tmpdir } from 'node:os';
import {
buildIndex,
frontCode,
isGenerated,
packPosting,
plainText,
slugifyHeading,
splitSections,
tokenize,
} from '../scripts/build-docs-search-index.mjs';
const ROOT = resolve(import.meta.dirname, '..');
const DOCS_DIR = resolve(ROOT, 'docs');
// The runtime is a browser script that assigns itself onto the global.
await import('../public/docs-search.js');
const engine = globalThis.twsDocsSearch;
const RUNTIME_SOURCE = readFileSync(resolve(ROOT, 'public/docs-search.js'), 'utf8');
const BUILDER_SOURCE = readFileSync(resolve(ROOT, 'scripts/build-docs-search-index.mjs'), 'utf8');
const DOCS_SHELL = readFileSync(resolve(ROOT, 'docs/index.html'), 'utf8');
describe('docs search: builder helpers', () => {
it('slugifies headings the way the docs reader assigns heading ids', () => {
expect(slugifyHeading('Master wallet vs agent wallet')).toBe('master-wallet-vs-agent-wallet');
expect(slugifyHeading('Step 3 & Fund the wallet')).toBe('step-3--fund-the-wallet');
expect(slugifyHeading(' Trailing and leading ')).toBe('trailing-and-leading');
expect(slugifyHeading('')).toBe('');
});
it('reduces markdown to the text a reader sees', () => {
expect(plainText('A [link](/docs/x) and `code` and **bold**')).toBe('A link and code and bold');
expect(plainText(' after')).toBe('after');
expect(plainText('| a | b |')).toBe('a b');
});
it('keeps compound technical terms and also indexes their parts', () => {
expect(tokenize('erc-8004')).toEqual(['erc-8004', 'erc', '8004']);
expect(tokenize('x402 payments')).toEqual(['x402', 'payments']);
// Punctuation at the edges is noise; inside a token it is meaning.
expect(tokenize('(x402),')).toEqual(['x402']);
});
it('drops stopwords, single characters, and identifier noise', () => {
expect(tokenize('the a of')).toEqual([]);
expect(tokenize('x')).toEqual([]);
// A transaction hash and a block number: unique, unsearchable, pure bloat.
expect(tokenize('deadbeefcafe1234')).toEqual([]);
expect(tokenize('1234567')).toEqual([]);
// Short numbers still matter (HTTP statuses, chain ids).
expect(tokenize('402')).toEqual(['402']);
});
it('detects generated aggregates by the marker generators already write', () => {
expect(isGenerated('<!-- GENERATED by scripts/combine-docs.mjs -->\n# Title')).toBe(true);
// A generator stamps its output at the top. The same words further down are
// prose about generators, not a claim that this file is one.
expect(isGenerated('# Title\n\nProse.\n\nMore.\n\n<!-- GENERATED files look like this -->')).toBe(false);
expect(isGenerated('# A normal document\n\nProse.')).toBe(false);
});
});
describe('docs search: section splitting', () => {
const sample = [
'---',
'title: front matter that must not be indexed',
'---',
'# Doc Title',
'',
'Lead paragraph that belongs to the document rather than to any section of it.',
'',
'## First Section',
'',
'Enough prose here to clear the minimum length that keeps stubs out of the index entirely.',
'',
'```js',
'const secretIdentifierNobodySearchesFor = 1;',
'```',
'',
'### Nested Section',
'',
'A third-level heading is a real destination in the reader, so it is a real section here too.',
'',
'## Stub',
'',
'Too short.',
].join('\n');
const parsed = splitSections(sample, 'fallback');
it('takes the document title from the H1 without making it a section', () => {
expect(parsed.title).toBe('Doc Title');
expect(parsed.sections.map((s) => s.heading)).toEqual(['', 'First Section', 'Nested Section']);
});
it('keeps the lead paragraph as the untitled opening section', () => {
expect(parsed.sections[0].text).toContain('Lead paragraph');
expect(parsed.sections[0].text).not.toContain('front matter');
});
it('never indexes code fences', () => {
const all = parsed.sections.map((s) => s.text).join(' ');
expect(all).not.toContain('secretIdentifierNobodySearchesFor');
});
it('drops sections too short to answer anything', () => {
expect(parsed.sections.some((s) => s.heading === 'Stub')).toBe(false);
});
it('falls back to a supplied title when the file has no H1', () => {
expect(splitSections('Body prose long enough to survive the minimum section length filter.', 'my doc').title).toBe(
'my doc',
);
});
});
describe('docs search: encoding round-trips', () => {
it('front codes and expands a sorted vocabulary losslessly', () => {
const terms = ['agent', 'agents', 'agentwallet', 'wallet', 'x402'];
expect(engine.expandTerms(frontCode(terms))).toEqual(terms);
});
it('front coding survives a shared prefix longer than one base-36 digit', () => {
const long = 'a'.repeat(40);
const terms = [long, `${long}b`];
expect(engine.expandTerms(frontCode(terms))).toEqual(terms);
});
it('packs a section gap and a term frequency into one integer', () => {
expect(packPosting(0, 1) % 16).toBe(1);
expect(Math.floor(packPosting(7, 3) / 16)).toBe(7);
// Frequencies above the 4-bit ceiling clamp rather than overflow into the
// gap, which would move the posting to a different section entirely.
expect(packPosting(2, 99) % 16).toBe(15);
expect(Math.floor(packPosting(2, 99) / 16)).toBe(2);
});
});
describe('docs search: builder and runtime agree', () => {
it('tokenizes identically on both sides', () => {
const samples = [
'How do I fund an agent wallet?',
'erc-8004 identity',
'x402 / MCP tools',
'Next.js embed',
'THE END',
'0xdeadbeefcafe',
'viseme_aa lip-sync',
];
for (const sample of samples) {
expect(engine.tokenize(sample), sample).toEqual(tokenize(sample));
}
});
it('shares one stopword list', () => {
const list = (source) => source.match(/\('(a an and[\s\S]*?your)'/)[1].replace(/'\s*\+\s*'/g, '');
expect(list(RUNTIME_SOURCE)).toBe(list(BUILDER_SOURCE));
});
it('slugifies headings the same way the docs reader does', () => {
// All three copies of these rules must match or emitted anchors stop
// resolving. Compared as normalized source, so a rewritten rule fails here.
const body = (source) =>
source
.match(/slugifyHeading\((?:text)\)\s*\{([\s\S]*?)\n\t*\}/)[1]
.replace(/\s+/g, ' ')
.trim();
expect(body(RUNTIME_SOURCE)).toBe(body(BUILDER_SOURCE));
expect(body(DOCS_SHELL)).toBe(body(BUILDER_SOURCE));
});
});
describe('docs search: the real corpus', () => {
let index;
let sourceOf;
beforeAll(() => {
index = buildIndex();
engine.load(index);
sourceOf = new Map();
});
it('indexes the documentation and skips generated aggregates', () => {
expect(index.docCount).toBeGreaterThan(200);
expect(index.sectionCount).toBeGreaterThan(1000);
expect(index.skippedGenerated).toContain('ALL');
expect(index.docs.some(([slug]) => slug === 'ALL')).toBe(false);
});
it('emits anchors that exist as headings in the document they point at', () => {
// Sample across the corpus rather than all 6,000 sections: this reads
// every sampled file from disk.
const step = Math.floor(index.sectionCount / 60) || 1;
for (let i = 0; i < index.sectionCount; i += step) {
const [docId, heading] = index.sections[i];
if (!heading) continue;
const slug = index.docs[docId][0];
if (!sourceOf.has(slug)) sourceOf.set(slug, readFileSync(resolve(ROOT, 'docs', `${slug}.md`), 'utf8'));
const anchor = slugifyHeading(heading);
const headings = sourceOf
.get(slug)
.split(/\r?\n/)
.filter((line) => /^#{1,3}\s/.test(line))
.map((line) => slugifyHeading(plainText(line.replace(/^#{1,3}\s+/, ''))));
expect(headings, `${slug} → ${anchor}`).toContain(anchor);
}
});
it('routes every result through the docs reader hash format', () => {
for (const hit of engine.query('agent wallet', 8)) {
expect(hit.url).toMatch(/^\/docs#[\w./-]+(@[\w-]*)?$/);
expect(hit.docTitle).toBeTruthy();
}
});
it('finds a section from its own heading', () => {
// Self-retrieval: the strongest relevance assertion that does not encode
// today's corpus into the test. A heading distinctive enough to be worth
// linking should retrieve its own section.
const candidates = [];
const step = Math.floor(index.sectionCount / 80) || 1;
for (let i = 0; i < index.sectionCount; i += step) {
const heading = index.sections[i][1];
if (heading && tokenize(heading).length >= 3) candidates.push(i);
}
expect(candidates.length).toBeGreaterThan(20);
let found = 0;
for (const sectionId of candidates) {
const [docId, heading] = index.sections[sectionId];
const slug = index.docs[docId][0];
const anchor = slugifyHeading(heading);
const hits = engine.query(heading, 5);
// The anchor, not the slug, is what has to come back. Long-form docs are
// published twice (FULL-ARTICLE plus per-chapter files) and the
// duplicate-heading suppression deliberately keeps one copy of each
// heading, so demanding the copy in one specific file would fail the
// sampler on whichever twin it happened to draw. Retrieving the section
// that owns that heading is the property under test either way.
if (hits.some((h) => h.anchor === anchor)) found++;
}
// Not 100%: a few headings are too generic to out-rank a near-identical one
// elsewhere in the corpus ("Step 6: See it in the browser").
expect(found / candidates.length).toBeGreaterThan(0.8);
});
it('returns nothing rather than everything for an unknown term', () => {
expect(engine.query('zzzzqqqxnotaword')).toEqual([]);
expect(engine.query('')).toEqual([]);
expect(engine.query(' ')).toEqual([]);
});
it('never returns more sections from one document than the cap allows', () => {
const perDoc = {};
for (const hit of engine.query('agent', 8)) perDoc[hit.slug] = (perDoc[hit.slug] || 0) + 1;
for (const slug of Object.keys(perDoc)) expect(perDoc[slug]).toBeLessThanOrEqual(2);
});
it('is deterministic, so --check can trust a byte comparison', () => {
// Both builds read a snapshot of the corpus, not docs/ itself. Comparing a
// build taken now against the one beforeAll took measures whether any agent
// sharing this worktree edited a doc in between, which they routinely do;
// determinism is a property of the builder over a fixed input, so the input
// has to be held still to test it.
const snapshot = mkdtempSync(join(tmpdir(), 'docs-search-'));
try {
cpSync(DOCS_DIR, snapshot, { recursive: true });
expect(JSON.stringify(buildIndex(snapshot))).toBe(JSON.stringify(buildIndex(snapshot)));
} finally {
rmSync(snapshot, { recursive: true, force: true });
}
});
});