Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

10.9 KB
Raw Blame
// Documentation search: builder + runtime.
//
// The two halves of this feature live in different files and different
// languages of the stack (a node ESM build script, a browser IIFE), but they
// share one encoding and one tokenizer. Every test here that looks pedantic is
// guarding a way those two can silently drift apart, because the failure mode
// is never an exception: it is a search that quietly stops finding things.

import { describe, it, expect, beforeAll } from 'vitest';
import { readFileSync, cpSync, mkdtempSync, rmSync } from 'node:fs';
import { resolve, join } from 'node:path';
import { tmpdir } from 'node:os';

import {
	buildIndex,
	frontCode,
	isGenerated,
	packPosting,
	plainText,
	slugifyHeading,
	splitSections,
	tokenize,
} from '../scripts/build-docs-search-index.mjs';

const ROOT = resolve(import.meta.dirname, '..');
const DOCS_DIR = resolve(ROOT, 'docs');

// The runtime is a browser script that assigns itself onto the global.
await import('../public/docs-search.js');
const engine = globalThis.twsDocsSearch;

const RUNTIME_SOURCE = readFileSync(resolve(ROOT, 'public/docs-search.js'), 'utf8');
const BUILDER_SOURCE = readFileSync(resolve(ROOT, 'scripts/build-docs-search-index.mjs'), 'utf8');
const DOCS_SHELL = readFileSync(resolve(ROOT, 'docs/index.html'), 'utf8');

describe('docs search: builder helpers', () => {
	it('slugifies headings the way the docs reader assigns heading ids', () => {
		expect(slugifyHeading('Master wallet vs agent wallet')).toBe('master-wallet-vs-agent-wallet');
		expect(slugifyHeading('Step 3 & Fund the wallet')).toBe('step-3--fund-the-wallet');
		expect(slugifyHeading('  Trailing and leading  ')).toBe('trailing-and-leading');
		expect(slugifyHeading('')).toBe('');
	});

	it('reduces markdown to the text a reader sees', () => {
		expect(plainText('A [link](/docs/x) and `code` and **bold**')).toBe('A link and code and bold');
		expect(plainText('![alt](/img.png) after')).toBe('after');
		expect(plainText('| a | b |')).toBe('a b');
	});

	it('keeps compound technical terms and also indexes their parts', () => {
		expect(tokenize('erc-8004')).toEqual(['erc-8004', 'erc', '8004']);
		expect(tokenize('x402 payments')).toEqual(['x402', 'payments']);
		// Punctuation at the edges is noise; inside a token it is meaning.
		expect(tokenize('(x402),')).toEqual(['x402']);
	});

	it('drops stopwords, single characters, and identifier noise', () => {
		expect(tokenize('the a of')).toEqual([]);
		expect(tokenize('x')).toEqual([]);
		// A transaction hash and a block number: unique, unsearchable, pure bloat.
		expect(tokenize('deadbeefcafe1234')).toEqual([]);
		expect(tokenize('1234567')).toEqual([]);
		// Short numbers still matter (HTTP statuses, chain ids).
		expect(tokenize('402')).toEqual(['402']);
	});

	it('detects generated aggregates by the marker generators already write', () => {
		expect(isGenerated('<!-- GENERATED by scripts/combine-docs.mjs -->\n# Title')).toBe(true);
		// A generator stamps its output at the top. The same words further down are
		// prose about generators, not a claim that this file is one.
		expect(isGenerated('# Title\n\nProse.\n\nMore.\n\n<!-- GENERATED files look like this -->')).toBe(false);
		expect(isGenerated('# A normal document\n\nProse.')).toBe(false);
	});
});

describe('docs search: section splitting', () => {
	const sample = [
		'---',
		'title: front matter that must not be indexed',
		'---',
		'# Doc Title',
		'',
		'Lead paragraph that belongs to the document rather than to any section of it.',
		'',
		'## First Section',
		'',
		'Enough prose here to clear the minimum length that keeps stubs out of the index entirely.',
		'',
		'```js',
		'const secretIdentifierNobodySearchesFor = 1;',
		'```',
		'',
		'### Nested Section',
		'',
		'A third-level heading is a real destination in the reader, so it is a real section here too.',
		'',
		'## Stub',
		'',
		'Too short.',
	].join('\n');

	const parsed = splitSections(sample, 'fallback');

	it('takes the document title from the H1 without making it a section', () => {
		expect(parsed.title).toBe('Doc Title');
		expect(parsed.sections.map((s) => s.heading)).toEqual(['', 'First Section', 'Nested Section']);
	});

	it('keeps the lead paragraph as the untitled opening section', () => {
		expect(parsed.sections[0].text).toContain('Lead paragraph');
		expect(parsed.sections[0].text).not.toContain('front matter');
	});

	it('never indexes code fences', () => {
		const all = parsed.sections.map((s) => s.text).join(' ');
		expect(all).not.toContain('secretIdentifierNobodySearchesFor');
	});

	it('drops sections too short to answer anything', () => {
		expect(parsed.sections.some((s) => s.heading === 'Stub')).toBe(false);
	});

	it('falls back to a supplied title when the file has no H1', () => {
		expect(splitSections('Body prose long enough to survive the minimum section length filter.', 'my doc').title).toBe(
			'my doc',
		);
	});
});

describe('docs search: encoding round-trips', () => {
	it('front codes and expands a sorted vocabulary losslessly', () => {
		const terms = ['agent', 'agents', 'agentwallet', 'wallet', 'x402'];
		expect(engine.expandTerms(frontCode(terms))).toEqual(terms);
	});

	it('front coding survives a shared prefix longer than one base-36 digit', () => {
		const long = 'a'.repeat(40);
		const terms = [long, `${long}b`];
		expect(engine.expandTerms(frontCode(terms))).toEqual(terms);
	});

	it('packs a section gap and a term frequency into one integer', () => {
		expect(packPosting(0, 1) % 16).toBe(1);
		expect(Math.floor(packPosting(7, 3) / 16)).toBe(7);
		// Frequencies above the 4-bit ceiling clamp rather than overflow into the
		// gap, which would move the posting to a different section entirely.
		expect(packPosting(2, 99) % 16).toBe(15);
		expect(Math.floor(packPosting(2, 99) / 16)).toBe(2);
	});
});

describe('docs search: builder and runtime agree', () => {
	it('tokenizes identically on both sides', () => {
		const samples = [
			'How do I fund an agent wallet?',
			'erc-8004 identity',
			'x402 / MCP tools',
			'Next.js embed',
			'THE END',
			'0xdeadbeefcafe',
			'viseme_aa lip-sync',
		];
		for (const sample of samples) {
			expect(engine.tokenize(sample), sample).toEqual(tokenize(sample));
		}
	});

	it('shares one stopword list', () => {
		const list = (source) => source.match(/\('(a an and[\s\S]*?your)'/)[1].replace(/'\s*\+\s*'/g, '');
		expect(list(RUNTIME_SOURCE)).toBe(list(BUILDER_SOURCE));
	});

	it('slugifies headings the same way the docs reader does', () => {
		// All three copies of these rules must match or emitted anchors stop
		// resolving. Compared as normalized source, so a rewritten rule fails here.
		const body = (source) =>
			source
				.match(/slugifyHeading\((?:text)\)\s*\{([\s\S]*?)\n\t*\}/)[1]
				.replace(/\s+/g, ' ')
				.trim();
		expect(body(RUNTIME_SOURCE)).toBe(body(BUILDER_SOURCE));
		expect(body(DOCS_SHELL)).toBe(body(BUILDER_SOURCE));
	});
});

describe('docs search: the real corpus', () => {
	let index;
	let sourceOf;

	beforeAll(() => {
		index = buildIndex();
		engine.load(index);
		sourceOf = new Map();
	});

	it('indexes the documentation and skips generated aggregates', () => {
		expect(index.docCount).toBeGreaterThan(200);
		expect(index.sectionCount).toBeGreaterThan(1000);
		expect(index.skippedGenerated).toContain('ALL');
		expect(index.docs.some(([slug]) => slug === 'ALL')).toBe(false);
	});

	it('emits anchors that exist as headings in the document they point at', () => {
		// Sample across the corpus rather than all 6,000 sections: this reads
		// every sampled file from disk.
		const step = Math.floor(index.sectionCount / 60) || 1;
		for (let i = 0; i < index.sectionCount; i += step) {
			const [docId, heading] = index.sections[i];
			if (!heading) continue;
			const slug = index.docs[docId][0];
			if (!sourceOf.has(slug)) sourceOf.set(slug, readFileSync(resolve(ROOT, 'docs', `${slug}.md`), 'utf8'));
			const anchor = slugifyHeading(heading);
			const headings = sourceOf
				.get(slug)
				.split(/\r?\n/)
				.filter((line) => /^#{1,3}\s/.test(line))
				.map((line) => slugifyHeading(plainText(line.replace(/^#{1,3}\s+/, ''))));
			expect(headings, `${slug} → ${anchor}`).toContain(anchor);
		}
	});

	it('routes every result through the docs reader hash format', () => {
		for (const hit of engine.query('agent wallet', 8)) {
			expect(hit.url).toMatch(/^\/docs#[\w./-]+(@[\w-]*)?$/);
			expect(hit.docTitle).toBeTruthy();
		}
	});

	it('finds a section from its own heading', () => {
		// Self-retrieval: the strongest relevance assertion that does not encode
		// today's corpus into the test. A heading distinctive enough to be worth
		// linking should retrieve its own section.
		const candidates = [];
		const step = Math.floor(index.sectionCount / 80) || 1;
		for (let i = 0; i < index.sectionCount; i += step) {
			const heading = index.sections[i][1];
			if (heading && tokenize(heading).length >= 3) candidates.push(i);
		}
		expect(candidates.length).toBeGreaterThan(20);

		let found = 0;
		for (const sectionId of candidates) {
			const [docId, heading] = index.sections[sectionId];
			const slug = index.docs[docId][0];
			const anchor = slugifyHeading(heading);
			const hits = engine.query(heading, 5);
			// The anchor, not the slug, is what has to come back. Long-form docs are
			// published twice (FULL-ARTICLE plus per-chapter files) and the
			// duplicate-heading suppression deliberately keeps one copy of each
			// heading, so demanding the copy in one specific file would fail the
			// sampler on whichever twin it happened to draw. Retrieving the section
			// that owns that heading is the property under test either way.
			if (hits.some((h) => h.anchor === anchor)) found++;
		}
		// Not 100%: a few headings are too generic to out-rank a near-identical one
		// elsewhere in the corpus ("Step 6: See it in the browser").
		expect(found / candidates.length).toBeGreaterThan(0.8);
	});

	it('returns nothing rather than everything for an unknown term', () => {
		expect(engine.query('zzzzqqqxnotaword')).toEqual([]);
		expect(engine.query('')).toEqual([]);
		expect(engine.query('   ')).toEqual([]);
	});

	it('never returns more sections from one document than the cap allows', () => {
		const perDoc = {};
		for (const hit of engine.query('agent', 8)) perDoc[hit.slug] = (perDoc[hit.slug] || 0) + 1;
		for (const slug of Object.keys(perDoc)) expect(perDoc[slug]).toBeLessThanOrEqual(2);
	});

	it('is deterministic, so --check can trust a byte comparison', () => {
		// Both builds read a snapshot of the corpus, not docs/ itself. Comparing a
		// build taken now against the one beforeAll took measures whether any agent
		// sharing this worktree edited a doc in between, which they routinely do;
		// determinism is a property of the builder over a fixed input, so the input
		// has to be held still to test it.
		const snapshot = mkdtempSync(join(tmpdir(), 'docs-search-'));
		try {
			cpSync(DOCS_DIR, snapshot, { recursive: true });
			expect(JSON.stringify(buildIndex(snapshot))).toBe(JSON.stringify(buildIndex(snapshot)));
		} finally {
			rmSync(snapshot, { recursive: true, force: true });
		}
	});
});