Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
test(crawl): cover the crawl library and the agent-crawler frontier
nirholas656f3cda76c3
diff --git a/tests/agent-crawler-frontier.test.js b/tests/agent-crawler-frontier.test.jsnew file mode 100644index 000000000..e2b5e7fdd--- /dev/null+++ b/tests/agent-crawler-frontier.test.js@@ -0,0 +1,109 @@+// The Crawl worker's link choice (workers/agent-crawler/frontier.js): pure, so+// the rules that decide where an agent walks are pinned here.+import { describe, expect, it } from 'vitest';+import {+ Frontier, MAX_PAGES_PER_DOMAIN, canonical, gistOf, isCrawlable, relevance, scoreLink, stem, thoughtFor, tokenize, topicTerms,+} from '../workers/agent-crawler/frontier.js';++const ctx = (over = {}) => ({ terms: topicTerms('solana validator economics'), fromHost: 'example.com', domainCounts: new Map(), visited: new Set(), ...over });++describe('terms', () => {+ it('stems plurals and gerunds and drops stopwords', () => {+ expect(stem('validators')).toBe('validator');+ expect(stem('staking')).toBe('stak');+ expect(stem('class')).toBe('class');+ expect(tokenize('The validators are staking')).toEqual(['validator', 'stak']);+ expect(topicTerms('Solana validator economics and the validators')).toEqual(['solana', 'validator', 'economic']);+ });+});++describe('isCrawlable / canonical', () => {+ it('refuses account walls, share intents, binaries and non-web links', () => {+ for (const u of [+ 'https://e.com/login', 'https://e.com/account/settings', 'https://e.com/cart?x=1', 'https://twitter.com/intent/tweet?u=1',+ 'https://e.com/paper.pdf', 'https://e.com/a.zip', 'mailto:[email protected]', 'javascript:void(0)', 'https://www.facebook.com/page',+ ]) expect(isCrawlable(u), u).toBe(false);+ expect(isCrawlable('https://e.com/blog/validator-economics')).toBe(true);+ });++ it('canonicalizes like the API so a revisit is caught', () => {+ expect(canonical('https://e.com/a/?utm_source=x&b=1#c')).toBe('https://e.com/a?b=1');+ expect(canonical('ftp://e.com')).toBeNull();+ });+});++describe('scoreLink', () => {+ it('prefers an on-topic descriptive anchor over boilerplate and noise', () => {+ const good = scoreLink({ href: 'https://example.com/blog/validator-rewards', text: 'How Solana validator rewards work', visible: true }, ctx());+ const meh = scoreLink({ href: 'https://example.com/about-us', text: 'About our team', visible: true }, ctx());+ expect(good).toBeGreaterThan(meh);+ expect(scoreLink({ href: 'https://example.com/privacy', text: 'Privacy policy' }, ctx())).toBe(-Infinity);+ });++ it('never revisits and stops at the per-domain cap', () => {+ const href = 'https://example.com/validators';+ expect(scoreLink({ href, text: 'validators' }, ctx({ visited: new Set([canonical(href)]) }))).toBe(-Infinity);+ const full = new Map([['other.org', MAX_PAGES_PER_DOMAIN]]);+ expect(scoreLink({ href: 'https://other.org/validators', text: 'validators' }, ctx({ domainCounts: full }))).toBe(-Infinity);+ });++ it('rewards a relevant new site to widen the corpus', () => {+ const fresh = scoreLink({ href: 'https://new.org/x', text: 'validator economics explained' }, ctx());+ const worn = scoreLink({ href: 'https://new.org/x', text: 'validator economics explained' }, ctx({ domainCounts: new Map([['new.org', 12]]) }));+ expect(fresh).toBeGreaterThan(worn);+ });+});++describe('relevance', () => {+ it('separates a page about the topic from a passing mention', () => {+ const terms = topicTerms('solana validator economics');+ const about = relevance(terms, 'Solana validator economics', 'Validators on Solana earn inflation rewards. Validator economics depend on stake. '.repeat(20));+ const passing = relevance(terms, 'Gardening tips', `${'Tomatoes need sun and water. '.repeat(60)} Someone mentioned Solana once.`);+ expect(about).toBeGreaterThan(0.6);+ expect(passing).toBeLessThan(0.2);+ expect(relevance([], 'x', 'y')).toBe(0);+ });+});++describe('Frontier', () => {+ it('pops best first, dedupes by canonical URL, keeps the higher score, and respects the cap', () => {+ const f = new Frontier(3);+ f.add('https://a.com/1', 1);+ f.add('https://a.com/2', 5);+ f.add('https://a.com/2#frag', 2); // same page, lower score: ignored+ f.add('https://a.com/3', 3);+ f.add('https://a.com/4', 4); // evicts the lowest (score 1)+ expect(f.size).toBe(3);+ expect(f.pop().url).toBe('https://a.com/2');+ expect(f.pop((i) => i.url !== 'https://a.com/4').url).toBe('https://a.com/3');+ expect(f.pop().url).toBe('https://a.com/4');+ expect(f.pop()).toBeNull();+ });++ it('decays old hopes', () => {+ const f = new Frontier();+ f.add('https://a.com/1', 10);+ f.decay(0.5);+ expect(f.items.get('https://a.com/1').score).toBe(5);+ });+});++describe('gistOf', () => {+ it('picks real sentences, prefers on-topic ones, and keeps reading order', () => {+ const text = 'Menu Home About\nValidators secure the Solana network by voting on blocks every slot. '+ + 'The weather in the city was mild and pleasant for most of the week. '+ + 'Validator economics come down to inflation rewards, fees and commission rates.';+ const g = gistOf(text, topicTerms('solana validator economics'), 480);+ expect(g.startsWith('Validators secure the Solana network')).toBe(true);+ expect(g).toContain('Validator economics come down');+ expect(gistOf('short. bits.', [], 480)).toBe('');+ });+});++describe('thoughtFor', () => {+ it('narrates what the agent is really doing', () => {+ expect(thoughtFor('reading', { title: 'Rewards', domain: 'e.com', topic: 'validators', relevance: 0.5 })).toMatch(/squarely about validators/);+ expect(thoughtFor('walking', { text: 'Next page', host: 'b.org', offsite: true })).toBe('Walking to "Next page", off to b.org.');+ expect(thoughtFor('blocked', { host: 'e.com', reason: 'answered 403' })).toBe('e.com answered 403. Finding another way.');+ });+});diff --git a/tests/crawl-lib.test.js b/tests/crawl-lib.test.jsnew file mode 100644index 000000000..83ae2bca5--- /dev/null+++ b/tests/crawl-lib.test.js@@ -0,0 +1,143 @@+// The Crawl: boundary rules shared by the API and the worker (api/_lib/crawl.js).+import { describe, expect, it } from 'vitest';+import {+ MAX_LINKS, MAX_SEEDS, MAX_TEXT_CHARS, domainOf, estimateTokens, isFrameB64, isPrivateHostname, normalizeUrl,+ pageMemory, sanitizeLinkRect, sanitizePage, sanitizeStep, urlHash, validateMission,+} from '../api/_lib/crawl.js';++describe('normalizeUrl', () => {+ it('strips fragments, tracking tags, default ports and trailing slashes', () => {+ expect(normalizeUrl('https://Example.com:443/a/b/?utm_source=x&b=2&a=1#top')).toBe('https://example.com/a/b?a=1&b=2');+ expect(normalizeUrl('http://example.com:80/')).toBe('http://example.com/');+ expect(normalizeUrl('https://example.com/p?fbclid=1&gclid=2')).toBe('https://example.com/p');+ });++ it('refuses non-web schemes and embedded credentials', () => {+ expect(normalizeUrl('javascript:alert(1)')).toBeNull();+ expect(normalizeUrl('ftp://example.com/x')).toBeNull();+ expect(normalizeUrl('https://user:[email protected]/')).toBeNull();+ expect(normalizeUrl('not a url')).toBeNull();+ });++ it('keys the same page identically however it was linked', () => {+ const a = urlHash(normalizeUrl('https://example.com/post/?utm_medium=feed'));+ const b = urlHash(normalizeUrl('https://EXAMPLE.com/post#comments'));+ expect(a).toBe(b);+ expect(a).toMatch(/^[0-9a-f]{64}$/);+ });+});++describe('small helpers', () => {+ it('domainOf drops www and lowercases', () => {+ expect(domainOf('https://WWW.Example.org/x')).toBe('example.org');+ expect(domainOf('nope')).toBe('');+ });++ it('estimateTokens is four characters a token, rounded up', () => {+ expect(estimateTokens('')).toBe(0);+ expect(estimateTokens('abcde')).toBe(2);+ });++ it('pageMemory names the page, the topic and the gist', () => {+ const m = pageMemory({ title: 'Spiders', url: 'https://e.com/s', gist: 'They spin.' }, 'arachnids');+ expect(m).toBe('Read "Spiders" (https://e.com/s) while crawling for arachnids: They spin.');+ });+});++describe('sanitizeLinkRect', () => {+ it('keeps a well-formed rect and clips it to the viewport', () => {+ expect(sanitizeLinkRect({ x: 0.9, y: 0.5, w: 0.3, h: 0.05, t: ' Read more ' })).toEqual({ x: 0.9, y: 0.5, w: expect.closeTo(0.1, 10), h: 0.05, t: 'Read more' });+ });++ it('drops degenerate or malformed rects', () => {+ expect(sanitizeLinkRect({ x: 0.2, y: 0.2, w: 0, h: 0.1 })).toBeNull();+ expect(sanitizeLinkRect({ x: 'a', y: 0, w: 0.1, h: 0.1 })).toBeNull();+ expect(sanitizeLinkRect({ x: 1, y: 0.5, w: 0.1, h: 0.1 })).toBeNull();+ expect(sanitizeLinkRect(null)).toBeNull();+ });+});++describe('sanitizeStep', () => {+ const link = { x: 0.1, y: 0.2, w: 0.2, h: 0.03, t: 'Next' };++ it('keeps target and nextUrl only when they point at a real link', () => {+ const s = sanitizeStep({ url: 'https://e.com/a', status: 'walking', links: [link], target: 0, nextUrl: 'https://e.com/b#x', seq: 7 });+ expect(s.target).toBe(0);+ expect(s.nextUrl).toBe('https://e.com/b');+ expect(s.domain).toBe('e.com');+ expect(s.seq).toBe(7);++ const bad = sanitizeStep({ url: 'https://e.com/a', links: [link], target: 3, nextUrl: 'https://e.com/b' });+ expect(bad.target).toBeNull();+ expect(bad.nextUrl).toBeNull();+ });++ it('bounds the link list, coerces unknown statuses, and clamps scroll', () => {+ const many = Array.from({ length: MAX_LINKS + 20 }, () => link);+ const s = sanitizeStep({ url: 'https://e.com', links: many, status: 'dancing', scrollY: 4, seq: -1 });+ expect(s.links).toHaveLength(MAX_LINKS);+ expect(s.status).toBe('reading');+ expect(s.scrollY).toBe(1);+ expect(s.seq).toBe(0);+ });++ it('rejects a missing step', () => {+ expect(sanitizeStep(null)).toBeNull();+ });+});++describe('sanitizePage', () => {+ it('bounds text and derives hash, domain and tokens', () => {+ const p = sanitizePage({ url: 'https://www.e.com/x/', title: 't', text: 'a'.repeat(MAX_TEXT_CHARS + 50), relevance: 3, linksOut: -2 });+ expect(p.url).toBe('https://www.e.com/x');+ expect(p.domain).toBe('e.com');+ expect(p.text).toHaveLength(MAX_TEXT_CHARS);+ expect(p.tokens).toBe(MAX_TEXT_CHARS / 4);+ expect(p.relevance).toBe(1);+ expect(p.linksOut).toBe(0);+ expect(p.hash).toBe(urlHash(p.url));+ });++ it('refuses a page with no usable URL', () => {+ expect(sanitizePage({ url: 'mailto:[email protected]' })).toBeNull();+ });+});++describe('isFrameB64', () => {+ it('accepts JPEG base64 only', () => {+ const jpeg = Buffer.concat([Buffer.from([0xff, 0xd8, 0xff, 0xe0]), Buffer.alloc(100, 7)]).toString('base64');+ expect(isFrameB64(jpeg)).toBe(true);+ const png = Buffer.concat([Buffer.from([0x89, 0x50, 0x4e, 0x47]), Buffer.alloc(100, 7)]).toString('base64');+ expect(isFrameB64(png)).toBe(false);+ expect(isFrameB64('/9j/<script>')).toBe(false);+ });+});++describe('validateMission', () => {+ it('accepts a topic with seeds as a list or a pasted string', () => {+ expect(validateMission({ topic: 'solana validators', seeds: 'docs.solana.com, https://solana.com/news' })).toEqual({+ topic: 'solana validators', seeds: ['https://docs.solana.com/', 'https://solana.com/news'], enabled: true,+ });+ expect(validateMission({ topic: 'x'.repeat(10), seeds: [], enabled: false }).enabled).toBe(false);+ });++ it('rejects bad topics, private hosts and too many seeds', () => {+ expect(validateMission({ topic: 'a' }).error).toMatch(/topic/);+ expect(validateMission({ topic: 'x'.repeat(121) }).error).toMatch(/topic/);+ expect(validateMission({ topic: 'ok topic', seeds: ['http://127.0.0.1/admin'] }).error).toMatch(/public/);+ expect(validateMission({ topic: 'ok topic', seeds: ['http://metadata.google.internal/'] }).error).toMatch(/public/);+ const seeds = Array.from({ length: MAX_SEEDS + 1 }, (_, i) => `https://e${i}.com`);+ expect(validateMission({ topic: 'ok topic', seeds }).error).toMatch(/at most/);+ });+});++describe('isPrivateHostname', () => {+ it.each([+ 'localhost', 'app.localhost', 'printer.local', 'db.internal', 'metadata', '10.1.2.3', '127.0.0.1', '169.254.169.254',+ '172.20.0.1', '192.168.1.1', '100.64.0.1', '0.0.0.0', '224.0.0.1', '[::1]', 'fd00::1', 'fe80::1', '::ffff:127.0.0.1', 'intranet',+ ])('treats %s as private', (h) => expect(isPrivateHostname(h)).toBe(true));++ it.each(['example.com', '8.8.8.8', '172.32.0.1', '2606:4700::1111'])('treats %s as public', (h) => {+ expect(isPrivateHostname(h)).toBe(false);+ });+});