Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

test(crawl): cover the crawl library and the agent-crawler frontier

nirholas656f3cda76c3
diff --git a/tests/agent-crawler-frontier.test.js b/tests/agent-crawler-frontier.test.js
new file mode 100644
index 000000000..e2b5e7fdd
--- /dev/null
+++ b/tests/agent-crawler-frontier.test.js
@@ -0,0 +1,109 @@
+// The Crawl worker's link choice (workers/agent-crawler/frontier.js): pure, so
+// the rules that decide where an agent walks are pinned here.
+import { describe, expect, it } from 'vitest';
+import {
+ Frontier, MAX_PAGES_PER_DOMAIN, canonical, gistOf, isCrawlable, relevance, scoreLink, stem, thoughtFor, tokenize, topicTerms,
+} from '../workers/agent-crawler/frontier.js';
+
+const ctx = (over = {}) => ({ terms: topicTerms('solana validator economics'), fromHost: 'example.com', domainCounts: new Map(), visited: new Set(), ...over });
+
+describe('terms', () => {
+ it('stems plurals and gerunds and drops stopwords', () => {
+ expect(stem('validators')).toBe('validator');
+ expect(stem('staking')).toBe('stak');
+ expect(stem('class')).toBe('class');
+ expect(tokenize('The validators are staking')).toEqual(['validator', 'stak']);
+ expect(topicTerms('Solana validator economics and the validators')).toEqual(['solana', 'validator', 'economic']);
+ });
+});
+
+describe('isCrawlable / canonical', () => {
+ it('refuses account walls, share intents, binaries and non-web links', () => {
+ for (const u of [
+ 'https://e.com/login', 'https://e.com/account/settings', 'https://e.com/cart?x=1', 'https://twitter.com/intent/tweet?u=1',
+ 'https://e.com/paper.pdf', 'https://e.com/a.zip', 'mailto:[email protected]', 'javascript:void(0)', 'https://www.facebook.com/page',
+ ]) expect(isCrawlable(u), u).toBe(false);
+ expect(isCrawlable('https://e.com/blog/validator-economics')).toBe(true);
+ });
+
+ it('canonicalizes like the API so a revisit is caught', () => {
+ expect(canonical('https://e.com/a/?utm_source=x&b=1#c')).toBe('https://e.com/a?b=1');
+ expect(canonical('ftp://e.com')).toBeNull();
+ });
+});
+
+describe('scoreLink', () => {
+ it('prefers an on-topic descriptive anchor over boilerplate and noise', () => {
+ const good = scoreLink({ href: 'https://example.com/blog/validator-rewards', text: 'How Solana validator rewards work', visible: true }, ctx());
+ const meh = scoreLink({ href: 'https://example.com/about-us', text: 'About our team', visible: true }, ctx());
+ expect(good).toBeGreaterThan(meh);
+ expect(scoreLink({ href: 'https://example.com/privacy', text: 'Privacy policy' }, ctx())).toBe(-Infinity);
+ });
+
+ it('never revisits and stops at the per-domain cap', () => {
+ const href = 'https://example.com/validators';
+ expect(scoreLink({ href, text: 'validators' }, ctx({ visited: new Set([canonical(href)]) }))).toBe(-Infinity);
+ const full = new Map([['other.org', MAX_PAGES_PER_DOMAIN]]);
+ expect(scoreLink({ href: 'https://other.org/validators', text: 'validators' }, ctx({ domainCounts: full }))).toBe(-Infinity);
+ });
+
+ it('rewards a relevant new site to widen the corpus', () => {
+ const fresh = scoreLink({ href: 'https://new.org/x', text: 'validator economics explained' }, ctx());
+ const worn = scoreLink({ href: 'https://new.org/x', text: 'validator economics explained' }, ctx({ domainCounts: new Map([['new.org', 12]]) }));
+ expect(fresh).toBeGreaterThan(worn);
+ });
+});
+
+describe('relevance', () => {
+ it('separates a page about the topic from a passing mention', () => {
+ const terms = topicTerms('solana validator economics');
+ const about = relevance(terms, 'Solana validator economics', 'Validators on Solana earn inflation rewards. Validator economics depend on stake. '.repeat(20));
+ const passing = relevance(terms, 'Gardening tips', `${'Tomatoes need sun and water. '.repeat(60)} Someone mentioned Solana once.`);
+ expect(about).toBeGreaterThan(0.6);
+ expect(passing).toBeLessThan(0.2);
+ expect(relevance([], 'x', 'y')).toBe(0);
+ });
+});
+
+describe('Frontier', () => {
+ it('pops best first, dedupes by canonical URL, keeps the higher score, and respects the cap', () => {
+ const f = new Frontier(3);
+ f.add('https://a.com/1', 1);
+ f.add('https://a.com/2', 5);
+ f.add('https://a.com/2#frag', 2); // same page, lower score: ignored
+ f.add('https://a.com/3', 3);
+ f.add('https://a.com/4', 4); // evicts the lowest (score 1)
+ expect(f.size).toBe(3);
+ expect(f.pop().url).toBe('https://a.com/2');
+ expect(f.pop((i) => i.url !== 'https://a.com/4').url).toBe('https://a.com/3');
+ expect(f.pop().url).toBe('https://a.com/4');
+ expect(f.pop()).toBeNull();
+ });
+
+ it('decays old hopes', () => {
+ const f = new Frontier();
+ f.add('https://a.com/1', 10);
+ f.decay(0.5);
+ expect(f.items.get('https://a.com/1').score).toBe(5);
+ });
+});
+
+describe('gistOf', () => {
+ it('picks real sentences, prefers on-topic ones, and keeps reading order', () => {
+ const text = 'Menu Home About\nValidators secure the Solana network by voting on blocks every slot. '
+ + 'The weather in the city was mild and pleasant for most of the week. '
+ + 'Validator economics come down to inflation rewards, fees and commission rates.';
+ const g = gistOf(text, topicTerms('solana validator economics'), 480);
+ expect(g.startsWith('Validators secure the Solana network')).toBe(true);
+ expect(g).toContain('Validator economics come down');
+ expect(gistOf('short. bits.', [], 480)).toBe('');
+ });
+});
+
+describe('thoughtFor', () => {
+ it('narrates what the agent is really doing', () => {
+ expect(thoughtFor('reading', { title: 'Rewards', domain: 'e.com', topic: 'validators', relevance: 0.5 })).toMatch(/squarely about validators/);
+ expect(thoughtFor('walking', { text: 'Next page', host: 'b.org', offsite: true })).toBe('Walking to "Next page", off to b.org.');
+ expect(thoughtFor('blocked', { host: 'e.com', reason: 'answered 403' })).toBe('e.com answered 403. Finding another way.');
+ });
+});
diff --git a/tests/crawl-lib.test.js b/tests/crawl-lib.test.js
new file mode 100644
index 000000000..83ae2bca5
--- /dev/null
+++ b/tests/crawl-lib.test.js
@@ -0,0 +1,143 @@
+// The Crawl: boundary rules shared by the API and the worker (api/_lib/crawl.js).
+import { describe, expect, it } from 'vitest';
+import {
+ MAX_LINKS, MAX_SEEDS, MAX_TEXT_CHARS, domainOf, estimateTokens, isFrameB64, isPrivateHostname, normalizeUrl,
+ pageMemory, sanitizeLinkRect, sanitizePage, sanitizeStep, urlHash, validateMission,
+} from '../api/_lib/crawl.js';
+
+describe('normalizeUrl', () => {
+ it('strips fragments, tracking tags, default ports and trailing slashes', () => {
+ expect(normalizeUrl('https://Example.com:443/a/b/?utm_source=x&b=2&a=1#top')).toBe('https://example.com/a/b?a=1&b=2');
+ expect(normalizeUrl('http://example.com:80/')).toBe('http://example.com/');
+ expect(normalizeUrl('https://example.com/p?fbclid=1&gclid=2')).toBe('https://example.com/p');
+ });
+
+ it('refuses non-web schemes and embedded credentials', () => {
+ expect(normalizeUrl('javascript:alert(1)')).toBeNull();
+ expect(normalizeUrl('ftp://example.com/x')).toBeNull();
+ expect(normalizeUrl('https://user:[email protected]/')).toBeNull();
+ expect(normalizeUrl('not a url')).toBeNull();
+ });
+
+ it('keys the same page identically however it was linked', () => {
+ const a = urlHash(normalizeUrl('https://example.com/post/?utm_medium=feed'));
+ const b = urlHash(normalizeUrl('https://EXAMPLE.com/post#comments'));
+ expect(a).toBe(b);
+ expect(a).toMatch(/^[0-9a-f]{64}$/);
+ });
+});
+
+describe('small helpers', () => {
+ it('domainOf drops www and lowercases', () => {
+ expect(domainOf('https://WWW.Example.org/x')).toBe('example.org');
+ expect(domainOf('nope')).toBe('');
+ });
+
+ it('estimateTokens is four characters a token, rounded up', () => {
+ expect(estimateTokens('')).toBe(0);
+ expect(estimateTokens('abcde')).toBe(2);
+ });
+
+ it('pageMemory names the page, the topic and the gist', () => {
+ const m = pageMemory({ title: 'Spiders', url: 'https://e.com/s', gist: 'They spin.' }, 'arachnids');
+ expect(m).toBe('Read "Spiders" (https://e.com/s) while crawling for arachnids: They spin.');
+ });
+});
+
+describe('sanitizeLinkRect', () => {
+ it('keeps a well-formed rect and clips it to the viewport', () => {
+ expect(sanitizeLinkRect({ x: 0.9, y: 0.5, w: 0.3, h: 0.05, t: ' Read more ' })).toEqual({ x: 0.9, y: 0.5, w: expect.closeTo(0.1, 10), h: 0.05, t: 'Read more' });
+ });
+
+ it('drops degenerate or malformed rects', () => {
+ expect(sanitizeLinkRect({ x: 0.2, y: 0.2, w: 0, h: 0.1 })).toBeNull();
+ expect(sanitizeLinkRect({ x: 'a', y: 0, w: 0.1, h: 0.1 })).toBeNull();
+ expect(sanitizeLinkRect({ x: 1, y: 0.5, w: 0.1, h: 0.1 })).toBeNull();
+ expect(sanitizeLinkRect(null)).toBeNull();
+ });
+});
+
+describe('sanitizeStep', () => {
+ const link = { x: 0.1, y: 0.2, w: 0.2, h: 0.03, t: 'Next' };
+
+ it('keeps target and nextUrl only when they point at a real link', () => {
+ const s = sanitizeStep({ url: 'https://e.com/a', status: 'walking', links: [link], target: 0, nextUrl: 'https://e.com/b#x', seq: 7 });
+ expect(s.target).toBe(0);
+ expect(s.nextUrl).toBe('https://e.com/b');
+ expect(s.domain).toBe('e.com');
+ expect(s.seq).toBe(7);
+
+ const bad = sanitizeStep({ url: 'https://e.com/a', links: [link], target: 3, nextUrl: 'https://e.com/b' });
+ expect(bad.target).toBeNull();
+ expect(bad.nextUrl).toBeNull();
+ });
+
+ it('bounds the link list, coerces unknown statuses, and clamps scroll', () => {
+ const many = Array.from({ length: MAX_LINKS + 20 }, () => link);
+ const s = sanitizeStep({ url: 'https://e.com', links: many, status: 'dancing', scrollY: 4, seq: -1 });
+ expect(s.links).toHaveLength(MAX_LINKS);
+ expect(s.status).toBe('reading');
+ expect(s.scrollY).toBe(1);
+ expect(s.seq).toBe(0);
+ });
+
+ it('rejects a missing step', () => {
+ expect(sanitizeStep(null)).toBeNull();
+ });
+});
+
+describe('sanitizePage', () => {
+ it('bounds text and derives hash, domain and tokens', () => {
+ const p = sanitizePage({ url: 'https://www.e.com/x/', title: 't', text: 'a'.repeat(MAX_TEXT_CHARS + 50), relevance: 3, linksOut: -2 });
+ expect(p.url).toBe('https://www.e.com/x');
+ expect(p.domain).toBe('e.com');
+ expect(p.text).toHaveLength(MAX_TEXT_CHARS);
+ expect(p.tokens).toBe(MAX_TEXT_CHARS / 4);
+ expect(p.relevance).toBe(1);
+ expect(p.linksOut).toBe(0);
+ expect(p.hash).toBe(urlHash(p.url));
+ });
+
+ it('refuses a page with no usable URL', () => {
+ expect(sanitizePage({ url: 'mailto:[email protected]' })).toBeNull();
+ });
+});
+
+describe('isFrameB64', () => {
+ it('accepts JPEG base64 only', () => {
+ const jpeg = Buffer.concat([Buffer.from([0xff, 0xd8, 0xff, 0xe0]), Buffer.alloc(100, 7)]).toString('base64');
+ expect(isFrameB64(jpeg)).toBe(true);
+ const png = Buffer.concat([Buffer.from([0x89, 0x50, 0x4e, 0x47]), Buffer.alloc(100, 7)]).toString('base64');
+ expect(isFrameB64(png)).toBe(false);
+ expect(isFrameB64('/9j/<script>')).toBe(false);
+ });
+});
+
+describe('validateMission', () => {
+ it('accepts a topic with seeds as a list or a pasted string', () => {
+ expect(validateMission({ topic: 'solana validators', seeds: 'docs.solana.com, https://solana.com/news' })).toEqual({
+ topic: 'solana validators', seeds: ['https://docs.solana.com/', 'https://solana.com/news'], enabled: true,
+ });
+ expect(validateMission({ topic: 'x'.repeat(10), seeds: [], enabled: false }).enabled).toBe(false);
+ });
+
+ it('rejects bad topics, private hosts and too many seeds', () => {
+ expect(validateMission({ topic: 'a' }).error).toMatch(/topic/);
+ expect(validateMission({ topic: 'x'.repeat(121) }).error).toMatch(/topic/);
+ expect(validateMission({ topic: 'ok topic', seeds: ['http://127.0.0.1/admin'] }).error).toMatch(/public/);
+ expect(validateMission({ topic: 'ok topic', seeds: ['http://metadata.google.internal/'] }).error).toMatch(/public/);
+ const seeds = Array.from({ length: MAX_SEEDS + 1 }, (_, i) => `https://e${i}.com`);
+ expect(validateMission({ topic: 'ok topic', seeds }).error).toMatch(/at most/);
+ });
+});
+
+describe('isPrivateHostname', () => {
+ it.each([
+ 'localhost', 'app.localhost', 'printer.local', 'db.internal', 'metadata', '10.1.2.3', '127.0.0.1', '169.254.169.254',
+ '172.20.0.1', '192.168.1.1', '100.64.0.1', '0.0.0.0', '224.0.0.1', '[::1]', 'fd00::1', 'fe80::1', '::ffff:127.0.0.1', 'intranet',
+ ])('treats %s as private', (h) => expect(isPrivateHostname(h)).toBe(true));
+
+ it.each(['example.com', '8.8.8.8', '172.32.0.1', '2606:4700::1111'])('treats %s as public', (h) => {
+ expect(isPrivateHostname(h)).toBe(false);
+ });
+});