Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

11.0 KB
Raw Blame
// Integrity coverage for api/_lib/news-sources.js — the crypto-news source
// registry. No network here: these assertions guard the registry's shape and
// the invariants the aggregator (api/_lib/news.js) relies on. Liveness is a
// separate, networked concern — see scripts/news-sources-probe.mjs.

import { describe, it, expect } from 'vitest';

const {
	NEWS_SOURCES,
	NEWS_CATEGORIES,
	NEWS_LANGUAGES,
	sourcesForCategory,
	sourcesForLanguage,
	sourcePriority,
	sourceKeyForLink,
} = await import('../api/_lib/news-sources.js');

const entries = Object.entries(NEWS_SOURCES);

describe('registry shape', () => {
	it('is substantial and every entry carries name, url, category', () => {
		expect(entries.length).toBeGreaterThan(150);
		for (const [key, src] of entries) {
			expect(key, `${key}: key must be snake_case`).toMatch(/^[a-z0-9_]+$/);
			expect(src.name, `${key}: name`).toBeTruthy();
			expect(src.category, `${key}: category`).toBeTruthy();
			expect(() => new URL(src.url), `${key}: url must parse`).not.toThrow();
			expect(new URL(src.url).protocol, `${key}: must be https`).toBe('https:');
		}
	});

	it('has no duplicate feed urls', () => {
		const seen = new Map();
		for (const [key, src] of entries) {
			const norm = src.url.replace(/\/+$/, '').toLowerCase();
			expect(seen.has(norm), `${key} duplicates ${seen.get(norm)}`).toBe(false);
			seen.set(norm, key);
		}
	});

	it('only uses categories declared in NEWS_CATEGORIES', () => {
		for (const [key, src] of entries) {
			expect(NEWS_CATEGORIES, `${key}: category "${src.category}"`).toContain(src.category);
		}
	});

	it('declares every category it lists', () => {
		const used = new Set(entries.map(([, s]) => s.category));
		for (const c of NEWS_CATEGORIES) expect(used, `category "${c}" has no sources`).toContain(c);
	});

	it('tags international feeds with a declared language', () => {
		for (const [key, src] of entries) {
			if (!src.language) continue;
			expect(NEWS_LANGUAGES, `${key}: language "${src.language}"`).toContain(src.language);
			expect(src.language, `${key}: ISO 639-1`).toMatch(/^[a-z]{2}$/);
		}
		// English feeds are the untagged default — never spelled out explicitly.
		expect(entries.filter(([, s]) => s.language === 'en')).toHaveLength(0);
	});

	// Untagged means "English" to sourcesForLanguage(), so a non-English
	// publisher that ships without a language tag silently interleaves its
	// headlines into the default /api/news/feed view. BlockTempo (a Traditional
	// Chinese Taiwanese outlet) did exactly that until 2026-08-14: it reads as
	// an English brand name, which is the trap. Pin the publishers whose feed
	// language cannot be inferred from their name.
	it('tags non-English publishers that carry an English brand name', () => {
		for (const key of ['blocktempo']) {
			expect(NEWS_SOURCES[key], `${key} left the registry`).toBeDefined();
			expect(NEWS_SOURCES[key].language, `${key} must not fall into the untagged English pool`).toBe('zh');
		}
		expect(sourcesForLanguage('en')).not.toContain('blocktempo');
	});
});

// The feed count is quoted as a literal in product copy, MCP tool descriptions,
// and docs, and every one of those copies went stale the moment six subreddits
// joined the registry: the markets hub advertised 192 feeds while the registry
// held 197, and three docs still said 191. A number restated in ten places and
// derived in none will always drift, so the registry is the single source of
// truth and this test fails the next time a feed lands without the copy moving.
describe('the advertised feed count matches the registry', () => {
	const QUOTING_FILES = [
		'api/_lib/news-sources.js',
		'src/markets-page.js',
		'src/dashboard-next/pages/data-api.js',
		'mcp-server/src/index.js',
		'mcp-server/src/tools/crypto-news.js',
		'mcp-server/README.md',
		'STRUCTURE.md',
		'docs/coin-pages.md',
		'docs/api-reference.md',
		'docs/agent-abilities/ABILITIES.md',
		'docs/agent-abilities/FULL-ARTICLE.md',
		'docs/agent-abilities/chapters/15-appendix.md',
	];
	// Both phrasings the repo uses for the CURRENT registry size. Deliberately
	// anchored on the word "publisher" (or the MCP tool title's "live, N feeds")
	// so the registry header's own history of what was dropped ("~450 feeds"
	// upstream, 28 substack, 43 medium) stays out: those are real counts of
	// something else, not stale copies of this one.
	const QUOTED = [
		/(\d{2,4})\s+(?:live\s+)?publisher(?:\s+RSS\/Atom)?\s+feeds\b/gi,
		/\blive,\s*(\d{2,4})\s+feeds\b/gi,
	];

	it('quotes the live registry size everywhere the number appears', async () => {
		const { readFileSync } = await import('node:fs');
		const expected = String(entries.length);
		const wrong = [];
		let quotes = 0;
		for (const file of QUOTING_FILES) {
			const text = readFileSync(file, 'utf8');
			for (const pattern of QUOTED) {
				for (const m of text.matchAll(pattern)) {
					quotes += 1;
					if (m[1] === expected) continue;
					const line = text.slice(0, m.index).split('\n').length;
					wrong.push(`${file}:${line} says "${m[0].trim()}", registry has ${expected}`);
				}
			}
		}
		expect(wrong, `stale feed counts:\n  ${wrong.join('\n  ')}`).toEqual([]);
		// A pattern that stops matching would pass this test silently while the
		// copy rots, so assert the sweep is still finding the copies it guards.
		expect(quotes, 'the count patterns matched nothing; the copy was reworded').toBeGreaterThanOrEqual(
			QUOTING_FILES.length,
		);
	});
});

describe('hosts we must not regress onto', () => {
	// substack.com and mirror.xyz sit behind a Cloudflare bot challenge: every
	// feed on them answers 403 to server-side fetches, so they can never be
	// served from Cloud Run. They read as plausible sources, which is exactly
	// why this guard exists.
	//
	// medium.com joined the list on 2026-07-10: 43 medium.com/feed/* sources were
	// added to the registry and every one answers 429 to a server-side fetch
	// (probed 5/5 from Cloud Run's egress, spaced seconds apart — it is a blanket
	// block on datacenter IPs, not per-feed throttling). They looked live in a
	// browser, which is the trap.
	it('lists no substack.com, mirror.xyz, or medium.com feeds', () => {
		const blocked = entries.filter(([, s]) =>
			/(^|\.)(substack\.com|mirror\.xyz|medium\.com)$/.test(new URL(s.url).hostname),
		);
		expect(blocked.map(([k]) => k)).toEqual([]);
	});

	it('lists no known parked domain', () => {
		const parked = entries.filter(([, s]) => new URL(s.url).hostname.replace(/^www\./, '') === 'legendarynames.com');
		expect(parked.map(([k]) => k)).toEqual([]);
	});
});

describe('non-RSS sources', () => {
	it('every kind:json source has an adapter in the aggregator', async () => {
		const jsonSources = entries.filter(([, s]) => s.kind === 'json').map(([k]) => k);
		if (!jsonSources.length) return;
		const src = await import('node:fs').then((fs) => fs.readFileSync('api/_lib/news.js', 'utf8'));
		for (const key of jsonSources) {
			expect(src, `JSON_ADAPTERS.${key} missing`).toContain(`${key}(data, key)`);
		}
	});

	it('marks only json sources with a kind', () => {
		for (const [key, src] of entries) {
			if (src.kind) expect(src.kind, `${key}`).toBe('json');
		}
	});
});

describe('sourcesForCategory', () => {
	it('returns every key for "all" and for no argument', () => {
		expect(sourcesForCategory('all')).toHaveLength(entries.length);
		expect(sourcesForCategory()).toHaveLength(entries.length);
	});

	it('filters to the requested category', () => {
		const defi = sourcesForCategory('defi');
		expect(defi.length).toBeGreaterThan(0);
		for (const k of defi) expect(NEWS_SOURCES[k].category).toBe('defi');
	});

	it('returns nothing for an unknown category', () => {
		expect(sourcesForCategory('does-not-exist')).toEqual([]);
	});
});

describe('sourcesForLanguage', () => {
	it('"en" selects exactly the untagged feeds', () => {
		const en = sourcesForLanguage('en');
		expect(en.length).toBeGreaterThan(0);
		for (const k of en) expect(NEWS_SOURCES[k].language).toBeUndefined();
	});

	it('selects tagged feeds by language', () => {
		for (const lang of NEWS_LANGUAGES) {
			const keys = sourcesForLanguage(lang);
			expect(keys.length, `no sources for ${lang}`).toBeGreaterThan(0);
			for (const k of keys) expect(NEWS_SOURCES[k].language).toBe(lang);
		}
	});

	it('"all" and no argument return everything', () => {
		expect(sourcesForLanguage('all')).toHaveLength(entries.length);
		expect(sourcesForLanguage()).toHaveLength(entries.length);
	});
});

describe('sourcePriority', () => {
	it('ranks every source into a finite band', () => {
		for (const [key] of entries) {
			const p = sourcePriority(key);
			expect(Number.isInteger(p), `${key}`).toBe(true);
			expect(p, `${key}`).toBeGreaterThanOrEqual(0);
			expect(p, `${key}`).toBeLessThanOrEqual(4);
		}
	});

	it('puts tier1 newsrooms ahead of the untiered long tail', () => {
		const tier1 = entries.filter(([, s]) => s.tier === 'tier1').map(([k]) => k);
		const untiered = entries.filter(([, s]) => !s.tier && !s.language).map(([k]) => k);
		expect(tier1.length).toBeGreaterThan(0);
		expect(untiered.length).toBeGreaterThan(0);
		const worstTier1 = Math.max(...tier1.map(sourcePriority));
		const bestUntiered = Math.min(...untiered.map(sourcePriority));
		expect(worstTier1).toBeLessThan(bestUntiered);
	});

	it('ranks international feeds behind the English long tail', () => {
		const intl = entries.filter(([, s]) => s.language && !s.tier).map(([k]) => k);
		if (!intl.length) return;
		for (const k of intl) expect(sourcePriority(k)).toBe(4);
	});

	it('is stable for an unknown key', () => {
		expect(sourcePriority('no_such_source')).toBe(3);
	});
});


// findArticle uses this to refresh the one feed that could hold a link instead
// of fanning out over the whole registry and truncating at a short deadline, so
// a wrong answer here turns a live news card into a cacheable 404.
describe('sourceKeyForLink', () => {
	it('maps an article URL back to the feed that publishes it', () => {
		for (const [key, src] of entries.slice(0, 40)) {
			const host = new URL(src.url).hostname;
			const resolved = sourceKeyForLink(`https://${host}/some-article`);
			// A host shared by two feeds resolves to whichever was registered first,
			// which is correct for this purpose: both carry the same articles.
			expect(resolved, `${key}`).toBeTruthy();
			expect(new URL(NEWS_SOURCES[resolved].url).hostname, `${key}`).toBe(host);
		}
	});

	it('resolves the www host of a feed served from the bare domain', () => {
		const bare = entries.find(([, s]) => !new URL(s.url).hostname.startsWith('www.'));
		const host = new URL(bare[1].url).hostname;
		expect(sourceKeyForLink(`https://www.${host}/story`)).toBe(sourceKeyForLink(`https://${host}/story`));
	});

	it('returns null for a host nobody here publishes, and for a non-URL', () => {
		expect(sourceKeyForLink('https://not-a-registered-publisher.example/x')).toBeNull();
		expect(sourceKeyForLink('')).toBeNull();
		expect(sourceKeyForLink('not a url')).toBeNull();
		expect(sourceKeyForLink(null)).toBeNull();
	});

	it('never resolves a bare public suffix, which would map every site to one feed', () => {
		for (const tld of ['https://com/x', 'https://io/x', 'https://net/x']) {
			expect(sourceKeyForLink(tld)).toBeNull();
		}
	});
});