Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

15.5 KB
Raw Blame
// GET /api/news/article?url=<article url>&title=&source=
// ---------------------------------------------------------------------------
// Rich article detail for the /markets/news reader. Runs the full extraction
// ladder (publisher page → Jina Reader → Wayback → SEC EDGAR → sibling coverage →
// publisher feed → preview, in
// api/_lib/article-extract.js), so a Cloudflare-blocked publisher still yields
// the full story instead of a one-line teaser. Layers analysis on top:
//   • paragraphs      BOUNDED lead excerpt of the publisher's body (see
//                     api/_lib/news-rights.js — we quote the lead and link out;
//                     the full body is never served from three.ws)
//   • summary         LLM summary via the platform chain; falls back to lead
//   • key_points      3–5 takeaways (LLM, or highest-signal sentences)
//   • entities        orgs / people / projects the story is about (LLM)
//   • sentiment       bullish / bearish / neutral
//   • trading         structured read: assets (exposure + direction), whether
//                     the news is priced in, horizon, and a mandatory
//                     counter-argument (LLM only; null on the extractive path)
//   • tickers         detected symbols, linkable to /coin/:id
//   • coins           live price + 7d sparkline for each mapped ticker
//   • related         live related coverage from the native aggregator
//
// Every fully-extracted, analyzed story is recorded to the durable news
// knowledge base (api/_lib/news-knowledge-store.js) — the corpus the 3D agents
// read crypto from — which also serves as a cross-instance cache so a story is
// only pulled from the (rate-limited, blockable) publisher once.
//
// The LLM layer is optional by design — when no provider key is present the
// endpoint still returns the full extracted article with heuristic analysis,
// clearly labelled via analysis_provider.

import { cors, json, method, wrap, error, rateLimited } from '../_lib/http.js';
import { limits, clientIp } from '../_lib/rate-limit.js';
import { getNews, findArticle, articleId, extractTickers, lexiconSentiment, stripJsonFence, metaContent, stripHtml } from '../_lib/news.js';
import { llmComplete, llmConfigured } from '../_lib/llm.js';
import { extractArticle } from '../_lib/article-extract.js';
import { enrichTickers } from '../_lib/news-coins.js';
import { recordExtraction, getExtraction } from '../_lib/news-knowledge-store.js';
import { suppression, excerptParagraphs, excerptText } from '../_lib/news-rights.js';
import { truncateChars } from '../_lib/safe-text.js';

const CACHE_TTL_MS = 30 * 60_000;

const _cache = new Map(); // url → { value, expiresAt }

// ── Analysis ─────────────────────────────────────────────────────────────────

const EXPOSURES = ['direct', 'indirect', 'sector'];
const DIRECTIONS = ['up', 'down', 'mixed'];
const PRICED_IN = ['yes', 'partly', 'no', 'unclear'];
const HORIZONS = ['intraday', 'days', 'weeks', 'months', 'long_term'];

/**
 * Validate the trading read an LLM returned. `priced_in`, `horizon` and
 * `counter` are required: a read with no counter-argument is a one-sided call,
 * so the whole read is rejected rather than shown half-formed. Assets with an
 * unknown shape are dropped individually.
 */
export function parseTradingRead(raw) {
	if (!raw || typeof raw !== 'object') return null;
	const counter = truncateChars(raw.counter, 500);
	if (!counter || !PRICED_IN.includes(raw.priced_in) || !HORIZONS.includes(raw.horizon)) return null;
	const assets = (Array.isArray(raw.assets) ? raw.assets : [])
		.map((a) => ({
			symbol: String(a?.symbol || '').toUpperCase().replace(/[^A-Z0-9]/g, '').slice(0, 12),
			exposure: a?.exposure,
			direction: a?.direction,
		}))
		.filter((a) => a.symbol && EXPOSURES.includes(a.exposure) && DIRECTIONS.includes(a.direction))
		.slice(0, 6);
	return {
		assets,
		priced_in: raw.priced_in,
		priced_in_reason: truncateChars(raw.priced_in_reason, 300) || null,
		horizon: raw.horizon,
		counter,
	};
}

// The platform LLM chain (api/_lib/llm.js) owns provider order, key discovery,
// failover, and spend tracking. Returns null when no provider is configured or
// every provider fails — the caller then serves the extractive analysis below.
async function llmAnalyze(content, title, source) {
	if (!llmConfigured()) return null;
	try {
		const { text, provider } = await llmComplete({
			system: 'You are a crypto news analyst. Respond only with valid JSON, no markdown fence. Base every claim on the article text; if the story names no tradable asset, return an empty assets array.',
			user: `Analyze this article. Respond as {"summary": "...2-3 sentence summary...", "key_points": ["...", "..."], "sentiment": "bullish|bearish|neutral", "entities": ["organizations, people, and projects the story is about"], "topics": ["1-3 word themes"], "assets": [{"symbol": "TICKER", "exposure": "${EXPOSURES.join('|')}", "direction": "${DIRECTIONS.join('|')}"}], "priced_in": "${PRICED_IN.join('|')}", "priced_in_reason": "one sentence on why", "horizon": "${HORIZONS.join('|')}", "counter": "the strongest argument against the obvious read of this news"}. "counter" is required and must be a real counter-argument, not a restatement.\n\nTitle: ${title}\nSource: ${source}\n\n${content.slice(0, 6000)}`,
			maxTokens: 1100,
			timeoutMs: 18_000,
			track: { tool: 'news_article' },
		});
		const parsed = JSON.parse(stripJsonFence(text));
		if (!parsed.summary) return null;
		const list = (v, n, cap) => (Array.isArray(v) ? v : []).map((p) => truncateChars(p, cap)).filter(Boolean).slice(0, n);
		return {
			summary: truncateChars(parsed.summary, 1200),
			key_points: list(parsed.key_points, 5, 300),
			entities: list(parsed.entities, 12, 80),
			topics: list(parsed.topics, 8, 40),
			sentiment: ['bullish', 'bearish', 'neutral'].includes(parsed.sentiment) ? parsed.sentiment : 'neutral',
			trading: parseTradingRead(parsed),
			provider,
		};
	} catch {
		return null; // extractive fallback covers it
	}
}

// The fallback when no LLM provider answers. Unlike the LLM path, this one is
// EXTRACTIVE — its summary and key points are the publisher's own sentences,
// not new prose. So it is held to the same quotation budget as the excerpt:
// a bounded lead, and a few short representative sentences. Without these
// bounds the "analysis" would quietly become a second copy of the article.
const HEURISTIC_KEY_POINTS = 3;
const HEURISTIC_POINT_CHARS = 200;

function heuristicAnalyze(paragraphs, title) {
	const lead = paragraphs.slice(0, 3).join(' ');
	const summary = excerptText(lead);
	// Highest-signal sentences: numbers, detected tickers, and early position.
	const sentences = paragraphs
		.join(' ')
		.split(/(?<=[.!?。])\s+/)
		.filter((s) => s.length > 50 && s.length < 320);
	const scored = sentences
		.map((s, i) => {
			let score = 0;
			if (/\d/.test(s)) score += 2;
			if (extractTickers(s).length) score += 2;
			if (/percent|%|\$|million|billion|亿|万/.test(s)) score += 1;
			score += Math.max(0, 2 - i * 0.1);
			return { s, score };
		})
		.sort((a, b) => b.score - a.score);
	const key_points = [];
	for (const { s } of scored) {
		if (key_points.length >= HEURISTIC_KEY_POINTS) break;
		if (key_points.some((k) => k.slice(0, 60) === s.slice(0, 60))) continue;
		key_points.push(s.length > HEURISTIC_POINT_CHARS ? `${truncateChars(s, HEURISTIC_POINT_CHARS).trimEnd()}…` : s);
	}
	const lex = lexiconSentiment(`${title} ${lead}`);
	const sentiment = lex.label.includes('positive') ? 'bullish' : lex.label.includes('negative') ? 'bearish' : 'neutral';
	return { summary, key_points, entities: [], topics: [], sentiment, trading: null, provider: 'heuristic' };
}

// ── Rights boundary ──────────────────────────────────────────────────────────

/**
 * The public shape of an extraction. The extraction ladder and the knowledge
 * base hold the full body because the analysis layer and the agents' RAG corpus
 * need it; the HTTP response does NOT get it. `paragraphs` is capped to a lead
 * excerpt here, at the one place every response passes through, so no caller —
 * reader, cached record, or third party hitting the endpoint directly — can
 * obtain the publisher's article body from us.
 */
function publicView(value) {
	const { paragraphs, truncated } = excerptParagraphs(value.paragraphs);
	return {
		...value,
		paragraphs,
		excerpt_truncated: truncated,
		// Where the rest of the story lives. The reader renders this as the
		// primary call to action whenever the body was truncated.
		full_text_url: value.url,
	};
}

// ─────────────────────────────────────────────────────────────────────────────

export default wrap(async (req, res) => {
	if (cors(req, res, { methods: 'GET,OPTIONS', origins: '*' })) return;
	if (!method(req, res, ['GET'])) return;

	const rl = await limits.marketFeedIp(clientIp(req));
	if (!rl.success) return rateLimited(res, rl);

	const params = new URL(req.url, 'http://x').searchParams;
	const rawUrl = (params.get('url') || '').replace(/^<!\[CDATA\[/, '').replace(/\]\]>$/, '').trim();
	if (!rawUrl) return error(res, 400, 'bad_url', 'url is required');
	let parsed;
	try {
		parsed = new URL(rawUrl);
	} catch {
		return error(res, 400, 'bad_url', 'url is not a valid absolute URL');
	}
	// An article link is a web link. `new URL()` happily parses javascript:,
	// file:, and data: URLs, and every rung of the extraction ladder then
	// refuses them one by one, which used to end in a 200 carrying an empty
	// "Untitled" record. Reject the scheme here instead: a non-web URL is bad
	// input, not an article we failed to read.
	if (parsed.protocol !== 'https:' && parsed.protocol !== 'http:') {
		return error(res, 400, 'bad_url', 'url must be an http(s) article link');
	}
	const target = parsed.toString();

	// Withdrawn publisher (or a taken-down story reached by its link) — refuse
	// before fetching anything. 410: the resource is permanently gone from here.
	const sup = suppression({ url: target });
	if (sup) {
		return error(res, 410, 'removed', `Coverage from ${sup.publisher || 'this publisher'} is no longer available on three.ws.`);
	}

	const cacheHeaders = { 'cache-control': 'public, max-age=600, s-maxage=1800, stale-while-revalidate=3600' };

	const hit = _cache.get(target);
	if (hit && hit.expiresAt > Date.now()) return json(res, 200, publicView(hit.value), cacheHeaders);

	const id = articleId(target);

	// Durable cache: a prior instance may have already done the expensive full
	// extraction. Reuse it when it carries real body text — never re-hammer a
	// rate-limited publisher for a story we already know in full.
	const stored = await getExtraction(id).catch(() => null);
	if (stored && ['page', 'reader', 'wayback'].includes(stored.extraction) && stored.content_chars > 400) {
		_cache.set(target, { value: stored, expiresAt: Date.now() + CACHE_TTL_MS });
		return json(res, 200, publicView(stored), cacheHeaders);
	}

	// The publisher's own feed copy — trusted metadata, and (for feeds that
	// ship content:encoded) a fuller body. Looked up first so a bot-blocked
	// page fetch still has a fallback body, never a dead end.
	const feedCopy = await findArticle({ link: target }).catch(() => null);

	// Run the extraction ladder: page → reader → wayback → primary → siblings →
	// feed → preview.
	const { paragraphs, extraction, blocked_reason, html, lane_sources } = await extractArticle(target, {
		feedContentText: feedCopy?.content_text || null,
		title: (params.get('title') || '').trim().slice(0, 300) || feedCopy?.title || null,
		publishedAt: feedCopy?.pub_date || null,
	});

	const title =
		(params.get('title') || '').trim().slice(0, 300) ||
		(html && metaContent(html, ['og:title', 'twitter:title'])) ||
		feedCopy?.title ||
		(html && stripHtml(html.match(/<title[^>]*>([\s\S]*?)<\/title>/i)?.[1] || '')) ||
		'Untitled';
	const source =
		(params.get('source') || '').trim().slice(0, 80) ||
		(html && metaContent(html, ['og:site_name'])) ||
		feedCopy?.source ||
		new URL(target).hostname.replace(/^www\./, '');

	const content = paragraphs.join('\n').slice(0, 8000);
	const analysis =
		(content.length > 300 ? await llmAnalyze(content, title, source) : null) ||
		heuristicAnalyze(paragraphs.length ? paragraphs : [feedCopy?.description || title], title);

	const tickers = extractTickers(`${title} ${content.slice(0, 2000)} ${feedCopy?.description || ''}`);

	// Live market snapshot for the mapped tickers, and related coverage — both
	// optional, fetched together so neither blocks the other.
	const [coins, related] = await Promise.all([
		enrichTickers(tickers).catch(() => []),
		(async () => {
			try {
				const relatedQuery = tickers[0] || title.split(/\s+/).slice(0, 3).join(' ');
				const rel = await getNews({ q: relatedQuery, limit: 7, curated: true });
				let out = rel.articles.filter((a) => a.link !== target).slice(0, 6);
				if (!out.length) {
					const fallback = await getNews({ category: feedCopy?.category, limit: 7, curated: true });
					out = fallback.articles.filter((a) => a.link !== target).slice(0, 6);
				}
				return out;
			} catch {
				return [];
			}
		})(),
	]);

	const value = {
		id,
		url: target,
		title,
		source,
		image: (html && metaContent(html, ['og:image', 'twitter:image'])) || feedCopy?.image || null,
		author: (html && metaContent(html, ['author', 'article:author', 'twitter:creator'])) || feedCopy?.author || null,
		published_at:
			(html && metaContent(html, ['article:published_time', 'og:article:published_time', 'date'])) ||
			feedCopy?.pub_date ||
			null,
		description:
			(html && metaContent(html, ['og:description', 'description', 'twitter:description'])) ||
			feedCopy?.description ||
			null,
		extraction, // 'page' | 'reader' | 'wayback' | 'primary' | 'siblings' | 'feed' | 'preview'
		lane_sources, // where a wayback/primary/siblings read came from, [] otherwise
		blocked_reason: extraction === 'preview' ? blocked_reason : null,
		paragraphs,
		content_chars: content.length,
		tickers,
		coins,
		summary: analysis.summary,
		key_points: analysis.key_points,
		entities: analysis.entities,
		topics: analysis.topics,
		sentiment: analysis.sentiment,
		trading: analysis.trading,
		analysis_provider: analysis.provider,
		market_context: feedCopy?.market_context || null,
		related,
		fetched_at: new Date().toISOString(),
	};

	_cache.set(target, { value, expiresAt: Date.now() + CACHE_TTL_MS });
	if (_cache.size > 256) _cache.delete(_cache.keys().next().value);

	// Record into the durable knowledge base the 3D agents read from. Fire-and-
	// forget: a persistence hiccup must never fail the reader response.
	//
	// Only rows carrying real body text are worth recording. A zero-content
	// preview (every rung of the ladder refused the URL) grounds no agent and
	// is never reused as the cross-instance cache above, which requires a
	// page/reader row over 400 chars, so writing it only adds noise to the
	// corpus. It also closes the corpus to drive-by writes: this endpoint is
	// public and unauthenticated, and the caller controls `url`, `title`, and
	// `source`, so an arbitrary link used to mint a permanent knowledge row
	// with an attacker-chosen headline behind a single GET.
	if (value.content_chars > 0) recordExtraction(value).catch(() => {});

	return json(res, 200, publicView(value), cacheHeaders);
});