Soul-driven AI agent with permission-hardened tools, token budgets, and multi-channel access. Runs 24/7 from CLI, Telegram or More.
import { tool, zodSchema } from 'ai';
import { z } from 'zod';
import { guardedFetch } from '../../utils/ssrf.js';
/**
* web_search: free, keyless search for the model.
*
* web → Parallel (search.parallel.ai, keyless MCP), then DuckDuckGo's
* html page, then its lite page: the three keyless web indexes
* that answer automated requests
* news → Parallel, then Bing News (direct article links), Google News (its
* links are Google redirects), Hacker News
*
* Every answer must mention the query (relevant()); an engine serving junk
* counts as empty and the next one is tried.
*
* Read-only and through the same SSRF guard as fetch_url. Returns titles,
* links, snippets and dates; the model follows up with fetch_url to read a
* page. Brave, Mojeek and Startpage refuse automated requests, and Bing's
* web RSS serves unrelated pages, so none of them are used.
*/
export interface SearchResult {
title: string;
url: string;
snippet: string;
source?: string;
date?: string;
}
const UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126 Safari/537.36';
const TIMEOUT_MS = 12_000;
function unescape(s: string): string {
return s
.replace(/</g, '<')
.replace(/>/g, '>')
.replace(/"/g, '"')
.replace(/'|'/g, "'")
.replace(/&#(\d+);/g, (_m, n) => String.fromCharCode(Number(n)))
.replace(/&/g, '&');
}
function decode(s: string): string {
// Feeds often carry escaped HTML inside a field (Google News descriptions):
// unescape, strip tags, then unescape what was inside the markup.
return unescape(unescape(s.replace(/<!\[CDATA\[([\s\S]*?)\]\]>/g, '$1')).replace(/<[^>]+>/g, ' '))
.replace(/ /g, ' ')
.replace(/<[^>]+>/g, ' ')
.replace(/\s+/g, ' ')
.trim();
}
function tag(item: string, name: string): string {
const m = new RegExp(`<${name}[^>]*>([\\s\\S]*?)</${name}>`, 'i').exec(item);
return m ? decode(m[1]) : '';
}
/**
* The publisher's own URL behind a news-feed redirect. Bing News wraps every
* article as bing.com/news/apiclick.aspx?…&url=<the article>; unwrap it so the
* owner gets a link that opens the story. (Google News links can't be
* unwrapped without running its JavaScript, which is why Bing News comes first.)
*/
export function directLink(link: string): string {
try {
const u = new URL(link);
if (/(^|\.)bing\.com$/.test(u.hostname) && /\/apiclick\.aspx$/i.test(u.pathname)) {
const target = u.searchParams.get('url');
if (target && /^https?:\/\//i.test(target)) return target;
}
} catch { /* not a URL: leave it */ }
return link;
}
export function parseRss(xml: string, max: number): SearchResult[] {
const out: SearchResult[] = [];
for (const m of xml.matchAll(/<item>([\s\S]*?)<\/item>/gi)) {
const item = m[1];
const url = directLink(tag(item, 'link'));
const title = tag(item, 'title');
if (!url || !title || /bing\.com(:\d+)?\/search\?/.test(url)) continue;
const source = tag(item, 'source');
out.push({
title: source && title.endsWith(` - ${source}`) ? title.slice(0, -(source.length + 3)) : title,
url,
snippet: tag(item, 'description').slice(0, 300),
source: source || undefined,
date: tag(item, 'pubDate') || undefined,
});
if (out.length >= max) break;
}
return out;
}
async function fetchText(url: string): Promise<string> {
const ac = new AbortController();
const timer = setTimeout(() => ac.abort(), TIMEOUT_MS);
try {
const res = await guardedFetch(url, ac.signal, { 'User-Agent': UA, Accept: 'application/rss+xml, application/xml, text/xml, */*' });
if (!res.ok) throw new Error(`HTTP ${res.status}`);
return await res.text();
} finally {
clearTimeout(timer);
}
}
/** Bing News RSS: relevant, recent, keyless. */
async function bingNews(query: string, max: number): Promise<SearchResult[]> {
if (!query.trim()) return [];
return parseRss(await fetchText(`https://www.bing.com/news/search?q=${encodeURIComponent(query)}&format=rss`), max);
}
/**
* Parallel's public search over MCP (https://search.parallel.ai/mcp): a real
* web index built for agents, keyless, answering in about a second with
* excerpts and publish dates. Plain JSON-RPC over HTTP: one tools/call to
* its web_search tool, no session needed. A fixed public HTTPS endpoint, so
* it is called directly rather than through guardedFetch (GET-only).
*/
export const PARALLEL_MCP_URL = 'https://search.parallel.ai/mcp';
interface ParallelHit { url?: string; title?: string; publish_date?: string | null; excerpts?: string[] }
/** The MCP reply, as plain JSON or as a server-sent-events body. */
export function parseParallel(body: string, max: number): SearchResult[] {
let text = body.trim();
if (!text.startsWith('{')) {
const data = text.split('\n').filter((l) => l.startsWith('data:')).map((l) => l.slice(5).trim()).filter(Boolean);
text = data[data.length - 1] ?? '';
}
const msg = JSON.parse(text) as { error?: { message?: string }; result?: { isError?: boolean; structuredContent?: { results?: ParallelHit[] }; content?: Array<{ type?: string; text?: string }> } };
if (msg.error) throw new Error(msg.error.message ?? 'MCP error');
if (msg.result?.isError) throw new Error(msg.result.content?.[0]?.text?.slice(0, 160) ?? 'search error');
let hits = msg.result?.structuredContent?.results;
if (!hits) {
const t = msg.result?.content?.find((c) => c.type === 'text')?.text;
hits = t ? (JSON.parse(t) as { results?: ParallelHit[] }).results : undefined;
}
return (hits ?? []).filter((h) => h.url && /^https?:\/\//.test(h.url)).slice(0, max).map((h) => ({
title: (h.title ?? h.url!).replace(/\s+/g, ' ').trim(),
url: h.url!,
// Excerpts are markdown cut from the page: flatten links and markup.
snippet: (h.excerpts?.[0] ?? '').replace(/\[([^\]]*)\]\([^)]*\)/g, '$1').replace(/[#*>`_]+/g, ' ').replace(/\s+/g, ' ').trim().slice(0, 300),
date: h.publish_date ?? undefined,
}));
}
async function parallelSearch(objective: string, queries: string[], max: number): Promise<SearchResult[]> {
const ac = new AbortController();
const timer = setTimeout(() => ac.abort(), TIMEOUT_MS);
try {
const res = await fetch(PARALLEL_MCP_URL, {
method: 'POST',
signal: ac.signal,
headers: { 'Content-Type': 'application/json', Accept: 'application/json, text/event-stream', 'MCP-Protocol-Version': '2025-06-18', 'User-Agent': 'Mercury-Agent' },
body: JSON.stringify({ jsonrpc: '2.0', id: 1, method: 'tools/call', params: { name: 'web_search', arguments: { objective: objective.slice(0, 400), search_queries: queries } } }),
});
if (res.status === 429) throw new Error('rate-limited (HTTP 429)');
if (!res.ok) throw new Error(`HTTP ${res.status}`);
return parseParallel(await res.text(), max);
} finally {
clearTimeout(timer);
}
}
/** Keyword queries for Parallel: the query itself, cut to its first eight words. */
function parallelQueries(query: string): string[] {
return [query.replace(/["']/g, '').split(/\s+/).filter(Boolean).slice(0, 8).join(' ')];
}
async function parallel(query: string, max: number): Promise<SearchResult[]> {
if (!query.trim()) return [];
return parallelSearch(query, parallelQueries(query), max);
}
/**
* A link to one story rather than a section front ("/kashmir", "/latest-news"):
* a dated path, or at least two path segments with a long slug. News answers
* should open the article.
*/
export function looksLikeArticle(url: string): boolean {
let path: string;
try { path = new URL(url).pathname; } catch { return false; }
const parts = path.split('/').filter(Boolean);
if (/\/(19|20)\d{2}\/\d{1,2}\//.test(path) || /\d{5,}/.test(path)) return true;
const last = parts[parts.length - 1] ?? '';
return parts.length >= 2 && (last.split('-').length >= 4 || last.length >= 30);
}
async function parallelNews(query: string, max: number): Promise<SearchResult[]> {
const q = query.trim() || 'top news today';
const hits = await parallelSearch(`Latest news articles about: ${q}. Prefer individual articles published in the last few days, from news sites.`, [...parallelQueries(q), `${parallelQueries(q)[0]} news`], max * 2);
return hits.filter((h) => looksLikeArticle(h.url)).slice(0, max);
}
/**
* DuckDuckGo's two no-JavaScript result pages, the keyless web index behind
* Parallel. Both wrap every result link as
* //duckduckgo.com/l/?uddg=<the page>&rut=…, and both mark ads: the html page
* with a `result--ad` block linking to duckduckgo.com/y.js, the lite page
* with a sponsored row. Parsed exactly from their markup:
*
* html.duckduckgo.com/html/ <div class="result results_links …">
* <a class="result__a" href=…>title</a>
* <a class="result__snippet" href=…>snippet</a>
* lite.duckduckgo.com/lite/ <a class='result-link' href=…>title</a>
* <td class='result-snippet'>snippet</td>
*
* DuckDuckGo answers some automated requests with a challenge (HTTP 202 or
* an "anomaly" page). That, and a page with no results and no "No results"
* notice (the markup changed), count as failures, so the trail says so.
*/
/** The page behind DuckDuckGo's redirect, or null for ads and DuckDuckGo's own links. */
export function ddgTarget(href: string): string | null {
const raw = unescape(href.trim());
let u: URL;
try { u = new URL(raw.startsWith('//') ? `https:${raw}` : raw, 'https://duckduckgo.com'); } catch { return null; }
const target = /(^|\.)duckduckgo\.com$/.test(u.hostname) ? u.searchParams.get('uddg') : u.href;
if (!target || !/^https?:\/\//i.test(target)) return null;
try {
const t = new URL(target);
return /(^|\.)duckduckgo\.com$/.test(t.hostname) ? null : t.href;
} catch {
return null;
}
}
export function parseDdgHtml(html: string, max: number): SearchResult[] {
const out: SearchResult[] = [];
const blocks = [...html.matchAll(/<div class="(result\b[^"]*)"/g)].filter((m) => /\b(results_links|web-result)\b/.test(m[1]));
for (let i = 0; i < blocks.length && out.length < max; i++) {
if (/\bresult--ad\b/.test(blocks[i][1])) continue;
const block = html.slice(blocks[i].index ?? 0, blocks[i + 1]?.index ?? html.length);
const a = /<a\b[^>]*class="result__a"[^>]*>([\s\S]*?)<\/a>/.exec(block);
const href = a && /href="([^"]*)"/.exec(a[0])?.[1];
const url = href ? ddgTarget(href) : null;
const title = a ? decode(a[1]) : '';
if (!url || !title) continue;
const snippet = /<(?:a|div)\b[^>]*class="result__snippet"[^>]*>([\s\S]*?)<\/(?:a|div)>/.exec(block)?.[1] ?? '';
out.push({ title, url, snippet: decode(snippet).slice(0, 300) });
}
return out;
}
export function parseDdgLite(html: string, max: number): SearchResult[] {
const out: SearchResult[] = [];
const links = [...html.matchAll(/<a\b[^>]*class=['"]result-link['"][^>]*>([\s\S]*?)<\/a>/g)];
for (let i = 0; i < links.length && out.length < max; i++) {
const at = links[i].index ?? 0;
// A sponsored result sits in a row marked as such.
if (/sponsored/i.test(html.slice(Math.max(0, html.lastIndexOf('<tr', at)), at))) continue;
const href = /href=['"]([^'"]*)['"]/.exec(links[i][0])?.[1];
const url = href ? ddgTarget(href) : null;
const title = decode(links[i][1]);
if (!url || !title) continue;
const scope = html.slice(at, links[i + 1]?.index ?? html.length);
const snippet = /<td\b[^>]*class=['"]result-snippet['"][^>]*>([\s\S]*?)<\/td>/.exec(scope)?.[1] ?? '';
out.push({ title, url, snippet: decode(snippet).slice(0, 300) });
}
return out;
}
/** Either page: the html page has result__a links, the lite page result-link. */
export function parseDdg(html: string, max: number): SearchResult[] {
return /class="result__a"/.test(html) ? parseDdgHtml(html, max) : parseDdgLite(html, max);
}
const DDG_CHALLENGE = /anomaly|challenge-form|bots use DuckDuckGo too/i;
const DDG_NO_RESULTS = /No results\.|no-results|No more results/i;
async function ddgOnce(kind: 'html' | 'lite', query: string, max: number): Promise<SearchResult[]> {
const ac = new AbortController();
const timer = setTimeout(() => ac.abort(), TIMEOUT_MS);
try {
const res = await guardedFetch(`https://${kind}.duckduckgo.com/${kind}/?q=${encodeURIComponent(query)}`, ac.signal, {
'User-Agent': UA,
Accept: 'text/html',
'Accept-Language': 'en-US,en;q=0.9',
Referer: `https://${kind}.duckduckgo.com/`,
});
const body = await res.text();
if (res.status === 202 || DDG_CHALLENGE.test(body)) throw new Error('bot challenge');
if (!res.ok) throw new Error(`HTTP ${res.status}`);
const results = kind === 'html' ? parseDdgHtml(body, max) : parseDdgLite(body, max);
if (!results.length && !DDG_NO_RESULTS.test(body)) throw new Error('page not recognised');
return results;
} finally {
clearTimeout(timer);
}
}
/** DuckDuckGo lets roughly every other automated request through: one retry after a short pause. */
const DDG_RETRY_MS = 1500;
async function ddg(kind: 'html' | 'lite', query: string, max: number): Promise<SearchResult[]> {
if (!query.trim()) return [];
try {
return await ddgOnce(kind, query, max);
} catch (err) {
if (!/challenge/.test((err as Error).message)) throw err;
await new Promise((r) => setTimeout(r, DDG_RETRY_MS));
return ddgOnce(kind, query, max);
}
}
const ddgHtml = (q: string, n: number) => ddg('html', q, n);
const ddgLite = (q: string, n: number) => ddg('lite', q, n);
async function googleNews(query: string, max: number): Promise<SearchResult[]> {
const url = query.trim()
? `https://news.google.com/rss/search?q=${encodeURIComponent(query)}&hl=en-US&gl=US&ceid=US:en`
: 'https://news.google.com/rss?hl=en-US&gl=US&ceid=US:en';
return parseRss(await fetchText(url), max);
}
async function fetchJson(url: string): Promise<unknown> {
return JSON.parse(await fetchText(url));
}
/** Hacker News via Algolia: keyless JSON, strong for software, startups and tech news. */
export async function hackerNews(query: string, max: number): Promise<SearchResult[]> {
if (!query.trim()) return [];
const data = await fetchJson(`https://hn.algolia.com/api/v1/search?query=${encodeURIComponent(query)}&tags=story&hitsPerPage=${Math.min(10, max)}`) as {
hits?: Array<{ title?: string; url?: string | null; objectID: string; created_at?: string; points?: number }>;
};
return (data.hits ?? []).filter((h) => h.title).map((h) => ({
title: h.title!,
url: h.url || `https://news.ycombinator.com/item?id=${h.objectID}`,
snippet: `Hacker News discussion · ${h.points ?? 0} points · https://news.ycombinator.com/item?id=${h.objectID}`,
source: 'Hacker News',
date: h.created_at,
}));
}
/**
* The search query behind a search-engine results URL, or null. Old skills
* (and models from habit) fetch DuckDuckGo/Google/Bing result pages with
* fetch_url; those answer bots with a challenge page or a wall of markup.
* fetch_url hands such a URL to web_search instead.
*/
export function searchQueryFromUrl(raw: string): string | null {
let u: URL;
try { u = new URL(raw); } catch { return null; }
const host = u.hostname.replace(/^www\./, '');
const isSearch =
(/^(html\.|lite\.)?duckduckgo\.com$/.test(host) && (u.pathname === '/' || /^\/(html|lite)\/?$/.test(u.pathname)))
|| (/^google\.[a-z.]+$/.test(host) && u.pathname === '/search')
|| (host === 'bing.com' && u.pathname === '/search' && u.searchParams.get('format') !== 'rss')
|| (host === 'search.yahoo.com' && u.pathname.startsWith('/search'))
|| (host === 'search.brave.com' && u.pathname === '/search')
|| (host === 'startpage.com' && u.pathname.startsWith('/do/search'))
|| (host === 'mojeek.com' && u.pathname === '/search');
if (!isSearch) return null;
const q = (u.searchParams.get('q') ?? u.searchParams.get('p') ?? u.searchParams.get('query') ?? '').trim();
return q || null;
}
/** web_search output as text, for the tool and for fetch_url's search redirect. */
export function formatResults(results: SearchResult[], engine: string, query: string, kind: 'web' | 'news'): string {
const lines = results.map((r, i) => {
const meta = [r.source, r.date ? new Date(r.date).toUTCString().slice(0, 16) : ''].filter(Boolean).join(' · ');
return `${i + 1}. ${r.title}${meta ? ` (${meta})` : ''}\n ${r.url}${r.snippet ? `\n ${r.snippet}` : ''}`;
});
const viaGoogle = results.some((r) => /^https?:\/\/news\.google\.com\/rss\/articles\//.test(r.url));
const note = viaGoogle ? '\n\nNote: news.google.com links are Google redirects, not the publisher\'s page. Say so if you share them, and name the source site.' : '';
return `${kind === 'news' ? 'News' : 'Web'} results from ${engine} for "${query || 'top stories'}":\n\n${lines.join('\n')}${note}`;
}
const STOP = new Set(['the', 'and', 'for', 'with', 'what', 'who', 'how', 'are', 'was', 'is', 'of', 'in', 'on', 'to', 'a', 'an', 'best', 'top', 'latest', 'new', 'news', 'does', 'did', 'can', 'from', 'about', 'near', 'me', 'my', 'vs']);
/**
* Keep results that mention the query: every significant word when there are
* one or two, at least half of them otherwise (title, snippet or URL). An
* engine that answers with unrelated pages — Bing's web RSS does this to
* automated requests — then counts as "no results" and the chain moves on
* instead of handing the model junk. A query with no usable words (empty,
* all stopwords) keeps everything.
*/
export function relevant(query: string, results: SearchResult[]): SearchResult[] {
const terms = [...new Set(query.toLowerCase().split(/[^\p{L}\p{N}]+/u).filter((w) => w.length >= 3 && !STOP.has(w)))];
if (terms.length === 0) return results;
const need = terms.length <= 2 ? terms.length : Math.ceil(terms.length / 2);
return results.filter((r) => {
const hay = `${r.title} ${r.snippet} ${decodeURIComponent(r.url.replace(/%(?![0-9a-f]{2})/gi, '%25'))}`.toLowerCase();
return terms.filter((t) => hay.includes(t)).length >= need;
});
}
/** Results that point at the same page (http/https, www, trailing slash, fragments) count once. */
function dedupe(results: SearchResult[]): SearchResult[] {
const seen = new Set<string>();
return results.filter((r) => {
const key = r.url.replace(/^https?:\/\/(www\.)?/, '').replace(/#.*$/, '').replace(/\/$/, '').toLowerCase();
if (seen.has(key)) return false;
seen.add(key);
return true;
});
}
export interface SearchOutcome {
results: SearchResult[];
/** The engines whose results are in the answer, e.g. "Parallel + DuckDuckGo". */
engine: string;
/** What each engine tried did: "Parallel: timed out", "DuckDuckGo: 6 results". */
trail: string[];
}
/** Results needed before the search stops asking more engines. */
const ENOUGH = 4;
/** A short reason a source failed, for the trail. */
function why(err: unknown): string {
const m = (err as Error)?.message ?? String(err);
if (/429|challenge|rate-limited|captcha/i.test(m)) return `blocked (${m})`;
if (/abort/i.test(m)) return 'timed out';
return m;
}
export async function webSearch(query: string, opts: { kind?: 'web' | 'news'; max?: number } = {}): Promise<SearchOutcome> {
const max = Math.max(1, Math.min(opts.max ?? 8, 15));
// Every engine is free and keyless; no library. Web: Parallel's search,
// then DuckDuckGo's html page, then its lite page (DuckDuckGo blocks some
// automated requests, so each page gets one retry). News: Parallel's
// article links, then Bing News (direct links), Google News, Hacker News.
//
// How a search is judged: an engine's answer counts only for the results
// that mention the query (relevant()). Engines are asked in order and their
// relevant results combined until there are ENOUGH, so one thin or blocked
// source never decides the answer. Every engine tried is recorded in the
// trail, so "nothing found" and "everything was blocked" read differently.
const engines: Array<[string, (q: string, n: number) => Promise<SearchResult[]>]> = opts.kind === 'news'
? [['Parallel', parallelNews], ['Bing News', bingNews], ['Google News', googleNews], ['Hacker News', hackerNews]]
: [['Parallel', parallel], ['DuckDuckGo', ddgHtml], ['DuckDuckGo Lite', ddgLite]];
const want = Math.min(ENOUGH, max);
const trail: string[] = [];
const used: string[] = [];
let gathered: SearchResult[] = [];
for (const [name, run] of engines) {
// DuckDuckGo Lite is the same index as DuckDuckGo: skip it once that answered.
if (name === 'DuckDuckGo Lite' && used.includes('DuckDuckGo')) continue;
try {
const raw = await run(query, max);
const fresh = relevant(query, raw);
const before = gathered.length;
gathered = dedupe([...gathered, ...fresh]);
const added = gathered.length - before;
if (!added) {
trail.push(`${name}: ${raw.length ? (fresh.length ? 'only duplicates' : 'only unrelated results') : 'no results'}`);
continue;
}
used.push(name);
trail.push(`${name}: ${added} result${added === 1 ? '' : 's'}`);
if (gathered.length >= want) break;
} catch (err) {
trail.push(`${name}: ${why(err)}`);
}
}
if (!gathered.length) throw new Error(`no source found anything (${trail.join('; ')})`);
return { results: gathered.slice(0, max), engine: used.join(' + '), trail };
}
export function createWebSearchTool() {
return tool({
description:
'Search the web for current information: news, facts, prices, people, products, anything recent. ' +
'kind "news" for headlines and recent events (query can be empty for top stories). ' +
'Returns titles, links, snippets and dates; use fetch_url on a link to read the full page. ' +
'Use this instead of guessing about anything that may have changed.',
inputSchema: zodSchema(z.object({
query: z.string().max(300).describe('What to search for. For top news, use kind "news" and an empty query.'),
kind: z.enum(['web', 'news']).default('web').optional(),
max: z.number().int().min(1).max(15).default(8).optional(),
})),
execute: async ({ query, kind, max }) => {
try {
const { results, engine, trail } = await webSearch(query, { kind: kind ?? 'web', max: max ?? 8 });
return `${formatResults(results, engine, query, kind ?? 'web')}\n\nSearched: ${trail.join(' → ')}`;
} catch (err) {
return `Error: web search found nothing: ${(err as Error).message}. Tell the owner which sources were tried and why each failed; try a shorter or rephrased query once.`;
}
},
});
}