Skip to content

A one-of-a-kind resume builder that keeps your privacy in mind. Completely secure, customizable, portable, open-source and free forever. Try it out today!

10.9 KB
Raw Blame
import type {
	ProviderRequest,
	ReadPageResult,
	WebAccessConnection,
	WebAccessContext,
	WebAccessFailure,
} from "./contracts";
import { env } from "@reactive-resume/env/server";
import { rateLimitConfig } from "@reactive-resume/utils/rate-limit";
import { abortable } from "@reactive-resume/utils/url-security.node";
import { createRateLimiter } from "../../redis";
import { assertPublicTarget, htmlToText, readBuiltinPage } from "./builtin";
import { MAX_SEARCH_QUERY_CHARS, MAX_SEARCH_RESULTS, WEB_ACCESS_DEADLINE_MS, WebAccessError } from "./contracts";
import { webAccessService } from "./credentials";
import { readExa, searchExa } from "./exa";
import { readFirecrawl, searchFirecrawl } from "./firecrawl";
import { searchSearxng } from "./searxng";
import { readTavily, searchTavily } from "./tavily";
import { normalizeSearch, pageContent, safeFailure } from "./transport";

const limiter = createRateLimiter("web-access", rateLimitConfig.orpc.aiRequest);

async function consume(userId: string, signal: AbortSignal) {
	signal.throwIfAborted();
	if (process.env.NODE_ENV !== "production" || env.FLAG_DISABLE_API_RATE_LIMIT) return;
	const { success } = await abortable(limiter.limit(userId), signal);
	if (!success) throw new WebAccessError("rate-limit");
}

const deadline = (signal?: AbortSignal) =>
	AbortSignal.any([...(signal ? [signal] : []), AbortSignal.timeout(WEB_ACCESS_DEADLINE_MS)]);

async function externalSearch(query: string, options: ProviderRequest) {
	try {
		switch (options.connection.provider) {
			case "firecrawl":
				return await searchFirecrawl(query, options);
			case "tavily":
				return await searchTavily(query, options);
			case "exa":
				return await searchExa(query, options);
		}
	} catch (error) {
		throw safeFailure(error, options.signal);
	}
}

async function externalRead(url: string, options: ProviderRequest) {
	try {
		switch (options.connection.provider) {
			case "firecrawl":
				return await readFirecrawl(url, options);
			case "tavily":
				return await readTavily(url, options);
			case "exa":
				return await readExa(url, options);
		}
	} catch (error) {
		throw safeFailure(error, options.signal);
	}
}

function log(operation: "search" | "read", provider: string, started: number, outcome: string, fallback?: string) {
	console.info("[web-access]", {
		provider,
		operation,
		durationMs: Date.now() - started,
		outcome,
		...(fallback ? { fallback } : {}),
	});
}

function searchInput(query: string) {
	const input = query.trim();
	if (!input || input.length > MAX_SEARCH_QUERY_CHARS) throw new WebAccessError("malformed");
	return input;
}

/** Deduplicates by URL and takes results round-robin in region order, so each region that returned anything shows. */
function interleave<T extends { url: string }>(lists: T[][]) {
	const seen = new Set<string>();
	const merged: T[] = [];
	const longest = Math.max(...lists.map((list) => list.length));
	for (let index = 0; index < longest; index++) {
		for (const list of lists) {
			const item = list[index];
			if (!item || seen.has(item.url)) continue;
			seen.add(item.url);
			merged.push(item);
		}
	}
	return merged.slice(0, MAX_SEARCH_RESULTS);
}

type SearchResults = Awaited<ReturnType<typeof externalSearch>>;

/** Where searches go: the operator's SearXNG when set, otherwise the web connection's provider. */
function searchBackend(context: WebAccessContext, signal: AbortSignal) {
	const baseUrl = env.WEB_SEARCH_SEARXNG_URL;
	if (baseUrl)
		return {
			name: "searxng",
			search: (query: string, region?: string): Promise<SearchResults> =>
				searchSearxng(query, { baseUrl, region, signal }),
		};
	const connection = context.connection;
	if (!connection) return null;
	return {
		name: connection.provider,
		search: (query: string, region?: string) =>
			externalSearch(query, region ? { connection, signal, region } : { connection, signal }),
	};
}

/**
 * One rate-limited search. Each region gets its own search call, with `queryFor(region)` as its query when given,
 * in parallel under one deadline. A failed region drops out unless every region fails.
 */
export async function searchWeb(query: string, context: WebAccessContext, queryFor?: (region: string) => string) {
	const signal = deadline(context.signal);
	const started = Date.now();
	const backend = searchBackend(context, signal);
	const provider = backend?.name ?? "builtin";
	let usedFallback = false;
	let failureRecorded: Promise<void> | undefined;
	const rememberFailure = async (error: unknown) => {
		if (
			!signal.aborted &&
			provider !== "searxng" &&
			error instanceof WebAccessError &&
			(error.reason === "auth" || error.reason === "quota")
		) {
			const result = { success: false, error: error.reason };
			await (failureRecorded ??= webAccessService
				.recordTest(context.userId, context.connection, { search: result, read: result })
				.catch(() => undefined));
		}
	};
	try {
		await consume(context.userId, signal);
		if (!backend) throw new WebAccessError("unavailable");
		const search = async (query: string, region?: string) => {
			try {
				return await backend.search(query, region);
			} catch (error) {
				if (
					signal.aborted ||
					!context.fallbackSearch ||
					!(error instanceof WebAccessError) ||
					(error.reason !== "auth" && error.reason !== "quota")
				)
					throw error;
				await rememberFailure(error);
				signal.throwIfAborted();
				usedFallback = true;
				return normalizeSearch(await context.fallbackSearch(query, region, signal));
			}
		};
		const input = searchInput(query);
		const searches = (context.regions ?? []).map((region) => ({
			region,
			query: queryFor ? searchInput(queryFor(region)) : input,
		}));
		let results: SearchResults;
		let partial: string | undefined;
		if (searches.length === 0) results = await search(input);
		else {
			const settled = await Promise.allSettled(searches.map((entry) => search(entry.query, entry.region)));
			const lists = settled.flatMap((outcome) => (outcome.status === "fulfilled" ? [outcome.value] : []));
			if (lists.length === 0) throw (settled[0] as PromiseRejectedResult).reason;
			const failed = settled.length - lists.length;
			if (failed) partial = `${failed}/${settled.length}-regions-failed`;
			results = interleave(lists);
		}
		signal.throwIfAborted();
		log("search", usedFallback ? `${provider}+ai-native` : provider, started, "success", partial);
		return results;
	} catch (error) {
		await rememberFailure(error);
		log(
			"search",
			usedFallback ? `${provider}+ai-native` : provider,
			started,
			signal.aborted ? "aborted" : error instanceof WebAccessError ? error.reason : "unreachable",
		);
		throw safeFailure(error, signal);
	}
}

/** One deadline covers DNS, the selected provider, and the direct fallback. Cancellation never falls back. */
export async function readPage(input: string, context: WebAccessContext): Promise<ReadPageResult> {
	const signal = deadline(context.signal);
	const started = Date.now();
	const provider = context.connection?.provider ?? "builtin";
	let fallback: WebAccessFailure | undefined;
	try {
		await consume(context.userId, signal);
		const url = await assertPublicTarget(input, signal);
		if (context.connection) {
			// Reserve five seconds of the same overall budget for the direct reader.
			const externalBudget = Math.max(1, WEB_ACCESS_DEADLINE_MS - (Date.now() - started) - 5_000);
			const externalSignal = AbortSignal.any([signal, AbortSignal.timeout(externalBudget)]);
			try {
				// Remote services must independently protect their own redirects and DNS lookups.
				const page = await externalRead(url.toString(), {
					connection: context.connection,
					signal: externalSignal,
				});
				if (page.resolvedUrl) await assertPublicTarget(page.resolvedUrl, externalSignal);
				const content = pageContent(page.content);
				externalSignal.throwIfAborted();
				log("read", provider, started, "success");
				return {
					...page,
					...content,
					requestedUrl: url.toString(),
					retrievedAt: new Date().toISOString(),
					method: context.connection.provider,
				};
			} catch (error) {
				signal.throwIfAborted();
				const failure = safeFailure(error, externalSignal);
				if (failure.reason === "unsafe-url") throw failure;
				fallback = failure.reason;
				if (failure.reason === "auth" || failure.reason === "quota") {
					const result = { success: false, error: failure.reason };
					await webAccessService
						.recordTest(context.userId, context.connection, { search: result, read: result })
						.catch(() => undefined);
				}
			}
		}
		const page = await readBuiltinPage(url.toString(), signal);
		const content = pageContent(htmlToText(page.html));
		signal.throwIfAborted();
		log("read", provider, started, "success", fallback ? `${fallback}:builtin` : undefined);
		return {
			...page,
			...content,
			requestedUrl: url.toString(),
			retrievedAt: new Date().toISOString(),
			method: "builtin",
			format: "text",
			...(fallback ? { fallbackReason: fallback } : {}),
		};
	} catch (error) {
		log(
			"read",
			provider,
			started,
			signal.aborted ? "aborted" : error instanceof WebAccessError ? error.reason : "unreachable",
			fallback ? `${fallback}:failed` : undefined,
		);
		throw safeFailure(error, signal);
	}
}

/**
 * Explicit settings test. Provider operations are independent and never use the built-in fallback. Search is tested
 * against SearXNG when the server sends searches there, and the log names which service each operation tested.
 * Without a connection, reading is not tested and reports `unavailable`: only the built-in reader serves it.
 */
export async function probeWebAccess(
	connection: WebAccessConnection | null,
	context: { userId: string; signal?: AbortSignal | undefined; region?: string | undefined },
) {
	const signal = deadline(context.signal);
	await consume(context.userId, signal);
	const searxng = env.WEB_SEARCH_SEARXNG_URL;
	const probe = async (operation: "search" | "read") => {
		const started = Date.now();
		const provider = operation === "search" && searxng ? "searxng" : (connection?.provider ?? "builtin");
		try {
			if (operation === "search" && searxng)
				await searchSearxng("example domain", { baseUrl: searxng, region: context.region, signal });
			else if (!connection) throw new WebAccessError("unavailable");
			else if (operation === "search")
				await externalSearch("example domain", { connection, signal, region: context.region });
			else {
				const url = await assertPublicTarget("https://example.com/", signal);
				const page = await externalRead(url.toString(), { connection, signal });
				pageContent(page.content);
				if (page.resolvedUrl) await assertPublicTarget(page.resolvedUrl, signal);
			}
			signal.throwIfAborted();
			log(operation, provider, started, "success");
			return { success: true };
		} catch (error) {
			const failure = safeFailure(error, signal);
			log(operation, provider, started, failure.reason);
			return { success: false, error: failure.reason };
		}
	};
	const [search, read] = await Promise.all([probe("search"), probe("read")]);
	return { search, read };
}