Open-source AI browser agent for Chrome and Firefox (monorepo) π§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
/**
* Safe fetch wrapper for the LM Studio plugin.
*
* Replaces the native `fetch` with a `node:http(s)` request that
* returns a standard `Response`, adding three security guarantees that
* are easy to get wrong with the bare API:
*
* 1. Every redirect target is re-validated through the URL guard,
* so a public URL can't 30x to `http://169.254.169.254/...` (cloud
* metadata) or any RFC1918 / loopback / link-local target. The
* bare `fetch(..., { redirect: "follow" })` follows redirects
* transparently without the caller getting a chance to intervene.
*
* 2. Hostnames are resolved by the socket's own `lookup` hook, which
* checks every returned address against private-IP ranges before
* handing them to the connection. This closes the bypass where
* `attacker.example A 127.0.0.1` passes a syntactic hostname check
* but resolves to localhost, and DNS rebinding: the address that
* was checked is the address that is dialled, so a TTL=0 record
* has no second lookup to answer differently. Global `fetch` can't
* do this without an undici Agent, which isn't importable here.
*
* 3. Response bodies are read with a hard byte cap so a malicious
* or accidental gigabyte response can't OOM the plugin. Chunks
* past the cap are dropped; the caller gets `truncated:true` and
* whatever fit.
*
* Cross-origin redirects also have `Authorization`, `Cookie`, and
* `Proxy-*` request headers stripped so credentials picked up from a
* page-injected `headers` arg can't leak to a different origin via an
* open-redirect chain.
*/
import { lookup as dnsLookup, type LookupAddress } from "node:dns";
import * as http from "node:http";
import * as https from "node:https";
import { isIP, type LookupFunction } from "node:net";
import { pipeline, Readable } from "node:stream";
import * as zlib from "node:zlib";
import {
assertSafeUrl,
isCloudMetadataHost,
isPrivateIpv4,
isPrivateIpv6,
PRIVATE_NETWORK_HINT,
type UrlGuardOptions,
} from "./urlGuard.js";
const DEFAULT_MAX_REDIRECTS = 5;
const DEFAULT_BODY_CAP_BYTES = 10 * 1024 * 1024; // 10 MB hard ceiling
/** Headers we never forward across origins. Lower-case for easy compare. */
const SENSITIVE_HEADERS = new Set([
"authorization",
"cookie",
"proxy-authorization",
"proxy-authenticate",
]);
/**
* Strip headers that shouldn't survive a cross-origin redirect.
* Same-origin redirects keep the full header set.
*/
function stripCrossOriginHeaders(
headers: Record<string, string>,
fromOrigin: string,
toOrigin: string,
): Record<string, string> {
if (fromOrigin === toOrigin) return headers;
const out: Record<string, string> = {};
for (const [k, v] of Object.entries(headers)) {
if (SENSITIVE_HEADERS.has(k.toLowerCase())) continue;
out[k] = v;
}
return out;
}
/**
* Reject if any resolved address is in a private range.
*
* Why we check ALL returned addresses rather than just the first one:
* Node's happy-eyeballs connect may try any of them, and it is
* defense-in-depth against accidentally-misconfigured dual-stack hosts.
*
* With allowPrivate the check still runs, because a hostname that
* resolves to a cloud metadata address stays blocked.
*/
function assertAddressesArePublic(host: string, addresses: LookupAddress[], opts: UrlGuardOptions = {}): void {
for (const { address, family } of addresses) {
if (!opts.allowPrivate) {
if (family === 4 && isPrivateIpv4(address)) {
throw new Error(
`Refusing ${host} β resolves to private IPv4 ${address}; ${PRIVATE_NETWORK_HINT}`,
);
}
if (family === 6 && isPrivateIpv6(address)) {
throw new Error(
`Refusing ${host} β resolves to private IPv6 ${address}; ${PRIVATE_NETWORK_HINT}`,
);
}
}
if (isCloudMetadataHost(address)) {
throw new Error(
`Refusing ${host} β resolves to cloud metadata endpoint ${address}, which stays blocked even with private network access on.`,
);
}
}
}
/**
* Literal-IP hosts never reach `guardedLookup`: Node connects to them
* directly without calling the socket's `lookup` hook. Classify them here
* on every hop so this layer does not depend on `assertSafeUrl` alone.
*/
function assertLiteralIpIsPublic(url: URL, opts: UrlGuardOptions = {}): void {
let host = url.hostname;
if (host.startsWith("[") && host.endsWith("]")) host = host.slice(1, -1);
const family = isIP(host);
if (family === 0) return;
if (!opts.allowPrivate && (family === 4 ? isPrivateIpv4(host) : isPrivateIpv6(host))) {
throw new Error(
`Refusing private IPv${family} ${host} β ${PRIVATE_NETWORK_HINT}`,
);
}
if (isCloudMetadataHost(host)) {
throw new Error(
`Refusing cloud metadata endpoint ${host} β blocked even with private network access on.`,
);
}
}
/**
* `lookup` hook for the request socket. Node calls it when connecting
* and dials only the addresses it hands back, so the check and the use
* are the same resolution β resolving first and then letting the HTTP
* client resolve the name again would let a TTL=0 record answer public
* to the check and private to the connection. Literal-IP hosts never
* reach a lookup; `assertLiteralIpIsPublic` covers them.
*/
function createGuardedLookup(opts: UrlGuardOptions): LookupFunction {
return (hostname, options, callback) => {
dnsLookup(hostname, { ...options, all: true }, (err, addresses) => {
if (err) {
callback(err, "");
return;
}
try {
if (addresses.length === 0) throw new Error(`No addresses found for ${hostname}`);
assertAddressesArePublic(hostname, addresses, opts);
} catch (e) {
callback(e as NodeJS.ErrnoException, "");
return;
}
if (options.all) callback(null, addresses);
else callback(null, addresses[0]!.address, addresses[0]!.family);
});
};
}
/** Statuses whose Response must have a null body (Fetch standard). */
const NULL_BODY_STATUSES = new Set([101, 103, 204, 205, 304]);
/** Lenient flushing, as fetch uses, so a truncated stream still yields what decoded. */
const ZLIB_OPTIONS = {
flush: zlib.constants.Z_SYNC_FLUSH,
finishFlush: zlib.constants.Z_SYNC_FLUSH,
};
const BROTLI_OPTIONS = {
flush: zlib.constants.BROTLI_OPERATION_FLUSH,
finishFlush: zlib.constants.BROTLI_OPERATION_FLUSH,
};
function abortReason(signal: AbortSignal): Error {
return signal.reason ??
Object.assign(new Error("This operation was aborted"), { name: "AbortError" });
}
interface HopInit {
method: string;
headers: Record<string, string>;
body: string | undefined;
signal?: AbortSignal;
}
/**
* Headers fetch adds on its own. Without them some servers refuse the
* request (no User-Agent) or skip compression. Caller values win.
*/
function withFetchDefaultHeaders(
headers: Record<string, string>,
body: string | undefined,
): Record<string, string> {
const out = { ...headers };
const present = new Set(Object.keys(out).map((k) => k.toLowerCase()));
const defaults: Array<[string, string]> = [
["accept", "*/*"],
["accept-encoding", "gzip, deflate, br"],
["user-agent", "node"],
];
if (body !== undefined) defaults.push(["content-type", "text/plain;charset=UTF-8"]);
for (const [k, v] of defaults) {
if (!present.has(k)) out[k] = v;
}
return out;
}
/**
* Undo Content-Encoding like fetch: gzip / deflate / br, last-applied
* first. An unknown coding leaves the body as it came.
*/
function decodeBody(incoming: http.IncomingMessage): Readable {
const codings = (incoming.headers["content-encoding"] || "")
.toLowerCase()
.split(",")
.map((c) => c.trim())
.filter(Boolean)
.reverse();
if (codings.length === 0) return incoming;
const decoders: NodeJS.ReadWriteStream[] = [];
for (const coding of codings) {
if (coding === "gzip" || coding === "x-gzip") decoders.push(zlib.createGunzip(ZLIB_OPTIONS));
else if (coding === "deflate") decoders.push(zlib.createInflate(ZLIB_OPTIONS));
else if (coding === "br") decoders.push(zlib.createBrotliDecompress(BROTLI_OPTIONS));
else return incoming;
}
// pipeline destroys every stage when one fails or the last is
// destroyed (e.g. readBodyCapped cancelling), which closes the socket.
return pipeline([incoming, ...decoders], () => {}) as unknown as Readable;
}
/**
* Send one request (no redirect following) over a connection whose
* address went through `guardedLookup`, and wrap the reply in a
* standard `Response` with `url` set, as fetch would return it.
*/
function pinnedFetch(url: URL, init: HopInit, opts: UrlGuardOptions): Promise<Response> {
return new Promise((resolve, reject) => {
const { signal } = init;
if (url.username || url.password) {
reject(new TypeError(
`Request cannot be constructed from a URL that includes credentials: ${url}`,
));
return;
}
if (signal?.aborted) {
reject(abortReason(signal));
return;
}
try { assertLiteralIpIsPublic(url, opts); } catch (e) { reject(e); return; }
let req: http.ClientRequest;
try {
req = (url.protocol === "https:" ? https : http).request(url, {
method: init.method,
headers: withFetchDefaultHeaders(init.headers, init.body),
// A fresh connection per request: a pooled keep-alive socket was
// resolved for whichever request opened it (possibly one with
// allowPrivate), so reusing it would skip this request's lookup.
agent: false,
lookup: createGuardedLookup(opts),
});
} catch (e) {
reject(e);
return;
}
const onAbort = () => req.destroy(abortReason(signal!));
signal?.addEventListener("abort", onAbort, { once: true });
// Kept for the request's lifetime: a late socket error with no
// listener would crash the plugin process.
req.on("error", (e) => {
signal?.removeEventListener("abort", onAbort);
reject(e);
});
req.once("response", (incoming) => {
signal?.removeEventListener("abort", onAbort);
try {
resolve(toFetchResponse(incoming, url, init.method, signal));
} catch (e) {
incoming.destroy();
reject(e);
}
});
req.end(init.body);
});
}
function toFetchResponse(
incoming: http.IncomingMessage,
url: URL,
method: string,
signal: AbortSignal | undefined,
): Response {
const status = incoming.statusCode ?? 0;
if (status < 200 || status > 599) {
throw new Error(`Unsupported HTTP status ${status} from ${url.origin}`);
}
const headers = new Headers();
for (let i = 0; i + 1 < incoming.rawHeaders.length; i += 2) {
headers.append(incoming.rawHeaders[i]!, incoming.rawHeaders[i + 1]!);
}
let body: ReadableStream<Uint8Array> | null = null;
if (method === "HEAD" || NULL_BODY_STATUSES.has(status)) {
incoming.on("error", () => {});
incoming.resume();
} else {
const decoded = decodeBody(incoming);
if (signal) {
// Aborting mid-body errors the stream with the abort reason, as fetch does.
const onAbort = () => decoded.destroy(abortReason(signal));
signal.addEventListener("abort", onAbort, { once: true });
decoded.once("close", () => signal.removeEventListener("abort", onAbort));
}
body = Readable.toWeb(decoded) as unknown as ReadableStream<Uint8Array>;
}
// Response's constructor rejects reason phrases fetch would accept;
// the phrase is cosmetic.
const statusMessage = incoming.statusMessage ?? "";
const statusText = /^[\t\x20-\x7e\x80-\xff]*$/.test(statusMessage) ? statusMessage : "";
const res = new Response(body, { status, statusText, headers });
// Only fetch() can set Response.url; callers report it as the final URL.
const finalUrl = new URL(url);
finalUrl.hash = "";
Object.defineProperty(res, "url", { value: finalUrl.toString() });
return res;
}
export interface SafeFetchInit {
method?: string;
headers?: Record<string, string>;
body?: string | undefined;
signal?: AbortSignal;
/**
* Allow private/loopback/link-local destinations. Default false. Set
* from the user's plugin setting, never from tool arguments. Cloud
* metadata endpoints stay blocked regardless.
*/
allowPrivate?: boolean;
/** Cap manual redirect hops. Default 5. */
maxRedirects?: number;
}
/**
* Fetch `initialUrl` with redirect-following that re-validates each
* hop through the URL guard. Returns a Response object whose body has
* NOT been consumed β caller is responsible for reading it (use
* `readBodyCapped` to apply the streaming size limit).
*
* Follows redirects manually so we can intercept each 3xx Location
* header; every hop connects through the same pinned, guarded lookup.
* On exhausting the hop budget, throws.
*/
export async function safeFetch(
initialUrl: string,
init: SafeFetchInit = {},
): Promise<Response> {
const allowPrivate = !!init.allowPrivate;
const maxRedirects = init.maxRedirects ?? DEFAULT_MAX_REDIRECTS;
// Initial URL goes through the sync structural check here; DNS
// resolution is checked when the connection is made (guardedLookup).
// Throws on a private hostname / IP / TLD.
let current: URL;
current = assertSafeUrl(initialUrl, { allowPrivate });
let originForCredentials = current.origin;
let headers = { ...(init.headers || {}) };
// Pass through method/body on the first hop only β 30x with body
// forwarding is a footgun (RFC 7231 says 301/302/303 should drop
// the body, 307/308 should preserve it; we conservatively drop on
// every redirect to keep the surface predictable).
let method = (init.method || "GET").toUpperCase();
let body: string | undefined = init.body;
// The same requests fetch() refused before node:http replaced it.
if (method === "CONNECT" || method === "TRACE" || method === "TRACK") {
throw new TypeError(`'${method}' HTTP method is unsupported.`);
}
if (body !== undefined && (method === "GET" || method === "HEAD")) {
throw new TypeError("Request with GET/HEAD method cannot have body.");
}
for (let hop = 0; hop <= maxRedirects; hop++) {
const res = await pinnedFetch(
current,
{ method, headers, body, ...(init.signal ? { signal: init.signal } : {}) },
{ allowPrivate },
);
// Not a redirect β return as-is. Caller reads the body.
if (res.status < 300 || res.status >= 400) return res;
const location = res.headers.get("location");
if (!location) return res; // 3xx without Location β let caller see it.
// Redirect bodies are never read; release the connection.
await res.body?.cancel().catch(() => {});
let next: URL;
try {
next = new URL(location, current);
} catch {
throw new Error(`Invalid redirect Location: ${location}`);
}
// Re-validate the redirect target: sync protocol / hostname-pattern
// checks here, DNS resolution when the next hop connects.
next = assertSafeUrl(next.toString(), { allowPrivate });
// Cross-origin redirect β drop sensitive headers.
headers = stripCrossOriginHeaders(headers, originForCredentials, next.origin);
// Drop the request body and downgrade method per RFC 7231 / 7538
// semantics for the common 301/302/303 case. 307/308 technically
// preserve, but body forwarding to a different origin is rarely
// what a fetch_url caller wants β we strip uniformly.
body = undefined;
if (res.status === 303) method = "GET";
current = next;
originForCredentials = next.origin;
}
throw new Error(
`Too many redirects (>${maxRedirects}) starting from ${initialUrl}`,
);
}
export interface CappedReadResult {
text: string;
/** True if the body was longer than `maxBytes` and we cut it short. */
truncated: boolean;
/** Bytes actually read (β€ maxBytes when truncated). */
bytesRead: number;
}
/**
* Read a Response body to a string with a hard byte cap. Decodes as
* UTF-8 (the default for text/html/json/etc.); binary callers should
* use res.arrayBuffer() and apply their own caps.
*
* On exceeding `maxBytes` we cancel the underlying reader so the
* server doesn't keep sending and we don't keep buffering.
*/
export async function readBodyCapped(
res: Response,
maxBytes: number = DEFAULT_BODY_CAP_BYTES,
): Promise<CappedReadResult> {
if (!res.body) return { text: "", truncated: false, bytesRead: 0 };
const reader = res.body.getReader();
const decoder = new TextDecoder("utf-8", { fatal: false });
let bytesRead = 0;
let truncated = false;
let out = "";
try {
while (true) {
const { done, value } = await reader.read();
if (done) break;
if (!value) continue;
const remaining = maxBytes - bytesRead;
if (value.length > remaining) {
// Take only what fits, decode the partial chunk with `stream:false`
// so the decoder flushes any pending lead bytes, then stop.
const slice = value.subarray(0, Math.max(0, remaining));
if (slice.length > 0) {
out += decoder.decode(slice, { stream: false });
bytesRead += slice.length;
}
truncated = true;
try { await reader.cancel(); } catch { /* ignore */ }
break;
}
out += decoder.decode(value, { stream: true });
bytesRead += value.length;
}
if (!truncated) {
// Final flush for any trailing multibyte sequence.
out += decoder.decode();
}
} catch (e) {
// Surface I/O errors to the caller β but only if we have nothing
// useful. If we have a partial body, return it with truncated:true.
if (out.length === 0) throw e;
truncated = true;
}
return { text: out, truncated, bytesRead };
}