⚡️⚡️⚡️ Self-hosted collaborative bookmark manager to collect, read, annotate, and fully preserve what matters, all in one place.
TypeScript bookmarkbookmark-managercollaborationnextjsreact-nativeread-it-laterself-hostedtypescript
implement safe fetch functionality across the app and the worker
daniel31x1323da4b8d79b3
diff --git a/.env.sample b/.env.sampleindex d607f617..fc71395a 100644--- a/.env.sample+++ b/.env.sample@@ -18,8 +18,6 @@ DISABLE_NEW_SSO_USERS=MAX_LINKS_PER_USER=ARCHIVE_TAKE_COUNT=BROWSER_TIMEOUT=-IGNORE_UNAUTHORIZED_CA=-IGNORE_HTTPS_ERRORS=IGNORE_URL_SIZE_LIMIT=NEXT_PUBLIC_DEMO=NEXT_PUBLIC_DEMO_USERNAME=@@ -43,6 +41,8 @@ TEXT_CONTENT_LIMIT=SEARCH_FILTER_LIMIT=INDEX_TAKE_COUNT=MEILI_TIMEOUT=+ALLOW_PRIVATE_NETWORK_ACCESS=+ALLOW_INSECURE_TLS=# AI SettingsNEXT_PUBLIC_OLLAMA_ENDPOINT_URL=diff --git a/apps/web/lib/api/controllers/links/postLink.ts b/apps/web/lib/api/controllers/links/postLink.tsindex 36d37e6a..89e22e5a 100644--- a/apps/web/lib/api/controllers/links/postLink.ts+++ b/apps/web/lib/api/controllers/links/postLink.ts@@ -7,6 +7,7 @@ import {PostLinkSchemaType,} from "@linkwarden/lib/schemaValidation";import { hasPassedLimit } from "@linkwarden/lib/verifyCapacity";+import { isUrlSafeForServerSideFetch } from "@linkwarden/lib/ssrf";export default async function postLink(body: PostLinkSchemaType,@@ -24,6 +25,9 @@ export default async function postLink(}const link = dataValidation.data;+ const shouldPreserveUrl = link.url+ ? await isUrlSafeForServerSideFetch(link.url)+ : false;const linkCollection = await setCollection({userId,@@ -71,12 +75,17 @@ export default async function postLink(};}- const { title = "", headers = new Headers() } = link.url- ? await fetchTitleAndHeaders(link.url)- : {};+ const { title = "", headers = new Headers() } =+ link.url && shouldPreserveUrl ? await fetchTitleAndHeaders(link.url) : {};const name =- link.name && link.name !== "" ? link.name : link.url ? title : "";+ link.name && link.name !== ""+ ? link.name+ : link.url+ ? shouldPreserveUrl && title+ ? title+ : link.url+ : "";const contentType = headers?.get("content-type");let linkType = "url";@@ -126,6 +135,17 @@ export default async function postLink(},})),},+ ...(!shouldPreserveUrl && link.url+ ? {+ lastPreserved: new Date().toISOString(),+ readable: "unavailable",+ image: "unavailable",+ monolith: "unavailable",+ pdf: "unavailable",+ preview: "unavailable",+ indexVersion: null,+ }+ : {}),},include: { tags: true, collection: true },});diff --git a/apps/web/lib/shared/fetchTitleAndHeaders.ts b/apps/web/lib/shared/fetchTitleAndHeaders.tsindex f133da1e..06503ccb 100644--- a/apps/web/lib/shared/fetchTitleAndHeaders.ts+++ b/apps/web/lib/shared/fetchTitleAndHeaders.ts@@ -1,47 +1,14 @@-import fetch from "node-fetch";-import https from "https";-import http from "http";-import { HttpsProxyAgent } from "https-proxy-agent";-import { SocksProxyAgent } from "socks-proxy-agent";+import { safeFetch } from "@linkwarden/lib/safeFetch";export default async function fetchTitleAndHeaders(url: string,content?: string) {- if (!url?.startsWith("http://") && !url?.startsWith("https://"))+ if (!content && !url?.startsWith("http://") && !url?.startsWith("https://"))return { title: "", headers: null };try {- const httpsAgent = url?.startsWith("http://")- ? new http.Agent({})- : new https.Agent({- rejectUnauthorized:- process.env.IGNORE_UNAUTHORIZED_CA === "true" ? false : true,- });-- // fetchOpts allows a proxy to be defined- let fetchOpts = {- agent: httpsAgent,- };-- if (process.env.PROXY) {- // parse proxy url- let proxy = new URL(process.env.PROXY);- // if authentication set, apply to proxy URL- if (process.env.PROXY_USERNAME) {- proxy.username = process.env.PROXY_USERNAME;- proxy.password = process.env.PROXY_PASSWORD || "";- }-- const proxyAgent = proxy.protocol.includes("http")- ? HttpsProxyAgent- : SocksProxyAgent;-- // add socks5/http/https proxy to fetchOpts- fetchOpts = { agent: new proxyAgent(proxy.toString()) };- }-- const responsePromise = fetch(url, fetchOpts);+ const responsePromise = content ? Promise.resolve(null) : safeFetch(url);const timeoutPromise = new Promise((_, reject) => {setTimeout(() => {reject(new Error("Fetch title timeout"));@@ -50,7 +17,7 @@ export default async function fetchTitleAndHeaders(const response = await Promise.race([responsePromise, timeoutPromise]);- if ((response as any)?.status) {+ if ((response as any)?.status || content) {let text: string;if (content) {@@ -59,7 +26,7 @@ export default async function fetchTitleAndHeaders(text = await (response as any).text();}- const headers = (response as Response)?.headers || null;+ const headers = (response as Response | null)?.headers || null;// regular expression to find the <title> taglet match = text.match(/<title.*>([^<]*)<\/title>/);diff --git a/apps/web/pages/api/v1/archives/index.ts b/apps/web/pages/api/v1/archives/index.tsindex e85688cb..7a67f5e3 100644--- a/apps/web/pages/api/v1/archives/index.ts+++ b/apps/web/pages/api/v1/archives/index.ts@@ -11,6 +11,7 @@ import isDemoMode from "@/lib/api/isDemoMode";import getSuffixFromFormat from "@/lib/shared/getSuffixFromFormat";import setCollection from "@/lib/api/setCollection";import fetchTitleAndHeaders from "@/lib/shared/fetchTitleAndHeaders";+import { isUrlSafeForServerSideFetch } from "@linkwarden/lib/ssrf";export const config = {api: {@@ -112,6 +113,9 @@ async function handlePost(req: NextApiRequest, res: NextApiResponse) {}const url = typeof fields.url === "string" ? fields.url : fields.url?.[0];+ const shouldFetchUrl = url+ ? await isUrlSafeForServerSideFetch(url)+ : false;// Validate input against Zod schemaconst dataValidation = UploadFileSchema.safeParse({@@ -153,7 +157,7 @@ async function handlePost(req: NextApiRequest, res: NextApiResponse) {const isImage = mimetype?.includes("image");const isHTML = mimetype === "text/html";- const { title = "" } = url+ const { title = "" } = url && (shouldFetchUrl || isHTML)? await fetchTitleAndHeaders(url,isHTML && !isPreview ? fileBuffer.toString("utf-8") : undefineddiff --git a/apps/web/pages/api/v1/rss/index.ts b/apps/web/pages/api/v1/rss/index.tsindex 59cc681e..ff186983 100644--- a/apps/web/pages/api/v1/rss/index.ts+++ b/apps/web/pages/api/v1/rss/index.ts@@ -5,6 +5,8 @@ import verifyUser from "@/lib/api/verifyUser";import { PostRssSubscriptionSchema } from "@linkwarden/lib/schemaValidation";import { NextApiRequest, NextApiResponse } from "next";import Parser from "rss-parser";+import { assertUrlIsSafeForServerSideFetch } from "@linkwarden/lib/ssrf";+import { safeFetch } from "@linkwarden/lib/safeFetch";export default async function handler(req: NextApiRequest,@@ -65,6 +67,14 @@ export default async function handler(const { name, url, collectionId, collectionName } = dataValidation.data;+ try {+ await assertUrlIsSafeForServerSideFetch(url);+ } catch (error: any) {+ return res.status(400).json({+ response: error?.message || "RSS URL is not allowed.",+ });+ }+const linkCollection = await setCollection({userId: user.id,collectionId: collectionId,@@ -104,8 +114,10 @@ export default async function handler(});const parser = new Parser();+ const xml = await safeFetch(response.url).then((res) => res.text());+ const feed = await parser.parseString(xml);- await rssHandler(response, parser);+ await rssHandler(response, feed);return res.status(200).json({ response });}diff --git a/apps/worker/lib/archiveHandler.ts b/apps/worker/lib/archiveHandler.tsindex a1b3e338..41861b1c 100644--- a/apps/worker/lib/archiveHandler.ts+++ b/apps/worker/lib/archiveHandler.ts@@ -15,6 +15,11 @@ import { LinkWithCollectionOwnerAndTags } from "@linkwarden/types/global";import { isArchivalTag } from "@linkwarden/lib/isArchivalTag";import { ArchivalSettings } from "@linkwarden/types/global";import { getDefaultContextOptions } from "./browser";+import {+ assertUrlIsSafeForServerSideFetch,+ UnsafeUrlError,+} from "@linkwarden/lib/ssrf";+import protectPageRequests from "./protectPageRequests";const BROWSER_TIMEOUT = Number(process.env.BROWSER_TIMEOUT) || 5;@@ -23,9 +28,22 @@ export default async function archiveHandler(browser: Browser) {const user = link.collection?.owner;+ let skipPreservation = process.env.DISABLE_PRESERVATION === "true";++ if (!skipPreservation && link.url) {+ try {+ await assertUrlIsSafeForServerSideFetch(link.url);+ } catch (error) {+ if (error instanceof UnsafeUrlError) {+ skipPreservation = true;+ } else {+ throw error;+ }+ }+ }if (- process.env.DISABLE_PRESERVATION === "true" ||+ skipPreservation ||(!link.url?.startsWith("http://") && !link.url?.startsWith("https://"))) {await prisma.link.update({@@ -72,6 +90,7 @@ export default async function archiveHandler(const contextOptions = getDefaultContextOptions();const context = await browser.newContext(contextOptions);+ await protectPageRequests(context);const page = await context.newPage();createFolder({ filePath: `archives/preview/${link.collectionId}` });diff --git a/apps/worker/lib/browser.ts b/apps/worker/lib/browser.tsindex 203b67f9..c5a40e81 100644--- a/apps/worker/lib/browser.ts+++ b/apps/worker/lib/browser.ts@@ -34,7 +34,9 @@ export function getBrowserOptions(): LaunchOptions {export function getDefaultContextOptions(): BrowserContextOptions {const base: BrowserContextOptions = {...devices["Desktop Chrome"],- ignoreHTTPSErrors: process.env.IGNORE_HTTPS_ERRORS === "true",+ ignoreHTTPSErrors:+ process.env.ALLOW_INSECURE_TLS === "true" ||+ process.env.IGNORE_HTTPS_ERRORS === "true",};if (process.env.PLAYWRIGHT_WS_URL) {diff --git a/apps/worker/lib/fetchHeaders.ts b/apps/worker/lib/fetchHeaders.tsindex 1f5f8563..2291ae12 100644--- a/apps/worker/lib/fetchHeaders.ts+++ b/apps/worker/lib/fetchHeaders.ts@@ -1,41 +1,10 @@-import fetch from "node-fetch";-import https from "https";-import http from "http";-import { HttpsProxyAgent } from 'https-proxy-agent';-import { SocksProxyAgent } from "socks-proxy-agent";+import { safeFetch } from "@linkwarden/lib/safeFetch";export default async function fetchHeaders(url: string) {if (process.env.IGNORE_URL_SIZE_LIMIT === "true") return null;try {- const httpsAgent = url.startsWith("http://")- ? new http.Agent({})- : new https.Agent({- rejectUnauthorized:- process.env.IGNORE_UNAUTHORIZED_CA === "true" ? false : true,- });-- let fetchOpts = {- method: "HEAD",- agent: httpsAgent,- };-- if (process.env.PROXY) {- let proxy = new URL(process.env.PROXY);- if (process.env.PROXY_USERNAME) {- proxy.username = process.env.PROXY_USERNAME;- proxy.password = process.env.PROXY_PASSWORD || "";- }-- const proxyAgent = proxy.protocol.includes("http") ? HttpsProxyAgent : SocksProxyAgent;-- fetchOpts = {- method: "HEAD",- agent: new proxyAgent(proxy.toString()),- };- }-- const responsePromise = fetch(url, fetchOpts);+ const responsePromise = safeFetch(url, { method: "HEAD" });const timeoutPromise = new Promise((_, reject) => {setTimeout(() => {diff --git a/apps/worker/lib/preservationScheme/handleArchivePreview.ts b/apps/worker/lib/preservationScheme/handleArchivePreview.tsindex 5e3a1bf6..126fd041 100644--- a/apps/worker/lib/preservationScheme/handleArchivePreview.ts+++ b/apps/worker/lib/preservationScheme/handleArchivePreview.ts@@ -3,6 +3,10 @@ import { Page } from "playwright";import { generatePreview } from "@linkwarden/lib/generatePreview";import { createFile } from "@linkwarden/filesystem";import { prisma } from "@linkwarden/prisma";+import {+ assertUrlIsSafeForServerSideFetch,+ UnsafeUrlError,+} from "@linkwarden/lib/ssrf";type LinksAndCollectionAndOwner = Link & {collection: Collection & {@@ -31,18 +35,25 @@ const handleArchivePreview = async (origin + (ogImageUrl.startsWith("/") ? ogImageUrl : "/" + ogImageUrl);}- const imageResponse = await page.goto(ogImageUrl);+ try {+ await assertUrlIsSafeForServerSideFetch(ogImageUrl);+ const imageResponse = await page.goto(ogImageUrl);- if (imageResponse && !link.preview?.startsWith("archive")) {- const buffer = await imageResponse.body();- previewGenerated = await generatePreview(- buffer,- link.collectionId,- link.id- );- }+ if (imageResponse && !link.preview?.startsWith("archive")) {+ const buffer = await imageResponse.body();+ previewGenerated = await generatePreview(+ buffer,+ link.collectionId,+ link.id+ );+ }- await page.goBack();+ await page.goBack();+ } catch (error) {+ if (!(error instanceof UnsafeUrlError)) {+ throw error;+ }+ }}if (!previewGenerated && !link.preview?.startsWith("archive")) {diff --git a/apps/worker/lib/preservationScheme/imageHandler.ts b/apps/worker/lib/preservationScheme/imageHandler.tsindex ed6f6162..78d2bf6b 100644--- a/apps/worker/lib/preservationScheme/imageHandler.ts+++ b/apps/worker/lib/preservationScheme/imageHandler.ts@@ -2,11 +2,10 @@ import { Link } from "@linkwarden/prisma/client";import { prisma } from "@linkwarden/prisma";import { createFile } from "@linkwarden/filesystem";import { generatePreview } from "@linkwarden/lib/generatePreview";+import { safeFetch } from "@linkwarden/lib/safeFetch";const imageHandler = async ({ url, id }: Link, extension: string) => {- const image = await fetch(url as string).then((res) => res.blob());-- const buffer = Buffer.from(await image.arrayBuffer());+ const buffer = await safeFetch(url as string).then((res) => res.buffer());if (Buffer.byteLength(buffer) >diff --git a/apps/worker/lib/preservationScheme/pdfHandler.ts b/apps/worker/lib/preservationScheme/pdfHandler.tsindex 01f6b830..ed766e33 100644--- a/apps/worker/lib/preservationScheme/pdfHandler.ts+++ b/apps/worker/lib/preservationScheme/pdfHandler.ts@@ -1,11 +1,10 @@import { Link } from "@linkwarden/prisma/client";import { prisma } from "@linkwarden/prisma";import { createFile } from "@linkwarden/filesystem";+import { safeFetch } from "@linkwarden/lib/safeFetch";const pdfHandler = async ({ url, id }: Link) => {- const pdf = await fetch(url as string).then((res) => res.blob());-- const buffer = Buffer.from(await pdf.arrayBuffer());+ const buffer = await safeFetch(url as string).then((res) => res.buffer());if (Buffer.byteLength(buffer) >diff --git a/apps/worker/lib/protectPageRequests.ts b/apps/worker/lib/protectPageRequests.tsnew file mode 100644index 00000000..65199998--- /dev/null+++ b/apps/worker/lib/protectPageRequests.ts@@ -0,0 +1,36 @@+import { BrowserContext, Route } from "playwright";+import {+ assertUrlIsSafeForServerSideFetch,+ UnsafeUrlError,+} from "@linkwarden/lib/ssrf";++function isNonNetworkUrl(url: string) {+ return (+ url.startsWith("about:") ||+ url.startsWith("blob:") ||+ url.startsWith("data:")+ );+}++export default async function protectPageRequests(context: BrowserContext) {+ await context.route("**/*", async (route: Route) => {+ const requestUrl = route.request().url();++ if (isNonNetworkUrl(requestUrl)) {+ await route.continue();+ return;+ }++ try {+ await assertUrlIsSafeForServerSideFetch(requestUrl);+ await route.continue();+ } catch (error) {+ if (error instanceof UnsafeUrlError) {+ await route.abort("blockedbyclient");+ return;+ }++ throw error;+ }+ });+}diff --git a/apps/worker/workers/rssPolling.ts b/apps/worker/workers/rssPolling.tsindex d01c442a..1aca03c7 100644--- a/apps/worker/workers/rssPolling.ts+++ b/apps/worker/workers/rssPolling.ts@@ -2,6 +2,8 @@ import { prisma } from "@linkwarden/prisma";import Parser from "rss-parser";import { delay } from "@linkwarden/lib/utils";import { rssHandler } from "@linkwarden/lib/rssHandler";+import { assertUrlIsSafeForServerSideFetch } from "@linkwarden/lib/ssrf";+import { safeFetch } from "@linkwarden/lib/safeFetch";const pollingIntervalInSeconds =(Number(process.env.NEXT_PUBLIC_RSS_POLLING_INTERVAL_MINUTES) || 60) * 60; // Default to one hour if not set@@ -15,7 +17,20 @@ export async function startRSSPolling() {await Promise.all(rssSubscriptions.map(async (rssSubscription) => {- rssHandler(rssSubscription, parser);+ try {+ await assertUrlIsSafeForServerSideFetch(rssSubscription.url);+ const xml = await safeFetch(rssSubscription.url).then((res) =>+ res.text()+ );+ const feed = await parser.parseString(xml);+ await rssHandler(rssSubscription, feed);+ } catch (error) {+ console.error(+ "\x1b[34m%s\x1b[0m",+ `Error processing RSS feed ${rssSubscription.url}:`,+ error+ );+ }}));await delay(pollingIntervalInSeconds);diff --git a/packages/lib/rssHandler.ts b/packages/lib/rssHandler.tsindex 00978184..b5b7b030 100644--- a/packages/lib/rssHandler.ts+++ b/packages/lib/rssHandler.ts@@ -1,17 +1,20 @@import { RssSubscription } from "@linkwarden/prisma/client";import { hasPassedLimit } from "./verifyCapacity";-import Parser from "rss-parser";import { prisma } from "@linkwarden/prisma";+import Parser from "rss-parser";+import { assertUrlIsSafeForServerSideFetch } from "./ssrf";export const rssHandler = async (rssSubscription: RssSubscription,- parser: Parser+ feed: Parser.Output<any>) => {try {- const feed = await parser.parseURL(rssSubscription.url);-+ const feedLastBuildDate = (feed as any).lastBuildDate as+ | string+ | Date+ | undefined;const feedLastPubDate =- feed.lastBuildDate ??+ feedLastBuildDate ??feed.items.reduce((acc, item) => {const itemPubDate = item.pubDate ? new Date(item.pubDate) : null;return itemPubDate && itemPubDate > acc ? itemPubDate : acc;@@ -53,6 +56,14 @@ export const rssHandler = async (// Create all links concurrentlyawait Promise.all(newItems.map(async (item) => {+ if (!item.link) return null;++ try {+ await assertUrlIsSafeForServerSideFetch(item.link);+ } catch {+ return null;+ }+return prisma.link.create({data: {name: item.title,diff --git a/packages/lib/safeFetch.ts b/packages/lib/safeFetch.tsnew file mode 100644index 00000000..b2610440--- /dev/null+++ b/packages/lib/safeFetch.ts@@ -0,0 +1,125 @@+import fetch, { RequestInit, Response } from "node-fetch";+import http from "http";+import https from "https";+import { HttpsProxyAgent } from "https-proxy-agent";+import { SocksProxyAgent } from "socks-proxy-agent";+import {+ assertUrlIsSafeForServerSideFetch,+ defaultHostnameLookup,+ resolveHostnameForServerSideFetch,+} from "@linkwarden/lib/ssrf";++type SafeFetchOptions = Omit<RequestInit, "agent" | "redirect"> & {+ maxRedirects?: number;+};++function createSafeLookup() {+ const lookup: any = (+ hostname: string,+ options: number | { family?: number | string; all?: boolean },+ callback: (+ error: NodeJS.ErrnoException | null,+ addressOrAddresses?:+ | string+ | ReadonlyArray<{ address: string; family: 4 | 6 }>,+ family?: number+ ) => void+ ) => {+ const normalizedOptions =+ typeof options === "number" ? { family: options } : options ?? {};+ const requestedFamily =+ normalizedOptions.family === "IPv4"+ ? 4+ : normalizedOptions.family === "IPv6"+ ? 6+ : normalizedOptions.family;++ void resolveHostnameForServerSideFetch(hostname, defaultHostnameLookup)+ .then((resolved) => {+ if (normalizedOptions.all) {+ callback(null, resolved);+ return;+ }++ const match =+ resolved.find(+ (entry) => !requestedFamily || entry.family === requestedFamily+ ) ?? resolved[0];++ callback(null, match.address, match.family);+ })+ .catch((error) => {+ callback(error as NodeJS.ErrnoException);+ });+ };++ return lookup;+}++function createAgent(target: URL) {+ if (process.env.PROXY) {+ const proxy = new URL(process.env.PROXY);++ if (process.env.PROXY_USERNAME) {+ proxy.username = process.env.PROXY_USERNAME;+ proxy.password = process.env.PROXY_PASSWORD || "";+ }++ const ProxyAgent = proxy.protocol.includes("http")+ ? HttpsProxyAgent+ : SocksProxyAgent;++ return new ProxyAgent(proxy.toString());+ }++ const lookup = createSafeLookup();++ if (target.protocol === "http:") {+ return new http.Agent({ lookup });+ }++ return new https.Agent({+ lookup,+ rejectUnauthorized:+ process.env.ALLOW_INSECURE_TLS === "true" ||+ process.env.IGNORE_UNAUTHORIZED_CA === "true"+ ? false+ : true,+ });+}++function isRedirectStatus(status: number) {+ return status >= 300 && status < 400;+}++export async function safeFetch(+ input: string,+ options: SafeFetchOptions = {}+): Promise<Response> {+ const { maxRedirects = 5, ...fetchOptions } = options;++ let currentUrl = input;++ for (let redirectCount = 0; redirectCount <= maxRedirects; redirectCount++) {+ const validatedUrl = await assertUrlIsSafeForServerSideFetch(currentUrl);+ const response = await fetch(validatedUrl.toString(), {+ ...fetchOptions,+ agent: createAgent(validatedUrl),+ redirect: "manual",+ });++ if (!isRedirectStatus(response.status)) {+ return response;+ }++ const location = response.headers.get("location");++ if (!location) {+ return response;+ }++ currentUrl = new URL(location, validatedUrl).toString();+ }++ throw new Error("Fetch redirected too many times.");+}diff --git a/packages/lib/ssrf.test.ts b/packages/lib/ssrf.test.tsnew file mode 100644index 00000000..e51f6469--- /dev/null+++ b/packages/lib/ssrf.test.ts@@ -0,0 +1,50 @@+import { describe, expect, it } from "vitest";+import {+ assertUrlIsSafeForServerSideFetch,+ isHostnameBlockedForServerSideFetch,+ isIpAddressBlockedForServerSideFetch,+ UnsafeUrlError,+} from "./ssrf";++describe("ssrf guard", () => {+ it("blocks localhost hostnames", () => {+ expect(isHostnameBlockedForServerSideFetch("localhost")).toBe(true);+ expect(isHostnameBlockedForServerSideFetch("subdomain.localhost")).toBe(+ true+ );+ });++ it("blocks single-label internal hostnames", () => {+ expect(isHostnameBlockedForServerSideFetch("meilisearch")).toBe(true);+ expect(isHostnameBlockedForServerSideFetch("postgres")).toBe(true);+ });++ it("blocks private and loopback IP literals", () => {+ expect(isIpAddressBlockedForServerSideFetch("127.0.0.1")).toBe(true);+ expect(isIpAddressBlockedForServerSideFetch("169.254.169.254")).toBe(true);+ expect(isIpAddressBlockedForServerSideFetch("::1")).toBe(true);+ expect(isIpAddressBlockedForServerSideFetch("fc00::1")).toBe(true);+ });++ it("allows public hostnames that resolve to public IPs", async () => {+ await expect(+ assertUrlIsSafeForServerSideFetch("https://example.com", {+ lookup: async () => [{ address: "93.184.216.34", family: 4 }],+ })+ ).resolves.toBeInstanceOf(URL);+ });++ it("blocks hostnames that resolve to private IPs", async () => {+ await expect(+ assertUrlIsSafeForServerSideFetch("https://example.com", {+ lookup: async () => [{ address: "10.0.0.8", family: 4 }],+ })+ ).rejects.toBeInstanceOf(UnsafeUrlError);+ });++ it("blocks IPv6 loopback literals", async () => {+ await expect(+ assertUrlIsSafeForServerSideFetch("http://[::1]/")+ ).rejects.toBeInstanceOf(UnsafeUrlError);+ });+});diff --git a/packages/lib/ssrf.ts b/packages/lib/ssrf.tsnew file mode 100644index 00000000..a27013b3--- /dev/null+++ b/packages/lib/ssrf.ts@@ -0,0 +1,348 @@+import { lookup as dnsLookup } from "node:dns/promises";+import { isIP } from "node:net";++export class UnsafeUrlError extends Error {+ code = "ERR_UNSAFE_URL";++ constructor(message: string) {+ super(message);+ this.name = "UnsafeUrlError";+ }+}++export type ResolvedAddress = {+ address: string;+ family: 4 | 6;+};++export type HostnameLookup = (+ hostname: string+) => Promise<ReadonlyArray<ResolvedAddress>>;++const BLOCKED_HOSTNAMES = new Set([+ "localhost",+ "localhost.localdomain",+ "ip6-localhost",+ "ip6-loopback",+ "broadcasthost",+]);++const BLOCKED_HOSTNAME_SUFFIXES = [+ ".localhost",+ ".local",+ ".localdomain",+ ".internal",+];++const IPV4_BLOCKED_CIDRS = [+ ["0.0.0.0", 8],+ ["10.0.0.0", 8],+ ["100.64.0.0", 10],+ ["127.0.0.0", 8],+ ["169.254.0.0", 16],+ ["172.16.0.0", 12],+ ["192.0.0.0", 24],+ ["192.0.2.0", 24],+ ["192.168.0.0", 16],+ ["198.18.0.0", 15],+ ["198.51.100.0", 24],+ ["203.0.113.0", 24],+ ["224.0.0.0", 4],+ ["240.0.0.0", 4],+] as const;++const IPV6_BLOCKED_CIDRS = [+ ["::", 128],+ ["::1", 128],+ ["fc00::", 7],+ ["fe80::", 10],+ ["ff00::", 8],+ ["2001:db8::", 32],+] as const;++const IPV4_BLOCKED_RANGES = IPV4_BLOCKED_CIDRS.map(([address, prefix]) => {+ const parsed = parseIPv4(address);++ if (parsed === null) {+ throw new Error(`Invalid blocked IPv4 CIDR base: ${address}/${prefix}`);+ }++ return {+ network: parsed,+ mask: ipv4Mask(prefix),+ };+});++const IPV6_BLOCKED_RANGES = IPV6_BLOCKED_CIDRS.map(([address, prefix]) => {+ const parsed = parseIPv6(address);++ if (parsed === null) {+ throw new Error(`Invalid blocked IPv6 CIDR base: ${address}/${prefix}`);+ }++ return {+ network: parsed,+ prefix,+ };+});++function ipv4Mask(prefixLength: number) {+ if (prefixLength === 0) return 0;+ return Math.floor(2 ** 32 - 2 ** (32 - prefixLength));+}++function normalizeHostname(hostname: string) {+ return hostname.trim().toLowerCase().replace(/\.+$/, "");+}++function parseIPv4(address: string) {+ const octets = address.split(".");++ if (octets.length !== 4) return null;++ let value = 0;++ for (const octet of octets) {+ if (!/^\d{1,3}$/.test(octet)) return null;++ const parsed = Number(octet);+ if (!Number.isInteger(parsed) || parsed < 0 || parsed > 255) return null;++ value = value * 256 + parsed;+ }++ return value;+}++function ipv4TailToHexSegments(address: string) {+ const parsed = parseIPv4(address);+ if (parsed === null) return null;++ const high = Math.floor(parsed / 65536)+ .toString(16)+ .padStart(4, "0");+ const low = (parsed % 65536).toString(16).padStart(4, "0");++ return [high, low];+}++function parseIPv6(address: string) {+ const normalized = address.toLowerCase().split("%")[0];++ if (!normalized) return null;++ const hasDoubleColon = normalized.includes("::");++ if (hasDoubleColon && normalized.indexOf("::") !== normalized.lastIndexOf("::"))+ return null;++ const [leftRaw, rightRaw = ""] = hasDoubleColon+ ? normalized.split("::")+ : [normalized, ""];++ const expandSide = (side: string) => {+ if (!side) return [] as string[];++ return side.split(":").flatMap((segment) => {+ if (!segment) return [segment];+ if (segment.includes(".")) {+ const converted = ipv4TailToHexSegments(segment);+ return converted ?? [segment];+ }+ return [segment];+ });+ };++ const left = expandSide(leftRaw);+ const right = expandSide(rightRaw);++ const segments = hasDoubleColon+ ? (() => {+ const zeroCount = 8 - (left.length + right.length);+ if (zeroCount < 1) return null;++ return [+ ...left,+ ...Array.from({ length: zeroCount }, () => "0"),+ ...right,+ ];+ })()+ : left;++ if (!segments || segments.length !== 8) return null;++ const parsedSegments: number[] = [];++ for (const segment of segments) {+ if (!/^[0-9a-f]{1,4}$/i.test(segment)) return null;+ parsedSegments.push(parseInt(segment, 16));+ }++ return parsedSegments;+}++function extractIPv4FromMappedIPv6(address: string) {+ const normalized = address.toLowerCase();++ if (!normalized.includes(".")) return null;++ const ipv4Candidate = normalized.slice(normalized.lastIndexOf(":") + 1);+ return parseIPv4(ipv4Candidate) === null ? null : ipv4Candidate;+}++export function isHostnameBlockedForServerSideFetch(hostname: string) {+ const normalized = normalizeHostname(hostname);++ if (!normalized) return true;+ if (BLOCKED_HOSTNAMES.has(normalized)) return true;+ if (BLOCKED_HOSTNAME_SUFFIXES.some((suffix) => normalized.endsWith(suffix)))+ return true;++ return !normalized.includes(".") && isIP(normalized) === 0;+}++export function isIpAddressBlockedForServerSideFetch(address: string) {+ const ipv4 =+ parseIPv4(address) ?? (() => {+ const mapped = extractIPv4FromMappedIPv6(address);+ return mapped ? parseIPv4(mapped) : null;+ })();++ if (ipv4 !== null) {+ return IPV4_BLOCKED_RANGES.some(+ ({ network, mask }) => (ipv4 & mask) === (network & mask)+ );+ }++ const ipv6 = parseIPv6(address);+ if (ipv6 === null) return true;++ return IPV6_BLOCKED_RANGES.some(({ network, prefix }) =>+ ipv6MatchesPrefix(ipv6, network, prefix)+ );+}++function ipv6MatchesPrefix(+ address: number[],+ network: number[],+ prefixLength: number+) {+ const fullSegments = Math.floor(prefixLength / 16);+ const remainingBits = prefixLength % 16;++ for (let index = 0; index < fullSegments; index++) {+ if (address[index] !== network[index]) return false;+ }++ if (remainingBits === 0) return true;++ const mask = (0xffff << (16 - remainingBits)) & 0xffff;+ return (+ (address[fullSegments] & mask) === (network[fullSegments] & mask)+ );+}++export const defaultHostnameLookup: HostnameLookup = async (hostname) => {+ const resolved = await dnsLookup(hostname, { all: true, verbatim: true });++ return resolved+ .filter(+ (entry): entry is { address: string; family: 4 | 6 } =>+ entry.family === 4 || entry.family === 6+ )+ .map((entry) => ({+ address: entry.address,+ family: entry.family,+ }));+};++export async function resolveHostnameForServerSideFetch(+ hostname: string,+ lookup: HostnameLookup = defaultHostnameLookup+) {+ const normalized = normalizeHostname(hostname);++ if (isHostnameBlockedForServerSideFetch(normalized)) {+ throw new UnsafeUrlError("URL resolves to a blocked internal hostname.");+ }++ const ipFamily = isIP(normalized);++ if (ipFamily === 4 || ipFamily === 6) {+ if (isIpAddressBlockedForServerSideFetch(normalized)) {+ throw new UnsafeUrlError("URL resolves to a blocked internal IP address.");+ }++ return [{ address: normalized, family: ipFamily }] as const;+ }++ let addresses: ReadonlyArray<ResolvedAddress>;++ try {+ addresses = await lookup(normalized);+ } catch (error: any) {+ throw new UnsafeUrlError(+ error?.code === "ENOTFOUND" || error?.code === "EAI_AGAIN"+ ? "URL hostname could not be resolved."+ : "URL hostname lookup failed."+ );+ }++ if (!addresses.length) {+ throw new UnsafeUrlError("URL hostname did not resolve to a public IP.");+ }++ const blockedAddress = addresses.find((entry) =>+ isIpAddressBlockedForServerSideFetch(entry.address)+ );++ if (blockedAddress) {+ throw new UnsafeUrlError("URL resolves to a blocked internal IP address.");+ }++ return addresses;+}++export async function assertUrlIsSafeForServerSideFetch(+ value: string,+ options?: {+ lookup?: HostnameLookup;+ allowPrivateNetworkAccess?: boolean;+ }+) {+ const url = new URL(value);++ if (url.protocol !== "http:" && url.protocol !== "https:") {+ throw new UnsafeUrlError("Only http:// and https:// URLs can be archived.");+ }++ if (+ options?.allowPrivateNetworkAccess ||+ process.env.ALLOW_PRIVATE_NETWORK_ACCESS === "true"+ ) {+ return url;+ }++ await resolveHostnameForServerSideFetch(url.hostname, options?.lookup);++ return url;+}++export async function isUrlSafeForServerSideFetch(+ value: string,+ options?: {+ lookup?: HostnameLookup;+ allowPrivateNetworkAccess?: boolean;+ }+) {+ try {+ await assertUrlIsSafeForServerSideFetch(value, options);+ return true;+ } catch (error) {+ if (error instanceof UnsafeUrlError) {+ return false;+ }++ throw error;+ }+}