Skip to content

⚡️⚡️⚡️ Self-hosted collaborative bookmark manager to collect, read, annotate, and fully preserve what matters, all in one place.

implement safe fetch functionality across the app and the worker

daniel31x1323da4b8d79b3
diff --git a/.env.sample b/.env.sample
index d607f617..fc71395a 100644
--- a/.env.sample
+++ b/.env.sample
@@ -18,8 +18,6 @@ DISABLE_NEW_SSO_USERS=
MAX_LINKS_PER_USER=
ARCHIVE_TAKE_COUNT=
BROWSER_TIMEOUT=
-IGNORE_UNAUTHORIZED_CA=
-IGNORE_HTTPS_ERRORS=
IGNORE_URL_SIZE_LIMIT=
NEXT_PUBLIC_DEMO=
NEXT_PUBLIC_DEMO_USERNAME=
@@ -43,6 +41,8 @@ TEXT_CONTENT_LIMIT=
SEARCH_FILTER_LIMIT=
INDEX_TAKE_COUNT=
MEILI_TIMEOUT=
+ALLOW_PRIVATE_NETWORK_ACCESS=
+ALLOW_INSECURE_TLS=
# AI Settings
NEXT_PUBLIC_OLLAMA_ENDPOINT_URL=
diff --git a/apps/web/lib/api/controllers/links/postLink.ts b/apps/web/lib/api/controllers/links/postLink.ts
index 36d37e6a..89e22e5a 100644
--- a/apps/web/lib/api/controllers/links/postLink.ts
+++ b/apps/web/lib/api/controllers/links/postLink.ts
@@ -7,6 +7,7 @@ import {
PostLinkSchemaType,
} from "@linkwarden/lib/schemaValidation";
import { hasPassedLimit } from "@linkwarden/lib/verifyCapacity";
+import { isUrlSafeForServerSideFetch } from "@linkwarden/lib/ssrf";
export default async function postLink(
body: PostLinkSchemaType,
@@ -24,6 +25,9 @@ export default async function postLink(
}
const link = dataValidation.data;
+ const shouldPreserveUrl = link.url
+ ? await isUrlSafeForServerSideFetch(link.url)
+ : false;
const linkCollection = await setCollection({
userId,
@@ -71,12 +75,17 @@ export default async function postLink(
};
}
- const { title = "", headers = new Headers() } = link.url
- ? await fetchTitleAndHeaders(link.url)
- : {};
+ const { title = "", headers = new Headers() } =
+ link.url && shouldPreserveUrl ? await fetchTitleAndHeaders(link.url) : {};
const name =
- link.name && link.name !== "" ? link.name : link.url ? title : "";
+ link.name && link.name !== ""
+ ? link.name
+ : link.url
+ ? shouldPreserveUrl && title
+ ? title
+ : link.url
+ : "";
const contentType = headers?.get("content-type");
let linkType = "url";
@@ -126,6 +135,17 @@ export default async function postLink(
},
})),
},
+ ...(!shouldPreserveUrl && link.url
+ ? {
+ lastPreserved: new Date().toISOString(),
+ readable: "unavailable",
+ image: "unavailable",
+ monolith: "unavailable",
+ pdf: "unavailable",
+ preview: "unavailable",
+ indexVersion: null,
+ }
+ : {}),
},
include: { tags: true, collection: true },
});
diff --git a/apps/web/lib/shared/fetchTitleAndHeaders.ts b/apps/web/lib/shared/fetchTitleAndHeaders.ts
index f133da1e..06503ccb 100644
--- a/apps/web/lib/shared/fetchTitleAndHeaders.ts
+++ b/apps/web/lib/shared/fetchTitleAndHeaders.ts
@@ -1,47 +1,14 @@
-import fetch from "node-fetch";
-import https from "https";
-import http from "http";
-import { HttpsProxyAgent } from "https-proxy-agent";
-import { SocksProxyAgent } from "socks-proxy-agent";
+import { safeFetch } from "@linkwarden/lib/safeFetch";
export default async function fetchTitleAndHeaders(
url: string,
content?: string
) {
- if (!url?.startsWith("http://") && !url?.startsWith("https://"))
+ if (!content && !url?.startsWith("http://") && !url?.startsWith("https://"))
return { title: "", headers: null };
try {
- const httpsAgent = url?.startsWith("http://")
- ? new http.Agent({})
- : new https.Agent({
- rejectUnauthorized:
- process.env.IGNORE_UNAUTHORIZED_CA === "true" ? false : true,
- });
-
- // fetchOpts allows a proxy to be defined
- let fetchOpts = {
- agent: httpsAgent,
- };
-
- if (process.env.PROXY) {
- // parse proxy url
- let proxy = new URL(process.env.PROXY);
- // if authentication set, apply to proxy URL
- if (process.env.PROXY_USERNAME) {
- proxy.username = process.env.PROXY_USERNAME;
- proxy.password = process.env.PROXY_PASSWORD || "";
- }
-
- const proxyAgent = proxy.protocol.includes("http")
- ? HttpsProxyAgent
- : SocksProxyAgent;
-
- // add socks5/http/https proxy to fetchOpts
- fetchOpts = { agent: new proxyAgent(proxy.toString()) };
- }
-
- const responsePromise = fetch(url, fetchOpts);
+ const responsePromise = content ? Promise.resolve(null) : safeFetch(url);
const timeoutPromise = new Promise((_, reject) => {
setTimeout(() => {
reject(new Error("Fetch title timeout"));
@@ -50,7 +17,7 @@ export default async function fetchTitleAndHeaders(
const response = await Promise.race([responsePromise, timeoutPromise]);
- if ((response as any)?.status) {
+ if ((response as any)?.status || content) {
let text: string;
if (content) {
@@ -59,7 +26,7 @@ export default async function fetchTitleAndHeaders(
text = await (response as any).text();
}
- const headers = (response as Response)?.headers || null;
+ const headers = (response as Response | null)?.headers || null;
// regular expression to find the <title> tag
let match = text.match(/<title.*>([^<]*)<\/title>/);
diff --git a/apps/web/pages/api/v1/archives/index.ts b/apps/web/pages/api/v1/archives/index.ts
index e85688cb..7a67f5e3 100644
--- a/apps/web/pages/api/v1/archives/index.ts
+++ b/apps/web/pages/api/v1/archives/index.ts
@@ -11,6 +11,7 @@ import isDemoMode from "@/lib/api/isDemoMode";
import getSuffixFromFormat from "@/lib/shared/getSuffixFromFormat";
import setCollection from "@/lib/api/setCollection";
import fetchTitleAndHeaders from "@/lib/shared/fetchTitleAndHeaders";
+import { isUrlSafeForServerSideFetch } from "@linkwarden/lib/ssrf";
export const config = {
api: {
@@ -112,6 +113,9 @@ async function handlePost(req: NextApiRequest, res: NextApiResponse) {
}
const url = typeof fields.url === "string" ? fields.url : fields.url?.[0];
+ const shouldFetchUrl = url
+ ? await isUrlSafeForServerSideFetch(url)
+ : false;
// Validate input against Zod schema
const dataValidation = UploadFileSchema.safeParse({
@@ -153,7 +157,7 @@ async function handlePost(req: NextApiRequest, res: NextApiResponse) {
const isImage = mimetype?.includes("image");
const isHTML = mimetype === "text/html";
- const { title = "" } = url
+ const { title = "" } = url && (shouldFetchUrl || isHTML)
? await fetchTitleAndHeaders(
url,
isHTML && !isPreview ? fileBuffer.toString("utf-8") : undefined
diff --git a/apps/web/pages/api/v1/rss/index.ts b/apps/web/pages/api/v1/rss/index.ts
index 59cc681e..ff186983 100644
--- a/apps/web/pages/api/v1/rss/index.ts
+++ b/apps/web/pages/api/v1/rss/index.ts
@@ -5,6 +5,8 @@ import verifyUser from "@/lib/api/verifyUser";
import { PostRssSubscriptionSchema } from "@linkwarden/lib/schemaValidation";
import { NextApiRequest, NextApiResponse } from "next";
import Parser from "rss-parser";
+import { assertUrlIsSafeForServerSideFetch } from "@linkwarden/lib/ssrf";
+import { safeFetch } from "@linkwarden/lib/safeFetch";
export default async function handler(
req: NextApiRequest,
@@ -65,6 +67,14 @@ export default async function handler(
const { name, url, collectionId, collectionName } = dataValidation.data;
+ try {
+ await assertUrlIsSafeForServerSideFetch(url);
+ } catch (error: any) {
+ return res.status(400).json({
+ response: error?.message || "RSS URL is not allowed.",
+ });
+ }
+
const linkCollection = await setCollection({
userId: user.id,
collectionId: collectionId,
@@ -104,8 +114,10 @@ export default async function handler(
});
const parser = new Parser();
+ const xml = await safeFetch(response.url).then((res) => res.text());
+ const feed = await parser.parseString(xml);
- await rssHandler(response, parser);
+ await rssHandler(response, feed);
return res.status(200).json({ response });
}
diff --git a/apps/worker/lib/archiveHandler.ts b/apps/worker/lib/archiveHandler.ts
index a1b3e338..41861b1c 100644
--- a/apps/worker/lib/archiveHandler.ts
+++ b/apps/worker/lib/archiveHandler.ts
@@ -15,6 +15,11 @@ import { LinkWithCollectionOwnerAndTags } from "@linkwarden/types/global";
import { isArchivalTag } from "@linkwarden/lib/isArchivalTag";
import { ArchivalSettings } from "@linkwarden/types/global";
import { getDefaultContextOptions } from "./browser";
+import {
+ assertUrlIsSafeForServerSideFetch,
+ UnsafeUrlError,
+} from "@linkwarden/lib/ssrf";
+import protectPageRequests from "./protectPageRequests";
const BROWSER_TIMEOUT = Number(process.env.BROWSER_TIMEOUT) || 5;
@@ -23,9 +28,22 @@ export default async function archiveHandler(
browser: Browser
) {
const user = link.collection?.owner;
+ let skipPreservation = process.env.DISABLE_PRESERVATION === "true";
+
+ if (!skipPreservation && link.url) {
+ try {
+ await assertUrlIsSafeForServerSideFetch(link.url);
+ } catch (error) {
+ if (error instanceof UnsafeUrlError) {
+ skipPreservation = true;
+ } else {
+ throw error;
+ }
+ }
+ }
if (
- process.env.DISABLE_PRESERVATION === "true" ||
+ skipPreservation ||
(!link.url?.startsWith("http://") && !link.url?.startsWith("https://"))
) {
await prisma.link.update({
@@ -72,6 +90,7 @@ export default async function archiveHandler(
const contextOptions = getDefaultContextOptions();
const context = await browser.newContext(contextOptions);
+ await protectPageRequests(context);
const page = await context.newPage();
createFolder({ filePath: `archives/preview/${link.collectionId}` });
diff --git a/apps/worker/lib/browser.ts b/apps/worker/lib/browser.ts
index 203b67f9..c5a40e81 100644
--- a/apps/worker/lib/browser.ts
+++ b/apps/worker/lib/browser.ts
@@ -34,7 +34,9 @@ export function getBrowserOptions(): LaunchOptions {
export function getDefaultContextOptions(): BrowserContextOptions {
const base: BrowserContextOptions = {
...devices["Desktop Chrome"],
- ignoreHTTPSErrors: process.env.IGNORE_HTTPS_ERRORS === "true",
+ ignoreHTTPSErrors:
+ process.env.ALLOW_INSECURE_TLS === "true" ||
+ process.env.IGNORE_HTTPS_ERRORS === "true",
};
if (process.env.PLAYWRIGHT_WS_URL) {
diff --git a/apps/worker/lib/fetchHeaders.ts b/apps/worker/lib/fetchHeaders.ts
index 1f5f8563..2291ae12 100644
--- a/apps/worker/lib/fetchHeaders.ts
+++ b/apps/worker/lib/fetchHeaders.ts
@@ -1,41 +1,10 @@
-import fetch from "node-fetch";
-import https from "https";
-import http from "http";
-import { HttpsProxyAgent } from 'https-proxy-agent';
-import { SocksProxyAgent } from "socks-proxy-agent";
+import { safeFetch } from "@linkwarden/lib/safeFetch";
export default async function fetchHeaders(url: string) {
if (process.env.IGNORE_URL_SIZE_LIMIT === "true") return null;
try {
- const httpsAgent = url.startsWith("http://")
- ? new http.Agent({})
- : new https.Agent({
- rejectUnauthorized:
- process.env.IGNORE_UNAUTHORIZED_CA === "true" ? false : true,
- });
-
- let fetchOpts = {
- method: "HEAD",
- agent: httpsAgent,
- };
-
- if (process.env.PROXY) {
- let proxy = new URL(process.env.PROXY);
- if (process.env.PROXY_USERNAME) {
- proxy.username = process.env.PROXY_USERNAME;
- proxy.password = process.env.PROXY_PASSWORD || "";
- }
-
- const proxyAgent = proxy.protocol.includes("http") ? HttpsProxyAgent : SocksProxyAgent;
-
- fetchOpts = {
- method: "HEAD",
- agent: new proxyAgent(proxy.toString()),
- };
- }
-
- const responsePromise = fetch(url, fetchOpts);
+ const responsePromise = safeFetch(url, { method: "HEAD" });
const timeoutPromise = new Promise((_, reject) => {
setTimeout(() => {
diff --git a/apps/worker/lib/preservationScheme/handleArchivePreview.ts b/apps/worker/lib/preservationScheme/handleArchivePreview.ts
index 5e3a1bf6..126fd041 100644
--- a/apps/worker/lib/preservationScheme/handleArchivePreview.ts
+++ b/apps/worker/lib/preservationScheme/handleArchivePreview.ts
@@ -3,6 +3,10 @@ import { Page } from "playwright";
import { generatePreview } from "@linkwarden/lib/generatePreview";
import { createFile } from "@linkwarden/filesystem";
import { prisma } from "@linkwarden/prisma";
+import {
+ assertUrlIsSafeForServerSideFetch,
+ UnsafeUrlError,
+} from "@linkwarden/lib/ssrf";
type LinksAndCollectionAndOwner = Link & {
collection: Collection & {
@@ -31,18 +35,25 @@ const handleArchivePreview = async (
origin + (ogImageUrl.startsWith("/") ? ogImageUrl : "/" + ogImageUrl);
}
- const imageResponse = await page.goto(ogImageUrl);
+ try {
+ await assertUrlIsSafeForServerSideFetch(ogImageUrl);
+ const imageResponse = await page.goto(ogImageUrl);
- if (imageResponse && !link.preview?.startsWith("archive")) {
- const buffer = await imageResponse.body();
- previewGenerated = await generatePreview(
- buffer,
- link.collectionId,
- link.id
- );
- }
+ if (imageResponse && !link.preview?.startsWith("archive")) {
+ const buffer = await imageResponse.body();
+ previewGenerated = await generatePreview(
+ buffer,
+ link.collectionId,
+ link.id
+ );
+ }
- await page.goBack();
+ await page.goBack();
+ } catch (error) {
+ if (!(error instanceof UnsafeUrlError)) {
+ throw error;
+ }
+ }
}
if (!previewGenerated && !link.preview?.startsWith("archive")) {
diff --git a/apps/worker/lib/preservationScheme/imageHandler.ts b/apps/worker/lib/preservationScheme/imageHandler.ts
index ed6f6162..78d2bf6b 100644
--- a/apps/worker/lib/preservationScheme/imageHandler.ts
+++ b/apps/worker/lib/preservationScheme/imageHandler.ts
@@ -2,11 +2,10 @@ import { Link } from "@linkwarden/prisma/client";
import { prisma } from "@linkwarden/prisma";
import { createFile } from "@linkwarden/filesystem";
import { generatePreview } from "@linkwarden/lib/generatePreview";
+import { safeFetch } from "@linkwarden/lib/safeFetch";
const imageHandler = async ({ url, id }: Link, extension: string) => {
- const image = await fetch(url as string).then((res) => res.blob());
-
- const buffer = Buffer.from(await image.arrayBuffer());
+ const buffer = await safeFetch(url as string).then((res) => res.buffer());
if (
Buffer.byteLength(buffer) >
diff --git a/apps/worker/lib/preservationScheme/pdfHandler.ts b/apps/worker/lib/preservationScheme/pdfHandler.ts
index 01f6b830..ed766e33 100644
--- a/apps/worker/lib/preservationScheme/pdfHandler.ts
+++ b/apps/worker/lib/preservationScheme/pdfHandler.ts
@@ -1,11 +1,10 @@
import { Link } from "@linkwarden/prisma/client";
import { prisma } from "@linkwarden/prisma";
import { createFile } from "@linkwarden/filesystem";
+import { safeFetch } from "@linkwarden/lib/safeFetch";
const pdfHandler = async ({ url, id }: Link) => {
- const pdf = await fetch(url as string).then((res) => res.blob());
-
- const buffer = Buffer.from(await pdf.arrayBuffer());
+ const buffer = await safeFetch(url as string).then((res) => res.buffer());
if (
Buffer.byteLength(buffer) >
diff --git a/apps/worker/lib/protectPageRequests.ts b/apps/worker/lib/protectPageRequests.ts
new file mode 100644
index 00000000..65199998
--- /dev/null
+++ b/apps/worker/lib/protectPageRequests.ts
@@ -0,0 +1,36 @@
+import { BrowserContext, Route } from "playwright";
+import {
+ assertUrlIsSafeForServerSideFetch,
+ UnsafeUrlError,
+} from "@linkwarden/lib/ssrf";
+
+function isNonNetworkUrl(url: string) {
+ return (
+ url.startsWith("about:") ||
+ url.startsWith("blob:") ||
+ url.startsWith("data:")
+ );
+}
+
+export default async function protectPageRequests(context: BrowserContext) {
+ await context.route("**/*", async (route: Route) => {
+ const requestUrl = route.request().url();
+
+ if (isNonNetworkUrl(requestUrl)) {
+ await route.continue();
+ return;
+ }
+
+ try {
+ await assertUrlIsSafeForServerSideFetch(requestUrl);
+ await route.continue();
+ } catch (error) {
+ if (error instanceof UnsafeUrlError) {
+ await route.abort("blockedbyclient");
+ return;
+ }
+
+ throw error;
+ }
+ });
+}
diff --git a/apps/worker/workers/rssPolling.ts b/apps/worker/workers/rssPolling.ts
index d01c442a..1aca03c7 100644
--- a/apps/worker/workers/rssPolling.ts
+++ b/apps/worker/workers/rssPolling.ts
@@ -2,6 +2,8 @@ import { prisma } from "@linkwarden/prisma";
import Parser from "rss-parser";
import { delay } from "@linkwarden/lib/utils";
import { rssHandler } from "@linkwarden/lib/rssHandler";
+import { assertUrlIsSafeForServerSideFetch } from "@linkwarden/lib/ssrf";
+import { safeFetch } from "@linkwarden/lib/safeFetch";
const pollingIntervalInSeconds =
(Number(process.env.NEXT_PUBLIC_RSS_POLLING_INTERVAL_MINUTES) || 60) * 60; // Default to one hour if not set
@@ -15,7 +17,20 @@ export async function startRSSPolling() {
await Promise.all(
rssSubscriptions.map(async (rssSubscription) => {
- rssHandler(rssSubscription, parser);
+ try {
+ await assertUrlIsSafeForServerSideFetch(rssSubscription.url);
+ const xml = await safeFetch(rssSubscription.url).then((res) =>
+ res.text()
+ );
+ const feed = await parser.parseString(xml);
+ await rssHandler(rssSubscription, feed);
+ } catch (error) {
+ console.error(
+ "\x1b[34m%s\x1b[0m",
+ `Error processing RSS feed ${rssSubscription.url}:`,
+ error
+ );
+ }
})
);
await delay(pollingIntervalInSeconds);
diff --git a/packages/lib/rssHandler.ts b/packages/lib/rssHandler.ts
index 00978184..b5b7b030 100644
--- a/packages/lib/rssHandler.ts
+++ b/packages/lib/rssHandler.ts
@@ -1,17 +1,20 @@
import { RssSubscription } from "@linkwarden/prisma/client";
import { hasPassedLimit } from "./verifyCapacity";
-import Parser from "rss-parser";
import { prisma } from "@linkwarden/prisma";
+import Parser from "rss-parser";
+import { assertUrlIsSafeForServerSideFetch } from "./ssrf";
export const rssHandler = async (
rssSubscription: RssSubscription,
- parser: Parser
+ feed: Parser.Output<any>
) => {
try {
- const feed = await parser.parseURL(rssSubscription.url);
-
+ const feedLastBuildDate = (feed as any).lastBuildDate as
+ | string
+ | Date
+ | undefined;
const feedLastPubDate =
- feed.lastBuildDate ??
+ feedLastBuildDate ??
feed.items.reduce((acc, item) => {
const itemPubDate = item.pubDate ? new Date(item.pubDate) : null;
return itemPubDate && itemPubDate > acc ? itemPubDate : acc;
@@ -53,6 +56,14 @@ export const rssHandler = async (
// Create all links concurrently
await Promise.all(
newItems.map(async (item) => {
+ if (!item.link) return null;
+
+ try {
+ await assertUrlIsSafeForServerSideFetch(item.link);
+ } catch {
+ return null;
+ }
+
return prisma.link.create({
data: {
name: item.title,
diff --git a/packages/lib/safeFetch.ts b/packages/lib/safeFetch.ts
new file mode 100644
index 00000000..b2610440
--- /dev/null
+++ b/packages/lib/safeFetch.ts
@@ -0,0 +1,125 @@
+import fetch, { RequestInit, Response } from "node-fetch";
+import http from "http";
+import https from "https";
+import { HttpsProxyAgent } from "https-proxy-agent";
+import { SocksProxyAgent } from "socks-proxy-agent";
+import {
+ assertUrlIsSafeForServerSideFetch,
+ defaultHostnameLookup,
+ resolveHostnameForServerSideFetch,
+} from "@linkwarden/lib/ssrf";
+
+type SafeFetchOptions = Omit<RequestInit, "agent" | "redirect"> & {
+ maxRedirects?: number;
+};
+
+function createSafeLookup() {
+ const lookup: any = (
+ hostname: string,
+ options: number | { family?: number | string; all?: boolean },
+ callback: (
+ error: NodeJS.ErrnoException | null,
+ addressOrAddresses?:
+ | string
+ | ReadonlyArray<{ address: string; family: 4 | 6 }>,
+ family?: number
+ ) => void
+ ) => {
+ const normalizedOptions =
+ typeof options === "number" ? { family: options } : options ?? {};
+ const requestedFamily =
+ normalizedOptions.family === "IPv4"
+ ? 4
+ : normalizedOptions.family === "IPv6"
+ ? 6
+ : normalizedOptions.family;
+
+ void resolveHostnameForServerSideFetch(hostname, defaultHostnameLookup)
+ .then((resolved) => {
+ if (normalizedOptions.all) {
+ callback(null, resolved);
+ return;
+ }
+
+ const match =
+ resolved.find(
+ (entry) => !requestedFamily || entry.family === requestedFamily
+ ) ?? resolved[0];
+
+ callback(null, match.address, match.family);
+ })
+ .catch((error) => {
+ callback(error as NodeJS.ErrnoException);
+ });
+ };
+
+ return lookup;
+}
+
+function createAgent(target: URL) {
+ if (process.env.PROXY) {
+ const proxy = new URL(process.env.PROXY);
+
+ if (process.env.PROXY_USERNAME) {
+ proxy.username = process.env.PROXY_USERNAME;
+ proxy.password = process.env.PROXY_PASSWORD || "";
+ }
+
+ const ProxyAgent = proxy.protocol.includes("http")
+ ? HttpsProxyAgent
+ : SocksProxyAgent;
+
+ return new ProxyAgent(proxy.toString());
+ }
+
+ const lookup = createSafeLookup();
+
+ if (target.protocol === "http:") {
+ return new http.Agent({ lookup });
+ }
+
+ return new https.Agent({
+ lookup,
+ rejectUnauthorized:
+ process.env.ALLOW_INSECURE_TLS === "true" ||
+ process.env.IGNORE_UNAUTHORIZED_CA === "true"
+ ? false
+ : true,
+ });
+}
+
+function isRedirectStatus(status: number) {
+ return status >= 300 && status < 400;
+}
+
+export async function safeFetch(
+ input: string,
+ options: SafeFetchOptions = {}
+): Promise<Response> {
+ const { maxRedirects = 5, ...fetchOptions } = options;
+
+ let currentUrl = input;
+
+ for (let redirectCount = 0; redirectCount <= maxRedirects; redirectCount++) {
+ const validatedUrl = await assertUrlIsSafeForServerSideFetch(currentUrl);
+ const response = await fetch(validatedUrl.toString(), {
+ ...fetchOptions,
+ agent: createAgent(validatedUrl),
+ redirect: "manual",
+ });
+
+ if (!isRedirectStatus(response.status)) {
+ return response;
+ }
+
+ const location = response.headers.get("location");
+
+ if (!location) {
+ return response;
+ }
+
+ currentUrl = new URL(location, validatedUrl).toString();
+ }
+
+ throw new Error("Fetch redirected too many times.");
+}
diff --git a/packages/lib/ssrf.test.ts b/packages/lib/ssrf.test.ts
new file mode 100644
index 00000000..e51f6469
--- /dev/null
+++ b/packages/lib/ssrf.test.ts
@@ -0,0 +1,50 @@
+import { describe, expect, it } from "vitest";
+import {
+ assertUrlIsSafeForServerSideFetch,
+ isHostnameBlockedForServerSideFetch,
+ isIpAddressBlockedForServerSideFetch,
+ UnsafeUrlError,
+} from "./ssrf";
+
+describe("ssrf guard", () => {
+ it("blocks localhost hostnames", () => {
+ expect(isHostnameBlockedForServerSideFetch("localhost")).toBe(true);
+ expect(isHostnameBlockedForServerSideFetch("subdomain.localhost")).toBe(
+ true
+ );
+ });
+
+ it("blocks single-label internal hostnames", () => {
+ expect(isHostnameBlockedForServerSideFetch("meilisearch")).toBe(true);
+ expect(isHostnameBlockedForServerSideFetch("postgres")).toBe(true);
+ });
+
+ it("blocks private and loopback IP literals", () => {
+ expect(isIpAddressBlockedForServerSideFetch("127.0.0.1")).toBe(true);
+ expect(isIpAddressBlockedForServerSideFetch("169.254.169.254")).toBe(true);
+ expect(isIpAddressBlockedForServerSideFetch("::1")).toBe(true);
+ expect(isIpAddressBlockedForServerSideFetch("fc00::1")).toBe(true);
+ });
+
+ it("allows public hostnames that resolve to public IPs", async () => {
+ await expect(
+ assertUrlIsSafeForServerSideFetch("https://example.com", {
+ lookup: async () => [{ address: "93.184.216.34", family: 4 }],
+ })
+ ).resolves.toBeInstanceOf(URL);
+ });
+
+ it("blocks hostnames that resolve to private IPs", async () => {
+ await expect(
+ assertUrlIsSafeForServerSideFetch("https://example.com", {
+ lookup: async () => [{ address: "10.0.0.8", family: 4 }],
+ })
+ ).rejects.toBeInstanceOf(UnsafeUrlError);
+ });
+
+ it("blocks IPv6 loopback literals", async () => {
+ await expect(
+ assertUrlIsSafeForServerSideFetch("http://[::1]/")
+ ).rejects.toBeInstanceOf(UnsafeUrlError);
+ });
+});
diff --git a/packages/lib/ssrf.ts b/packages/lib/ssrf.ts
new file mode 100644
index 00000000..a27013b3
--- /dev/null
+++ b/packages/lib/ssrf.ts
@@ -0,0 +1,348 @@
+import { lookup as dnsLookup } from "node:dns/promises";
+import { isIP } from "node:net";
+
+export class UnsafeUrlError extends Error {
+ code = "ERR_UNSAFE_URL";
+
+ constructor(message: string) {
+ super(message);
+ this.name = "UnsafeUrlError";
+ }
+}
+
+export type ResolvedAddress = {
+ address: string;
+ family: 4 | 6;
+};
+
+export type HostnameLookup = (
+ hostname: string
+) => Promise<ReadonlyArray<ResolvedAddress>>;
+
+const BLOCKED_HOSTNAMES = new Set([
+ "localhost",
+ "localhost.localdomain",
+ "ip6-localhost",
+ "ip6-loopback",
+ "broadcasthost",
+]);
+
+const BLOCKED_HOSTNAME_SUFFIXES = [
+ ".localhost",
+ ".local",
+ ".localdomain",
+ ".internal",
+];
+
+const IPV4_BLOCKED_CIDRS = [
+ ["0.0.0.0", 8],
+ ["10.0.0.0", 8],
+ ["100.64.0.0", 10],
+ ["127.0.0.0", 8],
+ ["169.254.0.0", 16],
+ ["172.16.0.0", 12],
+ ["192.0.0.0", 24],
+ ["192.0.2.0", 24],
+ ["192.168.0.0", 16],
+ ["198.18.0.0", 15],
+ ["198.51.100.0", 24],
+ ["203.0.113.0", 24],
+ ["224.0.0.0", 4],
+ ["240.0.0.0", 4],
+] as const;
+
+const IPV6_BLOCKED_CIDRS = [
+ ["::", 128],
+ ["::1", 128],
+ ["fc00::", 7],
+ ["fe80::", 10],
+ ["ff00::", 8],
+ ["2001:db8::", 32],
+] as const;
+
+const IPV4_BLOCKED_RANGES = IPV4_BLOCKED_CIDRS.map(([address, prefix]) => {
+ const parsed = parseIPv4(address);
+
+ if (parsed === null) {
+ throw new Error(`Invalid blocked IPv4 CIDR base: ${address}/${prefix}`);
+ }
+
+ return {
+ network: parsed,
+ mask: ipv4Mask(prefix),
+ };
+});
+
+const IPV6_BLOCKED_RANGES = IPV6_BLOCKED_CIDRS.map(([address, prefix]) => {
+ const parsed = parseIPv6(address);
+
+ if (parsed === null) {
+ throw new Error(`Invalid blocked IPv6 CIDR base: ${address}/${prefix}`);
+ }
+
+ return {
+ network: parsed,
+ prefix,
+ };
+});
+
+function ipv4Mask(prefixLength: number) {
+ if (prefixLength === 0) return 0;
+ return Math.floor(2 ** 32 - 2 ** (32 - prefixLength));
+}
+
+function normalizeHostname(hostname: string) {
+ return hostname.trim().toLowerCase().replace(/\.+$/, "");
+}
+
+function parseIPv4(address: string) {
+ const octets = address.split(".");
+
+ if (octets.length !== 4) return null;
+
+ let value = 0;
+
+ for (const octet of octets) {
+ if (!/^\d{1,3}$/.test(octet)) return null;
+
+ const parsed = Number(octet);
+ if (!Number.isInteger(parsed) || parsed < 0 || parsed > 255) return null;
+
+ value = value * 256 + parsed;
+ }
+
+ return value;
+}
+
+function ipv4TailToHexSegments(address: string) {
+ const parsed = parseIPv4(address);
+ if (parsed === null) return null;
+
+ const high = Math.floor(parsed / 65536)
+ .toString(16)
+ .padStart(4, "0");
+ const low = (parsed % 65536).toString(16).padStart(4, "0");
+
+ return [high, low];
+}
+
+function parseIPv6(address: string) {
+ const normalized = address.toLowerCase().split("%")[0];
+
+ if (!normalized) return null;
+
+ const hasDoubleColon = normalized.includes("::");
+
+ if (hasDoubleColon && normalized.indexOf("::") !== normalized.lastIndexOf("::"))
+ return null;
+
+ const [leftRaw, rightRaw = ""] = hasDoubleColon
+ ? normalized.split("::")
+ : [normalized, ""];
+
+ const expandSide = (side: string) => {
+ if (!side) return [] as string[];
+
+ return side.split(":").flatMap((segment) => {
+ if (!segment) return [segment];
+ if (segment.includes(".")) {
+ const converted = ipv4TailToHexSegments(segment);
+ return converted ?? [segment];
+ }
+ return [segment];
+ });
+ };
+
+ const left = expandSide(leftRaw);
+ const right = expandSide(rightRaw);
+
+ const segments = hasDoubleColon
+ ? (() => {
+ const zeroCount = 8 - (left.length + right.length);
+ if (zeroCount < 1) return null;
+
+ return [
+ ...left,
+ ...Array.from({ length: zeroCount }, () => "0"),
+ ...right,
+ ];
+ })()
+ : left;
+
+ if (!segments || segments.length !== 8) return null;
+
+ const parsedSegments: number[] = [];
+
+ for (const segment of segments) {
+ if (!/^[0-9a-f]{1,4}$/i.test(segment)) return null;
+ parsedSegments.push(parseInt(segment, 16));
+ }
+
+ return parsedSegments;
+}
+
+function extractIPv4FromMappedIPv6(address: string) {
+ const normalized = address.toLowerCase();
+
+ if (!normalized.includes(".")) return null;
+
+ const ipv4Candidate = normalized.slice(normalized.lastIndexOf(":") + 1);
+ return parseIPv4(ipv4Candidate) === null ? null : ipv4Candidate;
+}
+
+export function isHostnameBlockedForServerSideFetch(hostname: string) {
+ const normalized = normalizeHostname(hostname);
+
+ if (!normalized) return true;
+ if (BLOCKED_HOSTNAMES.has(normalized)) return true;
+ if (BLOCKED_HOSTNAME_SUFFIXES.some((suffix) => normalized.endsWith(suffix)))
+ return true;
+
+ return !normalized.includes(".") && isIP(normalized) === 0;
+}
+
+export function isIpAddressBlockedForServerSideFetch(address: string) {
+ const ipv4 =
+ parseIPv4(address) ?? (() => {
+ const mapped = extractIPv4FromMappedIPv6(address);
+ return mapped ? parseIPv4(mapped) : null;
+ })();
+
+ if (ipv4 !== null) {
+ return IPV4_BLOCKED_RANGES.some(
+ ({ network, mask }) => (ipv4 & mask) === (network & mask)
+ );
+ }
+
+ const ipv6 = parseIPv6(address);
+ if (ipv6 === null) return true;
+
+ return IPV6_BLOCKED_RANGES.some(({ network, prefix }) =>
+ ipv6MatchesPrefix(ipv6, network, prefix)
+ );
+}
+
+function ipv6MatchesPrefix(
+ address: number[],
+ network: number[],
+ prefixLength: number
+) {
+ const fullSegments = Math.floor(prefixLength / 16);
+ const remainingBits = prefixLength % 16;
+
+ for (let index = 0; index < fullSegments; index++) {
+ if (address[index] !== network[index]) return false;
+ }
+
+ if (remainingBits === 0) return true;
+
+ const mask = (0xffff << (16 - remainingBits)) & 0xffff;
+ return (
+ (address[fullSegments] & mask) === (network[fullSegments] & mask)
+ );
+}
+
+export const defaultHostnameLookup: HostnameLookup = async (hostname) => {
+ const resolved = await dnsLookup(hostname, { all: true, verbatim: true });
+
+ return resolved
+ .filter(
+ (entry): entry is { address: string; family: 4 | 6 } =>
+ entry.family === 4 || entry.family === 6
+ )
+ .map((entry) => ({
+ address: entry.address,
+ family: entry.family,
+ }));
+};
+
+export async function resolveHostnameForServerSideFetch(
+ hostname: string,
+ lookup: HostnameLookup = defaultHostnameLookup
+) {
+ const normalized = normalizeHostname(hostname);
+
+ if (isHostnameBlockedForServerSideFetch(normalized)) {
+ throw new UnsafeUrlError("URL resolves to a blocked internal hostname.");
+ }
+
+ const ipFamily = isIP(normalized);
+
+ if (ipFamily === 4 || ipFamily === 6) {
+ if (isIpAddressBlockedForServerSideFetch(normalized)) {
+ throw new UnsafeUrlError("URL resolves to a blocked internal IP address.");
+ }
+
+ return [{ address: normalized, family: ipFamily }] as const;
+ }
+
+ let addresses: ReadonlyArray<ResolvedAddress>;
+
+ try {
+ addresses = await lookup(normalized);
+ } catch (error: any) {
+ throw new UnsafeUrlError(
+ error?.code === "ENOTFOUND" || error?.code === "EAI_AGAIN"
+ ? "URL hostname could not be resolved."
+ : "URL hostname lookup failed."
+ );
+ }
+
+ if (!addresses.length) {
+ throw new UnsafeUrlError("URL hostname did not resolve to a public IP.");
+ }
+
+ const blockedAddress = addresses.find((entry) =>
+ isIpAddressBlockedForServerSideFetch(entry.address)
+ );
+
+ if (blockedAddress) {
+ throw new UnsafeUrlError("URL resolves to a blocked internal IP address.");
+ }
+
+ return addresses;
+}
+
+export async function assertUrlIsSafeForServerSideFetch(
+ value: string,
+ options?: {
+ lookup?: HostnameLookup;
+ allowPrivateNetworkAccess?: boolean;
+ }
+) {
+ const url = new URL(value);
+
+ if (url.protocol !== "http:" && url.protocol !== "https:") {
+ throw new UnsafeUrlError("Only http:// and https:// URLs can be archived.");
+ }
+
+ if (
+ options?.allowPrivateNetworkAccess ||
+ process.env.ALLOW_PRIVATE_NETWORK_ACCESS === "true"
+ ) {
+ return url;
+ }
+
+ await resolveHostnameForServerSideFetch(url.hostname, options?.lookup);
+
+ return url;
+}
+
+export async function isUrlSafeForServerSideFetch(
+ value: string,
+ options?: {
+ lookup?: HostnameLookup;
+ allowPrivateNetworkAccess?: boolean;
+ }
+) {
+ try {
+ await assertUrlIsSafeForServerSideFetch(value, options);
+ return true;
+ } catch (error) {
+ if (error instanceof UnsafeUrlError) {
+ return false;
+ }
+
+ throw error;
+ }
+}