The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.
TypeScript clickhousemysqlnextjsopen-sourceplanetscaleprismatailwindcsstinybirdtypescriptupstashvercel
add ssrf-safe fetch wrapper for user-supplied urls
Pedro Ladeiraf6e9c6990013
diff --git a/apps/web/app/api/links/iframeable/route.ts b/apps/web/app/api/links/iframeable/route.tsindex 7e6a447921..71202c24c9 100644--- a/apps/web/app/api/links/iframeable/route.ts+++ b/apps/web/app/api/links/iframeable/route.ts@@ -1,4 +1,5 @@import { handleAndReturnErrorResponse } from "@/lib/api/errors";+import { safeFetch } from "@/lib/api/safe-fetch";import { ratelimitOrThrow } from "@/lib/api/utils";import {getDomainQuerySchema,@@ -17,7 +18,11 @@ export async function GET(req: NextRequest) {await ratelimitOrThrow(req, "iframeable");- const iframeable = await isIframeable({ url, requestDomain: domain });+ const res = await safeFetch(url);+ const iframeable = isIframeable({+ headers: res.headers,+ requestDomain: domain,+ });return NextResponse.json({ iframeable });} catch (error) {diff --git a/apps/web/app/api/links/metatags/utils.ts b/apps/web/app/api/links/metatags/utils.tsindex 97e5705230..5da6d9f86e 100644--- a/apps/web/app/api/links/metatags/utils.ts+++ b/apps/web/app/api/links/metatags/utils.ts@@ -1,13 +1,14 @@+import { safeFetch } from "@/lib/api/safe-fetch";import { recordMetatags } from "@/lib/upstash";import { linkPreviewImageBase64PrefixRegex } from "@/lib/zod/schemas/images";-import { fetchWithTimeout, isValidUrl } from "@dub/utils";+import { isValidUrl } from "@dub/utils";import { waitUntil } from "@vercel/functions";import he from "he";import { parse } from "node-html-parser";export const getHtml = async (url: string) => {try {- const response = await fetchWithTimeout(url);+ const response = await safeFetch(url);if (!response.ok) {// If we get a 406 or other error, check if it's a Cloudflare-protected sitediff --git a/apps/web/app/api/providers/route.ts b/apps/web/app/api/providers/route.tsindex ec9ac97ef3..1c9bbbf3b2 100644--- a/apps/web/app/api/providers/route.ts+++ b/apps/web/app/api/providers/route.ts@@ -1,4 +1,5 @@import { handleAndReturnErrorResponse } from "@/lib/api/errors";+import { safeFetch } from "@/lib/api/safe-fetch";import { ratelimitOrThrow } from "@/lib/api/utils";import { getUrlQuerySchema } from "@/lib/zod/schemas/links";import { fetchWithTimeout } from "@dub/utils";@@ -41,12 +42,15 @@ export async function GET(req: NextRequest) {urlObject.pathname = "/xyz";- const headers = await fetchWithTimeout(urlObject.toString(), {- headers: {- method: "HEAD",+ const headers = await safeFetch(+ urlObject.toString(),+ {+ headers: {+ method: "HEAD",+ },},- redirect: "manual",- })+ { maxRedirects: 0 },+ ).then((r) => ({engine: r.headers.get("engine"),poweredBy: r.headers.get("x-powered-by"),diff --git a/apps/web/lib/api/safe-fetch.ts b/apps/web/lib/api/safe-fetch.tsnew file mode 100644index 0000000000..87f2de5b9a--- /dev/null+++ b/apps/web/lib/api/safe-fetch.ts@@ -0,0 +1,297 @@+/**+ * SSRF-safe outbound fetch.+ *+ * Wraps `fetch` with three protections against attacker-controlled URLs:+ * 1. Scheme allowlist: only `http:` and `https:` are permitted.+ * 2. Host check: the hostname is resolved (IPv4 + IPv6 via Google DoH) and+ * every returned address is checked against a deny-list of private,+ * loopback, link-local, cloud-metadata, multicast, and reserved ranges.+ * 3. Manual redirect handling: each hop's `Location` is re-validated, so a+ * whitelisted host cannot bounce us into an internal IP.+ */++import { DubApiError } from "@/lib/api/errors";++const ALLOWED_PROTOCOLS = new Set(["http:", "https:"]);+const DEFAULT_TIMEOUT_MS = 5000;+const DEFAULT_MAX_REDIRECTS = 5;++const IPV4_REGEX = /^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/;++const BIG_ZERO = BigInt(0);+const BIG_16 = BigInt(16);+const BIG_32 = BigInt(32);+const BIG_FFFF = BigInt(0xffff);+const BIG_FFFFFFFF = BigInt(0xffffffff);++const PRIVATE_IPV4_CIDRS = [+ "0.0.0.0/8", // current network+ "10.0.0.0/8", // RFC1918+ "100.64.0.0/10", // CGNAT+ "127.0.0.0/8", // loopback+ "169.254.0.0/16", // link-local + cloud metadata+ "172.16.0.0/12", // RFC1918+ "192.0.0.0/24", // IETF protocol assignments+ "192.168.0.0/16", // RFC1918+ "198.18.0.0/15", // benchmarking+ "224.0.0.0/4", // multicast+ "240.0.0.0/4", // reserved+];++const PRIVATE_IPV6_CIDRS = [+ "::/128", // unspecified+ "::1/128", // loopback+ "fc00::/7", // unique local+ "fe80::/10", // link-local+ "ff00::/8", // multicast+ "64:ff9b::/96", // IPv4/IPv6 translation+ "2001:db8::/32", // documentation+];++function ipv4ToInt(ip: string): number | null {+ const m = IPV4_REGEX.exec(ip);+ if (!m) return null;+ const octets = [m[1], m[2], m[3], m[4]].map((s) => parseInt(s, 10));+ if (octets.some((n) => Number.isNaN(n) || n < 0 || n > 255)) return null;+ return (+ ((octets[0] << 24) | (octets[1] << 16) | (octets[2] << 8) | octets[3]) >>> 0+ );+}++function ipv4InCidr(ipInt: number, cidr: string): boolean {+ const [rangeIp, prefix] = cidr.split("/");+ const prefixLen = parseInt(prefix, 10);+ const rangeInt = ipv4ToInt(rangeIp);+ if (rangeInt === null || prefixLen < 0 || prefixLen > 32) return false;+ if (prefixLen === 0) return true;+ const mask = (0xffffffff << (32 - prefixLen)) >>> 0;+ return (ipInt & mask) === (rangeInt & mask);+}++function isPrivateIpv4(ip: string): boolean {+ const ipInt = ipv4ToInt(ip);+ // Unparseable → fail closed.+ if (ipInt === null) return true;+ return PRIVATE_IPV4_CIDRS.some((c) => ipv4InCidr(ipInt, c));+}++/** Expand an IPv6 string (possibly with embedded IPv4) to a 128-bit bigint. */+function ipv6ToBigInt(ip: string): bigint | null {+ let value = ip.trim();+ if (value.startsWith("[") && value.endsWith("]")) {+ value = value.slice(1, -1);+ }++ // Handle embedded IPv4 (e.g. ::ffff:127.0.0.1) by converting the dotted+ // suffix into two hextets.+ const lastColon = value.lastIndexOf(":");+ if (lastColon !== -1) {+ const suffix = value.slice(lastColon + 1);+ if (suffix.includes(".")) {+ const v4Int = ipv4ToInt(suffix);+ if (v4Int === null) return null;+ const hi = ((v4Int >>> 16) & 0xffff).toString(16);+ const lo = (v4Int & 0xffff).toString(16);+ value = `${value.slice(0, lastColon)}:${hi}:${lo}`;+ }+ }++ let head: string[] = [];+ let tail: string[] = [];+ const doubleColonIdx = value.indexOf("::");+ if (doubleColonIdx !== -1) {+ // Reject more than one "::"+ if (value.indexOf("::", doubleColonIdx + 1) !== -1) return null;+ const left = value.slice(0, doubleColonIdx);+ const right = value.slice(doubleColonIdx + 2);+ head = left ? left.split(":") : [];+ tail = right ? right.split(":") : [];+ } else {+ head = value.split(":");+ }++ const missing = 8 - head.length - tail.length;+ if (missing < 0 || (doubleColonIdx === -1 && missing !== 0)) return null;++ const groups = [...head, ...Array(missing).fill("0"), ...tail];+ if (groups.length !== 8) return null;++ let result = BIG_ZERO;+ for (const g of groups) {+ if (!/^[0-9a-fA-F]{1,4}$/.test(g)) return null;+ result = (result << BIG_16) | BigInt(parseInt(g, 16));+ }+ return result;+}++function ipv6InCidr(ipBig: bigint, cidr: string): boolean {+ const [rangeIp, prefix] = cidr.split("/");+ const prefixLen = parseInt(prefix, 10);+ const rangeBig = ipv6ToBigInt(rangeIp);+ if (rangeBig === null || prefixLen < 0 || prefixLen > 128) return false;+ if (prefixLen === 0) return true;+ const shift = BigInt(128 - prefixLen);+ return ipBig >> shift === rangeBig >> shift;+}++function isPrivateIpv6(ip: string): boolean {+ const ipBig = ipv6ToBigInt(ip);+ if (ipBig === null) return true;++ // IPv4-mapped IPv6 (::ffff:0:0/96): unwrap the last 32 bits and re-check.+ if (ipBig >> BIG_32 === BIG_FFFF) {+ const v4Int = Number(ipBig & BIG_FFFFFFFF);+ const v4Str = `${(v4Int >>> 24) & 0xff}.${(v4Int >>> 16) & 0xff}.${+ (v4Int >>> 8) & 0xff+ }.${v4Int & 0xff}`;+ return isPrivateIpv4(v4Str);+ }++ return PRIVATE_IPV6_CIDRS.some((c) => ipv6InCidr(ipBig, c));+}++export function isPrivateIp(ip: string): boolean {+ if (IPV4_REGEX.test(ip)) return isPrivateIpv4(ip);+ if (ip.includes(":")) return isPrivateIpv6(ip);+ // Anything we cannot classify, treat as private (fail closed).+ return true;+}++/** Returns the IP literal if `hostname` is one, else null. */+function ipLiteralFromHostname(hostname: string): string | null {+ if (IPV4_REGEX.test(hostname)) return hostname;+ if (hostname.startsWith("[") && hostname.endsWith("]")) {+ return hostname.slice(1, -1);+ }+ return null;+}++async function resolveHostnameToIps(hostname: string): Promise<string[]> {+ const query = async (type: "A" | "AAAA"): Promise<string[]> => {+ try {+ const res = await fetch(+ `https://dns.google/resolve?name=${encodeURIComponent(hostname)}&type=${type}`,+ );+ if (!res.ok) return [];+ const data = (await res.json()) as {+ Answer?: { data: string; type: number }[];+ };+ // Type 1 = A, type 28 = AAAA. Ignore CNAME / other intermediate records.+ const wantedType = type === "A" ? 1 : 28;+ return (data.Answer ?? [])+ .filter((a) => a.type === wantedType)+ .map((a) => a.data);+ } catch {+ return [];+ }+ };++ const [a, aaaa] = await Promise.all([query("A"), query("AAAA")]);+ return [...a, ...aaaa];+}++async function assertUrlIsSafe(url: URL): Promise<void> {+ if (!ALLOWED_PROTOCOLS.has(url.protocol)) {+ throw new DubApiError({+ code: "unprocessable_entity",+ message: `URL protocol "${url.protocol}" is not allowed.`,+ });+ }++ const literal = ipLiteralFromHostname(url.hostname);+ if (literal !== null) {+ if (isPrivateIp(literal)) {+ throw new DubApiError({+ code: "unprocessable_entity",+ message: "URL resolves to a disallowed IP address.",+ });+ }+ return;+ }++ const ips = await resolveHostnameToIps(url.hostname);+ if (ips.length === 0) {+ throw new DubApiError({+ code: "unprocessable_entity",+ message: `Could not resolve hostname: ${url.hostname}`,+ });+ }+ if (ips.some((ip) => isPrivateIp(ip))) {+ throw new DubApiError({+ code: "unprocessable_entity",+ message: "URL resolves to a disallowed IP address.",+ });+ }+}++type SafeFetchOptions = {+ /** Total timeout across all redirect hops. Defaults to 5000ms. */+ timeoutMs?: number;+ /** Max number of redirects to follow. Defaults to 5. Set to 0 to disable. */+ maxRedirects?: number;+};++export async function safeFetch(+ url: string,+ init?: RequestInit,+ opts: SafeFetchOptions = {},+): Promise<Response> {+ const timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS;+ const maxRedirects = opts.maxRedirects ?? DEFAULT_MAX_REDIRECTS;++ let currentUrl: URL;+ try {+ currentUrl = new URL(url);+ } catch {+ throw new DubApiError({+ code: "unprocessable_entity",+ message: `Invalid URL: ${url}`,+ });+ }++ const deadline = Date.now() + timeoutMs;+ let hops = 0;++ while (true) {+ await assertUrlIsSafe(currentUrl);++ const remaining = deadline - Date.now();+ if (remaining <= 0) {+ throw new Error("Request timed out");+ }++ const controller = new AbortController();+ const timer = setTimeout(() => controller.abort(), remaining);++ let response: Response;+ try {+ response = await fetch(currentUrl.toString(), {+ ...init,+ redirect: "manual",+ signal: controller.signal,+ });+ } finally {+ clearTimeout(timer);+ }++ const isRedirect = response.status >= 300 && response.status < 400;+ const location = response.headers.get("location");+ if (!isRedirect || !location) {+ return response;+ }++ if (hops >= maxRedirects) {+ throw new Error("Too many redirects");+ }++ try {+ currentUrl = new URL(location, currentUrl);+ } catch {+ throw new DubApiError({+ code: "unprocessable_entity",+ message: `Invalid redirect location: ${location}`,+ });+ }+ hops += 1;+ }+}diff --git a/apps/web/tests/misc/safe-fetch.test.ts b/apps/web/tests/misc/safe-fetch.test.tsnew file mode 100644index 0000000000..fd366accd7--- /dev/null+++ b/apps/web/tests/misc/safe-fetch.test.ts@@ -0,0 +1,100 @@+import { describe, expect, it, vi } from "vitest";++// `server-only` throws on import outside of a Next.js server bundle. Stub it+// so we can import `safe-fetch.ts`, which depends transitively on it via+// `@/lib/api/errors`.+vi.mock("server-only", () => ({}));++import { isPrivateIp } from "@/lib/api/safe-fetch";++describe("isPrivateIp", () => {+ describe("IPv4", () => {+ it.each([+ "0.0.0.0",+ "10.0.0.1",+ "10.255.255.255",+ "100.64.0.1",+ "127.0.0.1",+ "127.255.255.254",+ "169.254.169.254", // AWS / GCP / Azure metadata+ "172.16.0.1",+ "172.31.255.254",+ "192.168.0.1",+ "192.168.1.1",+ "198.18.0.1",+ "224.0.0.1", // multicast+ "240.0.0.1", // reserved+ "255.255.255.255",+ ])("flags %s as private", (ip) => {+ expect(isPrivateIp(ip)).toBe(true);+ });++ it.each([+ "1.1.1.1",+ "8.8.8.8",+ "9.9.9.9",+ "11.0.0.1", // just outside 10.0.0.0/8+ "100.63.255.255", // just outside CGNAT+ "172.15.255.255", // just outside 172.16.0.0/12+ "172.32.0.1", // just outside 172.16.0.0/12+ "192.0.1.1", // just outside 192.0.0.0/24+ "192.167.255.255", // just outside 192.168.0.0/16+ "193.0.0.1",+ "223.255.255.255", // just outside multicast+ ])("does not flag %s as private", (ip) => {+ expect(isPrivateIp(ip)).toBe(false);+ });+ });++ describe("IPv6", () => {+ it.each([+ "::",+ "::1",+ "fc00::",+ "fc00::1",+ "fd00::1",+ "fdff::ffff",+ "fe80::1",+ "fe80::abcd:1234",+ "ff00::1",+ "ff02::1",+ "64:ff9b::1.2.3.4",+ "2001:db8::1",+ ])("flags %s as private", (ip) => {+ expect(isPrivateIp(ip)).toBe(true);+ });++ it.each([+ "::ffff:127.0.0.1", // IPv4-mapped loopback+ "::ffff:10.0.0.1", // IPv4-mapped RFC1918+ "::ffff:169.254.169.254", // IPv4-mapped metadata+ "::ffff:192.168.1.1",+ ])("flags IPv4-mapped %s as private", (ip) => {+ expect(isPrivateIp(ip)).toBe(true);+ });++ it.each([+ "2606:4700:4700::1111", // Cloudflare DNS+ "2001:4860:4860::8888", // Google DNS+ "2620:fe::fe", // Quad9+ "::ffff:1.1.1.1", // IPv4-mapped public+ "::ffff:8.8.8.8",+ ])("does not flag %s as private", (ip) => {+ expect(isPrivateIp(ip)).toBe(false);+ });+ });++ describe("invalid inputs", () => {+ it.each([+ "",+ "not-an-ip",+ "999.999.999.999",+ "256.0.0.1",+ "1.2.3",+ "::gggg",+ ":::1",+ ])("treats %s as private (fail closed)", (value) => {+ expect(isPrivateIp(value)).toBe(true);+ });+ });+});diff --git a/packages/utils/src/functions/is-iframeable.ts b/packages/utils/src/functions/is-iframeable.tsindex 7b6a26a3b2..0018e212fb 100644--- a/packages/utils/src/functions/is-iframeable.ts+++ b/packages/utils/src/functions/is-iframeable.ts@@ -1,14 +1,14 @@-// check if a link can be displayed in an iframe-export const isIframeable = async ({- url,+// Determine whether a fetched page can be displayed in an iframe by the+// requesting domain, given the response's CSP / X-Frame-Options headers.+// The caller is responsible for fetching the URL (use an SSRF-safe fetcher).+export const isIframeable = ({+ headers,requestDomain,}: {- url: string;+ headers: Headers;requestDomain: string;}) => {- const res = await fetch(url);-- const cspHeader = res.headers.get("content-security-policy");+ const cspHeader = headers.get("content-security-policy");if (cspHeader) {const frameAncestorsMatch = cspHeader.match(/frame-ancestors\s+([\s\S]+?)(?=;|$)/i,@@ -24,7 +24,7 @@ export const isIframeable = async ({}}- const xFrameOptions = res.headers.get("X-Frame-Options");+ const xFrameOptions = headers.get("X-Frame-Options");if (xFrameOptions === "DENY" || xFrameOptions === "SAMEORIGIN") {return false;}