The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.
TypeScript clickhousemysqlnextjsopen-sourceplanetscaleprismatailwindcsstinybirdtypescriptupstashvercel
Check IPs at connect time on the Node runtime
Rafi6ab6d3e49423
diff --git a/apps/web/lib/api/safe-fetch-node.ts b/apps/web/lib/api/safe-fetch-node.tsnew file mode 100644index 0000000000..a4cb0f3202--- /dev/null+++ b/apps/web/lib/api/safe-fetch-node.ts@@ -0,0 +1,59 @@+/**+ * Node version of `safeFetch`. It runs the request through an undici agent+ * whose `lookup` checks every resolved IP when the socket connects. The IP we+ * check is the IP we connect to, so DNS rebinding cannot pass the check.+ *+ * Only for the Node runtime. Edge routes use `safeFetch` from `safe-fetch.ts`.+ */++import dns from "node:dns";+import type { LookupFunction } from "node:net";+import { Agent, fetch as undiciFetch } from "undici";+import { createSafeFetch, isPrivateIp } from "./safe-fetch";++type AllAddressesCallback = (+ err: NodeJS.ErrnoException | null,+ addresses: dns.LookupAddress[],+) => void;++/** Exported for unit tests. */+export const safeLookup: LookupFunction = (hostname, options, callback) => {+ dns.lookup(hostname, { ...options, all: true }, (err, addresses) => {+ if (err) {+ callback(err, "", 0);+ return;+ }++ if (+ addresses.length === 0 ||+ addresses.some(({ address }) => isPrivateIp(address))+ ) {+ callback(+ new Error(`${hostname} resolves to a disallowed IP address.`),+ "",+ 0,+ );+ return;+ }++ // Newer Node versions call `lookup` with `all: true` (for example with+ // `autoSelectFamily`) and expect the full list back. The Node 18 types do+ // not know this form yet.+ if ((options as dns.LookupOptions).all) {+ (callback as unknown as AllAddressesCallback)(null, addresses);+ } else {+ callback(null, addresses[0].address, addresses[0].family);+ }+ });+};++const agent = new Agent({ connect: { lookup: safeLookup } });++export const safeFetchNode = createSafeFetch({+ fetch: (url, init) =>+ undiciFetch(url, {+ ...(init as Parameters<typeof undiciFetch>[1]),+ dispatcher: agent,+ }) as unknown as Promise<Response>,+ resolveHostnames: false,+});diff --git a/apps/web/lib/api/safe-fetch.ts b/apps/web/lib/api/safe-fetch.tsindex 7c66f5160e..a17615dc8e 100644--- a/apps/web/lib/api/safe-fetch.ts+++ b/apps/web/lib/api/safe-fetch.ts@@ -9,8 +9,10 @@* 3. Manual redirect handling: each hop's `Location` is re-validated, so a* whitelisted host cannot bounce us into an internal IP.*- * Node sibling is `Storage.assertSafeUrl` in `storage.ts`. This module uses- * DoH because it runs on the Edge runtime, which has no `node:dns`.+ * `safeFetch` uses DoH because the Edge runtime has no `node:dns`. The DoH+ * check and the connect use separate lookups, so a DNS server that answers+ * differently each time can still pass it. On the Node runtime, use+ * `safeFetchNode` from `safe-fetch-node.ts`, which checks the IPs at connect time.*/import { DubApiError } from "@/lib/api/errors";@@ -220,7 +222,11 @@ async function resolveHostnameToIps(return [...a, ...aaaa];}-async function assertUrlIsSafe(url: URL, signal: AbortSignal): Promise<void> {+async function assertUrlIsSafe(+ url: URL,+ signal: AbortSignal,+ resolveHostnames: boolean,+): Promise<void> {if (!ALLOWED_PROTOCOLS.has(url.protocol)) {throw new DubApiError({code: "unprocessable_entity",@@ -239,6 +245,9 @@ async function assertUrlIsSafe(url: URL, signal: AbortSignal): Promise<void> {return;}+ // The transport checks the resolved IPs itself when it connects.+ if (!resolveHostnames) return;+const ips = await resolveHostnameToIps(url.hostname, signal);if (signal.aborted) {throw new Error("Request timed out");@@ -267,7 +276,34 @@ type SafeFetchOptions = {maxRedirects?: number;};-export async function safeFetch(+type SafeFetchTransport = {+ fetch: (url: string, init: RequestInit) => Promise<Response>;+ /**+ * False when `fetch` checks every resolved IP at connect time, so the DoH+ * check is not needed. IP literals are always checked here, because no DNS+ * lookup runs for them.+ */+ resolveHostnames: boolean;+};++export type SafeFetch = (+ url: string,+ init?: RequestInit,+ opts?: SafeFetchOptions,+) => Promise<Response>;++/** Builds a `safeFetch` on top of another transport. Used by `safeFetchNode`. */+export function createSafeFetch(transport: SafeFetchTransport): SafeFetch {+ return (url, init, opts) => safeFetchWith(transport, url, init, opts);+}++export const safeFetch = createSafeFetch({+ fetch: (url, init) => fetch(url, init),+ resolveHostnames: true,+});++async function safeFetchWith(+ transport: SafeFetchTransport,url: string,init?: RequestInit,opts: SafeFetchOptions = {},@@ -291,6 +327,7 @@ export async function safeFetch(try {return await fetchWithRedirects(+ transport,currentUrl,init,maxRedirects,@@ -302,6 +339,7 @@ export async function safeFetch(}async function fetchWithRedirects(+ transport: SafeFetchTransport,url: URL,init: RequestInit | undefined,maxRedirects: number,@@ -312,9 +350,9 @@ async function fetchWithRedirects(let hops = 0;while (true) {- await assertUrlIsSafe(currentUrl, signal);+ await assertUrlIsSafe(currentUrl, signal, transport.resolveHostnames);- const response = await fetch(currentUrl.toString(), {+ const response = await transport.fetch(currentUrl.toString(), {...currentInit,redirect: "manual",signal,diff --git a/apps/web/lib/sitemaps/import-tracked-sitemaps.ts b/apps/web/lib/sitemaps/import-tracked-sitemaps.tsindex 0e44f243c3..a54e1b6173 100644--- a/apps/web/lib/sitemaps/import-tracked-sitemaps.ts+++ b/apps/web/lib/sitemaps/import-tracked-sitemaps.ts@@ -1,5 +1,5 @@import { bulkCreateLinks } from "@/lib/api/links/bulk-create-links";-import { safeFetch } from "@/lib/api/safe-fetch";+import { safeFetchNode } from "@/lib/api/safe-fetch-node";import { prisma } from "@/lib/prisma";import type { TrackedSitemap } from "@/lib/sitemaps/site-visit-tracking";import { ProcessedLinkProps } from "@/lib/types";@@ -72,7 +72,9 @@ async function decompressIfGzip(buffer: ArrayBuffer): Promise<string> {async function fetchAndParseSitemap(sitemapUrl: string,): Promise<SitemapXmlResult> {- const response = await safeFetch(sitemapUrl, undefined, { maxRedirects: 0 });+ const response = await safeFetchNode(sitemapUrl, undefined, {+ maxRedirects: 0,+ });if (!response.ok) {throw new Error(`Failed to fetch sitemap: ${response.status} ${response.statusText}`,diff --git a/apps/web/package.json b/apps/web/package.jsonindex b06e20d3f3..a20988582d 100644--- a/apps/web/package.json+++ b/apps/web/package.json@@ -137,6 +137,7 @@"svix": "^1.76.1","swr": "^2.1.5","tremendous": "4.12.0",+ "undici": "7.29.0","unique-names-generator": "^4.7.1","unsplash-js": "^7.0.18","use-debounce": "^10.0.4",diff --git a/apps/web/tests/misc/import-tracked-sitemaps.test.ts b/apps/web/tests/misc/import-tracked-sitemaps.test.tsindex fbf57e4a6c..96c94473a7 100644--- a/apps/web/tests/misc/import-tracked-sitemaps.test.ts+++ b/apps/web/tests/misc/import-tracked-sitemaps.test.ts@@ -8,6 +8,13 @@ import { gzipSync } from "zlib";vi.mock("server-only", () => ({}));+// Run the sitemap import through the Edge `safeFetch`, so the stubbed global+// `fetch` below handles both the DoH lookups and the sitemap requests.+vi.mock("@/lib/api/safe-fetch-node", async () => {+ const { safeFetch } = await import("@/lib/api/safe-fetch");+ return { safeFetchNode: safeFetch };+});+function toArrayBuffer(buf: Buffer): ArrayBuffer {const ab = new ArrayBuffer(buf.length);new Uint8Array(ab).set(buf);diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex ddc05419d7..1708022763 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -391,6 +391,9 @@ importers:tremendous:specifier: 4.12.0version: 4.12.0+ undici:+ specifier: 7.29.0+ version: 7.29.0unique-names-generator:specifier: ^4.7.1version: 4.7.1