Skip to content

armgit / dub

Public Imported from GitHub
0

The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.

Check IPs at connect time on the Node runtime

Rafi6ab6d3e49423
diff --git a/apps/web/lib/api/safe-fetch-node.ts b/apps/web/lib/api/safe-fetch-node.ts
new file mode 100644
index 0000000000..a4cb0f3202
--- /dev/null
+++ b/apps/web/lib/api/safe-fetch-node.ts
@@ -0,0 +1,59 @@
+/**
+ * Node version of `safeFetch`. It runs the request through an undici agent
+ * whose `lookup` checks every resolved IP when the socket connects. The IP we
+ * check is the IP we connect to, so DNS rebinding cannot pass the check.
+ *
+ * Only for the Node runtime. Edge routes use `safeFetch` from `safe-fetch.ts`.
+ */
+
+import dns from "node:dns";
+import type { LookupFunction } from "node:net";
+import { Agent, fetch as undiciFetch } from "undici";
+import { createSafeFetch, isPrivateIp } from "./safe-fetch";
+
+type AllAddressesCallback = (
+ err: NodeJS.ErrnoException | null,
+ addresses: dns.LookupAddress[],
+) => void;
+
+/** Exported for unit tests. */
+export const safeLookup: LookupFunction = (hostname, options, callback) => {
+ dns.lookup(hostname, { ...options, all: true }, (err, addresses) => {
+ if (err) {
+ callback(err, "", 0);
+ return;
+ }
+
+ if (
+ addresses.length === 0 ||
+ addresses.some(({ address }) => isPrivateIp(address))
+ ) {
+ callback(
+ new Error(`${hostname} resolves to a disallowed IP address.`),
+ "",
+ 0,
+ );
+ return;
+ }
+
+ // Newer Node versions call `lookup` with `all: true` (for example with
+ // `autoSelectFamily`) and expect the full list back. The Node 18 types do
+ // not know this form yet.
+ if ((options as dns.LookupOptions).all) {
+ (callback as unknown as AllAddressesCallback)(null, addresses);
+ } else {
+ callback(null, addresses[0].address, addresses[0].family);
+ }
+ });
+};
+
+const agent = new Agent({ connect: { lookup: safeLookup } });
+
+export const safeFetchNode = createSafeFetch({
+ fetch: (url, init) =>
+ undiciFetch(url, {
+ ...(init as Parameters<typeof undiciFetch>[1]),
+ dispatcher: agent,
+ }) as unknown as Promise<Response>,
+ resolveHostnames: false,
+});
diff --git a/apps/web/lib/api/safe-fetch.ts b/apps/web/lib/api/safe-fetch.ts
index 7c66f5160e..a17615dc8e 100644
--- a/apps/web/lib/api/safe-fetch.ts
+++ b/apps/web/lib/api/safe-fetch.ts
@@ -9,8 +9,10 @@
* 3. Manual redirect handling: each hop's `Location` is re-validated, so a
* whitelisted host cannot bounce us into an internal IP.
*
- * Node sibling is `Storage.assertSafeUrl` in `storage.ts`. This module uses
- * DoH because it runs on the Edge runtime, which has no `node:dns`.
+ * `safeFetch` uses DoH because the Edge runtime has no `node:dns`. The DoH
+ * check and the connect use separate lookups, so a DNS server that answers
+ * differently each time can still pass it. On the Node runtime, use
+ * `safeFetchNode` from `safe-fetch-node.ts`, which checks the IPs at connect time.
*/
import { DubApiError } from "@/lib/api/errors";
@@ -220,7 +222,11 @@ async function resolveHostnameToIps(
return [...a, ...aaaa];
}
-async function assertUrlIsSafe(url: URL, signal: AbortSignal): Promise<void> {
+async function assertUrlIsSafe(
+ url: URL,
+ signal: AbortSignal,
+ resolveHostnames: boolean,
+): Promise<void> {
if (!ALLOWED_PROTOCOLS.has(url.protocol)) {
throw new DubApiError({
code: "unprocessable_entity",
@@ -239,6 +245,9 @@ async function assertUrlIsSafe(url: URL, signal: AbortSignal): Promise<void> {
return;
}
+ // The transport checks the resolved IPs itself when it connects.
+ if (!resolveHostnames) return;
+
const ips = await resolveHostnameToIps(url.hostname, signal);
if (signal.aborted) {
throw new Error("Request timed out");
@@ -267,7 +276,34 @@ type SafeFetchOptions = {
maxRedirects?: number;
};
-export async function safeFetch(
+type SafeFetchTransport = {
+ fetch: (url: string, init: RequestInit) => Promise<Response>;
+ /**
+ * False when `fetch` checks every resolved IP at connect time, so the DoH
+ * check is not needed. IP literals are always checked here, because no DNS
+ * lookup runs for them.
+ */
+ resolveHostnames: boolean;
+};
+
+export type SafeFetch = (
+ url: string,
+ init?: RequestInit,
+ opts?: SafeFetchOptions,
+) => Promise<Response>;
+
+/** Builds a `safeFetch` on top of another transport. Used by `safeFetchNode`. */
+export function createSafeFetch(transport: SafeFetchTransport): SafeFetch {
+ return (url, init, opts) => safeFetchWith(transport, url, init, opts);
+}
+
+export const safeFetch = createSafeFetch({
+ fetch: (url, init) => fetch(url, init),
+ resolveHostnames: true,
+});
+
+async function safeFetchWith(
+ transport: SafeFetchTransport,
url: string,
init?: RequestInit,
opts: SafeFetchOptions = {},
@@ -291,6 +327,7 @@ export async function safeFetch(
try {
return await fetchWithRedirects(
+ transport,
currentUrl,
init,
maxRedirects,
@@ -302,6 +339,7 @@ export async function safeFetch(
}
async function fetchWithRedirects(
+ transport: SafeFetchTransport,
url: URL,
init: RequestInit | undefined,
maxRedirects: number,
@@ -312,9 +350,9 @@ async function fetchWithRedirects(
let hops = 0;
while (true) {
- await assertUrlIsSafe(currentUrl, signal);
+ await assertUrlIsSafe(currentUrl, signal, transport.resolveHostnames);
- const response = await fetch(currentUrl.toString(), {
+ const response = await transport.fetch(currentUrl.toString(), {
...currentInit,
redirect: "manual",
signal,
diff --git a/apps/web/lib/sitemaps/import-tracked-sitemaps.ts b/apps/web/lib/sitemaps/import-tracked-sitemaps.ts
index 0e44f243c3..a54e1b6173 100644
--- a/apps/web/lib/sitemaps/import-tracked-sitemaps.ts
+++ b/apps/web/lib/sitemaps/import-tracked-sitemaps.ts
@@ -1,5 +1,5 @@
import { bulkCreateLinks } from "@/lib/api/links/bulk-create-links";
-import { safeFetch } from "@/lib/api/safe-fetch";
+import { safeFetchNode } from "@/lib/api/safe-fetch-node";
import { prisma } from "@/lib/prisma";
import type { TrackedSitemap } from "@/lib/sitemaps/site-visit-tracking";
import { ProcessedLinkProps } from "@/lib/types";
@@ -72,7 +72,9 @@ async function decompressIfGzip(buffer: ArrayBuffer): Promise<string> {
async function fetchAndParseSitemap(
sitemapUrl: string,
): Promise<SitemapXmlResult> {
- const response = await safeFetch(sitemapUrl, undefined, { maxRedirects: 0 });
+ const response = await safeFetchNode(sitemapUrl, undefined, {
+ maxRedirects: 0,
+ });
if (!response.ok) {
throw new Error(
`Failed to fetch sitemap: ${response.status} ${response.statusText}`,
diff --git a/apps/web/package.json b/apps/web/package.json
index b06e20d3f3..a20988582d 100644
--- a/apps/web/package.json
+++ b/apps/web/package.json
@@ -137,6 +137,7 @@
"svix": "^1.76.1",
"swr": "^2.1.5",
"tremendous": "4.12.0",
+ "undici": "7.29.0",
"unique-names-generator": "^4.7.1",
"unsplash-js": "^7.0.18",
"use-debounce": "^10.0.4",
diff --git a/apps/web/tests/misc/import-tracked-sitemaps.test.ts b/apps/web/tests/misc/import-tracked-sitemaps.test.ts
index fbf57e4a6c..96c94473a7 100644
--- a/apps/web/tests/misc/import-tracked-sitemaps.test.ts
+++ b/apps/web/tests/misc/import-tracked-sitemaps.test.ts
@@ -8,6 +8,13 @@ import { gzipSync } from "zlib";
vi.mock("server-only", () => ({}));
+// Run the sitemap import through the Edge `safeFetch`, so the stubbed global
+// `fetch` below handles both the DoH lookups and the sitemap requests.
+vi.mock("@/lib/api/safe-fetch-node", async () => {
+ const { safeFetch } = await import("@/lib/api/safe-fetch");
+ return { safeFetchNode: safeFetch };
+});
+
function toArrayBuffer(buf: Buffer): ArrayBuffer {
const ab = new ArrayBuffer(buf.length);
new Uint8Array(ab).set(buf);
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index ddc05419d7..1708022763 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -391,6 +391,9 @@ importers:
tremendous:
specifier: 4.12.0
version: 4.12.0
+ undici:
+ specifier: 7.29.0
+ version: 7.29.0
unique-names-generator:
specifier: ^4.7.1
version: 4.7.1