A self-hostable bookmark-everything app (links, notes and images) with AI-based automatic tagging and full text search
[network] fix dns rebinding problems post internal hostname validation (#2945)
Mohamed Bassem978544dfb0f4
diff --git a/apps/workers/network.test.ts b/apps/workers/network.test.tsnew file mode 100644index 00000000..98936c4c--- /dev/null+++ b/apps/workers/network.test.ts@@ -0,0 +1,132 @@+import { describe, expect, test, vi } from "vitest";++vi.mock("@karakeep/shared/config", () => ({+ default: {+ allowedInternalHostnames: undefined,+ crawler: { ipValidation: { dnsResolverTimeoutSec: 1 } },+ proxy: {+ httpProxy: undefined,+ httpsProxy: undefined,+ noProxy: undefined,+ },+ },+}));++import { createPinnedLookup } from "./network";++describe("createPinnedLookup", () => {+ test("returns a previously validated address without another DNS lookup", async () => {+ const lookup = createPinnedLookup([+ "93.184.216.34",+ "2606:2800:220:1:248:1893:25c8:1946",+ ]);++ const result = await new Promise<{ address: string; family: number }>(+ (resolve, reject) => {+ lookup("rebind.example", {}, (error, address, family) => {+ if (error) {+ reject(error);+ } else if (typeof address !== "string" || family === undefined) {+ reject(new Error("Expected a single lookup result"));+ } else {+ resolve({ address, family });+ }+ });+ },+ );++ expect(result).toEqual({ address: "93.184.216.34", family: 4 });+ });++ test("honors the socket's requested address family", async () => {+ const lookup = createPinnedLookup([+ "93.184.216.34",+ "2606:2800:220:1:248:1893:25c8:1946",+ ]);++ const result = await new Promise<{ address: string; family: number }>(+ (resolve, reject) => {+ lookup("rebind.example", { family: 6 }, (error, address, family) => {+ if (error) {+ reject(error);+ } else if (typeof address !== "string" || family === undefined) {+ reject(new Error("Expected a single lookup result"));+ } else {+ resolve({ address, family });+ }+ });+ },+ );++ expect(result).toEqual({+ address: "2606:2800:220:1:248:1893:25c8:1946",+ family: 6,+ });+ });++ test("returns all validated addresses when requested by the socket", async () => {+ const lookup = createPinnedLookup([+ "93.184.216.34",+ "2606:2800:220:1:248:1893:25c8:1946",+ ]);++ const result = await new Promise<{ address: string; family: number }[]>(+ (resolve, reject) => {+ lookup("rebind.example", { all: true }, (error, addresses) => {+ if (error) {+ reject(error);+ } else if (typeof addresses === "string") {+ reject(new Error("Expected all lookup results"));+ } else {+ resolve(addresses);+ }+ });+ },+ );++ expect(result).toEqual([+ { address: "93.184.216.34", family: 4 },+ { address: "2606:2800:220:1:248:1893:25c8:1946", family: 6 },+ ]);+ });++ test("refuses to return a forbidden address", async () => {+ const lookup = createPinnedLookup(["127.0.0.1", "169.254.169.254"]);++ const error = await new Promise<NodeJS.ErrnoException>(+ (resolve, reject) => {+ lookup("rebind.example", {}, (lookupError) => {+ if (lookupError) {+ resolve(lookupError);+ } else {+ reject(+ new Error("Expected the lookup to reject forbidden addresses"),+ );+ }+ });+ },+ );++ expect(error.code).toBe("ENOTFOUND");+ });++ test("never selects a forbidden address from a mixed result set", async () => {+ const lookup = createPinnedLookup(["127.0.0.1", "93.184.216.34"]);++ const result = await new Promise<{ address: string; family: number }>(+ (resolve, reject) => {+ lookup("rebind.example", {}, (error, address, family) => {+ if (error) {+ reject(error);+ } else if (typeof address !== "string" || family === undefined) {+ reject(new Error("Expected a single lookup result"));+ } else {+ resolve({ address, family });+ }+ });+ },+ );++ expect(result).toEqual({ address: "93.184.216.34", family: 4 });+ });+});diff --git a/apps/workers/network.ts b/apps/workers/network.tsindex a5a83299..034bb0a5 100644--- a/apps/workers/network.ts+++ b/apps/workers/network.ts@@ -1,4 +1,7 @@import dns from "node:dns/promises";+import http from "node:http";+import https from "node:https";+import type { LookupFunction } from "node:net";import { Readable } from "node:stream";import type { HeadersInit, RequestInit, Response } from "node-fetch";import { HttpProxyAgent } from "http-proxy-agent";@@ -97,7 +100,7 @@ export function getBookmarkDomain(url?: string | null): string | undefined {}export type UrlValidationResult =- | { ok: true; url: URL }+ | { ok: true; url: URL; resolvedAddresses?: string[] }| { ok: false; reason: string };function hostnameMatchesAnyPattern(@@ -219,7 +222,62 @@ export async function validateUrl(}}- return { ok: true, url: parsedUrl } as const;+ return { ok: true, url: parsedUrl, resolvedAddresses: records } as const;+}++/**+ * Builds a socket lookup function from the addresses that validateUrl already+ * checked. Returning one of those addresses directly to the HTTP agent avoids+ * resolving the hostname again between validation and connection (a DNS+ * rebinding/TOCTOU vulnerability).+ */+export function createPinnedLookup(addresses: string[]): LookupFunction {+ // validateUrl already rejects the entire result set if any address is+ // forbidden. Keep the socket boundary defensive as well, so a future caller+ // cannot accidentally pin a connection to an unchecked address.+ const lookupAddresses = addresses+ .filter((address) => !isAddressForbidden(address))+ .map((address) => ({+ address,+ family: ipaddr.parse(address).kind() === "ipv4" ? 4 : 6,+ }));++ return (_hostname, options, callback) => {+ const requestedFamily =+ options.family === 4 || options.family === "IPv4"+ ? 4+ : options.family === 6 || options.family === "IPv6"+ ? 6+ : undefined;+ const matchingAddresses = requestedFamily+ ? lookupAddresses.filter(({ family }) => family === requestedFamily)+ : lookupAddresses;++ if (matchingAddresses.length === 0) {+ const error = new Error(+ requestedFamily+ ? `Validated DNS lookup returned no allowed IPv${requestedFamily} addresses`+ : "Validated DNS lookup returned no allowed addresses",+ ) as NodeJS.ErrnoException;+ error.code = "ENOTFOUND";+ callback(error, "", requestedFamily);+ return;+ }++ if (options.all) {+ callback(null, matchingAddresses);+ } else {+ const selectedAddress = matchingAddresses[0];+ callback(null, selectedAddress.address, selectedAddress.family);+ }+ };+}++function getPinnedDnsAgent(url: URL, addresses: string[]) {+ const options = { lookup: createPinnedLookup(addresses) };+ return url.protocol === "https:"+ ? new https.Agent(options)+ : new http.Agent(options);}export function getRandomProxy(proxyList: string[]): string {@@ -434,14 +492,19 @@ export const fetchWithProxy = async (let currentBody = preparedBody;while (true) {- const agent = getProxyAgent(currentUrl, runProxy);+ const proxyAgent = getProxyAgent(currentUrl, runProxy);- const validation = await validateUrl(currentUrl, !!agent);+ const validation = await validateUrl(currentUrl, !!proxyAgent);if (!validation.ok) {throw new Error(validation.reason);}const requestUrl = validation.url;currentUrl = requestUrl.toString();+ const agent =+ proxyAgent ??+ (validation.resolvedAddresses+ ? getPinnedDnsAgent(requestUrl, validation.resolvedAddresses)+ : undefined);const response = await fetch(currentUrl,@@ -508,14 +571,19 @@ export async function resolveValidatedRedirectUrl(options.signal as globalThis.AbortSignal,]): AbortSignal.timeout(5000);- const agent = getProxyAgent(currentUrl, runProxy);- const validation = await validateUrl(currentUrl, !!agent);+ const proxyAgent = getProxyAgent(currentUrl, runProxy);+ const validation = await validateUrl(currentUrl, !!proxyAgent);if (!validation.ok) {throw new Error(validation.reason);}const requestUrl = validation.url;currentUrl = requestUrl.toString();+ const agent =+ proxyAgent ??+ (validation.resolvedAddresses+ ? getPinnedDnsAgent(requestUrl, validation.resolvedAddresses)+ : undefined);const response = await fetch(currentUrl,