Skip to content

Papermark is the open-source DocSend alternative and secure data rooms with built-in analytics and custom domains.

13.9 KB
Raw Blame
import { lookup } from "node:dns/promises";
import type { IncomingHttpHeaders, IncomingMessage } from "node:http";
import { request as httpsRequest } from "node:https";
import { isIP } from "node:net";

const DEFAULT_MAX_REDIRECTS = 5;
const DEFAULT_CONNECT_TIMEOUT_MS = 30_000;
const DEFAULT_READ_TIMEOUT_MS = 60_000;

type ResolvedPublicAddress = {
  address: string;
  family: 4 | 6;
  hostname: string;
  hostnameIsIp: boolean;
};

type RemoteHttpResponse = {
  status: number;
  headers: IncomingHttpHeaders;
  body: IncomingMessage;
  clearReadTimer: () => void;
  refreshReadTimer: () => void;
};

export class ConnectionTimeoutError extends Error {
  readonly code = "CONNECT_TIMEOUT" as const;
  constructor(message = "Connection to URL timed out.") {
    super(message);
    this.name = "ConnectionTimeoutError";
  }
}

export class ReadTimeoutError extends Error {
  readonly code = "READ_TIMEOUT" as const;
  constructor(message = "Reading response body from URL timed out.") {
    super(message);
    this.name = "ReadTimeoutError";
  }
}

export type PublicHttpsDownload = {
  buffer: Buffer;
  contentType: string;
  finalUrl: URL;
  headers: IncomingHttpHeaders;
};

export function normalizeHostnameForSSRF(hostname: string): string {
  const normalized = hostname.trim().toLowerCase();
  const withoutBrackets =
    normalized.startsWith("[") && normalized.endsWith("]")
      ? normalized.slice(1, -1)
      : normalized;

  return withoutBrackets.endsWith(".")
    ? withoutBrackets.slice(0, -1)
    : withoutBrackets;
}

function parseIpv4Bytes(address: string): number[] | null {
  if (isIP(address) !== 4) return null;

  const octets = address.split(".").map((part) => Number(part));
  if (
    octets.length !== 4 ||
    octets.some((octet) => !Number.isInteger(octet) || octet < 0 || octet > 255)
  ) {
    return null;
  }

  return octets;
}

function parseIpv6Bytes(address: string): number[] | null {
  const normalized = normalizeHostnameForSSRF(address);
  if (isIP(normalized) !== 6) return null;

  let expanded = normalized;
  const ipv4Tail = normalized.match(/(?:^|:)(\d{1,3}(?:\.\d{1,3}){3})$/);
  if (ipv4Tail) {
    const ipv4Bytes = parseIpv4Bytes(ipv4Tail[1]);
    if (!ipv4Bytes) return null;

    const firstGroup = (ipv4Bytes[0] << 8) + ipv4Bytes[1];
    const secondGroup = (ipv4Bytes[2] << 8) + ipv4Bytes[3];
    expanded = `${normalized.slice(0, -ipv4Tail[1].length)}${firstGroup.toString(16)}:${secondGroup.toString(16)}`;
  }

  const halves = expanded.split("::");
  if (halves.length > 2) return null;

  const left = halves[0] ? halves[0].split(":").filter(Boolean) : [];
  const right = halves[1] ? halves[1].split(":").filter(Boolean) : [];
  const missingGroups = 8 - left.length - right.length;
  if (
    (halves.length === 1 && missingGroups !== 0) ||
    (halves.length === 2 && missingGroups < 1)
  ) {
    return null;
  }

  const groups = [...left, ...Array(missingGroups).fill("0"), ...right];
  if (groups.length !== 8) return null;

  const bytes: number[] = [];
  for (const group of groups) {
    if (!/^[0-9a-f]{1,4}$/i.test(group)) return null;
    const value = Number.parseInt(group, 16);
    bytes.push((value >> 8) & 0xff, value & 0xff);
  }

  return bytes;
}

function isReservedIpv4Bytes(bytes: number[]): boolean {
  const [first, second, third] = bytes;

  return (
    first === 0 ||
    first === 10 ||
    first === 127 ||
    (first === 100 && second >= 64 && second <= 127) ||
    (first === 169 && second === 254) ||
    (first === 172 && second >= 16 && second <= 31) ||
    (first === 192 && second === 0 && third === 0) ||
    (first === 192 && second === 0 && third === 2) ||
    (first === 192 && second === 88 && third === 99) ||
    (first === 192 && second === 168) ||
    (first === 198 && (second === 18 || second === 19)) ||
    (first === 198 && second === 51 && third === 100) ||
    (first === 203 && second === 0 && third === 113) ||
    first >= 224
  );
}

function hasIpv6Prefix(
  bytes: number[],
  prefixBytes: number[],
  prefixBits: number,
): boolean {
  for (let bit = 0; bit < prefixBits; bit++) {
    const byteIndex = Math.floor(bit / 8);
    const mask = 1 << (7 - (bit % 8));
    if ((bytes[byteIndex] & mask) !== ((prefixBytes[byteIndex] ?? 0) & mask)) {
      return false;
    }
  }

  return true;
}

function getIpv4MappedBytes(ipv6Bytes: number[]): number[] | null {
  const isMapped =
    ipv6Bytes.slice(0, 10).every((byte) => byte === 0) &&
    ipv6Bytes[10] === 0xff &&
    ipv6Bytes[11] === 0xff;

  return isMapped ? ipv6Bytes.slice(12, 16) : null;
}

function isPrivateOrReservedIpAddress(address: string): boolean {
  const normalized = normalizeHostnameForSSRF(address);

  const ipv4Bytes = parseIpv4Bytes(normalized);
  if (ipv4Bytes) {
    return isReservedIpv4Bytes(ipv4Bytes);
  }

  const ipv6Bytes = parseIpv6Bytes(normalized);
  if (!ipv6Bytes) return false;

  const mappedIpv4Bytes = getIpv4MappedBytes(ipv6Bytes);
  if (mappedIpv4Bytes) {
    return true;
  }

  const nat64MappedIpv4Bytes = hasIpv6Prefix(ipv6Bytes, [0x64, 0xff, 0x9b], 96)
    ? ipv6Bytes.slice(12, 16)
    : null;
  if (nat64MappedIpv4Bytes) {
    return isReservedIpv4Bytes(nat64MappedIpv4Bytes);
  }

  return (
    ipv6Bytes.every((byte) => byte === 0) ||
    (ipv6Bytes.slice(0, 15).every((byte) => byte === 0) &&
      ipv6Bytes[15] === 1) ||
    // Deprecated IPv4-compatible IPv6 form (::/96).
    hasIpv6Prefix(ipv6Bytes, [], 96) ||
    hasIpv6Prefix(ipv6Bytes, [0x01, 0x00], 64) ||
    hasIpv6Prefix(ipv6Bytes, [0x20, 0x01, 0x00, 0x02, 0x00, 0x00], 48) ||
    hasIpv6Prefix(ipv6Bytes, [0x20, 0x01, 0x0d, 0xb8], 32) ||
    hasIpv6Prefix(ipv6Bytes, [0x3f, 0xff], 20) ||
    (ipv6Bytes[0] & 0xfe) === 0xfc ||
    (ipv6Bytes[0] === 0xfe && (ipv6Bytes[1] & 0xc0) === 0x80) ||
    ipv6Bytes[0] === 0xff
  );
}

export function isPublicHostnameLiteral(hostname: string): boolean {
  const normalizedHostname = normalizeHostnameForSSRF(hostname);

  if (
    !normalizedHostname ||
    normalizedHostname === "localhost" ||
    normalizedHostname.endsWith(".localhost") ||
    normalizedHostname.endsWith(".local")
  ) {
    return false;
  }

  return !isPrivateOrReservedIpAddress(normalizedHostname);
}

async function resolvePublicHostname(
  hostname: string,
): Promise<ResolvedPublicAddress> {
  const normalizedHostname = normalizeHostnameForSSRF(hostname);

  if (!isPublicHostnameLiteral(normalizedHostname)) {
    throw new Error("URL must point to a public host.");
  }

  const hostnameIpVersion = isIP(normalizedHostname);
  if (hostnameIpVersion) {
    return {
      address: normalizedHostname,
      family: hostnameIpVersion as 4 | 6,
      hostname: normalizedHostname,
      hostnameIsIp: true,
    };
  }

  let addresses: Array<{ address: string; family: number }>;
  try {
    addresses = await lookup(normalizedHostname, { all: true, verbatim: true });
  } catch {
    throw new Error("URL hostname could not be resolved.");
  }

  if (addresses.length === 0) {
    throw new Error("URL hostname could not be resolved.");
  }

  if (addresses.some((entry) => isPrivateOrReservedIpAddress(entry.address))) {
    throw new Error("URL must not resolve to a private or local IP address.");
  }

  const selected = addresses[0];
  return {
    address: selected.address,
    family: selected.family as 4 | 6,
    hostname: normalizedHostname,
    hostnameIsIp: false,
  };
}

function getHeaderValue(headers: IncomingHttpHeaders, name: string): string {
  const value = headers[name.toLowerCase()];
  if (Array.isArray(value)) return value[0] ?? "";
  return value ?? "";
}

async function requestResolvedHttpsUrl(
  url: URL,
  resolvedAddress: ResolvedPublicAddress,
  {
    connectTimeoutMs = DEFAULT_CONNECT_TIMEOUT_MS,
    readTimeoutMs = DEFAULT_READ_TIMEOUT_MS,
  }: { connectTimeoutMs?: number; readTimeoutMs?: number } = {},
): Promise<RemoteHttpResponse> {
  return new Promise((resolve, reject) => {
    let settled = false;

    const request = httpsRequest(
      {
        protocol: "https:",
        host: resolvedAddress.address,
        family: resolvedAddress.family,
        port: url.port ? Number(url.port) : 443,
        path: `${url.pathname}${url.search}`,
        method: "GET",
        headers: {
          Host: url.host,
        },
        servername: resolvedAddress.hostnameIsIp
          ? undefined
          : resolvedAddress.hostname,
      },
      (response) => {
        if (settled) {
          response.destroy();
          return;
        }
        settled = true;
        // Disable the socket-level connect timeout now that headers are in.
        request.setTimeout(0);

        let readTimer: NodeJS.Timeout | null = null;

        const clearReadTimer = () => {
          if (readTimer) {
            clearTimeout(readTimer);
            readTimer = null;
          }
        };

        // Restart the inactivity deadline so an active transfer is not
        // mistakenly treated as a stalled one. Safe to call after clear.
        const refreshReadTimer = () => {
          if (readTimer) {
            clearTimeout(readTimer);
          }
          readTimer = setTimeout(() => {
            readTimer = null;
            response.destroy(new ReadTimeoutError());
          }, readTimeoutMs);
        };

        refreshReadTimer();

        // Safety net: clear the timer if the stream ends, closes, or errors
        // without going through readResponseBuffer (e.g. discarded redirects).
        response.once("close", clearReadTimer);
        response.once("error", clearReadTimer);

        resolve({
          status: response.statusCode ?? 0,
          headers: response.headers,
          body: response,
          clearReadTimer,
          refreshReadTimer,
        });
      },
    );

    request.setTimeout(connectTimeoutMs);
    request.once("timeout", () => {
      if (settled) return;
      settled = true;
      const error = new ConnectionTimeoutError();
      request.destroy(error);
      reject(error);
    });

    request.once("error", (error) => {
      if (settled) return;
      settled = true;
      reject(error);
    });

    request.end();
  });
}

async function readResponseBuffer(
  response: IncomingMessage,
  maxBytes: number,
  clearReadTimer: () => void,
  refreshReadTimer: () => void,
): Promise<Buffer> {
  const chunks: Buffer[] = [];
  let total = 0;

  return new Promise((resolve, reject) => {
    let settled = false;

    const fail = (error: Error) => {
      if (settled) return;
      settled = true;
      clearReadTimer();
      response.destroy();
      reject(error);
    };

    response.on("data", (chunk: Buffer | string) => {
      refreshReadTimer();
      const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
      total += buffer.byteLength;
      if (total > maxBytes) {
        fail(new Error("URL response exceeds the maximum allowed size."));
        return;
      }

      chunks.push(buffer);
    });

    response.on("end", () => {
      if (settled) return;
      settled = true;
      clearReadTimer();
      resolve(Buffer.concat(chunks, total));
    });

    response.on("error", (error) => {
      fail(error);
    });
  });
}

function discardResponseBody(response: IncomingMessage): void {
  if (!response.destroyed) {
    response.resume();
  }
}

/**
 * Fetches an HTTPS URL while protecting against SSRF.
 *
 * @param url - Absolute HTTPS URL to download.
 * @param options.maxBytes - Maximum response size in bytes. Must be a finite
 *   positive integer; `NaN`, `Infinity`, `0`, or negative values are rejected
 *   so they cannot bypass the size checks performed against Content-Length and
 *   the streamed body.
 */
export async function fetchPublicHttpsUrlToBuffer(
  url: string,
  {
    maxBytes,
    maxRedirects = DEFAULT_MAX_REDIRECTS,
    connectTimeoutMs,
    readTimeoutMs,
  }: {
    maxBytes: number;
    maxRedirects?: number;
    connectTimeoutMs?: number;
    readTimeoutMs?: number;
  },
): Promise<PublicHttpsDownload> {
  if (!Number.isInteger(maxBytes) || maxBytes <= 0) {
    throw new RangeError("maxBytes must be a finite positive integer.");
  }

  let current = new URL(url);

  for (let redirects = 0; redirects <= maxRedirects; redirects++) {
    if (current.protocol !== "https:") {
      throw new Error("URL must use HTTPS.");
    }

    if (current.username || current.password) {
      throw new Error("URL must not contain embedded credentials.");
    }

    if (!isPublicHostnameLiteral(current.hostname)) {
      throw new Error("URL must point to a public host.");
    }

    const resolvedAddress = await resolvePublicHostname(current.hostname);
    const response = await requestResolvedHttpsUrl(current, resolvedAddress, {
      connectTimeoutMs,
      readTimeoutMs,
    });

    if ([301, 302, 303, 307, 308].includes(response.status)) {
      const location = getHeaderValue(response.headers, "location");
      response.clearReadTimer();
      discardResponseBody(response.body);
      if (!location) {
        throw new Error("Redirect response was missing a Location header.");
      }

      current = new URL(location, current);
      continue;
    }

    if (response.status < 200 || response.status >= 300) {
      response.clearReadTimer();
      discardResponseBody(response.body);
      throw new Error(`URL could not be downloaded (HTTP ${response.status}).`);
    }

    const contentLengthHeader = getHeaderValue(
      response.headers,
      "content-length",
    );
    const contentLength = contentLengthHeader
      ? Number(contentLengthHeader)
      : NaN;
    if (Number.isFinite(contentLength) && contentLength > maxBytes) {
      response.clearReadTimer();
      response.body.destroy();
      throw new Error("URL response exceeds the maximum allowed size.");
    }

    const buffer = await readResponseBuffer(
      response.body,
      maxBytes,
      response.clearReadTimer,
      response.refreshReadTimer,
    );
    return {
      buffer,
      contentType: getHeaderValue(response.headers, "content-type"),
      finalUrl: current,
      headers: response.headers,
    };
  }

  throw new Error("URL followed too many redirects.");
}