Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

9.4 KB
Raw Blame
import { lookup } from "node:dns/promises";
import { request as httpRequest, type IncomingHttpHeaders } from "node:http";
import { request as httpsRequest } from "node:https";
import { isIP, type LookupFunction } from "node:net";

const BLOCKED_HOST_SUFFIXES = [".localhost", ".local", ".internal", ".home", ".lan", ".test", ".invalid"];
const MAX_REDIRECTS = 3;

export const MAX_PUBLIC_RESPONSE_BYTES = 1_000_000;

export class PublicUrlError extends Error {
  constructor(message: string) {
    super(message);
    this.name = "PublicUrlError";
  }
}

interface ResolvedAddress {
  address: string;
  family: 4 | 6;
}

export interface PublicTextResponse {
  body: string;
  finalUrl: string;
  headers: IncomingHttpHeaders;
  redirectCount: number;
  statusCode: number;
}

function ipv4ToNumber(address: string): number | null {
  const octets = address.split(".").map(Number);
  if (octets.length !== 4 || octets.some(octet => !Number.isInteger(octet) || octet < 0 || octet > 255)) {
    return null;
  }

  return (((octets[0] << 24) >>> 0) + (octets[1] << 16) + (octets[2] << 8) + octets[3]) >>> 0;
}

function isIpv4Range(address: string, network: string, prefix: number): boolean {
  const value = ipv4ToNumber(address);
  const base = ipv4ToNumber(network);
  if (value === null || base === null) return true;
  const mask = prefix === 0 ? 0 : (0xffffffff << (32 - prefix)) >>> 0;
  return (value & mask) === (base & mask);
}

function parseIpv6(address: string): number[] | null {
  let normalized = address.toLowerCase().split("%")[0];
  const embeddedIpv4 = normalized.match(/(?:^|:)(\d+\.\d+\.\d+\.\d+)$/)?.[1];

  if (embeddedIpv4) {
    const value = ipv4ToNumber(embeddedIpv4);
    if (value === null) return null;
    normalized =
      normalized.slice(0, -embeddedIpv4.length) + `${(value >>> 16).toString(16)}:${(value & 0xffff).toString(16)}`;
  }

  const halves = normalized.split("::");
  if (halves.length > 2) return null;
  const left = halves[0] ? halves[0].split(":") : [];
  const right = halves[1] ? halves[1].split(":") : [];
  const missing = 8 - left.length - right.length;
  if ((halves.length === 1 && missing !== 0) || missing < 0) return null;

  const groups = [...left, ...Array(missing).fill("0"), ...right].map(group => Number.parseInt(group || "0", 16));
  if (groups.length !== 8 || groups.some(group => !Number.isInteger(group) || group < 0 || group > 0xffff)) {
    return null;
  }
  return groups;
}

function isPrivateOrReservedIp(address: string): boolean {
  const family = isIP(address);
  if (family === 4) {
    return [
      ["0.0.0.0", 8],
      ["10.0.0.0", 8],
      ["100.64.0.0", 10],
      ["127.0.0.0", 8],
      ["169.254.0.0", 16],
      ["172.16.0.0", 12],
      ["192.0.0.0", 24],
      ["192.0.2.0", 24],
      ["192.168.0.0", 16],
      ["198.18.0.0", 15],
      ["198.51.100.0", 24],
      ["203.0.113.0", 24],
      ["224.0.0.0", 4],
      ["240.0.0.0", 4],
    ].some(([network, prefix]) => isIpv4Range(address, network as string, prefix as number));
  }

  if (family === 6) {
    const groups = parseIpv6(address);
    if (!groups) return true;
    const allZero = groups.every(group => group === 0);
    const loopback = groups.slice(0, 7).every(group => group === 0) && groups[7] === 1;
    const mappedIpv4 = groups.slice(0, 5).every(group => group === 0) && groups[5] === 0xffff;
    const compatibleIpv4 = groups.slice(0, 6).every(group => group === 0);
    if (mappedIpv4) {
      const mapped = `${groups[6] >>> 8}.${groups[6] & 255}.${groups[7] >>> 8}.${groups[7] & 255}`;
      return isPrivateOrReservedIp(mapped);
    }
    if (compatibleIpv4) {
      const compatible = `${groups[6] >>> 8}.${groups[6] & 255}.${groups[7] >>> 8}.${groups[7] & 255}`;
      return isPrivateOrReservedIp(compatible);
    }

    return (
      allZero ||
      loopback ||
      (groups[0] & 0xfe00) === 0xfc00 ||
      (groups[0] & 0xffc0) === 0xfe80 ||
      (groups[0] & 0xffc0) === 0xfec0 ||
      (groups[0] & 0xff00) === 0xff00 ||
      (groups[0] === 0x0064 && groups[1] === 0xff9b) ||
      (groups[0] === 0x2001 && groups[1] === 0x0000) ||
      (groups[0] === 0x2001 && groups[1] === 0x0db8) ||
      (groups[0] === 0x2001 && (groups[1] & 0xfff0) === 0x0010)
    );
  }

  return true;
}

export async function validatePublicHttpUrl(input: string): Promise<{ url: URL; addresses: ResolvedAddress[] }> {
  let url: URL;
  try {
    url = new URL(input);
  } catch {
    throw new PublicUrlError("Enter a complete URL beginning with http:// or https://.");
  }

  if (url.protocol !== "http:" && url.protocol !== "https:") {
    throw new PublicUrlError("Only public HTTP and HTTPS URLs can be checked.");
  }
  if (url.username || url.password) {
    throw new PublicUrlError("URLs containing credentials are not supported.");
  }
  if (
    url.port &&
    !((url.protocol === "http:" && url.port === "80") || (url.protocol === "https:" && url.port === "443"))
  ) {
    throw new PublicUrlError("Only standard web ports (80 and 443) can be checked.");
  }

  const hostname = url.hostname
    .toLowerCase()
    .replace(/\.$/, "")
    .replace(/^\[(.*)\]$/, "$1");
  if (
    hostname === "localhost" ||
    BLOCKED_HOST_SUFFIXES.some(suffix => hostname.endsWith(suffix)) ||
    hostname.length === 0
  ) {
    throw new PublicUrlError("Local and internal network addresses cannot be checked.");
  }

  let addresses: ResolvedAddress[];
  if (isIP(hostname)) {
    addresses = [{ address: hostname, family: isIP(hostname) as 4 | 6 }];
  } else {
    try {
      const resolved = await lookup(hostname, { all: true, verbatim: true });
      addresses = resolved.map(({ address, family }) => ({ address, family: family as 4 | 6 }));
    } catch {
      throw new PublicUrlError("That hostname could not be resolved.");
    }
  }

  if (addresses.length === 0 || addresses.some(({ address }) => isPrivateOrReservedIp(address))) {
    throw new PublicUrlError("Local, private, and reserved network addresses cannot be checked.");
  }

  url.hash = "";
  return { url, addresses };
}

function requestOnce(
  url: URL,
  addresses: ResolvedAddress[],
  timeoutMs: number,
  maxBytes: number
): Promise<{ body: string; headers: IncomingHttpHeaders; statusCode: number }> {
  return new Promise((resolve, reject) => {
    let settled = false;
    const settle = <T>(callback: (value: T) => void, value: T) => {
      if (settled) return;
      settled = true;
      clearTimeout(absoluteTimer);
      callback(value);
    };
    let nextAddress = 0;
    const pinnedLookup: LookupFunction = (_hostname, options, callback) => {
      if (options.all) {
        callback(null, addresses);
        return;
      }
      const selected = addresses[nextAddress++ % addresses.length];
      callback(null, selected.address, selected.family);
    };

    const request = (url.protocol === "https:" ? httpsRequest : httpRequest)(url, {
      headers: {
        Accept: "text/html,application/xhtml+xml;q=0.9,text/plain;q=0.5",
        "Accept-Encoding": "identity",
        "User-Agent": "RybbitToolScanner/1.0 (+https://rybbit.com/tools)",
      },
      lookup: pinnedLookup,
      method: "GET",
    });

    const absoluteTimer = setTimeout(
      () => request.destroy(new PublicUrlError("The website took too long to respond.")),
      timeoutMs
    );
    request.setTimeout(timeoutMs, () => request.destroy(new PublicUrlError("The website took too long to respond.")));
    request.on("error", error => settle(reject, error));
    request.on("response", response => {
      const contentLength = Number(response.headers["content-length"] || 0);
      if (contentLength > maxBytes) {
        response.destroy(new PublicUrlError("The response is too large to scan safely."));
        return;
      }

      const chunks: Buffer[] = [];
      let bytes = 0;
      response.on("data", (chunk: Buffer) => {
        bytes += chunk.length;
        if (bytes > maxBytes) {
          response.destroy(new PublicUrlError("The response is too large to scan safely."));
          return;
        }
        chunks.push(chunk);
      });
      response.on("error", error => settle(reject, error));
      response.on("end", () => {
        settle(resolve, {
          body: Buffer.concat(chunks).toString("utf8"),
          headers: response.headers,
          statusCode: response.statusCode || 0,
        });
      });
    });
    request.end();
  });
}

export async function fetchPublicText(
  input: string,
  options: { maxBytes?: number; timeoutMs?: number } = {}
): Promise<PublicTextResponse> {
  const maxBytes = options.maxBytes ?? MAX_PUBLIC_RESPONSE_BYTES;
  const timeoutMs = options.timeoutMs ?? 10_000;
  const deadline = Date.now() + timeoutMs;
  let current = input;

  for (let redirectCount = 0; redirectCount <= MAX_REDIRECTS; redirectCount += 1) {
    const { url, addresses } = await validatePublicHttpUrl(current);
    const remaining = deadline - Date.now();
    if (remaining <= 0) throw new PublicUrlError("The website took too long to respond.");

    const response = await requestOnce(url, addresses, remaining, maxBytes);
    const location = response.headers.location;
    if (response.statusCode >= 300 && response.statusCode < 400 && location) {
      if (redirectCount === MAX_REDIRECTS) throw new PublicUrlError("The website redirected too many times.");
      current = new URL(location, url).toString();
      continue;
    }

    return {
      ...response,
      finalUrl: url.toString(),
      redirectCount,
    };
  }

  throw new PublicUrlError("The website redirected too many times.");
}