Skip to content

Open source alternative to Semrush and Ahrefs

6.4 KB
Raw Blame
/**
 * URL normalization and utility functions for the site audit crawler.
 */

/**
 * Normalize a URL for deduplication:
 * - Resolve relative URLs against a base
 * - Strip fragments (#...)
 * - Sort query parameters
 * - Lowercase the hostname
 * - Preserve trailing slashes. A trailing slash is the canonical form on most
 *   CMS platforms (WordPress etc.), which 301-redirect the non-slash version to
 *   it. Stripping it here would rewrite the canonical URL into its own redirect
 *   source, so the crawler would follow /path -> /path/ and strip back to /path
 *   forever — a 508 loop. Keeping /path and /path/ distinct lets the redirect
 *   resolve normally.
 */
export function normalizeUrl(url: string, base?: string): string | null {
  try {
    const parsed = new URL(url, base);

    // Only crawl HTTP(S)
    if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
      return null;
    }

    // Strip fragment
    parsed.hash = "";

    // Sort query params for consistent dedup
    parsed.searchParams.sort();

    // Lowercase hostname
    parsed.hostname = parsed.hostname.toLowerCase();

    return parsed.toString();
  } catch {
    return null;
  }
}

/**
 * A canonical key for URL equality checks that should survive the common
 * redirect patterns a site uses to reach its canonical form:
 *   - trailing-slash redirects   (/services -> /services/)
 *   - www <-> non-www            (www.example.com -> example.com)
 *   - http -> https upgrades
 *
 * Forces https, drops a leading "www.", lowercases the hostname, sorts query
 * params, and strips the fragment. Trailing slashes are intentionally left
 * intact so two genuinely different paths never collapse together; this key is
 * only for "is this effectively the same page as the start URL" comparisons
 * (e.g. picking the homepage for the Lighthouse sample), not for crawl dedup.
 */
export function canonicalUrlKey(url: string): string {
  try {
    const parsed = new URL(url);
    parsed.protocol = "https:";
    parsed.hostname = parsed.hostname.toLowerCase().replace(/^www\./, "");
    parsed.hash = "";
    parsed.searchParams.sort();
    return parsed.toString();
  } catch {
    return url.toLowerCase();
  }
}

function getEffectivePort(parsed: URL): string {
  if (parsed.port) return parsed.port;
  return parsed.protocol === "https:" ? "443" : "80";
}

function areEquivalentHostnames(a: string, b: string): boolean {
  const hostA = a.toLowerCase();
  const hostB = b.toLowerCase();
  if (hostA === hostB) return true;
  return hostA === `www.${hostB}` || hostB === `www.${hostA}`;
}

/**
 * Check if a URL belongs to the same crawl boundary as the crawl target.
 *
 * Rules:
 * - Hostname must match exactly.
 * - Same protocol/port is always allowed.
 * - http -> https upgrade on default ports is allowed.
 */
export function isSameOrigin(url: string, origin: string): boolean {
  try {
    const parsedUrl = new URL(url);
    const parsedOrigin = new URL(origin);

    if (!areEquivalentHostnames(parsedUrl.hostname, parsedOrigin.hostname)) {
      return false;
    }

    const originProtocol = parsedOrigin.protocol.toLowerCase();
    const urlProtocol = parsedUrl.protocol.toLowerCase();

    const originPort = getEffectivePort(parsedOrigin);
    const urlPort = getEffectivePort(parsedUrl);

    if (originProtocol === urlProtocol) {
      return originPort === urlPort;
    }

    const isHttpToHttpsUpgrade =
      originProtocol === "http:" &&
      urlProtocol === "https:" &&
      originPort === "80" &&
      urlPort === "443";

    return isHttpToHttpsUpgrade;
  } catch {
    return false;
  }
}

/**
 * Detect a URL template pattern by replacing path segments that look like
 * dynamic values (IDs, slugs, dates) with `:param`.
 *
 * Examples:
 *   /blog/my-great-post      → /blog/:slug
 *   /products/12345           → /products/:id
 *   /users/john-doe/settings  → /users/:slug/settings
 */
export function detectUrlTemplate(pathname: string): string {
  const segments = pathname.split("/").filter(Boolean);

  const normalized = segments.map((segment) => {
    // Pure numeric IDs
    if (/^\d+$/.test(segment)) return ":id";
    // UUIDs
    if (
      /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(
        segment,
      )
    )
      return ":uuid";
    // Date-like segments (2024-01-15)
    if (/^\d{4}-\d{2}-\d{2}$/.test(segment)) return ":date";
    // Slug-like: contains hyphens and is more than 2 segments (to avoid short
    // path parts like "my-account" that are likely fixed routes)
    if (segment.includes("-") && segment.split("-").length > 2) return ":slug";

    return segment;
  });

  return "/" + normalized.join("/");
}

/**
 * Extract the origin (protocol + hostname + port) from a URL string.
 */
export function getOrigin(url: string): string {
  return new URL(url).origin;
}

/** Files a crawl can follow a link to but cannot audit as a page. */
const NON_DOCUMENT_EXTENSIONS =
  /\.(?:jpe?g|png|gif|webp|avif|svg|ico|bmp|tiff?|heic|woff2?|ttf|otf|eot|mp4|webm|m4v|mov|avi|mp3|wav|ogg|m4a|flac|css|js|mjs|json|xml|txt|map|zip|gz|tgz|tar|rar|7z|dmg|exe)$/i;

/**
 * Shopify routes that are never storefront pages and that its edge refuses to
 * crawlers, with an optional locale prefix (`/en/`, `/pt-br/`). The cart,
 * checkout and account routes need no entry: Shopify's robots.txt disallows
 * them.
 */
const SHOPIFY_NON_PAGE_PATHS =
  /^\/(?:[a-z]{2}(?:-[a-z]{2})?\/)?(?:customer_authentication\/|customer_identity\/|discount\/|password(?:\/|$))/i;

/**
 * Whether a same-origin URL is worth a page fetch. Links to images, fonts,
 * scripts and other files pass the SSRF check but are not pages: Shopify
 * themes link every gallery photo on the storefront origin, and one store's
 * audit spent 1,986 of its 2,608 fetches on them. Unknown extensions pass.
 * The trade: a broken link to such a file is no longer reported.
 *
 * Cloudflare's reserved `/cdn-cgi/` paths are skipped too. Its email
 * obfuscation rewrites every mailto link to `/cdn-cgi/l/email-protection`,
 * which answers crawlers with a 404, so a site with an email in its footer
 * got one false broken link per page.
 */
export function isAuditablePageUrl(
  url: string,
  sitePlatform?: "shopify",
): boolean {
  let parsed: URL;
  try {
    parsed = new URL(url);
  } catch {
    return false;
  }
  if (NON_DOCUMENT_EXTENSIONS.test(parsed.pathname)) return false;
  if (parsed.pathname.startsWith("/cdn-cgi/")) return false;
  return !(
    sitePlatform === "shopify" && SHOPIFY_NON_PAGE_PATHS.test(parsed.pathname)
  );
}