Skip to content

Open source alternative to Semrush and Ahrefs

11.4 KB
Raw Blame
/**
 * robots.txt and sitemap.xml discovery for the site audit crawler.
 */
import robotsParser from "robots-parser";
import { XMLParser } from "fast-xml-parser";
import { readBodyCapped } from "./html-response";
import { isSameOrigin, normalizeUrl } from "./url-utils";
import { isCrawlableUrl } from "./url-policy";
import { publicFetch } from "@/server/lib/public-fetch";
import { createRateLimitWait, type RateLimitWait } from "./crawl-throttle";
import type { CrawlerAccess } from "@/shared/crawler-access";
import {
  auditRequestHeaders,
  CRAWL_USER_AGENT,
} from "@/server/lib/audit/web-bot-auth";

const SITEMAP_FETCH_TIMEOUT_MS = 15_000;
// robots.txt is checkpointed as durable Workflow step state (~1MiB cap, shared
// with the rest of the step's return). RFC 9309 requires parsers to handle at
// least 500 KiB and permits ignoring anything beyond it — Google does exactly
// that — so this cap matches standard crawler behavior while keeping a
// misbehaving server (e.g. HTML at /robots.txt) from blowing the step limit.
const MAX_ROBOTS_TXT_BYTES = 500 * 1024;
const MAX_SITEMAP_DEPTH = 3;
const MAX_SITEMAP_DOCS = 300;
const SITEMAP_CONCURRENCY = 5;
const SITEMAP_RETRIES = 1;
// Sitemap shards can legally reach 50 MB and SITEMAP_CONCURRENCY of them are
// read at once, so unbounded reads can exhaust Worker memory. Oversized
// shards are skipped whole — truncated XML would not parse anyway, and real
// generators shard far below this.
const MAX_SITEMAP_BYTES = 10 * 1024 * 1024;
const xmlParser = new XMLParser({
  ignoreAttributes: false,
  isArray: (name) => name === "sitemap" || name === "url",
});

export interface RobotsResult {
  isAllowed: (url: string) => boolean;
  sitemapUrls: string[];
}

/**
 * Fetch the raw robots.txt body (null = missing/unreachable). Kept separate
 * from parsing so Workflows can checkpoint the text as durable step state and
 * re-derive the parsed result deterministically on replay.
 */
async function fetchRobotsTxtText(
  origin: string,
  access: CrawlerAccess | null | undefined,
  waitForRateLimit: RateLimitWait,
): Promise<string | null> {
  try {
    const fetchRobots = () =>
      fetchFollowingRedirects(`${origin}/robots.txt`, 10_000, access);
    let fetched = await fetchRobots();
    if (
      fetched?.response.status === 429 &&
      (await waitForRateLimit(fetched.response))
    ) {
      fetched = await fetchRobots();
    }
    if (!fetched?.response.ok) return null;
    return (await fetched.response.text()).slice(0, MAX_ROBOTS_TXT_BYTES);
  } catch (error) {
    console.warn("Failed to fetch robots.txt:", error);
    return null;
  }
}

const MAX_DISCOVERY_REDIRECT_HOPS = 5;

/**
 * Redirects are followed by hand so each hop is revalidated against the crawl
 * policy and crawler-access headers — bot-protection credentials — are
 * re-matched against the hop's host instead of riding along to another site.
 */
async function fetchFollowingRedirects(
  url: string,
  timeoutMs: number,
  access: CrawlerAccess | null | undefined,
): Promise<{ response: Response; finalUrl: string } | null> {
  // One budget for the whole chain, as the automatic follow had.
  const deadline = Date.now() + timeoutMs;
  let current = url;
  for (let hop = 0; hop <= MAX_DISCOVERY_REDIRECT_HOPS; hop++) {
    const response = await publicFetch(current, {
      headers: await auditRequestHeaders(current, access),
      redirect: "manual",
      signal: AbortSignal.timeout(Math.max(1, deadline - Date.now())),
    });

    if (response.status < 300 || response.status >= 400) {
      return { response, finalUrl: current };
    }

    const location = response.headers.get("location");
    await response.body?.cancel();
    if (!location) return null;

    let next: string;
    try {
      next = new URL(location, current).toString();
    } catch {
      return null;
    }
    if (!isCrawlableUrl(next)) return null;
    current = next;
  }
  return null;
}

/** Deterministic: same text in, same result out. Null = everything allowed. */
export function parseRobotsTxt(
  origin: string,
  text: string | null,
): RobotsResult {
  if (text === null) {
    return { isAllowed: () => true, sitemapUrls: [] };
  }

  const robots = robotsParser(`${origin}/robots.txt`, text);
  return {
    // Our UA so an `OpenSEO-Audit` group wins over `*`.
    isAllowed: (url: string) => robots.isAllowed(url, CRAWL_USER_AGENT) ?? true,
    sitemapUrls: robots.getSitemaps(),
  };
}

/**
 * Fetch and parse a sitemap (supports sitemap index recursion).
 * Returns a flat list of page URLs found.
 */
function isProbablySitemapXml(
  contentType: string | null,
  body: string,
): boolean {
  if (contentType?.toLowerCase().includes("xml")) {
    return true;
  }

  const trimmed = body.trimStart().toLowerCase();
  return (
    trimmed.startsWith("<?xml") ||
    trimmed.startsWith("<urlset") ||
    trimmed.startsWith("<sitemapindex")
  );
}

function getSitemapLocations(input: unknown): string[] {
  if (!input) return [];
  const entries = Array.isArray(input) ? input : [input];
  return entries
    .map((entry) => {
      if (isRecord(entry)) {
        const loc = entry["loc"];
        return typeof loc === "string" ? loc : null;
      }
      return null;
    })
    .filter((loc): loc is string => typeof loc === "string");
}

function isRecord(value: unknown): value is Record<string, unknown> {
  return !!value && typeof value === "object";
}

function getParsedSitemapSections(parsed: unknown): {
  sitemap: unknown;
  url: unknown;
} {
  if (!parsed || typeof parsed !== "object") {
    return { sitemap: undefined, url: undefined };
  }

  const root = parsed as {
    sitemapindex?: { sitemap?: unknown };
    urlset?: { url?: unknown };
  };

  return {
    sitemap: root.sitemapindex?.sitemap,
    url: root.urlset?.url,
  };
}

function isTimeoutError(error: unknown): boolean {
  if (!error || typeof error !== "object") return false;
  return "name" in error && error.name === "TimeoutError";
}

async function fetchSitemapDocumentWithRetry(
  sitemapUrl: string,
  access: CrawlerAccess | null | undefined,
  waitForRateLimit: RateLimitWait,
): Promise<{
  nestedSitemaps: string[];
  pageUrls: string[];
  timedOut: boolean;
}> {
  const normalizedSitemapUrl = normalizeUrl(sitemapUrl);
  if (!normalizedSitemapUrl) {
    return { nestedSitemaps: [], pageUrls: [], timedOut: false };
  }

  let lastError: unknown = null;

  for (let attempt = 0; attempt <= SITEMAP_RETRIES; attempt++) {
    try {
      const fetched = await fetchFollowingRedirects(
        normalizedSitemapUrl,
        SITEMAP_FETCH_TIMEOUT_MS,
        access,
      );
      if (!fetched) {
        return { nestedSitemaps: [], pageUrls: [], timedOut: false };
      }
      const { response } = fetched;

      const finalUrl = normalizeUrl(fetched.finalUrl, normalizedSitemapUrl);
      if (!finalUrl || !isSameOrigin(finalUrl, normalizedSitemapUrl)) {
        return { nestedSitemaps: [], pageUrls: [], timedOut: false };
      }

      if (
        response.status === 429 &&
        attempt < SITEMAP_RETRIES &&
        (await waitForRateLimit(response))
      ) {
        continue;
      }
      if (!response.ok) {
        return { nestedSitemaps: [], pageUrls: [], timedOut: false };
      }

      const body = await readBodyCapped(response, MAX_SITEMAP_BYTES);
      if (
        body === null ||
        !isProbablySitemapXml(response.headers.get("content-type"), body)
      ) {
        return { nestedSitemaps: [], pageUrls: [], timedOut: false };
      }

      const parsed = xmlParser.parse(body) as unknown;
      const sections = getParsedSitemapSections(parsed);
      const nestedSitemaps = getSitemapLocations(sections.sitemap)
        .map((loc) => normalizeUrl(loc, finalUrl))
        .filter((loc): loc is string => loc !== null);
      const pageUrls = getSitemapLocations(sections.url)
        .map((loc) => normalizeUrl(loc, finalUrl))
        .filter((loc): loc is string => loc !== null);

      return { nestedSitemaps, pageUrls, timedOut: false };
    } catch (error) {
      lastError = error;
      if (!isTimeoutError(error) || attempt === SITEMAP_RETRIES) {
        break;
      }
    }
  }

  return {
    nestedSitemaps: [],
    pageUrls: [],
    timedOut: isTimeoutError(lastError),
  };
}

/**
 * Discover all page URLs from robots.txt + sitemaps for an origin.
 * Also tries the default /sitemap.xml if not listed in robots.txt.
 */
export async function discoverUrls(
  origin: string,
  maxPages = 50,
  access?: CrawlerAccess | null,
): Promise<{ urls: string[]; robotsText: string | null }> {
  const waitForRateLimit = createRateLimitWait();
  const robotsText = await fetchRobotsTxtText(origin, access, waitForRateLimit);
  const robots = parseRobotsTxt(origin, robotsText);

  // Collect sitemap URLs: from robots.txt + default location
  const sitemapSources = new Set(robots.sitemapUrls);
  sitemapSources.add(`${origin}/sitemap.xml`);

  const maxDiscoveredUrls = Math.min(Math.max(maxPages * 20, 500), 50_000);
  const allUrls = new Set<string>();

  const queue: Array<{ url: string; depth: number }> = Array.from(
    sitemapSources,
  )
    .map((url) => normalizeUrl(url, origin))
    .filter((url): url is string => url !== null)
    .filter((url) => isSameOrigin(url, origin))
    .map((url) => ({ url, depth: MAX_SITEMAP_DEPTH }));
  const seenSitemapDocs = new Set<string>();
  let fetchedDocs = 0;
  let failedDocs = 0;
  let timedOutDocs = 0;

  while (queue.length > 0 && allUrls.size < maxDiscoveredUrls) {
    if (fetchedDocs >= MAX_SITEMAP_DOCS) {
      break;
    }
    const batch = queue.splice(0, SITEMAP_CONCURRENCY);
    await Promise.all(
      batch.map(async ({ url, depth }) => {
        const normalizedUrl = normalizeUrl(url);
        if (
          !normalizedUrl ||
          !isSameOrigin(normalizedUrl, origin) ||
          depth <= 0 ||
          seenSitemapDocs.has(normalizedUrl)
        ) {
          return;
        }

        seenSitemapDocs.add(normalizedUrl);
        fetchedDocs += 1;

        const result = await fetchSitemapDocumentWithRetry(
          normalizedUrl,
          access,
          waitForRateLimit,
        );
        if (
          result.pageUrls.length === 0 &&
          result.nestedSitemaps.length === 0
        ) {
          failedDocs += 1;
          if (result.timedOut) {
            timedOutDocs += 1;
          }
          return;
        }

        for (const pageUrl of result.pageUrls) {
          if (!isSameOrigin(pageUrl, origin)) continue;
          if (allUrls.size >= maxDiscoveredUrls) break;
          allUrls.add(pageUrl);
        }

        if (depth <= 1) return;

        for (const nestedUrl of result.nestedSitemaps) {
          if (!isSameOrigin(nestedUrl, origin)) continue;
          if (!seenSitemapDocs.has(nestedUrl)) {
            queue.push({ url: nestedUrl, depth: depth - 1 });
          }
        }
      }),
    );
  }

  if (failedDocs > 0) {
    console.warn(
      `Sitemap discovery completed with partial failures for ${origin}: fetched=${fetchedDocs}, failed=${failedDocs}, timedOut=${timedOutDocs}, discoveredUrls=${allUrls.size}`,
    );
  }

  // Every discovered URL is returned, not just the first maxPages. Seeding is
  // also what records each page's in-sitemap flag on the frontier, so
  // truncating here made every sitemap URL past the budget report
  // inSitemap: false once the crawl reached it through a link. The set is
  // already bounded by maxDiscoveredUrls, seeds go straight into the
  // scratchpad DO rather than through durable step state, and the crawl still
  // stops at maxPages.
  return {
    urls: Array.from(allUrls),
    robotsText,
  };
}