Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

27.7 KB
Raw Blame
import { createHash } from 'node:crypto';
import type { Duplex } from 'node:stream';
import { PassThrough, pipeline, Readable, Transform } from 'node:stream';
import { StringDecoder } from 'node:string_decoder';
import { createGunzip } from 'node:zlib';

import { FetchHttpClient } from '@crawlee/http-client';
import type { BaseHttpClient } from '@crawlee/http-client';
import type { CrawleeLogger } from '@crawlee/types';
// Imported as a type only so `sax` (a fairly heavy parser) isn't loaded eagerly with @crawlee/utils.
import type * as sax from 'sax';
import { MIMEType } from 'whatwg-mimetype';

import { mergeAsyncIterables } from './iterables.js';
import { RobotsTxtFile } from './robots.js';
import { type EnqueueStrategy, filterUrl } from './url.js';

interface SitemapUrlData {
    loc: string;
    lastmod?: Date;
    changefreq?: 'always' | 'hourly' | 'daily' | 'weekly' | 'monthly' | 'yearly' | 'never';
    priority?: number;
}

export type SitemapUrl = SitemapUrlData & {
    originSitemapUrl: string;
};

interface NestedSitemap {
    loc: string;
    originSitemapUrl: null;
}

type SitemapSource = ({ type: 'url'; url: string } | { type: 'raw'; content: string }) & { depth?: number };
type SitemapItem = ({ type: 'url' } & SitemapUrlData) | { type: 'sitemapUrl'; url: string };

class SitemapTxtParser extends Transform {
    #decoder: StringDecoder = new StringDecoder('utf8');
    #buffer = '';

    constructor() {
        super({
            readableObjectMode: true,
            transform: (chunk, _encoding, callback) => {
                this.#processBuffer(this.#decoder.write(chunk), false);
                callback();
            },
            flush: (callback) => {
                this.#processBuffer(this.#decoder.end(), true);
                callback();
            },
        });
    }

    #processBuffer(input: string, finalize: boolean): void {
        const lines = (this.#buffer + input).split('\n');

        // The text after the last line break may be a partial line, so it waits for the next chunk.
        this.#buffer = finalize ? '' : lines.pop()!;

        for (const line of lines) {
            const url = line.trim();
            if (url.length > 0) {
                this.push({ type: 'url', loc: url } satisfies SitemapItem);
            }
        }
    }
}

/** Namespaces of the sitemap protocol (https://www.sitemaps.org/protocol.html), including Google's legacy ones. */
const SITEMAP_NAMESPACES = new Set([
    'http://www.sitemaps.org/schemas/sitemap/0.9',
    'https://www.sitemaps.org/schemas/sitemap/0.9',
    'http://www.google.com/schemas/sitemap/0.9',
    'http://www.google.com/schemas/sitemap/0.84',
]);

class SitemapXmlParser extends Transform {
    #decoder: StringDecoder = new StringDecoder('utf8');
    #parser: sax.SAXParser;

    #rootTagName?: 'sitemapindex' | 'urlset';
    #currentTag?: 'loc' | 'lastmod' | 'changefreq' | 'priority' = undefined;
    #url: Partial<SitemapUrl> = {};
    /** Prefixes bound to the sitemap namespace, e.g. `ns0` for `<ns0:urlset xmlns:ns0="...">`. */
    #sitemapPrefixes = new Set<string>();

    static async create(): Promise<SitemapXmlParser> {
        const { SAXParser } = (await import('sax')).default;
        return new SitemapXmlParser(new SAXParser(true));
    }

    private constructor(parser: sax.SAXParser) {
        super({
            readableObjectMode: true,
            transform: (chunk, _encoding, callback) => {
                this.#parser.write(this.#decoder.write(chunk));
                callback();
            },
            flush: (callback) => {
                const rest = this.#decoder.end();
                if (rest.length > 0) {
                    this.#parser.write(rest);
                }

                this.#parser.end();
                callback();
            },
        });

        this.#parser = parser;
        this.#parser.onopentag = this.#onOpenTag.bind(this);
        this.#parser.onclosetag = this.#onCloseTag.bind(this);

        this.#parser.ontext = this.#onText.bind(this);
        this.#parser.oncdata = this.#onText.bind(this);

        this.#parser.onerror = this.destroy.bind(this);
    }

    /**
     * Returns the local name of a sitemap protocol element, or `undefined` for an element of another
     * namespace (such as `image:loc`). Unprefixed elements are sitemap elements, as before.
     */
    #sitemapTagName(qualifiedName: string): string | undefined {
        const colon = qualifiedName.indexOf(':');
        if (colon === -1) return qualifiedName;
        return this.#sitemapPrefixes.has(qualifiedName.slice(0, colon)) ? qualifiedName.slice(colon + 1) : undefined;
    }

    #onOpenTag(node: sax.Tag | sax.QualifiedTag) {
        for (const [attribute, value] of Object.entries(node.attributes)) {
            if (attribute.startsWith('xmlns:') && SITEMAP_NAMESPACES.has(String(value).trim())) {
                this.#sitemapPrefixes.add(attribute.slice('xmlns:'.length));
            }
        }

        const name = this.#sitemapTagName(node.name);

        if (this.#rootTagName !== undefined) {
            if (name === 'loc' || name === 'lastmod' || name === 'priority' || name === 'changefreq') {
                this.#currentTag = name;
            }
        }
        if (name === 'urlset') {
            this.#rootTagName = 'urlset';
        }
        if (name === 'sitemapindex') {
            this.#rootTagName = 'sitemapindex';
        }
    }

    #onCloseTag(qualifiedName: string) {
        const name = this.#sitemapTagName(qualifiedName);

        if (name === 'loc' || name === 'lastmod' || name === 'priority' || name === 'changefreq') {
            this.#currentTag = undefined;
        }

        if (name === 'url') {
            if (this.#url.loc !== undefined) {
                this.push({ type: 'url', ...this.#url, loc: this.#url.loc } satisfies SitemapItem);
            }
            this.#url = {};
        }
    }

    #onText(text: string) {
        text = text.trim();

        // Whitespace around a CDATA section arrives as separate text events.
        if (text === '') {
            return;
        }

        if (this.#currentTag === 'loc') {
            if (this.#rootTagName === 'sitemapindex') {
                this.push({ type: 'sitemapUrl', url: text } satisfies SitemapItem);
            }

            if (this.#rootTagName === 'urlset') {
                this.#url ??= {};
                this.#url.loc = text;
            }
        }

        if (this.#currentTag === 'lastmod') {
            const lastmod = new Date(text);
            if (!Number.isNaN(lastmod.getTime())) {
                this.#url.lastmod = lastmod;
            }
        }

        if (this.#currentTag === 'priority') {
            this.#url.priority = Number(text);
        }

        if (this.#currentTag === 'changefreq') {
            if (['always', 'hourly', 'daily', 'weekly', 'monthly', 'yearly', 'never'].includes(text)) {
                this.#url.changefreq = text as SitemapUrl['changefreq'];
            }
        }
    }
}

export interface ParseSitemapOptions {
    /**
     * If set to `true`, elements referring to other sitemaps will be emitted as special objects with `originSitemapUrl` set to `null`.
     */
    emitNestedSitemaps?: true | false;
    /**
     * Maximum depth of nested sitemaps to follow.
     */
    maxDepth?: number;
    /**
     * Number of retries for fetching sitemaps. The counter resets for each nested sitemap.
     */
    sitemapRetries?: number;
    /**
     * Timeout settings for network requests when fetching sitemaps. By default this is `30000` milliseconds (30 seconds).
     */
    timeoutMillis?: number;
    /**
     * If true, the parser will log a warning if it fails to fetch a sitemap due to a network error
     * @default true
     */
    reportNetworkErrors?: boolean;
    /**
     * Custom HTTP client to be used for fetching sitemaps.
     */
    httpClient?: BaseHttpClient;
    /**
     * Optional filter for nested sitemap URLs discovered in sitemap index files.
     * Called with the URL of each child sitemap before it is fetched.
     * Return `true` to include the sitemap, `false` to skip it.
     * If not provided, all nested sitemaps are followed.
     */
    nestedSitemapFilter?: (sitemapUrl: string) => boolean;
    /**
     * Keep only sitemap-derived URLs (nested `<sitemap>` and `<url>` entries) matching this strategy
     * relative to the parent sitemap URL; non-`http(s)` schemes are always dropped. Skipped for raw string
     * sources (no parent URL). Pass `'all'` to disable host filtering.
     * @default 'same-hostname'
     */
    enqueueStrategy?: EnqueueStrategy | `${EnqueueStrategy}`;
    /**
     * Optional logger for reporting warnings during sitemap parsing.
     */
    logger?: CrawleeLogger;
}

export async function* parseSitemap<T extends ParseSitemapOptions>(
    initialSources: SitemapSource[],
    proxyUrl?: string,
    options?: T,
): AsyncIterable<T['emitNestedSitemaps'] extends true ? SitemapUrl | NestedSitemap : SitemapUrl> {
    const {
        httpClient = new FetchHttpClient(),
        emitNestedSitemaps = false,
        maxDepth = Infinity,
        sitemapRetries = 3,
        timeoutMillis: timeout = 30000,
        reportNetworkErrors = true,
        nestedSitemapFilter,
        enqueueStrategy = 'same-hostname',
        logger,
    } = options ?? {};

    const sources = [...initialSources];
    const visitedSitemapUrls = new Set<string>();

    const createParser = async (contentType = '', url?: URL): Promise<Duplex> => {
        let mimeType: MIMEType | null;

        try {
            mimeType = new MIMEType(contentType);
        } catch {
            mimeType = null;
        }

        if (mimeType?.isXML() || url?.pathname.endsWith('.xml')) {
            return SitemapXmlParser.create();
        }

        if (mimeType?.essence === 'text/plain' || url?.pathname.endsWith('.txt')) {
            return new SitemapTxtParser();
        }

        // A name like `sitemap.gz` says nothing about what was compressed. Sitemaps are XML unless named otherwise.
        if (mimeType?.essence === 'application/gzip') {
            return SitemapXmlParser.create();
        }

        throw new Error(`Unsupported sitemap content type (contentType = ${contentType}, url = ${url?.toString()})`);
    };

    while (sources.length > 0) {
        const source = sources.shift()!;

        if ((source?.depth ?? 0) > maxDepth) {
            continue;
        }

        let items: AsyncIterable<SitemapItem> | null = null;

        // Parent URL, parsed once and reused as the origin for the strategy checks below.
        let sitemapUrl: URL | undefined;

        if (source.type === 'url') {
            sitemapUrl = new URL(source.url);
            visitedSitemapUrls.add(sitemapUrl.toString());
            let retriesLeft = sitemapRetries + 1;

            while (retriesLeft-- > 0) {
                try {
                    let sitemapResponse: Response | null;

                    try {
                        sitemapResponse = await httpClient.sendRequest(
                            new Request(sitemapUrl, {
                                method: 'GET',
                                headers: {
                                    accept: '*/*',
                                },
                            }),
                            {
                                proxyUrl,
                                timeoutMillis: timeout,
                            },
                        );
                    } catch (error: any) {
                        sitemapResponse = null;
                    }

                    let error: { error: Error; type: 'fetch' | 'parser' } | null = null;

                    if (sitemapResponse && sitemapResponse.status >= 200 && sitemapResponse.status < 300) {
                        let contentType = sitemapResponse.headers.get('content-type');

                        if (sitemapResponse.body === null) {
                            break;
                        }
                        const { fileTypeStream } = await import('file-type');
                        const streamWithType = await fileTypeStream(Readable.fromWeb(sitemapResponse.body as any));
                        if (streamWithType.fileType !== undefined) {
                            contentType = streamWithType.fileType.mime;
                        }

                        let isGzipped = false;

                        if (
                            contentType !== undefined
                                ? contentType === 'application/gzip'
                                : sitemapUrl.pathname.endsWith('.gz')
                        ) {
                            isGzipped = true;

                            if (sitemapUrl.pathname.endsWith('.gz')) {
                                sitemapUrl.pathname = sitemapUrl.pathname.substring(0, sitemapUrl.pathname.length - 3);
                            }
                        }

                        items = pipeline(
                            streamWithType,
                            isGzipped ? createGunzip() : new PassThrough(),
                            await createParser(contentType ?? undefined, sitemapUrl),
                            (e) => {
                                if (e !== undefined && e !== null) {
                                    error = { type: 'parser', error: e };
                                }
                            },
                        );
                    } else {
                        error = {
                            type: 'fetch',
                            error: new Error(
                                `Failed to fetch sitemap: ${sitemapUrl}, status code: ${sitemapResponse?.status}`,
                            ),
                        };
                    }

                    if (error !== null) {
                        const shouldIgnoreError = error.type === 'fetch' && !reportNetworkErrors;
                        if (!shouldIgnoreError) {
                            throw error.error;
                        }
                    } else {
                        break;
                    }
                } catch (e) {
                    logger?.warning(
                        `Malformed sitemap content: ${sitemapUrl}, ${retriesLeft === 0 ? 'no retries left.' : 'retrying...'} (${e})`,
                    );
                }
            }
        } else if (source.type === 'raw') {
            items = pipeline(Readable.from([source.content]), await createParser('text/xml'), (error) => {
                if (error !== undefined) {
                    logger?.warning(`Malformed sitemap content: ${error}`);
                }
            });
        }

        if (items === null) {
            continue;
        }

        // URL entries dropped by the enqueue strategy filter, reported in one warning per sitemap after
        // the loop (per-entry warnings could flood the log; individual drops are logged at debug level).
        let droppedUrlEntries = 0;

        // The same for every entry of this source, so compute it once.
        const originSitemapUrl =
            source.type === 'url'
                ? source.url
                : `raw://${createHash('sha256').update(source.content).digest('base64')}`;

        const iterator = items[Symbol.asyncIterator]();
        try {
            while (true) {
                let next: IteratorResult<SitemapItem>;
                try {
                    next = await iterator.next();
                } catch (e) {
                    // Parsing failed: give up on this sitemap, keep the others.
                    // Raw sources already logged the error in their pipeline callback.
                    if (source.type !== 'raw') {
                        logger?.warning(`Malformed sitemap content: ${source.url} (${e})`);
                    }
                    break;
                }
                if (next.done) break;
                const item = next.value;

                if (item.type === 'sitemapUrl' && !visitedSitemapUrls.has(item.url)) {
                    if (nestedSitemapFilter && !nestedSitemapFilter(item.url)) {
                        logger?.debug(`Skipping sitemap ${item.url} due to nestedSitemapFilter.`);
                        continue;
                    }

                    // Keep only nested sitemaps matching the strategy (and using http(s)) relative to the
                    // parent. Raw string sources have no parent URL, so the check is skipped.
                    if (source.type === 'url') {
                        const { allowed, reason } = filterUrl(item.url, sitemapUrl!, enqueueStrategy);
                        if (!allowed) {
                            logger?.warning(`Skipping nested sitemap ${item.url} (parent ${source.url}): ${reason}.`);
                            continue;
                        }
                    }

                    sources.push({ type: 'url', url: item.url, depth: (source.depth ?? 0) + 1 });
                    if (emitNestedSitemaps) {
                        yield { loc: item.url, originSitemapUrl: null } as any;
                    }
                }

                if (item.type === 'url') {
                    // Keep only URL entries that match the enqueue strategy relative to the parent (see above).
                    if (source.type === 'url') {
                        const { allowed, reason } = filterUrl(item.loc, sitemapUrl!, enqueueStrategy);
                        if (!allowed) {
                            droppedUrlEntries++;
                            logger?.debug(`Skipping sitemap URL ${item.loc} (parent ${source.url}): ${reason}.`);
                            continue;
                        }
                    }

                    yield {
                        ...item,
                        originSitemapUrl,
                    };
                }
            }
        } finally {
            // Mirrors `for await` cleanup: destroys the stream when we stop early or the consumer bails out.
            await iterator.return?.();
        }

        if (droppedUrlEntries > 0 && source.type === 'url') {
            logger?.warning(
                `Skipped ${droppedUrlEntries} URL(s) from sitemap ${source.url} not matching enqueue strategy '${enqueueStrategy}' (or using a non-http(s) scheme). Enable debug logs to see each skipped URL.`,
            );
        }
    }
}

/**
 * Loads one or more sitemaps from given URLs, following references in sitemap index files, and exposes the contained URLs.
 *
 * **Example usage:**
 * ```javascript
 * // Load a sitemap
 * const sitemap = await Sitemap.load(['https://example.com/sitemap.xml', 'https://example.com/sitemap_2.xml.gz']);
 *
 * // Enqueue all the contained URLs (including those from sub-sitemaps from sitemap indexes)
 * await crawler.addRequests(sitemap.urls);
 * ```
 */
export class Sitemap {
    constructor(readonly urls: string[]) {}

    /**
     * Try to load sitemap from the most common locations - `/sitemap.xml` and `/sitemap.txt`.
     * For loading based on `Sitemap` entries in `robots.txt`, the {@apilink RobotsTxtFile} class should be used.
     * @param url The domain URL to fetch the sitemap for.
     * @param proxyUrl A proxy to be used for fetching the sitemap file.
     */
    static async tryCommonNames(
        url: string,
        proxyUrl?: string,
        parseSitemapOptions?: ParseSitemapOptions,
    ): Promise<Sitemap> {
        const sitemapUrls: string[] = [];

        const sitemapUrl = new URL(url);
        sitemapUrl.search = '';

        sitemapUrl.pathname = '/sitemap.xml';
        sitemapUrls.push(sitemapUrl.toString());

        sitemapUrl.pathname = '/sitemap.txt';
        sitemapUrls.push(sitemapUrl.toString());

        return Sitemap.load(sitemapUrls, proxyUrl, { reportNetworkErrors: false, ...parseSitemapOptions });
    }

    /**
     * Fetch sitemap content from given URL or URLs and return URLs of referenced pages.
     * @param urls sitemap URL(s)
     * @param proxyUrl URL of a proxy to be used for fetching sitemap contents
     */
    static async load(
        urls: string | string[],
        proxyUrl?: string,
        parseSitemapOptions?: ParseSitemapOptions,
    ): Promise<Sitemap> {
        return await this.#parse(
            (Array.isArray(urls) ? urls : [urls]).map((url) => ({ type: 'url', url })),
            proxyUrl,
            parseSitemapOptions,
        );
    }

    /**
     * Parse XML sitemap content from a string and return URLs of referenced pages. If the sitemap references other sitemaps, they will be loaded via HTTP.
     * @param content XML sitemap content
     * @param proxyUrl URL of a proxy to be used for fetching sitemap contents
     */
    static async fromXmlString(
        content: string,
        proxyUrl?: string,
        parseSitemapOptions?: ParseSitemapOptions,
    ): Promise<Sitemap> {
        return await this.#parse([{ type: 'raw', content }], proxyUrl, parseSitemapOptions);
    }

    static async #parse(
        sources: SitemapSource[],
        proxyUrl?: string,
        parseSitemapOptions?: ParseSitemapOptions,
    ): Promise<Sitemap> {
        const urls: string[] = [];

        try {
            for await (const item of parseSitemap(sources, proxyUrl, parseSitemapOptions)) {
                urls.push(item.loc);
            }
        } catch (e) {
            parseSitemapOptions?.logger?.warning(
                `Sitemap.load: Failed to load sitemap, returning empty result. (${e})`,
            );
            return new Sitemap([]);
        }

        return new Sitemap(urls);
    }
}

/**
 * Given a list of URLs, discover related sitemap files for these domains by checking the `robots.txt` file,
 * the default `sitemap.xml` & `sitemap.txt` files and the URLs themselves.
 * @param `urls` The list of URLs to discover sitemaps for.
 * @param `options` Options for sitemap discovery
 * @returns An async iterable with the discovered sitemap URLs.
 */
export async function* discoverValidSitemaps(
    urls: string[],
    options: {
        /**
         * Proxy URL to be used for network requests.
         */
        proxyUrl?: string;
        /**
         * Timeout in milliseconds for the entire `discoverValidSitemaps` call.
         * An `AbortController` is created internally and its signal is passed to every HTTP request,
         * so the whole discovery operation is cancelled once the timeout elapses.
         * Defaults to `60_000` ms (60 seconds) to prevent indefinite hangs.
         */
        timeoutMillis?: number;
        /**
         * An external `AbortSignal` to cancel the entire discovery operation.
         * If both `signal` and `timeout` are provided, the operation is cancelled
         * when either the signal is aborted or the timeout elapses (whichever comes first).
         */
        signal?: AbortSignal;
        /**
         * Timeout in milliseconds for each individual HTTP request during discovery.
         * Defaults to `20000` ms (20 seconds).
         */
        requestTimeoutMillis?: number;
        /**
         * HTTP client to be used for network requests.
         */
        httpClient?: BaseHttpClient;
        /**
         * Optional logger for reporting warnings during sitemap discovery.
         */
        logger?: CrawleeLogger;
    } = {},
): AsyncIterable<string> {
    const {
        proxyUrl,
        timeoutMillis = 60_000,
        signal: externalSignal,
        requestTimeoutMillis = 20_000,
        httpClient = new FetchHttpClient(),
        logger,
    } = options;
    const controller = new AbortController();

    const timeoutHandle = setTimeout(() => controller.abort(), timeoutMillis);
    const onExternalAbort = () => controller.abort();
    if (externalSignal) {
        if (externalSignal.aborted) {
            controller.abort();
        } else {
            externalSignal.addEventListener('abort', onExternalAbort, { once: true });
        }
    }

    const signal = controller.signal;
    const sitemapUrls = new Set<string>();

    const addSitemapUrl = (url: string): string | undefined => {
        const sizeBefore = sitemapUrls.size;

        sitemapUrls.add(url);

        if (sitemapUrls.size > sizeBefore) {
            return url;
        }

        return undefined;
    };

    const urlExists = async (url: string): Promise<boolean> => {
        if (!httpClient) {
            return false;
        }
        try {
            const response = await httpClient.sendRequest(new Request(url, { method: 'HEAD' }), {
                proxyUrl,
                timeoutMillis: requestTimeoutMillis,
                signal,
            });
            return response.status >= 200 && response.status < 400;
        } catch {
            return false;
        }
    };

    const discoverSitemapsForDomainUrls = async function* (hostname: string, domainUrls: string[]) {
        if (!hostname) {
            return;
        }

        try {
            const robotsFile = await RobotsTxtFile.find(domainUrls[0], {
                proxyUrl,
                timeoutMillis: requestTimeoutMillis,
                signal,
                httpClient,
                logger,
            });
            // Surface all referenced sitemaps, including cross-host; scoping happens at load time.
            for (const sitemapUrl of robotsFile.getSitemaps({ enqueueStrategy: 'all' })) {
                if (addSitemapUrl(sitemapUrl)) {
                    yield sitemapUrl;
                }
            }
        } catch (err) {
            logger?.warning(`Failed to fetch robots.txt file for ${hostname}`, { error: err });
        }

        const sitemapUrl = domainUrls.find((url) => /sitemap(?:_index)?\.(?:xml|txt)(?:\.gz)?$/i.test(url));

        if (sitemapUrl !== undefined) {
            if (addSitemapUrl(sitemapUrl)) {
                yield sitemapUrl;
            }
        } else {
            const firstUrl = new URL(domainUrls[0]);
            // The guessed candidates are well-known paths on the same origin, so the query string
            // and fragment of the input URL are not part of them - carrying them over probes (and
            // yields) URLs like `/sitemap.xml?page=2`, which servers may well answer with a 404.
            firstUrl.search = '';
            firstUrl.hash = '';
            const possibleSitemapPathnames = ['/sitemap.xml', '/sitemap.txt', '/sitemap_index.xml'];
            const candidateSitemapUrls = possibleSitemapPathnames.map((pathname) => {
                firstUrl.pathname = pathname;
                return firstUrl.toString();
            });
            const candidateResults = await Promise.allSettled(candidateSitemapUrls.map(urlExists));

            for (const [index, result] of candidateResults.entries()) {
                const candidateSitemapUrl = candidateSitemapUrls[index];

                if (result.status === 'fulfilled') {
                    if (result.value && addSitemapUrl(candidateSitemapUrl)) {
                        yield candidateSitemapUrl;
                    }
                } else {
                    logger?.debug(`Failed to check sitemap candidate ${candidateSitemapUrl} for ${hostname}`, {
                        error: result.reason,
                    });
                }
            }
        }
    };

    const groupedUrls = urls.reduce(
        (acc, url) => {
            const hostname = new URL(url)?.hostname ?? '';
            acc[hostname] ??= [];
            acc[hostname].push(url);
            return acc;
        },
        {} as Record<string, string[]>,
    );

    const iterables = Object.entries(groupedUrls).map(([hostname, domainUrls]) =>
        discoverSitemapsForDomainUrls(hostname, domainUrls),
    );

    const discoveredUrls = new Set<string>();

    try {
        for await (const url of mergeAsyncIterables(...iterables)) {
            if (discoveredUrls.has(url)) {
                continue;
            }
            discoveredUrls.add(url);
            yield url;
        }
    } finally {
        clearTimeout(timeoutHandle);
        externalSignal?.removeEventListener('abort', onExternalAbort);
    }
}