Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix: filter sitemap-derived URLs by enqueue strategy (#3797)

Vlada Dusek9dc3160e6b83
diff --git a/packages/core/src/enqueue_links/enqueue_links.ts b/packages/core/src/enqueue_links/enqueue_links.ts
index 764700005..b13f09508 100644
--- a/packages/core/src/enqueue_links/enqueue_links.ts
+++ b/packages/core/src/enqueue_links/enqueue_links.ts
@@ -1,5 +1,5 @@
import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';
-import { type RobotsTxtFile } from '@crawlee/utils';
+import { EnqueueStrategy, type RobotsTxtFile } from '@crawlee/utils';
import ow from 'ow';
import { getDomain } from 'tldts';
import type { SetRequired } from 'type-fest';
@@ -31,6 +31,8 @@ import {
filterRequestsByPatterns,
} from './shared';
+export { EnqueueStrategy };
+
export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
/** Limit the amount of actually enqueued URLs to this number. Useful for testing across the entire crawling scope. */
limit?: number;
@@ -199,61 +201,6 @@ export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
onSkippedRequest?: SkippedRequestCallback;
}
-/**
- * The different enqueueing strategies available.
- *
- * Depending on the strategy you select, we will only check certain parts of the URLs found. Here is a diagram of each URL part and their name:
- *
- * ```md
- * Protocol Domain
- * ┌────┐ ┌─────────┐
- * https://example.crawlee.dev/...
- * │ └─────────────────┤
- * │ Hostname │
- * │ │
- * └─────────────────────────┘
- * Origin
- *```
- *
- * - The `Protocol` is usually `http` or `https`
- * - The `Domain` represents the path without any possible subdomains to a website. For example, `crawlee.dev` is the domain of `https://example.crawlee.dev/`
- * - The `Hostname` is the full path to a website, including any subdomains. For example, `example.crawlee.dev` is the hostname of `https://example.crawlee.dev/`
- * - The `Origin` is the combination of the `Protocol` and `Hostname`. For example, `https://example.crawlee.dev` is the origin of `https://example.crawlee.dev/`
- */
-export enum EnqueueStrategy {
- /**
- * Matches any URLs found
- */
- All = 'all',
-
- /**
- * Matches any URLs that have the same hostname.
- * For example, `https://wow.example.com/hello` will be matched for a base url of `https://wow.example.com/`, but
- * `https://example.com/hello` will not be matched.
- *
- * > This strategy will match both `http` and `https` protocols regardless of the base URL protocol.
- */
- SameHostname = 'same-hostname',
-
- /**
- * Matches any URLs that have the same domain as the base URL.
- * For example, `https://wow.an.example.com` and `https://example.com` will both be matched for a base url of
- * `https://example.com`.
- *
- * > This strategy will match both `http` and `https` protocols regardless of the base URL protocol.
- */
- SameDomain = 'same-domain',
-
- /**
- * Matches any URLs that have the same hostname and protocol.
- * For example, `https://wow.example.com/hello` will be matched for a base url of `https://wow.example.com/`, but
- * `http://wow.example.com/hello` will not be matched.
- *
- * > This strategy will ensure the protocol of the base URL is the same as the protocol of the URL to be enqueued.
- */
- SameOrigin = 'same-origin',
-}
-
/**
* This function enqueues the urls provided to the {@apilink RequestQueue} provided. If you want to automatically find and enqueue links,
* you should use the context-aware `enqueueLinks` function provided on the crawler contexts.
diff --git a/packages/core/src/storages/sitemap_request_list.ts b/packages/core/src/storages/sitemap_request_list.ts
index 87e06bca8..1b9903f05 100644
--- a/packages/core/src/storages/sitemap_request_list.ts
+++ b/packages/core/src/storages/sitemap_request_list.ts
@@ -9,7 +9,7 @@ import defaultLog from '@apify/log';
import { Configuration } from '../configuration';
import type { GlobInput, RegExpInput, UrlPatternObject } from '../enqueue_links';
-import { constructGlobObjectsFromGlobs, constructRegExpObjectsFromRegExps } from '../enqueue_links';
+import { constructGlobObjectsFromGlobs, constructRegExpObjectsFromRegExps, EnqueueStrategy } from '../enqueue_links';
import { type EventManager, EventType } from '../events/event_manager';
import { Request } from '../request';
import { KeyValueStore } from './key_value_store';
@@ -95,6 +95,13 @@ export interface SitemapRequestListOptions extends UrlConstraints {
* @default 200
*/
maxBufferSize?: number;
+ /**
+ * Keep only sitemap-derived URLs matching this strategy relative to the parent sitemap URL; non-`http(s)`
+ * schemes are always dropped. The filtering stays enforced after navigation (e.g. across redirects).
+ * Pass `'all'` to disable host filtering.
+ * @default EnqueueStrategy.SameHostname
+ */
+ enqueueStrategy?: EnqueueStrategy | `${EnqueueStrategy}`;
/**
* Advanced options for the underlying `parseSitemap` call.
*/
@@ -192,6 +199,11 @@ export class SitemapRequestList implements IRequestList {
*/
private proxyUrl: string | undefined;
+ /**
+ * Enqueue strategy applied to sitemap-derived URLs and stamped onto the emitted `Request` objects.
+ */
+ private enqueueStrategy: EnqueueStrategy | `${EnqueueStrategy}`;
+
/**
* Logger instance.
*/
@@ -216,6 +228,7 @@ export class SitemapRequestList implements IRequestList {
signal: ow.optional.any(),
timeoutMillis: ow.optional.number,
maxBufferSize: ow.optional.number,
+ enqueueStrategy: ow.optional.string.oneOf(Object.values(EnqueueStrategy)),
parseSitemapOptions: ow.optional.object,
globs: ow.optional.array.ofType(ow.any(ow.string, ow.object.hasKeys('glob'))),
exclude: ow.optional.array.ofType(
@@ -251,6 +264,7 @@ export class SitemapRequestList implements IRequestList {
this.persistenceOptions = { enable: true, ...options.persistenceOptions };
this.proxyUrl = options.proxyUrl;
+ this.enqueueStrategy = options.enqueueStrategy ?? EnqueueStrategy.SameHostname;
this.urlQueueStream = this.createNewStream(options.maxBufferSize ?? 200);
@@ -382,6 +396,7 @@ export class SitemapRequestList implements IRequestList {
...parseSitemapOptions,
maxDepth: 0,
emitNestedSitemaps: true,
+ enqueueStrategy: this.enqueueStrategy,
})) {
if (!item.originSitemapUrl) {
// This is a nested sitemap
@@ -561,7 +576,7 @@ export class SitemapRequestList implements IRequestList {
if (!nextUrl) {
return null;
}
- this.requestData.set(nextUrl, new Request({ url: nextUrl }));
+ this.requestData.set(nextUrl, new Request({ url: nextUrl, enqueueStrategy: this.enqueueStrategy }));
}
this.inProgress.add(nextUrl);
diff --git a/packages/utils/package.json b/packages/utils/package.json
index bec96c7e2..78b54f36a 100644
--- a/packages/utils/package.json
+++ b/packages/utils/package.json
@@ -57,6 +57,7 @@
"ow": "^0.28.1",
"robots-parser": "^3.0.1",
"sax": "^1.4.1",
+ "tldts": "^7.0.0",
"tslib": "^2.4.0",
"whatwg-mimetype": "^4.0.0"
},
diff --git a/packages/utils/src/internals/robots.ts b/packages/utils/src/internals/robots.ts
index a16af1b08..939bdb952 100644
--- a/packages/utils/src/internals/robots.ts
+++ b/packages/utils/src/internals/robots.ts
@@ -3,11 +3,23 @@ import type { HTTPError as HTTPErrorClass } from 'got-scraping';
import type { Robot } from 'robots-parser';
import robotsParser from 'robots-parser';
+import log from '@apify/log';
+
import { gotScraping } from './gotScraping';
import { Sitemap } from './sitemap';
+import { type EnqueueStrategy, filterUrl } from './url';
let HTTPError: typeof HTTPErrorClass;
+export interface RobotsTxtFileSitemapsOptions {
+ /**
+ * Keep only sitemap URLs matching this strategy relative to the robots.txt host; non-`http(s)` schemes
+ * are always dropped. Pass `'all'` to disable host filtering.
+ * @default 'same-hostname'
+ */
+ enqueueStrategy?: EnqueueStrategy | `${EnqueueStrategy}`;
+}
+
/**
* Loads and queries information from a [robots.txt file](https://en.wikipedia.org/wiki/Robots.txt).
*
@@ -28,6 +40,7 @@ let HTTPError: typeof HTTPErrorClass;
*/
export class RobotsTxtFile {
private constructor(
+ private url: string,
private robots: Pick<Robot, 'isAllowed' | 'getSitemaps'>,
private proxyUrl?: string,
) {}
@@ -59,7 +72,7 @@ export class RobotsTxtFile {
* @param [proxyUrl] a proxy to be used for fetching the robots.txt file
*/
static from(url: string, content: string, proxyUrl?: string): RobotsTxtFile {
- return new RobotsTxtFile(robotsParser(url, content), proxyUrl);
+ return new RobotsTxtFile(url, robotsParser(url, content), proxyUrl);
}
protected static async load(
@@ -81,10 +94,11 @@ export class RobotsTxtFile {
...(options?.timeoutMillis ? { timeout: { request: options.timeoutMillis } } : {}),
});
- return new RobotsTxtFile(robotsParser(url.toString(), response.body), proxyUrl);
+ return new RobotsTxtFile(url, robotsParser(url.toString(), response.body), proxyUrl);
} catch (e) {
if (e instanceof HTTPError && e.response.statusCode === 404) {
return new RobotsTxtFile(
+ url,
{
isAllowed() {
return true;
@@ -110,24 +124,41 @@ export class RobotsTxtFile {
}
/**
- * Get URLs of sitemaps referenced in the robots file.
+ * Get URLs of sitemaps referenced in the robots file, filtered by `options.enqueueStrategy` relative to
+ * the robots.txt host (default `'same-hostname'`; pass `'all'` to disable). Non-`http(s)` schemes are
+ * always dropped.
*/
- getSitemaps(): string[] {
- return this.robots.getSitemaps();
+ getSitemaps(options: RobotsTxtFileSitemapsOptions = {}): string[] {
+ const { enqueueStrategy = 'same-hostname' } = options;
+ const sitemaps: string[] = [];
+
+ for (const sitemapUrl of this.robots.getSitemaps()) {
+ // `filterUrl` tolerates an unparseable origin (returns not-allowed) rather than throwing.
+ const { allowed, reason } = filterUrl(sitemapUrl, this.url, enqueueStrategy);
+ if (!allowed) {
+ log.warning(`Skipping sitemap ${sitemapUrl} listed in robots.txt at ${this.url}: ${reason}.`);
+ continue;
+ }
+ sitemaps.push(sitemapUrl);
+ }
+
+ return sitemaps;
}
/**
- * Parse all the sitemaps referenced in the robots file.
+ * Parse all the sitemaps referenced in the robots file. `options` are forwarded to `getSitemaps`
+ * and the sitemap parser.
*/
- async parseSitemaps(): Promise<Sitemap> {
- return Sitemap.load(this.robots.getSitemaps(), this.proxyUrl);
+ async parseSitemaps(options: RobotsTxtFileSitemapsOptions = {}): Promise<Sitemap> {
+ return Sitemap.load(this.getSitemaps(options), this.proxyUrl, options);
}
/**
* Get all URLs from all the sitemaps referenced in the robots file. A shorthand for `(await robots.parseSitemaps()).urls`.
+ * `options` are forwarded to `parseSitemaps`.
*/
- async parseUrlsFromSitemaps(): Promise<string[]> {
- return (await this.parseSitemaps()).urls;
+ async parseUrlsFromSitemaps(options: RobotsTxtFileSitemapsOptions = {}): Promise<string[]> {
+ return (await this.parseSitemaps(options)).urls;
}
}
diff --git a/packages/utils/src/internals/sitemap.ts b/packages/utils/src/internals/sitemap.ts
index e44948e4e..3139d8e24 100644
--- a/packages/utils/src/internals/sitemap.ts
+++ b/packages/utils/src/internals/sitemap.ts
@@ -13,6 +13,7 @@ import log from '@apify/log';
import { mergeAsyncIterables } from './iterables';
import { RobotsFile } from './robots';
+import { type EnqueueStrategy, filterUrl } from './url';
interface SitemapUrlData {
loc: string;
@@ -206,6 +207,13 @@ export interface ParseSitemapOptions {
* If not provided, all nested sitemaps are followed.
*/
nestedSitemapFilter?: (sitemapUrl: string) => boolean;
+ /**
+ * Keep only sitemap-derived URLs (nested `<sitemap>` and `<url>` entries) matching this strategy
+ * relative to the parent sitemap URL; non-`http(s)` schemes are always dropped. Skipped for raw string
+ * sources (no parent URL). Pass `'all'` to disable host filtering.
+ * @default 'same-hostname'
+ */
+ enqueueStrategy?: EnqueueStrategy | `${EnqueueStrategy}`;
}
export async function* parseSitemap<T extends ParseSitemapOptions>(
@@ -222,6 +230,7 @@ export async function* parseSitemap<T extends ParseSitemapOptions>(
networkTimeouts,
reportNetworkErrors = true,
nestedSitemapFilter,
+ enqueueStrategy = 'same-hostname',
} = options ?? {};
const sources = [...initialSources];
@@ -259,8 +268,11 @@ export async function* parseSitemap<T extends ParseSitemapOptions>(
let items: AsyncIterable<SitemapItem> | null = null;
+ // Parent URL, parsed once and reused as the origin for the strategy checks below.
+ let sitemapUrl: URL | undefined;
+
if (source.type === 'url') {
- const sitemapUrl = new URL(source.url);
+ sitemapUrl = new URL(source.url);
visitedSitemapUrls.add(sitemapUrl.toString());
let retriesLeft = sitemapRetries + 1;
@@ -351,6 +363,10 @@ export async function* parseSitemap<T extends ParseSitemapOptions>(
continue;
}
+ // URL entries dropped by the enqueue strategy filter, reported in one warning per sitemap after
+ // the loop (per-entry warnings could flood the log; individual drops are logged at debug level).
+ let droppedUrlEntries = 0;
+
for await (const item of items) {
if (item.type === 'sitemapUrl' && !visitedSitemapUrls.has(item.url)) {
if (nestedSitemapFilter && !nestedSitemapFilter(item.url)) {
@@ -358,6 +374,16 @@ export async function* parseSitemap<T extends ParseSitemapOptions>(
continue;
}
+ // Keep only nested sitemaps matching the strategy (and using http(s)) relative to the
+ // parent. Raw string sources have no parent URL, so the check is skipped.
+ if (source.type === 'url') {
+ const { allowed, reason } = filterUrl(item.url, sitemapUrl!, enqueueStrategy);
+ if (!allowed) {
+ log.warning(`Skipping nested sitemap ${item.url} (parent ${source.url}): ${reason}.`);
+ continue;
+ }
+ }
+
sources.push({ type: 'url', url: item.url, depth: (source.depth ?? 0) + 1 });
if (emitNestedSitemaps) {
yield { loc: item.url, originSitemapUrl: null } as any;
@@ -365,6 +391,16 @@ export async function* parseSitemap<T extends ParseSitemapOptions>(
}
if (item.type === 'url') {
+ // Keep only URL entries that match the enqueue strategy relative to the parent (see above).
+ if (source.type === 'url') {
+ const { allowed, reason } = filterUrl(item.loc, sitemapUrl!, enqueueStrategy);
+ if (!allowed) {
+ droppedUrlEntries++;
+ log.debug(`Skipping sitemap URL ${item.loc} (parent ${source.url}): ${reason}.`);
+ continue;
+ }
+ }
+
yield {
...item,
originSitemapUrl:
@@ -374,6 +410,12 @@ export async function* parseSitemap<T extends ParseSitemapOptions>(
};
}
}
+
+ if (droppedUrlEntries > 0 && source.type === 'url') {
+ log.warning(
+ `Skipped ${droppedUrlEntries} URL(s) from sitemap ${source.url} not matching enqueue strategy '${enqueueStrategy}' (or using a non-http(s) scheme). Enable debug logs to see each skipped URL.`,
+ );
+ }
}
}
@@ -546,7 +588,8 @@ export async function* discoverValidSitemaps(
timeoutMillis: requestTimeoutMillis,
signal,
});
- for (const sitemapUrl of robotsFile.getSitemaps()) {
+ // Surface all referenced sitemaps, including cross-host; scoping happens at load time.
+ for (const sitemapUrl of robotsFile.getSitemaps({ enqueueStrategy: 'all' })) {
if (addSitemapUrl(sitemapUrl)) {
yield sitemapUrl;
}
diff --git a/packages/utils/src/internals/url.ts b/packages/utils/src/internals/url.ts
index a8b0fe6d4..9d6fd1f45 100644
--- a/packages/utils/src/internals/url.ts
+++ b/packages/utils/src/internals/url.ts
@@ -1,5 +1,151 @@
+import { getDomain } from 'tldts';
+
export type SearchParams = string | URLSearchParams | Record<string, string | number | boolean | null | undefined>;
+/**
+ * The different enqueueing strategies available.
+ *
+ * Depending on the strategy you select, we will only check certain parts of the URLs found. Here is a diagram of each URL part and their name:
+ *
+ * ```md
+ * Protocol Domain
+ * ┌────┐ ┌─────────┐
+ * https://example.crawlee.dev/...
+ * │ └─────────────────┤
+ * │ Hostname │
+ * │ │
+ * └─────────────────────────┘
+ * Origin
+ *```
+ *
+ * - The `Protocol` is usually `http` or `https`
+ * - The `Domain` represents the path without any possible subdomains to a website. For example, `crawlee.dev` is the domain of `https://example.crawlee.dev/`
+ * - The `Hostname` is the full path to a website, including any subdomains. For example, `example.crawlee.dev` is the hostname of `https://example.crawlee.dev/`
+ * - The `Origin` is the combination of the `Protocol` and `Hostname`. For example, `https://example.crawlee.dev` is the origin of `https://example.crawlee.dev/`
+ */
+export enum EnqueueStrategy {
+ /**
+ * Matches any URLs found
+ */
+ All = 'all',
+
+ /**
+ * Matches any URLs that have the same hostname.
+ * For example, `https://wow.example.com/hello` will be matched for a base url of `https://wow.example.com/`, but
+ * `https://example.com/hello` will not be matched.
+ *
+ * > This strategy will match both `http` and `https` protocols regardless of the base URL protocol.
+ */
+ SameHostname = 'same-hostname',
+
+ /**
+ * Matches any URLs that have the same domain as the base URL.
+ * For example, `https://wow.an.example.com` and `https://example.com` will both be matched for a base url of
+ * `https://example.com`.
+ *
+ * > This strategy will match both `http` and `https` protocols regardless of the base URL protocol.
+ */
+ SameDomain = 'same-domain',
+
+ /**
+ * Matches any URLs that have the same hostname and protocol.
+ * For example, `https://wow.example.com/hello` will be matched for a base url of `https://wow.example.com/`, but
+ * `http://wow.example.com/hello` will not be matched.
+ *
+ * > This strategy will ensure the protocol of the base URL is the same as the protocol of the URL to be enqueued.
+ */
+ SameOrigin = 'same-origin',
+}
+
+/** Reusable suffix for log messages explaining why a non-`http(s)` URL was rejected. */
+export const UNSUPPORTED_SCHEME_MESSAGE = 'unsupported URL scheme (only http and https are allowed)';
+
+const ALLOWED_SCHEMES = new Set(['http:', 'https:']);
+
+function toUrl(value: string | URL): URL | null {
+ if (value instanceof URL) {
+ return value;
+ }
+
+ try {
+ return new URL(value);
+ } catch {
+ return null;
+ }
+}
+
+/** Strip a trailing dot so `example.com.` equals `example.com`. */
+function normalizeHostname(hostname: string): string {
+ return hostname.endsWith('.') ? hostname.slice(0, -1) : hostname;
+}
+
+/**
+ * Check whether `target` matches `origin` under the given enqueue `strategy`. The URL scheme is not
+ * considered here (use {@apilink filterUrl} for the combined scheme + strategy check).
+ *
+ * The `enqueueLinks` implementation in `@crawlee/core` matches the same strategies via glob patterns
+ * (see `packages/core/src/enqueue_links/enqueue_links.ts`) — keep the two in sync when changing either.
+ */
+export function matchesEnqueueStrategy(
+ strategy: EnqueueStrategy | `${EnqueueStrategy}`,
+ target: URL,
+ origin: URL,
+): boolean {
+ switch (strategy) {
+ case 'all':
+ return true;
+ case 'same-hostname':
+ return normalizeHostname(target.hostname) === normalizeHostname(origin.hostname);
+ case 'same-domain': {
+ const originDomain = getDomain(origin.hostname, { mixedInputs: false });
+
+ if (originDomain) {
+ return originDomain === getDomain(target.hostname, { mixedInputs: false });
+ }
+
+ // No registrable domain (e.g. an IP address), fall back to comparing origins.
+ return target.origin === origin.origin;
+ }
+ case 'same-origin':
+ // Compare scheme/host/port directly so a trailing-dot host is normalized.
+ return (
+ target.protocol === origin.protocol &&
+ normalizeHostname(target.hostname) === normalizeHostname(origin.hostname) &&
+ target.port === origin.port
+ );
+ default:
+ throw new Error(`Unknown enqueue strategy '${strategy satisfies never}'.`);
+ }
+}
+
+/**
+ * Check whether `target` may be enqueued under `strategy` relative to `origin`: it must use an `http(s)`
+ * scheme and match the strategy. On rejection, `reason` is a human-readable message for log output.
+ */
+export function filterUrl(
+ target: string | URL,
+ origin: string | URL,
+ strategy: EnqueueStrategy | `${EnqueueStrategy}`,
+): { allowed: boolean; reason?: string } {
+ const targetUrl = toUrl(target);
+
+ if (targetUrl === null || !ALLOWED_SCHEMES.has(targetUrl.protocol)) {
+ return { allowed: false, reason: UNSUPPORTED_SCHEME_MESSAGE };
+ }
+
+ const originUrl = toUrl(origin);
+
+ if (originUrl === null) {
+ return { allowed: false, reason: 'invalid origin URL' };
+ }
+
+ if (!matchesEnqueueStrategy(strategy, targetUrl, originUrl)) {
+ return { allowed: false, reason: `does not match enqueue strategy '${strategy}'` };
+ }
+
+ return { allowed: true };
+}
+
/**
* Appends search (query string) parameters to a URL, replacing the original value (if any).
*
diff --git a/packages/utils/test/robots.test.ts b/packages/utils/test/robots.test.ts
index f678e3cc9..3dcaaa717 100644
--- a/packages/utils/test/robots.test.ts
+++ b/packages/utils/test/robots.test.ts
@@ -99,6 +99,38 @@ describe('RobotsTxtFile', () => {
expect(end - start).toBeLessThanOrEqual(500);
});
+ it('drops cross-host and non-http(s) sitemap directives under the default same-hostname strategy', () => {
+ const content = [
+ 'User-agent: *',
+ 'Sitemap: http://not-exists.com/legit.xml',
+ 'Sitemap: http://other.test/cross.xml',
+ 'Sitemap: mailto:[email protected]',
+ 'Sitemap: ftp://not-exists.com/cross.xml',
+ ].join('\n');
+ const robots = RobotsTxtFile.from('http://not-exists.com/robots.txt', content);
+ expect(robots.getSitemaps()).toEqual(['http://not-exists.com/legit.xml']);
+ });
+
+ it('keeps cross-host http sitemaps but still drops non-http(s) ones with enqueueStrategy "all"', () => {
+ const content = [
+ 'User-agent: *',
+ 'Sitemap: http://not-exists.com/legit.xml',
+ 'Sitemap: http://other.test/cross.xml',
+ 'Sitemap: mailto:[email protected]',
+ 'Sitemap: ftp://not-exists.com/cross.xml',
+ ].join('\n');
+ const robots = RobotsTxtFile.from('http://not-exists.com/robots.txt', content);
+ expect(new Set(robots.getSitemaps({ enqueueStrategy: 'all' }))).toEqual(
+ new Set(['http://not-exists.com/legit.xml', 'http://other.test/cross.xml']),
+ );
+ });
+
+ it('getSitemaps does not throw when the robots.txt url is not absolute', () => {
+ // A non-absolute url must not crash getSitemaps().
+ const robots = RobotsTxtFile.from('not-exists.com/robots.txt', 'Sitemap: http://not-exists.com/legit.xml');
+ expect(() => robots.getSitemaps()).not.toThrow();
+ });
+
it('parses allow/deny directives from explicitly provided robots.txt contents', async () => {
const contents = `User-agent: *',
Disallow: *deny_all/
diff --git a/packages/utils/test/sitemap.test.ts b/packages/utils/test/sitemap.test.ts
index 52831ff04..62ba689ef 100644
--- a/packages/utils/test/sitemap.test.ts
+++ b/packages/utils/test/sitemap.test.ts
@@ -217,6 +217,60 @@ describe('Sitemap', () => {
'http://not-exists.com/catalog?item=79&desc=vacation_somalia',
].join('\n'),
)
+ // urlset mixing a same-host and a cross-host URL entry (for enqueue-strategy filtering)
+ .get('/cross_host_urls.xml')
+ .reply(
+ 200,
+ [
+ '<?xml version="1.0" encoding="UTF-8"?>',
+ '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">',
+ '<url><loc>http://not-exists.com/local-page</loc></url>',
+ '<url><loc>http://other.test/cross-page</loc></url>',
+ '</urlset>',
+ ].join('\n'),
+ )
+ // sitemap index pointing at a cross-host nested sitemap
+ .get('/cross_host_index.xml')
+ .reply(
+ 200,
+ [
+ '<?xml version="1.0" encoding="UTF-8"?>',
+ '<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">',
+ '<sitemap><loc>http://other.test/child.xml</loc></sitemap>',
+ '</sitemapindex>',
+ ].join('\n'),
+ )
+ // urlset mixing a valid http URL with non-http(s) schemes
+ .get('/bad_schemes.xml')
+ .reply(
+ 200,
+ [
+ '<?xml version="1.0" encoding="UTF-8"?>',
+ '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">',
+ '<url><loc>http://not-exists.com/ok</loc></url>',
+ '<url><loc>mailto:[email protected]</loc></url>',
+ '<url><loc>javascript:alert(1)</loc></url>',
+ '<url><loc>ftp://example.com/file.txt</loc></url>',
+ '</urlset>',
+ ].join('\n'),
+ )
+ .get('*')
+ .reply(404);
+
+ // A cross-host server whose child sitemap is served, so a *followed* nested reference would surface
+ // `child-page`. Under the default `same-hostname` strategy it must never be fetched.
+ nock('http://other.test')
+ .persist()
+ .get('/child.xml')
+ .reply(
+ 200,
+ [
+ '<?xml version="1.0" encoding="UTF-8"?>',
+ '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">',
+ '<url><loc>http://other.test/child-page</loc></url>',
+ '</urlset>',
+ ].join('\n'),
+ )
.get('*')
.reply(404);
@@ -508,6 +562,51 @@ describe('Sitemap', () => {
]),
);
});
+
+ it('drops cross-host URL entries under the default same-hostname strategy', async () => {
+ const sitemap = await Sitemap.load('http://not-exists.com/cross_host_urls.xml');
+ expect(sitemap.urls).toEqual(['http://not-exists.com/local-page']);
+ });
+
+ it('logs one aggregated warning per sitemap for dropped URL entries', async () => {
+ const spy = vi.spyOn(log, 'warning');
+
+ await Sitemap.load('http://not-exists.com/cross_host_urls.xml');
+
+ expect(spy).toHaveBeenCalledTimes(1);
+ expect(spy).toHaveBeenCalledWith(
+ expect.stringContaining('Skipped 1 URL(s) from sitemap http://not-exists.com/cross_host_urls.xml'),
+ );
+ });
+
+ it('keeps cross-host URL entries when enqueueStrategy is "all"', async () => {
+ const sitemap = await Sitemap.load('http://not-exists.com/cross_host_urls.xml', undefined, {
+ enqueueStrategy: 'all',
+ });
+ expect(new Set(sitemap.urls)).toEqual(
+ new Set(['http://not-exists.com/local-page', 'http://other.test/cross-page']),
+ );
+ });
+
+ it('does not fetch cross-host nested sitemaps under the default same-hostname strategy', async () => {
+ const sitemap = await Sitemap.load('http://not-exists.com/cross_host_index.xml');
+ // If the cross-host nested sitemap were fetched, `child-page` would appear in the result.
+ expect(sitemap.urls).toEqual([]);
+ });
+
+ it('follows cross-host nested sitemaps when enqueueStrategy is "all"', async () => {
+ const sitemap = await Sitemap.load('http://not-exists.com/cross_host_index.xml', undefined, {
+ enqueueStrategy: 'all',
+ });
+ expect(sitemap.urls).toEqual(['http://other.test/child-page']);
+ });
+
+ it('drops non-http(s) scheme entries even when enqueueStrategy is "all"', async () => {
+ const sitemap = await Sitemap.load('http://not-exists.com/bad_schemes.xml', undefined, {
+ enqueueStrategy: 'all',
+ });
+ expect(sitemap.urls).toEqual(['http://not-exists.com/ok']);
+ });
});
describe('discoverValidSitemaps', () => {
@@ -541,6 +640,26 @@ describe('discoverValidSitemaps', () => {
expect(urls).toEqual(['http://sitemap-discovery.com/some-sitemap.xml']);
});
+ it('discovers cross-host sitemaps referenced in robots.txt', async () => {
+ nock('http://sitemap-discovery.com')
+ .get('/robots.txt')
+ .reply(200, 'Sitemap: http://cdn.other-host.com/some-sitemap.xml')
+ .head('/sitemap.xml')
+ .reply(404, '')
+ .head('/sitemap.txt')
+ .reply(404, '')
+ .head('/sitemap_index.xml')
+ .reply(404, '');
+
+ const urls = [];
+ for await (const url of discoverValidSitemaps(['http://sitemap-discovery.com'])) {
+ urls.push(url);
+ }
+
+ // Cross-host sitemap is surfaced; host scoping is applied at load time.
+ expect(urls).toEqual(['http://cdn.other-host.com/some-sitemap.xml']);
+ });
+
it('extracts sitemap from well-known paths if robots.txt is missing', async () => {
nock('http://sitemap-discovery.com')
.get('/robots.txt')
diff --git a/packages/utils/test/url.test.ts b/packages/utils/test/url.test.ts
new file mode 100644
index 000000000..1988a3fb0
--- /dev/null
+++ b/packages/utils/test/url.test.ts
@@ -0,0 +1,49 @@
+import { describe, expect, it } from 'vitest';
+
+import { filterUrl, matchesEnqueueStrategy } from '../src/internals/url';
+
+describe('matchesEnqueueStrategy', () => {
+ const match = (strategy: Parameters<typeof matchesEnqueueStrategy>[0], target: string, origin: string) =>
+ matchesEnqueueStrategy(strategy, new URL(target), new URL(origin));
+
+ it('all matches regardless of host, domain, or scheme', () => {
+ expect(match('all', 'https://other.test/page', 'http://example.com/sitemap.xml')).toBe(true);
+ });
+
+ it('same-hostname matches only the exact host', () => {
+ expect(match('same-hostname', 'http://example.com/a', 'http://example.com/sitemap.xml')).toBe(true);
+ expect(match('same-hostname', 'http://www.example.com/a', 'http://example.com/sitemap.xml')).toBe(false);
+ });
+
+ it('same-domain matches across subdomains but not across registrable domains', () => {
+ expect(match('same-domain', 'http://www.example.com/a', 'http://example.com/sitemap.xml')).toBe(true);
+ expect(match('same-domain', 'http://example.org/a', 'http://example.com/sitemap.xml')).toBe(false);
+ });
+
+ it('same-origin requires matching scheme, host, and port', () => {
+ expect(match('same-origin', 'http://example.com/a', 'http://example.com/sitemap.xml')).toBe(true);
+ expect(match('same-origin', 'https://example.com/a', 'http://example.com/sitemap.xml')).toBe(false);
+ });
+
+ it('treats a trailing-dot FQDN host as equal to the bare host across strategies', () => {
+ for (const strategy of ['same-hostname', 'same-domain', 'same-origin'] as const) {
+ expect(match(strategy, 'http://example.com./page', 'http://example.com/sitemap.xml')).toBe(true);
+ }
+ });
+});
+
+describe('filterUrl', () => {
+ it('drops non-http(s) schemes regardless of strategy', () => {
+ for (const target of ['mailto:[email protected]', 'ftp://example.com/f.txt']) {
+ expect(filterUrl(target, 'http://example.com/sitemap.xml', 'all').allowed).toBe(false);
+ }
+ });
+
+ it('drops gracefully (no throw) when the origin is not parseable', () => {
+ expect(() => filterUrl('http://example.com/a', 'not-a-url', 'same-hostname')).not.toThrow();
+ expect(filterUrl('http://example.com/a', 'not-a-url', 'same-hostname')).toEqual({
+ allowed: false,
+ reason: 'invalid origin URL',
+ });
+ });
+});
diff --git a/test/core/sitemap_request_list.test.ts b/test/core/sitemap_request_list.test.ts
index ed2282f3f..5daca1aec 100644
--- a/test/core/sitemap_request_list.test.ts
+++ b/test/core/sitemap_request_list.test.ts
@@ -193,6 +193,69 @@ beforeAll(async () => {
res.end();
});
+
+ // --- Fixtures for the enqueue-strategy filtering tests ---
+ // The server answers on both `localhost` and `127.0.0.1` (distinct hostnames), so the `127.0.0.1`
+ // variant is a reachable "cross-host" target — a dropped entry is distinguishable from a failed fetch.
+
+ // urlset mixing a same-host and a cross-host URL entry
+ app.get('/cross-host-content.xml', async (req, res) => {
+ const cross = url.replace('localhost', '127.0.0.1');
+ res.setHeader('content-type', 'text/xml');
+ res.end(
+ [
+ '<?xml version="1.0" encoding="UTF-8"?>',
+ '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">',
+ `<url><loc>${url}/same-host-page</loc></url>`,
+ `<url><loc>${cross}/cross-host-page</loc></url>`,
+ '</urlset>',
+ ].join('\n'),
+ );
+ });
+
+ // sitemap index pointing at a cross-host nested sitemap
+ app.get('/cross-host-index.xml', async (req, res) => {
+ const cross = url.replace('localhost', '127.0.0.1');
+ res.setHeader('content-type', 'text/xml');
+ res.end(
+ [
+ '<?xml version="1.0" encoding="UTF-8"?>',
+ '<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">',
+ `<sitemap><loc>${cross}/cross-host-child.xml</loc></sitemap>`,
+ '</sitemapindex>',
+ ].join('\n'),
+ );
+ });
+
+ // nested sitemap referenced by the cross-host index; its URL appears only if the index is followed
+ app.get('/cross-host-child.xml', async (req, res) => {
+ const cross = url.replace('localhost', '127.0.0.1');
+ res.setHeader('content-type', 'text/xml');
+ res.end(
+ [
+ '<?xml version="1.0" encoding="UTF-8"?>',
+ '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">',
+ `<url><loc>${cross}/child-page</loc></url>`,
+ '</urlset>',
+ ].join('\n'),
+ );
+ });
+
+ // urlset mixing a valid http URL with non-http(s) schemes
+ app.get('/mixed-scheme.xml', async (req, res) => {
+ res.setHeader('content-type', 'text/xml');
+ res.end(
+ [
+ '<?xml version="1.0" encoding="UTF-8"?>',
+ '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">',
+ `<url><loc>${url}/ok</loc></url>`,
+ '<url><loc>mailto:[email protected]</loc></url>',
+ '<url><loc>javascript:alert(1)</loc></url>',
+ '<url><loc>ftp://example.com/file.txt</loc></url>',
+ '</urlset>',
+ ].join('\n'),
+ );
+ });
});
afterAll(async () => {
@@ -212,7 +275,10 @@ afterAll(async () => {
describe('SitemapRequestList', () => {
test('requests are available before the sitemap is fully loaded', async () => {
- const list = await SitemapRequestList.open({ sitemapUrls: [`${url}/sitemap-stream.xml`] });
+ const list = await SitemapRequestList.open({
+ sitemapUrls: [`${url}/sitemap-stream.xml`],
+ enqueueStrategy: 'all',
+ });
while (await list.isEmpty()) {
await sleep(20);
@@ -232,7 +298,10 @@ describe('SitemapRequestList', () => {
});
test('retry sitemap load on error', async () => {
- const list = await SitemapRequestList.open({ sitemapUrls: [`${url}/sitemap-unreliable.xml`] });
+ const list = await SitemapRequestList.open({
+ sitemapUrls: [`${url}/sitemap-unreliable.xml`],
+ enqueueStrategy: 'all',
+ });
for await (const request of list) {
await list.markRequestHandled(request);
@@ -242,7 +311,10 @@ describe('SitemapRequestList', () => {
});
test('broken off sitemap load resurrects correctly and does not duplicate / lose requests', async () => {
- const list = await SitemapRequestList.open({ sitemapUrls: [`${url}/sitemap-unreliable-break-off.xml`] });
+ const list = await SitemapRequestList.open({
+ sitemapUrls: [`${url}/sitemap-unreliable-break-off.xml`],
+ enqueueStrategy: 'all',
+ });
const urls = new Set<string>();
@@ -264,7 +336,10 @@ describe('SitemapRequestList', () => {
});
test('teardown works', async () => {
- const list = await SitemapRequestList.open({ sitemapUrls: [`${url}/sitemap-index.xml`] });
+ const list = await SitemapRequestList.open({
+ sitemapUrls: [`${url}/sitemap-index.xml`],
+ enqueueStrategy: 'all',
+ });
for await (const request of list) {
await list.markRequestHandled(request);
@@ -283,6 +358,7 @@ describe('SitemapRequestList', () => {
const list = await SitemapRequestList.open({
sitemapUrls: [`${url}/sitemap.xml`],
globs: ['http://not-exists.com/catalog**'],
+ enqueueStrategy: 'all',
});
for await (const request of list) {
@@ -296,6 +372,7 @@ describe('SitemapRequestList', () => {
const list = await SitemapRequestList.open({
sitemapUrls: [`${url}/sitemap.xml`],
regexps: [/desc=vacation_new.+/],
+ enqueueStrategy: 'all',
});
for await (const request of list) {
@@ -309,6 +386,7 @@ describe('SitemapRequestList', () => {
const list = await SitemapRequestList.open({
sitemapUrls: [`${url}/sitemap.xml`],
exclude: [/desc=vacation_new/],
+ enqueueStrategy: 'all',
});
for await (const request of list) {
@@ -319,7 +397,10 @@ describe('SitemapRequestList', () => {
});
test('draining the request list between sitemaps', async () => {
- const list = await SitemapRequestList.open({ sitemapUrls: [`${url}/sitemap-index.xml`] });
+ const list = await SitemapRequestList.open({
+ sitemapUrls: [`${url}/sitemap-index.xml`],
+ enqueueStrategy: 'all',
+ });
while (await list.isEmpty()) {
await sleep(20);
@@ -354,7 +435,10 @@ describe('SitemapRequestList', () => {
});
test('for..await syntax works with SitemapRequestList', async () => {
- const list = await SitemapRequestList.open({ sitemapUrls: [`${url}/sitemap-index.xml`] });
+ const list = await SitemapRequestList.open({
+ sitemapUrls: [`${url}/sitemap-index.xml`],
+ enqueueStrategy: 'all',
+ });
for await (const request of list) {
await list.markRequestHandled(request);
@@ -370,6 +454,7 @@ describe('SitemapRequestList', () => {
const list = await SitemapRequestList.open({
sitemapUrls: [`${url}/sitemap-index.xml`],
signal: controller.signal,
+ enqueueStrategy: 'all',
});
await sleep(50); // Loads the first sub-sitemap, but not the second
@@ -388,6 +473,7 @@ describe('SitemapRequestList', () => {
const list = await SitemapRequestList.open({
sitemapUrls: [`${url}/sitemap-index.xml`],
timeoutMillis: 50, // Loads the first sub-sitemap, but not the second
+ enqueueStrategy: 'all',
});
for await (const request of list) {
@@ -404,6 +490,7 @@ describe('SitemapRequestList', () => {
sitemapUrls: [`${url}/sitemap-index.xml`],
persistStateKey: 'resurrection-abort',
timeoutMillis: 50,
+ enqueueStrategy: 'all' as const,
};
{
@@ -424,7 +511,7 @@ describe('SitemapRequestList', () => {
});
test('processing the whole list', async () => {
- const list = await SitemapRequestList.open({ sitemapUrls: [`${url}/sitemap.xml`] });
+ const list = await SitemapRequestList.open({ sitemapUrls: [`${url}/sitemap.xml`], enqueueStrategy: 'all' });
const requests: Request[] = [];
await expect(list.isFinished()).resolves.toBe(false);
@@ -448,7 +535,7 @@ describe('SitemapRequestList', () => {
});
test('processing the whole list with reclaiming', async () => {
- const list = await SitemapRequestList.open({ sitemapUrls: [`${url}/sitemap.xml`] });
+ const list = await SitemapRequestList.open({ sitemapUrls: [`${url}/sitemap.xml`], enqueueStrategy: 'all' });
const requests: Request[] = [];
await expect(list.isFinished()).resolves.toBe(false);
@@ -482,7 +569,11 @@ describe('SitemapRequestList', () => {
});
test('persists state', async () => {
- const options = { sitemapUrls: [`${url}/sitemap-stream.xml`], persistStateKey: 'some-key' };
+ const options = {
+ sitemapUrls: [`${url}/sitemap-stream.xml`],
+ persistStateKey: 'some-key',
+ enqueueStrategy: 'all' as const,
+ };
const list = await SitemapRequestList.open(options);
const firstRequest = await list.fetchNextRequest();
@@ -503,7 +594,7 @@ describe('SitemapRequestList', () => {
});
test("calling `persistState` doesn't throw", async () => {
- const list = await SitemapRequestList.open({ sitemapUrls: [`${url}/sitemap.xml`] });
+ const list = await SitemapRequestList.open({ sitemapUrls: [`${url}/sitemap.xml`], enqueueStrategy: 'all' });
for await (const request of list) {
await list.markRequestHandled(request);
@@ -518,6 +609,7 @@ describe('SitemapRequestList', () => {
const options = {
sitemapUrls: [`${url}/sitemap-stream.xml`],
persistStateKey: 'persist-user-changes',
+ enqueueStrategy: 'all' as const,
};
const userDataPayload = { some: 'data' };
@@ -538,6 +630,64 @@ describe('SitemapRequestList', () => {
const restoredRequest = await newList.fetchNextRequest();
expect(restoredRequest!.url).toEqual(firstLoadedUrl);
- expect(restoredRequest!.userData).toEqual(userDataPayload);
+ // `toMatchObject` (not `toEqual`): the request also carries internal `__crawlee` bookkeeping (the stamped strategy).
+ expect(restoredRequest!.userData).toMatchObject(userDataPayload);
+ });
+
+ async function collectUrls(list: SitemapRequestList): Promise<string[]> {
+ const urls: string[] = [];
+ for await (const request of list) {
+ urls.push(request.url);
+ await list.markRequestHandled(request);
+ }
+ return urls;
+ }
+
+ test('default `same-hostname` strategy drops cross-host URL entries', async () => {
+ const list = await SitemapRequestList.open({ sitemapUrls: [`${url}/cross-host-content.xml`] });
+ expect(await collectUrls(list)).toEqual([`${url}/same-host-page`]);
+ });
+
+ test('`enqueueStrategy: all` keeps cross-host URL entries', async () => {
+ const cross = url.replace('localhost', '127.0.0.1');
+ const list = await SitemapRequestList.open({
+ sitemapUrls: [`${url}/cross-host-content.xml`],
+ enqueueStrategy: 'all',
+ });
+ expect(new Set(await collectUrls(list))).toEqual(
+ new Set([`${url}/same-host-page`, `${cross}/cross-host-page`]),
+ );
+ });
+
+ test('default `same-hostname` strategy drops cross-host nested sitemaps before fetching them', async () => {
+ const list = await SitemapRequestList.open({ sitemapUrls: [`${url}/cross-host-index.xml`] });
+ // The cross-host nested sitemap is never fetched, so its `child-page` URL is absent.
+ expect(await collectUrls(list)).toEqual([]);
+ });
+
+ test('`enqueueStrategy: all` follows cross-host nested sitemaps', async () => {
+ const cross = url.replace('localhost', '127.0.0.1');
+ const list = await SitemapRequestList.open({
+ sitemapUrls: [`${url}/cross-host-index.xml`],
+ enqueueStrategy: 'all',
+ });
+ expect(await collectUrls(list)).toEqual([`${cross}/child-page`]);
+ });
+
+ test('non-http(s) schemes are dropped even with `enqueueStrategy: all`', async () => {
+ const list = await SitemapRequestList.open({
+ sitemapUrls: [`${url}/mixed-scheme.xml`],
+ enqueueStrategy: 'all',
+ });
+ expect(await collectUrls(list)).toEqual([`${url}/ok`]);
+ });
+
+ test('the selected enqueue strategy is stamped onto emitted requests', async () => {
+ const list = await SitemapRequestList.open({ sitemapUrls: [`${url}/cross-host-content.xml`] });
+ const request = await list.fetchNextRequest();
+
+ expect(request).not.toBe(null);
+ // The strategy is persisted on the request so it keeps being enforced after navigation.
+ expect((request as any).enqueueStrategy).toBe('same-hostname');
});
});
diff --git a/yarn.lock b/yarn.lock
index e51a95b8e..270132c01 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -1218,6 +1218,7 @@ __metadata:
ow: "npm:^0.28.1"
robots-parser: "npm:^3.0.1"
sax: "npm:^1.4.1"
+ tldts: "npm:^7.0.0"
tslib: "npm:^2.4.0"
whatwg-mimetype: "npm:^4.0.0"
languageName: unknown