Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix(basic): report skips from background add batches (#4078)

Michael Bujnovský2436868127ec
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.ts
index a3d839947..618f41e30 100644
--- a/packages/basic-crawler/src/internals/basic-crawler.ts
+++ b/packages/basic-crawler/src/internals/basic-crawler.ts
@@ -2108,6 +2108,41 @@ export class BasicCrawler<
this.#onSkippedRequest !== undefined || options.onSkippedRequest !== undefined;
const keepSkippedSource = (source: Source) => (hasSkippedRequestCallback ? source : source.url!);
+ const reportSkippedRequests = async () => {
+ const skippedRequests = allSkipped.splice(0);
+ if (skippedRequests.length === 0) {
+ return;
+ }
+
+ const skippedRobotsUrls = skippedRequests
+ .filter((s) => s.reason === 'robotsTxt')
+ .map(({ source }) => (typeof source === 'string' ? source : source.url!));
+ if (skippedRobotsUrls.length > 0) {
+ this.log.warning(
+ `Some requests were skipped because they were disallowed based on the robots.txt file`,
+ { skipped: skippedRobotsUrls },
+ );
+ }
+
+ // Only log the limit message when an explicit `limit` was passed (not the internal
+ // `maxRequestsPerCrawl`-derived one), and only once per call.
+ if (options.limit !== undefined && skippedRequests.some((s) => s.reason === 'limit')) {
+ this.log.info(
+ requestLimit === options.limit
+ ? `Skipping requests in this call due to the enqueueLinks limit of ${options.limit}.`
+ : `Skipping requests in this call due to the remaining maxRequestsPerCrawl budget of ${requestLimit}, which is lower than the enqueueLinks limit of ${options.limit}.`,
+ );
+ }
+
+ await Promise.all(
+ skippedRequests.map(async ({ source, reason }) => {
+ const args = createSkippedRequestArgs(source, reason);
+ await this.#handleSkippedRequest(args);
+ await options.onSkippedRequest?.(args);
+ }),
+ );
+ };
+
async function* filteredRequests() {
for await (const request of requests) {
const [requestOptions] = createRequestOptions(
@@ -2186,37 +2221,17 @@ export class BasicCrawler<
allSkipped.push({ source: request, reason: 'limit' });
}
- if (allSkipped.length > 0) {
- const skippedRobotsUrls = allSkipped
- .filter((s) => s.reason === 'robotsTxt')
- .map(({ source }) => (typeof source === 'string' ? source : source.url!));
- if (skippedRobotsUrls.length > 0) {
- this.log.warning(
- `Some requests were skipped because they were disallowed based on the robots.txt file`,
- { skipped: skippedRobotsUrls },
- );
- }
+ await reportSkippedRequests();
- // Only log the limit message when an explicit `limit` was passed (not the internal
- // `maxRequestsPerCrawl`-derived one), and only once per call.
- if (options.limit !== undefined && allSkipped.some((s) => s.reason === 'limit')) {
- this.log.info(
- requestLimit === options.limit
- ? `Skipping requests in this call due to the enqueueLinks limit of ${options.limit}.`
- : `Skipping requests in this call due to the remaining maxRequestsPerCrawl budget of ${requestLimit}, which is lower than the enqueueLinks limit of ${options.limit}.`,
- );
- }
-
- await Promise.all(
- allSkipped.map(async ({ source, reason }) => {
- const args = createSkippedRequestArgs(source, reason);
- await this.#handleSkippedRequest(args);
- await options.onSkippedRequest?.(args);
- }),
- );
- }
+ const waitForAllRequestsToBeAdded = result.waitForAllRequestsToBeAdded.then(async (addedRequests) => {
+ await reportSkippedRequests();
+ return addedRequests;
+ });
+ // Keep callback failures observable to callers that await this promise without emitting an unhandled rejection
+ // when callers intentionally leave background additions running, matching `drainRequestBatches` behavior.
+ void waitForAllRequestsToBeAdded.catch(() => {});
- return result;
+ return { ...result, waitForAllRequestsToBeAdded };
}
/**
diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.ts
index b85f01afa..cf6988f32 100644
--- a/test/core/crawlers/basic_crawler.test.ts
+++ b/test/core/crawlers/basic_crawler.test.ts
@@ -600,6 +600,142 @@ describe('BasicCrawler', () => {
});
});
+ describe('addRequests() background skip reporting', () => {
+ test('reports filtered requests discovered in background batches', async () => {
+ const onSkippedRequest = vitest.fn();
+ const crawler = new BasicCrawler({
+ requestHandler: async () => {},
+ onSkippedRequest,
+ });
+
+ const inScope = Array.from({ length: 2_000 }, (_, i) => `https://example.com/ok/${i}`);
+ const outOfScope = Array.from({ length: 50 }, (_, i) => `https://other.com/no/${i}`);
+
+ const { addedRequests, waitForAllRequestsToBeAdded } = await crawler.addRequests(
+ [...inScope, ...outOfScope],
+ {
+ waitBetweenBatchesMillis: 0,
+ include: ['https://example.com/**'],
+ },
+ );
+ const backgroundAddedRequests = await waitForAllRequestsToBeAdded;
+
+ expect(addedRequests).toHaveLength(1_000);
+ expect(backgroundAddedRequests).toHaveLength(1_000);
+ expect(onSkippedRequest.mock.calls.map(([{ request, reason }]) => ({ url: request.url, reason }))).toEqual(
+ outOfScope.map((url) => ({ url, reason: 'filters' })),
+ );
+ });
+
+ test('background onSkippedRequest rejection rejects the background completion promise', async () => {
+ const crawler = new BasicCrawler({
+ requestHandler: async () => {},
+ onSkippedRequest: async () => {
+ throw new Error('onSkippedRequest failed');
+ },
+ });
+
+ const { addedRequests, waitForAllRequestsToBeAdded } = await crawler.addRequests(
+ [
+ 'https://example.com/1',
+ 'https://example.com/2',
+ 'https://example.com/3',
+ 'https://example.com/4',
+ 'https://other.com/no',
+ ],
+ {
+ batchSize: 2,
+ waitBetweenBatchesMillis: 0,
+ include: ['https://example.com/**'],
+ },
+ );
+
+ expect(addedRequests).toHaveLength(2);
+ await expect(waitForAllRequestsToBeAdded).rejects.toThrow('onSkippedRequest failed');
+ });
+
+ test('a background onSkippedRequest rejection stays handled when nobody awaits the addition', async () => {
+ const reportStarted = Promise.withResolvers<void>();
+ const crawler = new BasicCrawler({
+ requestHandler: async () => {},
+ onSkippedRequest: async () => {
+ reportStarted.resolve();
+ throw new Error('onSkippedRequest failed');
+ },
+ });
+
+ const unhandled: unknown[] = [];
+ const collectUnhandled = (reason: unknown) => unhandled.push(reason);
+ process.on('unhandledRejection', collectUnhandled);
+
+ try {
+ // Deliberately dropping `waitForAllRequestsToBeAdded`: that is the default way to call
+ // `addRequests`, and the background skip report rejecting there must not take the process down.
+ await crawler.addRequests(
+ [
+ 'https://example.com/1',
+ 'https://example.com/2',
+ 'https://example.com/3',
+ 'https://example.com/4',
+ 'https://other.com/no',
+ ],
+ {
+ batchSize: 2,
+ waitBetweenBatchesMillis: 0,
+ include: ['https://example.com/**'],
+ },
+ );
+
+ await reportStarted.promise;
+ // Node only flags a rejection as unhandled once the tick's microtasks have drained.
+ const nextTick = Promise.withResolvers<void>();
+ setImmediate(nextTick.resolve);
+ await nextTick.promise;
+ } finally {
+ process.off('unhandledRejection', collectUnhandled);
+ }
+
+ expect(unhandled).toEqual([]);
+ });
+
+ test('drains foreground and background robots.txt skips exactly once', async () => {
+ const onSkippedRequest = vitest.fn();
+ const crawler = new BasicCrawler({
+ requestHandler: async () => {},
+ onSkippedRequest,
+ });
+ vitest
+ .spyOn(crawler as any, 'isAllowedBasedOnRobotsTxtFile')
+ .mockImplementation(async (url: unknown) => !String(url).includes('/denied'));
+ const warningSpy = vitest.spyOn(crawler.log, 'warning');
+
+ const foregroundDenied = 'https://example.com/denied/foreground';
+ const backgroundDenied = 'https://example.com/denied/background';
+ const { waitForAllRequestsToBeAdded } = await crawler.addRequests(
+ [
+ foregroundDenied,
+ 'https://example.com/1',
+ 'https://example.com/2',
+ 'https://example.com/3',
+ 'https://example.com/4',
+ backgroundDenied,
+ ],
+ { batchSize: 2, waitBetweenBatchesMillis: 0 },
+ );
+ await waitForAllRequestsToBeAdded;
+
+ expect(onSkippedRequest.mock.calls.map(([{ request, reason }]) => ({ url: request.url, reason }))).toEqual([
+ { url: foregroundDenied, reason: 'robotsTxt' },
+ { url: backgroundDenied, reason: 'robotsTxt' },
+ ]);
+ expect(
+ warningSpy.mock.calls
+ .filter(([message]) => message.includes('robots.txt'))
+ .map(([, details]) => details),
+ ).toEqual([{ skipped: [foregroundDenied] }, { skipped: [backgroundDenied] }]);
+ });
+ });
+
it('addCrawlDepthRequestGenerator() should generate requests with maxCrawlDepth', async () => {
type AddCrawlDepthWrapperOptions = Parameters<BasicCrawler['addCrawlDepthRequestGenerator']>;
class TestCrawler extends BasicCrawler {