Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix(core): log background request-batch failures instead of swallowing them (#4103)
florinBC89c7032308b273
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.tsindex ca3a4c440..84827eaed 100644--- a/packages/basic-crawler/src/internals/basic-crawler.ts+++ b/packages/basic-crawler/src/internals/basic-crawler.ts@@ -2228,6 +2228,7 @@ export class BasicCrawler<});// Keep callback failures observable to callers that await this promise without emitting an unhandled rejection// when callers intentionally leave background additions running, matching `drainRequestBatches` behavior.+ // Background failures are already logged by `drainRequestBatches`.void waitForAllRequestsToBeAdded.catch(() => {});return { ...result, waitForAllRequestsToBeAdded };diff --git a/packages/basic-crawler/src/internals/throttling-request-manager.ts b/packages/basic-crawler/src/internals/throttling-request-manager.tsindex 7eae6a649..f88758ffd 100644--- a/packages/basic-crawler/src/internals/throttling-request-manager.ts+++ b/packages/basic-crawler/src/internals/throttling-request-manager.ts@@ -815,6 +815,7 @@ export class ThrottlingRequestManager<T extends IRequestManager = IRequestManagewaitBetweenBatchesMillis: options.waitBetweenBatchesMillis ?? 1000,waitForAllRequestsToBeAdded: options.waitForAllRequestsToBeAdded ?? false,maxNewRequests: options.maxNewRequests,+ log: this.#log,// Routing is the only thing this manager adds; the targets do the batching, validation and// deduplication themselves.diff --git a/packages/core/src/storages/batched-adds.ts b/packages/core/src/storages/batched-adds.tsindex 8719c41dd..c3cac2320 100644--- a/packages/core/src/storages/batched-adds.ts+++ b/packages/core/src/storages/batched-adds.ts@@ -2,6 +2,7 @@ import { setTimeout as sleep } from 'node:timers/promises';import type { ProcessedRequest } from '@crawlee/types';import { chunkedAsyncIterable, peekableAsyncIterable } from '../iterables.js';+import type { CrawleeLogger } from '../log.js';import type { Source } from '../request.js';import type { AddRequestsBatchedResult } from './request-queue.js';import { activeStorageTransaction, withDirectStorageAccess } from './transaction.js';@@ -18,6 +19,8 @@ export interface DrainRequestBatchesOptions<TItem extends Source> {waitForAllRequestsToBeAdded: boolean;maxNewRequests?: number;+ log: CrawleeLogger;+/*** Adds a single chunk and reports what it processed.*@@ -52,6 +55,7 @@ export async function drainRequestBatches<TItem extends Source>(waitBetweenBatchesMillis,waitForAllRequestsToBeAdded,maxNewRequests,+ log,processChunk,trackBackgroundBatches,} = options;@@ -125,8 +129,15 @@ export async function drainRequestBatches<TItem extends Source>(const remainder = awaitsRemainder ? processRemainingChunks() : withDirectStorageAccess(processRemainingChunks);// The caller is not obliged to await `remainder`, so give it a handler of its own - an unhandled- // rejection here would otherwise take the process down.- trackBackgroundBatches?.(remainder.catch(() => {}));+ // rejection here would otherwise take the process down. Awaited failures reach the caller, so only+ // background ones get logged.+ trackBackgroundBatches?.(+ remainder.catch((error) => {+ if (!awaitsRemainder) {+ log.exception(error as Error, 'A batch of requests could not be added to the queue in the background');+ }+ }),+ );if (awaitsRemainder) {addedRequests.push(...(await remainder));diff --git a/packages/core/src/storages/request-queue.ts b/packages/core/src/storages/request-queue.tsindex a4a9370fe..d077c5c3b 100644--- a/packages/core/src/storages/request-queue.ts+++ b/packages/core/src/storages/request-queue.ts@@ -640,6 +640,7 @@ export class RequestQueue implements IStorage, IRequestManager {waitBetweenBatchesMillis,waitForAllRequestsToBeAdded,maxNewRequests,+ log: this.log,/*** Requests the backend reports as unprocessed are warned about and skipped rather than retried:diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yamlindex 8af6cebb2..f91617313 100644--- a/pnpm-workspace.yaml+++ b/pnpm-workspace.yaml@@ -48,6 +48,7 @@ overrides:# proceed when new build-requesting deps appear without a manual entry).allowBuilds:"@apify/ui-icons": true+ '@google/genai': set this to true or false"@playwright/browser-chromium": true"@playwright/browser-firefox": true"@playwright/browser-webkit": truediff --git a/test/core/storages/batched-adds.test.ts b/test/core/storages/batched-adds.test.tsnew file mode 100644index 000000000..7f6560fb7--- /dev/null+++ b/test/core/storages/batched-adds.test.ts@@ -0,0 +1,57 @@+import type { ProcessedRequest } from '@crawlee/types';+import { describe, expect, it, vitest } from 'vitest';++import { drainRequestBatches } from '../../../packages/core/src/storages/batched-adds.js';+import type { CrawleeLogger } from '../../../packages/core/src/log.js';++async function* twoItems() {+ yield { url: 'https://example.com/1' };+ yield { url: 'https://example.com/2' };+}++function setup(waitForAllRequestsToBeAdded: boolean) {+ const exception = vitest.fn();+ let tracked: Promise<unknown> | undefined;++ const result = drainRequestBatches({+ items: twoItems(),+ batchSize: 1,+ waitBetweenBatchesMillis: 0,+ waitForAllRequestsToBeAdded,+ log: { exception } as unknown as CrawleeLogger,+ processChunk: async (chunk, isInitial) => {+ if (!isInitial) throw new Error('boom');+ return chunk.map((): ProcessedRequest => ({+ uniqueKey: '',+ requestId: '',+ wasAlreadyPresent: false,+ wasAlreadyHandled: false,+ }));+ },+ trackBackgroundBatches: (batches) => {+ tracked = batches;+ },+ });++ return { exception, result, tracked: () => tracked };+}++describe('drainRequestBatches', () => {+ it('logs a failed background batch', async () => {+ const { exception, result, tracked } = setup(false);++ expect((await result).addedRequests).toHaveLength(1);+ await tracked();++ expect(exception).toHaveBeenCalledOnce();+ });++ it('rethrows an awaited batch failure without logging it', async () => {+ const { exception, result, tracked } = setup(true);++ await expect(result).rejects.toThrow('boom');+ await tracked();++ expect(exception).not.toHaveBeenCalled();+ });+});