Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix: respect falsy transformRequestFunction return in context enqueueLinks (#3925)
Martin Adámek6cb4fc392f43
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.tsindex 6663e784d..0b48a3080 100644--- a/packages/basic-crawler/src/internals/basic-crawler.ts+++ b/packages/basic-crawler/src/internals/basic-crawler.ts@@ -1764,7 +1764,8 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext}// After injecting the crawlDepth, we call the user-provided transform function, if there is one.- return options.transformRequestFunction?.(newRequest) ?? newRequest;+ // Its return value is passed through as is, so a falsy one still skips the request.+ return options.transformRequestFunction ? options.transformRequestFunction(newRequest) : newRequest;};// Create a request-scoped callback that logs enqueueLimit once per request handler calldiff --git a/packages/core/src/enqueue_links/shared.ts b/packages/core/src/enqueue_links/shared.tsindex eae760313..93fea17c6 100644--- a/packages/core/src/enqueue_links/shared.ts+++ b/packages/core/src/enqueue_links/shared.ts@@ -299,7 +299,7 @@ function createPatternObjectMatcher(urlPatternObject: UrlPatternObject) {export interface RequestTransform {/*** @param original Request options to be modified.- * @returns The modified request options to enqueue.+ * @returns The modified request options to enqueue, or any falsy value to skip the request.*/(original: RequestOptions): RequestOptions | false | undefined | null;}diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.tsindex 9844470e2..d2324e6cc 100644--- a/test/core/crawlers/basic_crawler.test.ts+++ b/test/core/crawlers/basic_crawler.test.ts@@ -307,6 +307,25 @@ describe('BasicCrawler', () => {expect(requests).toHaveLength(2);expect(requests[0]).toMatchObject({ url: 'https://example.com/1/', crawlDepth: 3 });});++ it.each([+ null,+ undefined,+ false,+ ] as const)('should skip requests when transformRequestFunction returns %s', async (returnValue) => {+ const transformRequestFunction = vi.fn(() => returnValue);+ const optionsWithTransform = { ...options, transformRequestFunction };++ await crawler.exposedEnqueueLinksWithCrawlDepth(optionsWithTransform, request, requestQueue);++ const requests = addRequestsBatchedMock.mock.calls[0][0];+ expect(requests).toHaveLength(0);++ const skippedRequests = onSkippedRequestMock.mock.calls.map((call) => call[0]);+ expect(skippedRequests).toHaveLength(2);+ expect(skippedRequests[0]).toStrictEqual({ url: 'https://example.com/1/', reason: 'filters' });+ expect(skippedRequests[1]).toStrictEqual({ url: 'https://example.com/2/', reason: 'filters' });+ });});it('addCrawlDepthRequestGenerator() should generate requests with maxCrawlDepth', async () => {