Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix: respect `autoscaledPoolOptions.isTaskReadyFunction` option (#2948)
Martin Adámekfe2d206b46af
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.tsindex 538696b22..973ce0557 100644--- a/packages/basic-crawler/src/internals/basic-crawler.ts+++ b/packages/basic-crawler/src/internals/basic-crawler.ts@@ -54,8 +54,6 @@ import type { Awaitable, BatchAddRequestsResult, Dictionary, SetStatusMessageOptimport { RobotsTxtFile, ROTATE_PROXY_ERRORS } from '@crawlee/utils';import { stringify } from 'csv-stringify/sync';import { ensureDir, writeFile, writeJSON } from 'fs-extra';-// @ts-expect-error This throws a compilation error due to got-scraping being ESM only but we only import types, so its alllll gooooood-import type { GotResponse, Method, OptionsInit } from 'got-scraping';import ow, { ArgumentError } from 'ow';import { getDomain } from 'tldts';import type { SetRequired } from 'type-fest';@@ -741,7 +739,8 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContextlet shouldLogMaxPagesExceeded = true;const isMaxPagesExceeded = () => maxRequestsPerCrawl && maxRequestsPerCrawl <= this.handledRequestsCount;- let { isFinishedFunction } = autoscaledPoolOptions;+ // eslint-disable-next-line prefer-const+ let { isFinishedFunction, isTaskReadyFunction } = autoscaledPoolOptions;// override even if `isFinishedFunction` provided by user - `keepAlive` has higher priorityif (keepAlive) {@@ -765,7 +764,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContextreturn false;}- return this._isTaskReadyFunction();+ return isTaskReadyFunction ? await isTaskReadyFunction() : await this._isTaskReadyFunction();},isFinishedFunction: async () => {if (isMaxPagesExceeded()) {diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.tsindex 713144109..5777dc248 100644--- a/test/core/crawlers/basic_crawler.test.ts+++ b/test/core/crawlers/basic_crawler.test.ts@@ -734,11 +734,13 @@ describe('BasicCrawler', () => {expect(await crawler._isTaskReadyFunction()).toBe(false);});- test('should be possible to override isFinishedFunction of underlying AutoscaledPool', async () => {+ test('should be possible to override isFinishedFunction and isTaskReadyFunction of underlying AutoscaledPool', async () => {const requestQueue = new RequestQueue({ id: 'xxx', client: Configuration.getStorageClient() });const processed: Request[] = [];const queue: Request[] = [];let isFinished = false;+ let isFinishedFunctionCalled = false;+ let isTaskReadyFunctionCalled = false;const basicCrawler = new BasicCrawler({requestQueue,@@ -746,8 +748,13 @@ describe('BasicCrawler', () => {minConcurrency: 1,maxConcurrency: 1,isFinishedFunction: async () => {+ isFinishedFunctionCalled = true;return Promise.resolve(isFinished);},+ isTaskReadyFunction: async () => {+ isTaskReadyFunctionCalled = true;+ return Promise.resolve(!isFinished);+ },},requestHandler: async ({ request }) => {await sleep(10);@@ -783,6 +790,8 @@ describe('BasicCrawler', () => {expect(markRequestHandled).toBeCalledWith(request0);expect(markRequestHandled).toBeCalledWith(request1);expect(isFinishedOrig).not.toBeCalled();+ expect(isFinishedFunctionCalled).toBe(true);+ expect(isTaskReadyFunctionCalled).toBe(true);// TODO: see why the request1 was passed as a second parameter to includesexpect(processed.includes(request0)).toBe(true);