Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix: respect `autoscaledPoolOptions.isTaskReadyFunction` option (#2948)

Martin Adámekfe2d206b46af
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.ts
index 538696b22..973ce0557 100644
--- a/packages/basic-crawler/src/internals/basic-crawler.ts
+++ b/packages/basic-crawler/src/internals/basic-crawler.ts
@@ -54,8 +54,6 @@ import type { Awaitable, BatchAddRequestsResult, Dictionary, SetStatusMessageOpt
import { RobotsTxtFile, ROTATE_PROXY_ERRORS } from '@crawlee/utils';
import { stringify } from 'csv-stringify/sync';
import { ensureDir, writeFile, writeJSON } from 'fs-extra';
-// @ts-expect-error This throws a compilation error due to got-scraping being ESM only but we only import types, so its alllll gooooood
-import type { GotResponse, Method, OptionsInit } from 'got-scraping';
import ow, { ArgumentError } from 'ow';
import { getDomain } from 'tldts';
import type { SetRequired } from 'type-fest';
@@ -741,7 +739,8 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
let shouldLogMaxPagesExceeded = true;
const isMaxPagesExceeded = () => maxRequestsPerCrawl && maxRequestsPerCrawl <= this.handledRequestsCount;
- let { isFinishedFunction } = autoscaledPoolOptions;
+ // eslint-disable-next-line prefer-const
+ let { isFinishedFunction, isTaskReadyFunction } = autoscaledPoolOptions;
// override even if `isFinishedFunction` provided by user - `keepAlive` has higher priority
if (keepAlive) {
@@ -765,7 +764,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
return false;
}
- return this._isTaskReadyFunction();
+ return isTaskReadyFunction ? await isTaskReadyFunction() : await this._isTaskReadyFunction();
},
isFinishedFunction: async () => {
if (isMaxPagesExceeded()) {
diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.ts
index 713144109..5777dc248 100644
--- a/test/core/crawlers/basic_crawler.test.ts
+++ b/test/core/crawlers/basic_crawler.test.ts
@@ -734,11 +734,13 @@ describe('BasicCrawler', () => {
expect(await crawler._isTaskReadyFunction()).toBe(false);
});
- test('should be possible to override isFinishedFunction of underlying AutoscaledPool', async () => {
+ test('should be possible to override isFinishedFunction and isTaskReadyFunction of underlying AutoscaledPool', async () => {
const requestQueue = new RequestQueue({ id: 'xxx', client: Configuration.getStorageClient() });
const processed: Request[] = [];
const queue: Request[] = [];
let isFinished = false;
+ let isFinishedFunctionCalled = false;
+ let isTaskReadyFunctionCalled = false;
const basicCrawler = new BasicCrawler({
requestQueue,
@@ -746,8 +748,13 @@ describe('BasicCrawler', () => {
minConcurrency: 1,
maxConcurrency: 1,
isFinishedFunction: async () => {
+ isFinishedFunctionCalled = true;
return Promise.resolve(isFinished);
},
+ isTaskReadyFunction: async () => {
+ isTaskReadyFunctionCalled = true;
+ return Promise.resolve(!isFinished);
+ },
},
requestHandler: async ({ request }) => {
await sleep(10);
@@ -783,6 +790,8 @@ describe('BasicCrawler', () => {
expect(markRequestHandled).toBeCalledWith(request0);
expect(markRequestHandled).toBeCalledWith(request1);
expect(isFinishedOrig).not.toBeCalled();
+ expect(isFinishedFunctionCalled).toBe(true);
+ expect(isTaskReadyFunctionCalled).toBe(true);
// TODO: see why the request1 was passed as a second parameter to includes
expect(processed.includes(request0)).toBe(true);