Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
chore: Re-delete maybeRunIntervalSecs (#4269)
Jan Bucharc69d2898ae21
diff --git a/docs/public-api/crawlee-basic.api.md b/docs/public-api/crawlee-basic.api.mdindex 6052c6d8c..2cc4b0aac 100644--- a/docs/public-api/crawlee-basic.api.md+++ b/docs/public-api/crawlee-basic.api.md@@ -150,7 +150,7 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingCstatusMessageCallback?: StatusMessageCallback;statusMessageLoggingInterval?: number;storageBackend?: StorageBackend;- taskLoopOptions?: TaskLoopOptions;+ taskLoopOptions?: TaskLoopPredicates;transactionalStorage?: boolean | Partial<StorageWritePolicy>;}@@ -1001,11 +1001,6 @@ export interface StorageBackendLoadSignalOptions {snapshotIntervalSecs?: number;}-// @public (undocumented)-export interface TaskLoopOptions extends TaskLoopPredicates {- maybeRunIntervalSecs?: number;-}-// @publicexport interface TaskLoopPredicates {isFinishedFunction?: () => Promise<boolean>;diff --git a/packages/basic-crawler/src/internals/autoscaling/autoscaled-pool.ts b/packages/basic-crawler/src/internals/autoscaling/autoscaled-pool.tsindex de9148564..534dbef1c 100644--- a/packages/basic-crawler/src/internals/autoscaling/autoscaled-pool.ts+++ b/packages/basic-crawler/src/internals/autoscaling/autoscaled-pool.ts@@ -52,16 +52,14 @@ export interface TaskLoopPredicates {isFinishedFunction?: () => Promise<boolean>;}-export interface TaskLoopOptions extends TaskLoopPredicates {+/** @internal */+export interface AutoscaledPoolOptions extends TaskLoopPredicates {/*** How often the pool should check if a new task is ready, in seconds.* @default 0.5*/maybeRunIntervalSecs?: number;-}-/** @internal */-export interface AutoscaledPoolOptions extends TaskLoopOptions {/*** The governor that decides whether there is free compute for one more task. Typically a* {@apilink ConcurrencySystem}, but any {@apilink IConcurrencySystem} works. Share a single instance acrossdiff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.tsindex afbc8e636..fdf396938 100644--- a/packages/basic-crawler/src/internals/basic-crawler.ts+++ b/packages/basic-crawler/src/internals/basic-crawler.ts@@ -60,7 +60,7 @@ import { LruCache } from '@apify/datastructures';import { addTimeoutToPromise, extendTimeout, storage as timeoutStorage, TimeoutError, tryCancel } from '@apify/timeout';import { cryptoRandomObjectId } from '@apify/utilities';-import type { AutoscaledPoolOptions, TaskLoopOptions } from './autoscaling/autoscaled-pool.js';+import type { AutoscaledPoolOptions, TaskLoopPredicates } from './autoscaling/autoscaled-pool.js';import type { ConcurrencySystemOptions, IConcurrencySystem } from './autoscaling/concurrency-system.js';import { ConcurrencySystem } from './autoscaling/concurrency-system.js';import type { FinalStatistics } from './autoscaling/system-status.js';@@ -376,7 +376,7 @@ export interface BasicCrawlerOptions<* Concurrency is configured elsewhere — through the `minConcurrency`/`maxConcurrency`/`maxRequestsPerMinute`* shortcuts, or a {@apilink BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} for finer control.*/- taskLoopOptions?: TaskLoopOptions;+ taskLoopOptions?: TaskLoopPredicates;/*** A pre-configured concurrency governor — the component that decides whether there is free compute for one morediff --git a/test/core/crawlers/basic-crawler.test.ts b/test/core/crawlers/basic-crawler.test.tsindex d8683847a..6f53b395d 100644--- a/test/core/crawlers/basic-crawler.test.ts+++ b/test/core/crawlers/basic-crawler.test.ts@@ -1586,7 +1586,6 @@ describe('BasicCrawler', () => {requestHandler: async ({ request }) => {handled.push(request.url);},- taskLoopOptions: { maybeRunIntervalSecs: 0.05 },});const run = crawler.run();@@ -1619,7 +1618,6 @@ describe('BasicCrawler', () => {isTaskReadyFunctionCalled = true;return Promise.resolve(!isFinished);},- maybeRunIntervalSecs: 0.05,},requestHandler: async ({ request }) => {await sleep(10);@@ -1675,9 +1673,6 @@ describe('BasicCrawler', () => {const basicCrawler = new BasicCrawler({requestQueue,keepAlive: true,- taskLoopOptions: {- maybeRunIntervalSecs: 0.05,- },requestHandler: async ({ request }) => {await sleep(10);processed.push(request);