Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

chore: Re-delete maybeRunIntervalSecs (#4269)

Jan Bucharc69d2898ae21
diff --git a/docs/public-api/crawlee-basic.api.md b/docs/public-api/crawlee-basic.api.md
index 6052c6d8c..2cc4b0aac 100644
--- a/docs/public-api/crawlee-basic.api.md
+++ b/docs/public-api/crawlee-basic.api.md
@@ -150,7 +150,7 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
statusMessageCallback?: StatusMessageCallback;
statusMessageLoggingInterval?: number;
storageBackend?: StorageBackend;
- taskLoopOptions?: TaskLoopOptions;
+ taskLoopOptions?: TaskLoopPredicates;
transactionalStorage?: boolean | Partial<StorageWritePolicy>;
}
@@ -1001,11 +1001,6 @@ export interface StorageBackendLoadSignalOptions {
snapshotIntervalSecs?: number;
}
-// @public (undocumented)
-export interface TaskLoopOptions extends TaskLoopPredicates {
- maybeRunIntervalSecs?: number;
-}
-
// @public
export interface TaskLoopPredicates {
isFinishedFunction?: () => Promise<boolean>;
diff --git a/packages/basic-crawler/src/internals/autoscaling/autoscaled-pool.ts b/packages/basic-crawler/src/internals/autoscaling/autoscaled-pool.ts
index de9148564..534dbef1c 100644
--- a/packages/basic-crawler/src/internals/autoscaling/autoscaled-pool.ts
+++ b/packages/basic-crawler/src/internals/autoscaling/autoscaled-pool.ts
@@ -52,16 +52,14 @@ export interface TaskLoopPredicates {
isFinishedFunction?: () => Promise<boolean>;
}
-export interface TaskLoopOptions extends TaskLoopPredicates {
+/** @internal */
+export interface AutoscaledPoolOptions extends TaskLoopPredicates {
/**
* How often the pool should check if a new task is ready, in seconds.
* @default 0.5
*/
maybeRunIntervalSecs?: number;
-}
-/** @internal */
-export interface AutoscaledPoolOptions extends TaskLoopOptions {
/**
* The governor that decides whether there is free compute for one more task. Typically a
* {@apilink ConcurrencySystem}, but any {@apilink IConcurrencySystem} works. Share a single instance across
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.ts
index afbc8e636..fdf396938 100644
--- a/packages/basic-crawler/src/internals/basic-crawler.ts
+++ b/packages/basic-crawler/src/internals/basic-crawler.ts
@@ -60,7 +60,7 @@ import { LruCache } from '@apify/datastructures';
import { addTimeoutToPromise, extendTimeout, storage as timeoutStorage, TimeoutError, tryCancel } from '@apify/timeout';
import { cryptoRandomObjectId } from '@apify/utilities';
-import type { AutoscaledPoolOptions, TaskLoopOptions } from './autoscaling/autoscaled-pool.js';
+import type { AutoscaledPoolOptions, TaskLoopPredicates } from './autoscaling/autoscaled-pool.js';
import type { ConcurrencySystemOptions, IConcurrencySystem } from './autoscaling/concurrency-system.js';
import { ConcurrencySystem } from './autoscaling/concurrency-system.js';
import type { FinalStatistics } from './autoscaling/system-status.js';
@@ -376,7 +376,7 @@ export interface BasicCrawlerOptions<
* Concurrency is configured elsewhere — through the `minConcurrency`/`maxConcurrency`/`maxRequestsPerMinute`
* shortcuts, or a {@apilink BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} for finer control.
*/
- taskLoopOptions?: TaskLoopOptions;
+ taskLoopOptions?: TaskLoopPredicates;
/**
* A pre-configured concurrency governor — the component that decides whether there is free compute for one more
diff --git a/test/core/crawlers/basic-crawler.test.ts b/test/core/crawlers/basic-crawler.test.ts
index d8683847a..6f53b395d 100644
--- a/test/core/crawlers/basic-crawler.test.ts
+++ b/test/core/crawlers/basic-crawler.test.ts
@@ -1586,7 +1586,6 @@ describe('BasicCrawler', () => {
requestHandler: async ({ request }) => {
handled.push(request.url);
},
- taskLoopOptions: { maybeRunIntervalSecs: 0.05 },
});
const run = crawler.run();
@@ -1619,7 +1618,6 @@ describe('BasicCrawler', () => {
isTaskReadyFunctionCalled = true;
return Promise.resolve(!isFinished);
},
- maybeRunIntervalSecs: 0.05,
},
requestHandler: async ({ request }) => {
await sleep(10);
@@ -1675,9 +1673,6 @@ describe('BasicCrawler', () => {
const basicCrawler = new BasicCrawler({
requestQueue,
keepAlive: true,
- taskLoopOptions: {
- maybeRunIntervalSecs: 0.05,
- },
requestHandler: async ({ request }) => {
await sleep(10);
processed.push(request);