Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
chore: Remove the -Secs suffix from lock methods on RequestQueueBackend interface (#4276)
Jan Bucharabddcfe2152e
diff --git a/docs/public-api/crawlee-basic.api.md b/docs/public-api/crawlee-basic.api.mdindex eaecde2d7..c4629dafb 100644--- a/docs/public-api/crawlee-basic.api.md+++ b/docs/public-api/crawlee-basic.api.md@@ -1019,7 +1019,7 @@ export class ThrottlingRequestManager<T extends IRequestManager = IRequestManageaddRequest(requestLike: Source, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo>;addRequestsBatched(requests: RequestsLike, options?: AddRequestsBatchedOptions): Promise<AddRequestsBatchedResult>;checkReadiness(): Promise<RequestSourceStatus>;- extendRequestProcessingTimeSecs(request: Request_2, secs: number): Promise<boolean>;+ extendRequestProcessingTime(request: Request_2, extensionSecs: number): Promise<boolean>;fetchNextRequest<R extends Dictionary = Dictionary>(): Promise<Request_2<R> | null>;// (undocumented)getHandledCount(): Promise<number>;@@ -1034,7 +1034,7 @@ export class ThrottlingRequestManager<T extends IRequestManager = IRequestManagereclaimRequest(request: Request_2, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo | null>;recordPacingSignal(signal: PacingSignal): boolean;// (undocumented)- setExpectedRequestProcessingTimeSecs(secs: number): Promise<void>;+ setExpectedRequestProcessingTime(processingTimeSecs: number): Promise<void>;}// @publicdiff --git a/docs/public-api/crawlee-core.api.md b/docs/public-api/crawlee-core.api.mdindex 7d55bfd05..5e6aee79d 100644--- a/docs/public-api/crawlee-core.api.md+++ b/docs/public-api/crawlee-core.api.md@@ -391,11 +391,11 @@ export interface IRequestManager extends IRequestLoader {addRequest(requestLike: Source, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo>;// (undocumented)addRequestsBatched(requests: RequestsLike, options?: AddRequestsBatchedOptions): Promise<AddRequestsBatchedResult>;- extendRequestProcessingTimeSecs?(request: Request_2, secs: number): Promise<boolean>;+ extendRequestProcessingTime?(request: Request_2, extensionSecs: number): Promise<boolean>;purge?(): Promise<void>;reclaimRequest(request: Request_2, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo | null>;recordPacingSignal(signal: PacingSignal): boolean;- setExpectedRequestProcessingTimeSecs?(secs: number): Promise<void>;+ setExpectedRequestProcessingTime?(processingTimeSecs: number): Promise<void>;}// @public@@ -718,7 +718,7 @@ export class RequestManagerTandem implements IRequestManager {addRequestsBatched(requests: RequestsLike, options?: AddRequestsBatchedOptions): Promise<AddRequestsBatchedResult>;checkReadiness(): Promise<RequestSourceStatus>;// (undocumented)- extendRequestProcessingTimeSecs(request: Request_2, secs: number): Promise<boolean>;+ extendRequestProcessingTime(request: Request_2, extensionSecs: number): Promise<boolean>;fetchNextRequest<T extends Dictionary = Dictionary>(): Promise<Request_2<T> | null>;// (undocumented)getHandledCount(): Promise<number>;@@ -732,7 +732,7 @@ export class RequestManagerTandem implements IRequestManager {// (undocumented)reclaimRequest(request: Request_2, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo | null>;recordPacingSignal(signal: PacingSignal): boolean;- setExpectedRequestProcessingTimeSecs(secs: number): Promise<void>;+ setExpectedRequestProcessingTime(processingTimeSecs: number): Promise<void>;}// @public@@ -765,7 +765,7 @@ export class RequestQueue implements IStorage, IRequestManager {readonly backend: RequestQueueBackend;checkReadiness(): Promise<RequestLoaderStatus>;drop(): Promise<void>;- extendRequestProcessingTimeSecs(request: Request_2, secs: number): Promise<boolean>;+ extendRequestProcessingTime(request: Request_2, extensionSecs: number): Promise<boolean>;fetchNextRequest<T extends Dictionary = Dictionary>(): Promise<Request_2<T> | null>;getHandledCount(): Promise<number>;getInfo(): Promise<RequestQueueInfo>;@@ -783,7 +783,7 @@ export class RequestQueue implements IStorage, IRequestManager {purge(): Promise<void>;reclaimRequest(request: Request_2, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo | null>;recordPacingSignal(_signal: PacingSignal): boolean;- setExpectedRequestProcessingTimeSecs(secs: number): Promise<void>;+ setExpectedRequestProcessingTime(processingTimeSecs: number): Promise<void>;get stats(): RequestQueueStats;}diff --git a/docs/public-api/crawlee-types.api.md b/docs/public-api/crawlee-types.api.mdindex 53a0bee9c..08dba030a 100644--- a/docs/public-api/crawlee-types.api.md+++ b/docs/public-api/crawlee-types.api.md@@ -351,7 +351,7 @@ export interface QueueOperationInfo {export interface RequestQueueBackend {addBatchOfRequests(requests: RequestSchema[], options?: RequestQueueOperationOptions): Promise<BatchAddRequestsResult>;drop(): Promise<void>;- extendRequestProcessingTimeSecs?(requestId: string, secs: number): Promise<boolean>;+ extendRequestProcessingTime?(requestId: string, extensionSecs: number): Promise<boolean>;fetchNextRequest(): Promise<UpdateRequestSchema | undefined>;getMetadata(): Promise<RequestQueueInfo>;getRequest(uniqueKey: string): Promise<UpdateRequestSchema | undefined>;@@ -360,7 +360,7 @@ export interface RequestQueueBackend {markRequestAsHandled(request: UpdateRequestSchema): Promise<QueueOperationInfo | undefined>;purge(): Promise<void>;reclaimRequest(request: UpdateRequestSchema, options?: RequestQueueOperationOptions): Promise<QueueOperationInfo | undefined>;- setExpectedRequestProcessingTimeSecs?(secs: number): Promise<void>;+ setExpectedRequestProcessingTime?(processingTimeSecs: number): Promise<void>;}// @public (undocumented)diff --git a/docs/upgrading/upgrading-v4/storage-backends.md b/docs/upgrading/upgrading-v4/storage-backends.mdindex 4a764b1af..8c257c00f 100644--- a/docs/upgrading/upgrading-v4/storage-backends.md+++ b/docs/upgrading/upgrading-v4/storage-backends.md@@ -74,7 +74,7 @@ The request queue backend's surface was reshaped. The frontend-owned distributed| `updateRequest(request, opts?)` | `markRequestAsHandled(request)` / `reclaimRequest(request, opts?)` || `listHead(opts?)` | `fetchNextRequest()` (returns a single request, marks it in progress) || `listAndLockHead(opts)` | Removed (locking is internal to the client) |-| `prolongRequestLock(id, opts)` | `extendRequestProcessingTimeSecs(requestId, secs)` (optional — per-request lock extension for locking backends; wired to `context.extendTimeout`) |+| `prolongRequestLock(id, opts)` | `extendRequestProcessingTime(requestId, extensionSecs)` (optional — per-request lock extension for locking backends; wired to `context.extendTimeout`) || `deleteRequestLock(id, opts?)` | Removed || `deleteRequest(id)` | Removed || _(n/a)_ | `isEmpty()` (new — `true` when no pending requests are left to fetch) |@@ -108,13 +108,13 @@ The `requestLocking` crawler experiment has been removed, along with the `experiThe `RequestQueue.requestLockSecs` property has been removed. Because request locking is now internal to the storage backend, the lock duration is no longer configured on the queue. When you run a crawler, it automatically tells the queue how long it expects to hold a request (based on `requestHandlerTimeoutMillis`), so a long-running request handler will not have its request handed out a second time — you usually don't need to configure anything.-If you use a `RequestQueue` outside of a crawler and your processing may exceed the 3-minute default lock, call `setExpectedRequestProcessingTimeSecs(secs)` on the queue to raise it:+If you use a `RequestQueue` outside of a crawler and your processing may exceed the 3-minute default lock, call `setExpectedRequestProcessingTime(processingTimeSecs)` on the queue to raise it:```typescriptimport { RequestQueue } from 'crawlee';const queue = await RequestQueue.open();-await queue.setExpectedRequestProcessingTimeSecs(600);+await queue.setExpectedRequestProcessingTime(600);```The `RequestQueue.internalTimeoutMillis` property and the associated "stuck queue" self-recovery have been removed. In v3 the `RequestQueue` frontend kept its own copy of the queue head and in-progress set, which could drift out of sync with the backing storage (an eventual-consistency hazard on the Apify platform); `isFinished()` watched for inactivity exceeding `internalTimeoutMillis` and reset that frontend state to recover. In v4 the frontend no longer holds any such bookkeeping — the storage backend is the single source of truth — so there is nothing for a reset to fix, and stuck request locks now self-heal on expiry. Any consistency-recovery logic that is genuinely specific to the Apify platform's distributed storage belongs in the Apify SDK's client implementation instead, and is tracked in [apify/crawlee#3328](https://github.com/apify/crawlee/issues/3328).diff --git a/docs/upgrading/upgrading-v4/upgrading-v4.md b/docs/upgrading/upgrading-v4/upgrading-v4.mdindex 1866a0b89..4a93e8cec 100644--- a/docs/upgrading/upgrading-v4/upgrading-v4.md+++ b/docs/upgrading/upgrading-v4/upgrading-v4.md@@ -939,7 +939,7 @@ const inMemory = new MemoryStorageBackend();#### No request lock expiry in `MemoryStorageBackend`-Because the in-memory queue lives entirely within a single process and is never shared with another consumer, `MemoryStorageBackend`'s request queue no longer uses an expiring, cross-process lock. A fetched request simply stays *in progress* until it is handled or reclaimed; it never becomes fetchable again on its own after a timeout. `setExpectedRequestProcessingTimeSecs()` is therefore a no-op for in-memory storage. (Disk-backed `FileSystemStorageBackend` keeps the lock-with-expiry behavior.)+Because the in-memory queue lives entirely within a single process and is never shared with another consumer, `MemoryStorageBackend`'s request queue no longer uses an expiring, cross-process lock. A fetched request simply stays *in progress* until it is handled or reclaimed; it never becomes fetchable again on its own after a timeout. `setExpectedRequestProcessingTime()` is therefore a no-op for in-memory storage. (Disk-backed `FileSystemStorageBackend` keeps the lock-with-expiry behavior.)#### `writeMetadata` option removeddiff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.tsindex 61a400937..954bfda24 100644--- a/packages/basic-crawler/src/internals/basic-crawler.ts+++ b/packages/basic-crawler/src/internals/basic-crawler.ts@@ -1473,7 +1473,7 @@ export class BasicCrawler<}// Extension failure must not fail the request that asked for more time.- this.requestManager?.extendRequestProcessingTimeSecs?.(context.request, secs)?.catch((error) => {+ this.requestManager?.extendRequestProcessingTime?.(context.request, secs)?.catch((error) => {this.log.debug('Extending the request processing time failed', { url: context.request.url, error });});},@@ -1915,7 +1915,7 @@ export class BasicCrawler<}// Apply the processing-time hint here (an async lifecycle point) rather than in the constructor,- // now that `setExpectedRequestProcessingTimeSecs` is async. The hint is raise-only and idempotent,+ // now that `setExpectedRequestProcessingTime` is async. The hint is raise-only and idempotent,// but guard so we do not re-issue it on every call.if (!this.#requestManagerTimeoutsApplied) {this.#requestManagerTimeoutsApplied = true;@@ -1958,7 +1958,7 @@ export class BasicCrawler<const maxRouteTimeoutSecs = (this.requestHandler as Partial<RouterHandler>).getMaxTimeoutSecs?.() ?? 0;const handlerTimeoutSecs = Math.max(this.#requestHandlerTimeoutMillis / 1000, maxRouteTimeoutSecs);- await requestManager.setExpectedRequestProcessingTimeSecs?.(Math.max(handlerTimeoutSecs + 5, 60));+ await requestManager.setExpectedRequestProcessingTime?.(Math.max(handlerTimeoutSecs + 5, 60));}/**diff --git a/packages/basic-crawler/src/internals/throttling-request-manager.ts b/packages/basic-crawler/src/internals/throttling-request-manager.tsindex 915956681..ab13228eb 100644--- a/packages/basic-crawler/src/internals/throttling-request-manager.ts+++ b/packages/basic-crawler/src/internals/throttling-request-manager.ts@@ -331,7 +331,7 @@ export class ThrottlingRequestManager<T extends IRequestManager = IRequestManage/** Requests moved into a sub-queue, which both managers count. Subtracted below; not persisted. */#migratedFromInner = 0;- /** The latest {@link setExpectedRequestProcessingTimeSecs} hint, kept for a wrapped manager resolved later. */+ /** The latest {@link setExpectedRequestProcessingTime} hint, kept for a wrapped manager resolved later. */#expectedRequestProcessingSecs?: number;readonly #warnedAbout = new Set<string>();@@ -432,7 +432,7 @@ export class ThrottlingRequestManager<T extends IRequestManager = IRequestManagethis.#resolvedInner = await this.#innerPromise;if (this.#expectedRequestProcessingSecs !== undefined) {- await this.#resolvedInner.setExpectedRequestProcessingTimeSecs?.(this.#expectedRequestProcessingSecs);+ await this.#resolvedInner.setExpectedRequestProcessingTime?.(this.#expectedRequestProcessingSecs);}}@@ -1064,10 +1064,10 @@ export class ThrottlingRequestManager<T extends IRequestManager = IRequestManage}}- async setExpectedRequestProcessingTimeSecs(secs: number): Promise<void> {+ async setExpectedRequestProcessingTime(processingTimeSecs: number): Promise<void> {// Remembered so a manager opened later still gets the hint, without opening one now just to pass it along.- this.#expectedRequestProcessingSecs = secs;- await this.#forEachManager((manager) => manager.setExpectedRequestProcessingTimeSecs?.(secs));+ this.#expectedRequestProcessingSecs = processingTimeSecs;+ await this.#forEachManager((manager) => manager.setExpectedRequestProcessingTime?.(processingTimeSecs));}/**@@ -1075,14 +1075,14 @@ export class ThrottlingRequestManager<T extends IRequestManager = IRequestManage* Extending a lock must not consume the in-flight marker that later routes* {@link ThrottlingRequestManager.markRequestAsHandled} / {@link ThrottlingRequestManager.reclaimRequest}.*/- async extendRequestProcessingTimeSecs(request: Request, secs: number): Promise<boolean> {+ async extendRequestProcessingTime(request: Request, extensionSecs: number): Promise<boolean> {const key = request.id ?? request.uniqueKey;const manager = this.#inFlightFromInner.has(key)? this.#resolvedInner!: await this.#selectManagerOrThrow(request.url);- return (await manager.extendRequestProcessingTimeSecs?.(request, secs)) ?? false;+ return (await manager.extendRequestProcessingTime?.(request, extensionSecs)) ?? false;}/**diff --git a/packages/core/src/storages/request-manager-tandem.ts b/packages/core/src/storages/request-manager-tandem.tsindex b514037cd..81a374f8a 100644--- a/packages/core/src/storages/request-manager-tandem.ts+++ b/packages/core/src/storages/request-manager-tandem.ts@@ -27,7 +27,7 @@ export class RequestManagerTandem implements IRequestManager {#requestManagerFactory: () => IRequestManager | Promise<IRequestManager>;/**- * The latest expected request-processing time hinted via {@link setExpectedRequestProcessingTimeSecs}.+ * The latest expected request-processing time hinted via {@link setExpectedRequestProcessingTime}.* Remembered so it can be applied to the writable manager once it is lazily resolved.*/#expectedRequestProcessingSecs?: number;@@ -64,7 +64,7 @@ export class RequestManagerTandem implements IRequestManager {// Apply any hint received before the manager was resolved.if (this.#expectedRequestProcessingSecs !== undefined) {- await this.#resolvedRequestManager.setExpectedRequestProcessingTimeSecs?.(+ await this.#resolvedRequestManager.setExpectedRequestProcessingTime?.(this.#expectedRequestProcessingSecs,);}@@ -229,9 +229,9 @@ export class RequestManagerTandem implements IRequestManager {* reserved. The manager is opened lazily, so the value is remembered and applied once it resolves.* @inheritdoc*/- async setExpectedRequestProcessingTimeSecs(secs: number): Promise<void> {- this.#expectedRequestProcessingSecs = secs;- await this.#resolvedRequestManager?.setExpectedRequestProcessingTimeSecs?.(secs);+ async setExpectedRequestProcessingTime(processingTimeSecs: number): Promise<void> {+ this.#expectedRequestProcessingSecs = processingTimeSecs;+ await this.#resolvedRequestManager?.setExpectedRequestProcessingTime?.(processingTimeSecs);}/**@@ -244,7 +244,7 @@ export class RequestManagerTandem implements IRequestManager {return this.#resolvedRequestManager?.recordPacingSignal(signal) ?? false;}- async extendRequestProcessingTimeSecs(request: Request, secs: number): Promise<boolean> {- return (await this.#getRequestManager()).extendRequestProcessingTimeSecs?.(request, secs) ?? false;+ async extendRequestProcessingTime(request: Request, extensionSecs: number): Promise<boolean> {+ return (await this.#getRequestManager()).extendRequestProcessingTime?.(request, extensionSecs) ?? false;}}diff --git a/packages/core/src/storages/request-manager.ts b/packages/core/src/storages/request-manager.tsindex abb524f26..f930dcf77 100644--- a/packages/core/src/storages/request-manager.ts+++ b/packages/core/src/storages/request-manager.ts@@ -42,7 +42,7 @@ export interface IRequestManager extends IRequestLoader {* the same request out again while it is still being processed. Implementations that do not need* this hint may leave it `undefined`.*/- setExpectedRequestProcessingTimeSecs?(secs: number): Promise<void>;+ setExpectedRequestProcessingTime?(processingTimeSecs: number): Promise<void>;/*** Records something said about the pace requests should go out at, so that a manager which paces its own@@ -64,7 +64,7 @@ export interface IRequestManager extends IRequestLoader {* requests or no longer holds this one; non-locking implementations may leave it `undefined`,* which callers treat as `false`.*/- extendRequestProcessingTimeSecs?(request: Request, secs: number): Promise<boolean>;+ extendRequestProcessingTime?(request: Request, extensionSecs: number): Promise<boolean>;}/**diff --git a/packages/core/src/storages/request-queue.ts b/packages/core/src/storages/request-queue.tsindex 52cd50d99..2104a0ef1 100644--- a/packages/core/src/storages/request-queue.ts+++ b/packages/core/src/storages/request-queue.ts@@ -150,7 +150,7 @@ export class RequestQueue implements IStorage, IRequestManager {/*** The largest expected request-processing time (in seconds) seen so far via- * {@link setExpectedRequestProcessingTimeSecs}. Used to ensure that value is only ever raised, never+ * {@link setExpectedRequestProcessingTime}. Used to ensure that value is only ever raised, never* lowered, before being forwarded to the storage backend.*/#expectedRequestProcessingSecs = 0;@@ -859,26 +859,26 @@ export class RequestQueue implements IStorage, IRequestManager {* ever raise the reservation duration, never lower it — otherwise a short-lived consumer could cut* short the reservation of a long-lived one and have its in-flight request stolen.*/- async setExpectedRequestProcessingTimeSecs(secs: number): Promise<void> {- if (secs <= this.#expectedRequestProcessingSecs) {+ async setExpectedRequestProcessingTime(processingTimeSecs: number): Promise<void> {+ if (processingTimeSecs <= this.#expectedRequestProcessingSecs) {return;}- this.#expectedRequestProcessingSecs = secs;- await this.backend.setExpectedRequestProcessingTimeSecs?.(secs);+ this.#expectedRequestProcessingSecs = processingTimeSecs;+ await this.backend.setExpectedRequestProcessingTime?.(processingTimeSecs);}/*** @inheritdoc- * Unlike {@link RequestQueue.setExpectedRequestProcessingTimeSecs}, which sizes every future lock,+ * Unlike {@link RequestQueue.setExpectedRequestProcessingTime}, which sizes every future lock,* this only touches the one request it is given.*/- async extendRequestProcessingTimeSecs(request: Request, secs: number): Promise<boolean> {+ async extendRequestProcessingTime(request: Request, extensionSecs: number): Promise<boolean> {if (!request.id) {return false;}- return (await this.backend.extendRequestProcessingTimeSecs?.(request.id, secs)) ?? false;+ return (await this.backend.extendRequestProcessingTime?.(request.id, extensionSecs)) ?? false;}/**@@ -923,7 +923,7 @@ export class RequestQueue implements IStorage, IRequestManager {this.#inProgressRequestBatchCount = 0;// Reset the expected-processing-time high-water mark too, otherwise the monotonic-raise guard- // in `setExpectedRequestProcessingTimeSecs` would let a value raised in an earlier run leak into a+ // in `setExpectedRequestProcessingTime` would let a value raised in an earlier run leak into a// later one and silently swallow a lower hint (the queue is meant to be reusable across runs).this.#expectedRequestProcessingSecs = 0;}diff --git a/packages/fs-storage/src/resource-clients/request-queue.ts b/packages/fs-storage/src/resource-clients/request-queue.tsindex b72ec36b0..6dc9ba476 100644--- a/packages/fs-storage/src/resource-clients/request-queue.ts+++ b/packages/fs-storage/src/resource-clients/request-queue.ts@@ -71,16 +71,16 @@ export class RequestQueueBackend extends CachedIdClient implements storage.Reque* Tells the native client how long (in seconds) a fetched request stays locked before it becomes* available again.*/- async setExpectedRequestProcessingTimeSecs(secs: number): Promise<void> {- await this.#nativeBackend.setExpectedRequestProcessingTime(secs);+ async setExpectedRequestProcessingTime(processingTimeSecs: number): Promise<void> {+ await this.#nativeBackend.setExpectedRequestProcessingTime(processingTimeSecs);}/*** @inheritdoc* The lock is owned and extended by the native client; `false` means it no longer holds the request.*/- async extendRequestProcessingTimeSecs(requestId: string, secs: number): Promise<boolean> {- return this.#nativeBackend.prolongRequestLock(requestId, secs);+ async extendRequestProcessingTime(requestId: string, extensionSecs: number): Promise<boolean> {+ return this.#nativeBackend.prolongRequestLock(requestId, extensionSecs);}async getMetadata(): Promise<storage.RequestQueueInfo> {diff --git a/packages/types/src/storages.ts b/packages/types/src/storages.tsindex e2f766682..c4dbaf3c0 100644--- a/packages/types/src/storages.ts+++ b/packages/types/src/storages.ts@@ -392,7 +392,7 @@ export interface RequestQueueBackend {* this long, so that a long-running consumer does not have its request handed out again while it is* still being processed. Clients that do not lock may ignore it.*/- setExpectedRequestProcessingTimeSecs?(secs: number): Promise<void>;+ setExpectedRequestProcessingTime?(processingTimeSecs: number): Promise<void>;/*** Extends the lock on a request previously handed out by {@link fetchNextRequest}, for a consumer@@ -402,7 +402,7 @@ export interface RequestQueueBackend {* does not lock at all). `false` is information, not an error: the request may be processed twice.* Non-locking backends leave this unimplemented.*/- extendRequestProcessingTimeSecs?(requestId: string, secs: number): Promise<boolean>;+ extendRequestProcessingTime?(requestId: string, extensionSecs: number): Promise<boolean>;}/**diff --git a/test/basic-crawler/throttling-request-manager.test.ts b/test/basic-crawler/throttling-request-manager.test.tsindex eb292cb03..5c3c8346c 100644--- a/test/basic-crawler/throttling-request-manager.test.ts+++ b/test/basic-crawler/throttling-request-manager.test.ts@@ -423,7 +423,7 @@ describe('ThrottlingRequestManager', () => {const manager = new ThrottlingRequestManager({ ...throttling, inner: factory });await manager.purge();- await manager.setExpectedRequestProcessingTimeSecs(600);+ await manager.setExpectedRequestProcessingTime(600);await manager.drop();expect(factory).not.toHaveBeenCalled();@@ -432,10 +432,10 @@ describe('ThrottlingRequestManager', () => {test('receives an expected-processing-time hint given before it was resolved', async () => {const inner = await createQueue();- const setHint = vitest.spyOn(inner, 'setExpectedRequestProcessingTimeSecs');+ const setHint = vitest.spyOn(inner, 'setExpectedRequestProcessingTime');const manager = new ThrottlingRequestManager({ ...throttling, inner: () => inner });- await manager.setExpectedRequestProcessingTimeSecs(600);+ await manager.setExpectedRequestProcessingTime(600);expect(setHint).not.toHaveBeenCalled();await manager.fetchNextRequest();@@ -517,7 +517,7 @@ describe('ThrottlingRequestManager', () => {});});- test('extendRequestProcessingTimeSecs reaches the manager holding the request without disturbing its routing', async () => {+ test('extendRequestProcessingTime reaches the manager holding the request without disturbing its routing', async () => {// The file-system backend actually locks fetched requests, so the spies observe// the real forwarding without replacing it.const tmpLocation = resolve(import.meta.dirname, './tmp/extend-routing');@@ -547,14 +547,14 @@ describe('ThrottlingRequestManager', () => {// Re-opening the deterministic alias yields the same cached frontend the manager routes to.const subQueue = await RequestQueue.open({ alias: 'throttled-example.com' }, { storageBackend });- const innerSpy = vi.spyOn(inner.backend, 'extendRequestProcessingTimeSecs');- const subSpy = vi.spyOn(subQueue.backend, 'extendRequestProcessingTimeSecs');+ const innerSpy = vi.spyOn(inner.backend, 'extendRequestProcessingTime');+ const subSpy = vi.spyOn(subQueue.backend, 'extendRequestProcessingTime');- await expect(manager.extendRequestProcessingTimeSecs(routed, 30)).resolves.toBe(true);+ await expect(manager.extendRequestProcessingTime(routed, 30)).resolves.toBe(true);expect(subSpy).toHaveBeenCalledExactlyOnceWith(routed.id, 30);expect(innerSpy).not.toHaveBeenCalled();- await expect(manager.extendRequestProcessingTimeSecs(fromInner, 30)).resolves.toBe(true);+ await expect(manager.extendRequestProcessingTime(fromInner, 30)).resolves.toBe(true);expect(innerSpy).toHaveBeenCalledExactlyOnceWith(fromInner.id, 30);expect(subSpy).toHaveBeenCalledTimes(1);diff --git a/test/core/crawlers/adaptive-playwright-crawler.test.ts b/test/core/crawlers/adaptive-playwright-crawler.test.tsindex cf7f7bcf8..2e40f88c0 100644--- a/test/core/crawlers/adaptive-playwright-crawler.test.ts+++ b/test/core/crawlers/adaptive-playwright-crawler.test.ts@@ -1408,7 +1408,7 @@ describe('AdaptivePlaywrightCrawler', () => {test('reserves a request for the longest route timeout override', async () => {const requestQueue = await RequestQueue.open(`rq-adaptive-${Math.random() * 10000}`);- const hintSpy = vitest.spyOn(requestQueue, 'setExpectedRequestProcessingTimeSecs');+ const hintSpy = vitest.spyOn(requestQueue, 'setExpectedRequestProcessingTime');const router = createAdaptivePlaywrightRouter();router.addHandler('LIST', async () => {}, { requestHandlerTimeoutSecs: 300 });diff --git a/test/core/crawlers/basic-crawler.test.ts b/test/core/crawlers/basic-crawler.test.tsindex 9e38a29b4..2d937cd84 100644--- a/test/core/crawlers/basic-crawler.test.ts+++ b/test/core/crawlers/basic-crawler.test.ts@@ -2014,7 +2014,7 @@ describe('BasicCrawler', () => {const requestQueue = await RequestQueue.open(null, {storageBackend: new FileSystemStorageBackend({ localDataDirectory: tmpLocation }),});- const extendSpy = vitest.spyOn(requestQueue.backend, 'extendRequestProcessingTimeSecs');+ const extendSpy = vitest.spyOn(requestQueue.backend, 'extendRequestProcessingTime');try {let sawFailure = false;diff --git a/test/core/request-manager-tandem.test.ts b/test/core/request-manager-tandem.test.tsindex 0812be445..140e8a58c 100644--- a/test/core/request-manager-tandem.test.ts+++ b/test/core/request-manager-tandem.test.ts@@ -333,30 +333,30 @@ describe('RequestManagerTandem', () => {).toBe(false);});- test('setExpectedRequestProcessingTimeSecs forwards to an already-resolved manager', async () => {+ test('setExpectedRequestProcessingTime forwards to an already-resolved manager', async () => {const requestList = await RequestList.open(null, [{ url: 'https://example.com/1' }]);const requestQueue = await RequestQueue.open();- const hintSpy = vi.spyOn(requestQueue, 'setExpectedRequestProcessingTimeSecs');+ const hintSpy = vi.spyOn(requestQueue, 'setExpectedRequestProcessingTime');const tandem = new RequestManagerTandem(requestList, requestQueue);// Resolve the manager first (the queue was passed eagerly, but make the dependency explicit).await tandem.fetchNextRequest();- await tandem.setExpectedRequestProcessingTimeSecs(600);+ await tandem.setExpectedRequestProcessingTime(600);expect(hintSpy).toHaveBeenCalledWith(600);});- test('setExpectedRequestProcessingTimeSecs applies a hint set before the manager is lazily resolved', async () => {+ test('setExpectedRequestProcessingTime applies a hint set before the manager is lazily resolved', async () => {const requestList = await RequestList.open(null, [{ url: 'https://example.com/1' }]);const requestQueue = await RequestQueue.open();- const hintSpy = vi.spyOn(requestQueue, 'setExpectedRequestProcessingTimeSecs');+ const hintSpy = vi.spyOn(requestQueue, 'setExpectedRequestProcessingTime');// Provide the manager lazily so it is not resolved at construction time.const tandem = new RequestManagerTandem(requestList, () => requestQueue);// Hint arrives before anything resolves the manager — nothing forwarded yet.- await tandem.setExpectedRequestProcessingTimeSecs(600);+ await tandem.setExpectedRequestProcessingTime(600);expect(hintSpy).not.toHaveBeenCalled();// Resolving the manager (via any operation) applies the remembered hint.@@ -364,7 +364,7 @@ describe('RequestManagerTandem', () => {expect(hintSpy).toHaveBeenCalledWith(600);});- test('extendRequestProcessingTimeSecs forwards to the writable request manager', async () => {+ test('extendRequestProcessingTime forwards to the writable request manager', async () => {const requestList = await RequestList.open(null, []);const requestQueue = await RequestQueue.open();await requestQueue.addRequest({ url: 'https://example.com/1' });@@ -372,11 +372,11 @@ describe('RequestManagerTandem', () => {expect(fetched).not.toBeNull();const stub = vi.fn(async () => true);- (requestQueue.backend as any).extendRequestProcessingTimeSecs = stub;+ (requestQueue.backend as any).extendRequestProcessingTime = stub;const tandem = new RequestManagerTandem(requestList, requestQueue);- await expect(tandem.extendRequestProcessingTimeSecs!(fetched!, 30)).resolves.toBe(true);+ await expect(tandem.extendRequestProcessingTime!(fetched!, 30)).resolves.toBe(true);expect(stub).toHaveBeenCalledExactlyOnceWith(fetched!.id, 30);});});diff --git a/test/core/storages/request-queue.test.ts b/test/core/storages/request-queue.test.tsindex c8bb84f5d..d1c1107e5 100644--- a/test/core/storages/request-queue.test.ts+++ b/test/core/storages/request-queue.test.ts@@ -391,30 +391,30 @@ describe('RequestQueue remote', () => {expect(second.sessionId).toBeUndefined();});- describe('setExpectedRequestProcessingTimeSecs', () => {+ describe('setExpectedRequestProcessingTime', () => {test('forwards the value to the client, but only ever raises it', async () => {const queue = await RequestQueue.open();// The in-memory client does not implement this optional hint (it has no request locking to// tune), so attach a stub to verify the frontend's raise-only forwarding logic in isolation.const spy = vitest.fn();- (queue.backend as any).setExpectedRequestProcessingTimeSecs = spy;+ (queue.backend as any).setExpectedRequestProcessingTime = spy;// First hint is forwarded.- await queue.setExpectedRequestProcessingTimeSecs(60);+ await queue.setExpectedRequestProcessingTime(60);expect(spy).toHaveBeenLastCalledWith(60);// A larger hint is forwarded.- await queue.setExpectedRequestProcessingTimeSecs(120);+ await queue.setExpectedRequestProcessingTime(120);expect(spy).toHaveBeenLastCalledWith(120);// A smaller (or equal) hint must not shorten the reservation, so it is not forwarded.- await queue.setExpectedRequestProcessingTimeSecs(30);- await queue.setExpectedRequestProcessingTimeSecs(120);+ await queue.setExpectedRequestProcessingTime(30);+ await queue.setExpectedRequestProcessingTime(120);expect(spy).toHaveBeenCalledTimes(2);});});- describe('extendRequestProcessingTimeSecs', () => {+ describe('extendRequestProcessingTime', () => {test('forwards the request id and extension to the backend', async () => {// The file-system backend actually locks fetched requests; the spy observes// the real forwarding and the real lock extension.@@ -427,9 +427,9 @@ describe('RequestQueue remote', () => {const fetched = await queue.fetchNextRequest();expect(fetched).not.toBeNull();- const stub = vitest.spyOn(queue.backend, 'extendRequestProcessingTimeSecs');+ const stub = vitest.spyOn(queue.backend, 'extendRequestProcessingTime');- await expect(queue.extendRequestProcessingTimeSecs(fetched!, 30)).resolves.toBe(true);+ await expect(queue.extendRequestProcessingTime(fetched!, 30)).resolves.toBe(true);expect(stub).toHaveBeenCalledExactlyOnceWith(fetched!.id, 30);} finally {await rm(tmpLocation, { force: true, recursive: true });@@ -442,7 +442,7 @@ describe('RequestQueue remote', () => {const fetched = await queue.fetchNextRequest();expect(fetched).not.toBeNull();- await expect(queue.extendRequestProcessingTimeSecs(fetched!, 30)).resolves.toBe(false);+ await expect(queue.extendRequestProcessingTime(fetched!, 30)).resolves.toBe(false);});});diff --git a/test/fs-storage/request-queue/prolong-request-lock.test.ts b/test/fs-storage/request-queue/prolong-request-lock.test.tsindex 845d4a983..01f84b506 100644--- a/test/fs-storage/request-queue/prolong-request-lock.test.ts+++ b/test/fs-storage/request-queue/prolong-request-lock.test.ts@@ -3,7 +3,7 @@ import { resolve } from 'node:path';import { FileSystemStorageBackend } from '@crawlee/fs-storage';-describe('FileSystemStorageBackend extendRequestProcessingTimeSecs', () => {+describe('FileSystemStorageBackend extendRequestProcessingTime', () => {const tmpLocation = resolve(import.meta.dirname, './tmp/prolong-request-lock');afterEach(async () => {@@ -22,8 +22,8 @@ describe('FileSystemStorageBackend extendRequestProcessingTimeSecs', () => {const locked = await queue.fetchNextRequest();expect(locked).toBeDefined();- expect(await queue.extendRequestProcessingTimeSecs!(locked!.id!, 30)).toBe(true);- expect(await queue.extendRequestProcessingTimeSecs!('no-such-request', 30)).toBe(false);+ expect(await queue.extendRequestProcessingTime!(locked!.id!, 30)).toBe(true);+ expect(await queue.extendRequestProcessingTime!('no-such-request', 30)).toBe(false);await queue.drop();});