Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

feat: extend the request lock from `extendTimeout` (#4041)

Atirna1b31982541e4
diff --git a/docs/public-api/crawlee-basic.api.md b/docs/public-api/crawlee-basic.api.md
index 60fb0fa96..c74f9e01a 100644
--- a/docs/public-api/crawlee-basic.api.md
+++ b/docs/public-api/crawlee-basic.api.md
@@ -1140,6 +1140,7 @@ export class ThrottlingRequestManager<T extends IRequestManager = IRequestManage
checkReadiness(): Promise<RequestSourceStatus>;
// (undocumented)
drop(): Promise<void>;
+ extendRequestProcessingTimeSecs(request: Request_2, secs: number): Promise<boolean>;
fetchNextRequest<R extends Dictionary = Dictionary>(): Promise<Request_2<R> | null>;
// (undocumented)
getHandledCount(): Promise<number>;
diff --git a/docs/public-api/crawlee-core.api.md b/docs/public-api/crawlee-core.api.md
index e44dfa09d..04f285520 100644
--- a/docs/public-api/crawlee-core.api.md
+++ b/docs/public-api/crawlee-core.api.md
@@ -427,6 +427,7 @@ export interface IRequestManager extends IRequestLoader {
addRequest(requestLike: Source, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo>;
// (undocumented)
addRequestsBatched(requests: RequestsLike, options?: AddRequestsBatchedOptions): Promise<AddRequestsBatchedResult>;
+ extendRequestProcessingTimeSecs?(request: Request_2, secs: number): Promise<boolean>;
purge?(): Promise<void>;
reclaimRequest(request: Request_2, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo | null>;
recordPacingSignal(signal: PacingSignal): boolean;
@@ -787,6 +788,8 @@ export class RequestManagerTandem implements IRequestManager {
// (undocumented)
addRequestsBatched(requests: RequestsLike, options?: AddRequestsBatchedOptions): Promise<AddRequestsBatchedResult>;
checkReadiness(): Promise<RequestSourceStatus>;
+ // (undocumented)
+ extendRequestProcessingTimeSecs(request: Request_2, secs: number): Promise<boolean>;
fetchNextRequest<T extends Dictionary = Dictionary>(): Promise<Request_2<T> | null>;
// (undocumented)
getHandledCount(): Promise<number>;
@@ -833,6 +836,7 @@ export class RequestQueue implements IStorage, IRequestManager {
readonly backend: RequestQueueBackend;
checkReadiness(): Promise<RequestLoaderStatus>;
drop(): Promise<void>;
+ extendRequestProcessingTimeSecs(request: Request_2, secs: number): Promise<boolean>;
fetchNextRequest<T extends Dictionary = Dictionary>(): Promise<Request_2<T> | null>;
getHandledCount(): Promise<number>;
getInfo(): Promise<RequestQueueInfo>;
diff --git a/docs/public-api/crawlee-types.api.md b/docs/public-api/crawlee-types.api.md
index 8b1b7ad43..e5fd04ede 100644
--- a/docs/public-api/crawlee-types.api.md
+++ b/docs/public-api/crawlee-types.api.md
@@ -374,6 +374,7 @@ export type RedirectHandler = (redirectResponse: Response, updatedRequest: {
export interface RequestQueueBackend {
addBatchOfRequests(requests: RequestSchema[], options?: RequestQueueOperationOptions): Promise<BatchAddRequestsResult>;
drop(): Promise<void>;
+ extendRequestProcessingTimeSecs?(requestId: string, secs: number): Promise<boolean>;
fetchNextRequest(): Promise<UpdateRequestSchema | undefined>;
getMetadata(): Promise<RequestQueueInfo>;
getRequest(uniqueKey: string): Promise<UpdateRequestSchema | undefined>;
diff --git a/docs/upgrading/upgrading_v4.md b/docs/upgrading/upgrading_v4.md
index dfd08c622..be633dc36 100644
--- a/docs/upgrading/upgrading_v4.md
+++ b/docs/upgrading/upgrading_v4.md
@@ -1717,7 +1717,7 @@ The sub-backend interfaces (`DatasetBackend`, `KeyValueStoreBackend`, `RequestQu
**`RequestQueueBackend`:**
-The request queue backend was reduced from 12 methods to 10. The distributed-locking protocol (`listAndLockHead` → `prolongRequestLock` → `deleteRequestLock`) and the queue-head/consistency bookkeeping that used to live in the `RequestQueue` frontend have been removed from the interface; coordinating multiple clients accessing the same queue (e.g. request locking on the Apify platform) is now an internal concern of the backend implementation.
+The request queue backend's surface was reshaped. The frontend-owned distributed-locking protocol (`listAndLockHead` → `prolongRequestLock` → `deleteRequestLock`) was removed. Queue-head and consistency bookkeeping are now internal concerns of the backend implementation.
| Before (v3) | After (v4) |
|---|---|
@@ -1727,7 +1727,7 @@ The request queue backend was reduced from 12 methods to 10. The distributed-loc
| `updateRequest(request, opts?)` | `markRequestAsHandled(request)` / `reclaimRequest(request, opts?)` |
| `listHead(opts?)` | `fetchNextRequest()` (returns a single request, marks it in progress) |
| `listAndLockHead(opts)` | Removed (locking is internal to the client) |
-| `prolongRequestLock(id, opts)` | Removed |
+| `prolongRequestLock(id, opts)` | `extendRequestProcessingTimeSecs(requestId, secs)` (optional — per-request lock extension for locking backends; wired to `context.extendTimeout`) |
| `deleteRequestLock(id, opts?)` | Removed |
| `deleteRequest(id)` | Removed |
| _(n/a)_ | `isEmpty()` (new — `true` when no pending requests are left to fetch) |
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.ts
index 8d515afbc..3960b3b50 100644
--- a/packages/basic-crawler/src/internals/basic-crawler.ts
+++ b/packages/basic-crawler/src/internals/basic-crawler.ts
@@ -1499,6 +1499,11 @@ export class BasicCrawler<
if (context[navigationDeadlineKey] !== undefined) {
context[navigationDeadlineKey] += extraMillis;
}
+
+ // Extension failure must not fail the request that asked for more time.
+ this.requestManager?.extendRequestProcessingTimeSecs?.(context.request, secs)?.catch((error) => {
+ this.log.debug('Extending the request processing time failed', { url: context.request.url, error });
+ });
},
};
}
diff --git a/packages/basic-crawler/src/internals/crawlers/crawler_commons.ts b/packages/basic-crawler/src/internals/crawlers/crawler_commons.ts
index 8843397b2..024436fba 100644
--- a/packages/basic-crawler/src/internals/crawlers/crawler_commons.ts
+++ b/packages/basic-crawler/src/internals/crawlers/crawler_commons.ts
@@ -251,8 +251,9 @@ export interface CrawlingContext<UserData extends Dictionary = Dictionary> exten
* ```
*
* Extends the request handler's own timeout and the crawler's internal one together, so the extension
- * is not immediately undone by the latter. Calling it from a handler that has already timed out does
- * nothing.
+ * is not immediately undone by the latter. On a locking storage backend the request's lock is
+ * extended by the same amount, so the extra time is not spent while the queue considers the request
+ * free to hand out again. Calling it from a handler that has already timed out does nothing.
*/
extendTimeout(secs: number): void;
}
diff --git a/packages/basic-crawler/src/internals/throttling_request_manager.ts b/packages/basic-crawler/src/internals/throttling_request_manager.ts
index a88f54bb3..17fa5d8eb 100644
--- a/packages/basic-crawler/src/internals/throttling_request_manager.ts
+++ b/packages/basic-crawler/src/internals/throttling_request_manager.ts
@@ -1042,6 +1042,21 @@ export class ThrottlingRequestManager<T extends IRequestManager = IRequestManage
await this.#forEachManager((manager) => manager.setExpectedRequestProcessingTimeSecs?.(secs));
}
+ /**
+ * @inheritdoc
+ * Extending a lock must not consume the in-flight marker that later routes
+ * {@link ThrottlingRequestManager.markRequestAsHandled} / {@link ThrottlingRequestManager.reclaimRequest}.
+ */
+ async extendRequestProcessingTimeSecs(request: Request, secs: number): Promise<boolean> {
+ const key = request.id ?? request.uniqueKey;
+
+ const manager = this.#inFlightFromInner.has(key)
+ ? this.#resolvedInner!
+ : await this.#selectManagerOrThrow(request.url);
+
+ return (await manager.extendRequestProcessingTimeSecs?.(request, secs)) ?? false;
+ }
+
/**
* Runs `fn` over the sub-queues and, if it has been resolved, the wrapped manager - bookkeeping never forces
* a lazily-opened `inner`, since there is no point opening a queue purely to tell it something.
diff --git a/packages/basic-crawler/test/throttling_request_manager.test.ts b/packages/basic-crawler/test/throttling_request_manager.test.ts
index 09bed0c79..92494a6e7 100644
--- a/packages/basic-crawler/test/throttling_request_manager.test.ts
+++ b/packages/basic-crawler/test/throttling_request_manager.test.ts
@@ -1,6 +1,11 @@
+import { rm } from 'node:fs/promises';
+import { resolve } from 'node:path';
+
import type { ThrottlingRequestManagerOptions } from '@crawlee/basic';
import { ThrottlingRequestManager } from '@crawlee/basic';
import type { AddRequestsBatchedResult, StorageIdentifier } from '@crawlee/core';
+
+import { vi } from 'vitest';
import {
KeyValueStore,
MemoryStorageBackend,
@@ -9,6 +14,7 @@ import {
withStorageTransaction,
} from '@crawlee/core';
import { sleep } from '@crawlee/utils';
+import { FileSystemStorageBackend } from '@crawlee/fs-storage';
describe('ThrottlingRequestManager', () => {
beforeEach(() => {
@@ -499,6 +505,52 @@ describe('ThrottlingRequestManager', () => {
});
});
+ test('extendRequestProcessingTimeSecs reaches the manager holding the request without disturbing its routing', async () => {
+ // The file-system backend actually locks fetched requests, so the spies observe
+ // the real forwarding without replacing it.
+ const tmpLocation = resolve(import.meta.dirname, './tmp/extend-routing');
+ const storageBackend = new FileSystemStorageBackend({ localDataDirectory: tmpLocation });
+ try {
+ await withLockingRoutingTest();
+ } finally {
+ await rm(tmpLocation, { force: true, recursive: true });
+ }
+
+ async function withLockingRoutingTest() {
+ const inner = await RequestQueue.open({ name: 'inner-queue' }, { storageBackend });
+ const manager = new ThrottlingRequestManager({
+ inner,
+ domains: ['example.com'],
+ requestManagerOpener: async (identifier, options) =>
+ RequestQueue.open(identifier, { ...options, storageBackend }),
+ });
+
+ await manager.addRequest({ url: 'https://example.com/routed' });
+ await inner.addRequest({ url: 'https://example.com/inner' });
+
+ const routed = (await manager.fetchNextRequest())!;
+ expect(routed.url).toBe('https://example.com/routed');
+ const fromInner = (await manager.fetchNextRequest())!;
+ expect(fromInner.url).toBe('https://example.com/inner');
+
+ // Re-opening the deterministic alias yields the same cached frontend the manager routes to.
+ const subQueue = await RequestQueue.open({ alias: 'throttled-example.com' }, { storageBackend });
+ const innerSpy = vi.spyOn(inner.backend, 'extendRequestProcessingTimeSecs');
+ const subSpy = vi.spyOn(subQueue.backend, 'extendRequestProcessingTimeSecs');
+
+ await expect(manager.extendRequestProcessingTimeSecs(routed, 30)).resolves.toBe(true);
+ expect(subSpy).toHaveBeenCalledExactlyOnceWith(routed.id, 30);
+ expect(innerSpy).not.toHaveBeenCalled();
+
+ await expect(manager.extendRequestProcessingTimeSecs(fromInner, 30)).resolves.toBe(true);
+ expect(innerSpy).toHaveBeenCalledExactlyOnceWith(fromInner.id, 30);
+ expect(subSpy).toHaveBeenCalledTimes(1);
+
+ await manager.markRequestAsHandled(fromInner);
+ expect(await inner.getPendingCount()).toBe(0);
+ }
+ });
+
test('recordPacingSignal enforces throttling and fair scheduling', async () => {
const inner = await createQueue();
const manager = new ThrottlingRequestManager({
diff --git a/packages/core/src/storages/request_manager.ts b/packages/core/src/storages/request_manager.ts
index cfa20dcfd..6c3c80f72 100644
--- a/packages/core/src/storages/request_manager.ts
+++ b/packages/core/src/storages/request_manager.ts
@@ -54,6 +54,17 @@ export interface IRequestManager extends IRequestLoader {
* @returns `true` if anything in the composition took responsibility for the signal.
*/
recordPacingSignal(signal: PacingSignal): boolean;
+
+ /**
+ * Extends the lock on a request previously handed out by `fetchNextRequest()` and still being
+ * processed, on storage backends that reserve requests via locking (e.g. via
+ * {@apilink CrawlingContext.extendTimeout|`context.extendTimeout`}).
+ *
+ * @returns `true` when the lock was prolonged, `false` when this manager does not lock
+ * requests or no longer holds this one; non-locking implementations may leave it `undefined`,
+ * which callers treat as `false`.
+ */
+ extendRequestProcessingTimeSecs?(request: Request, secs: number): Promise<boolean>;
}
/**
diff --git a/packages/core/src/storages/request_manager_tandem.ts b/packages/core/src/storages/request_manager_tandem.ts
index 1a5c4c607..c101a5ddb 100644
--- a/packages/core/src/storages/request_manager_tandem.ts
+++ b/packages/core/src/storages/request_manager_tandem.ts
@@ -245,4 +245,8 @@ export class RequestManagerTandem implements IRequestManager {
recordPacingSignal(signal: PacingSignal): boolean {
return this.#resolvedRequestManager?.recordPacingSignal(signal) ?? false;
}
+
+ async extendRequestProcessingTimeSecs(request: Request, secs: number): Promise<boolean> {
+ return (await this.getRequestManager()).extendRequestProcessingTimeSecs?.(request, secs) ?? false;
+ }
}
diff --git a/packages/core/src/storages/request_queue.ts b/packages/core/src/storages/request_queue.ts
index 2dbcb5ea9..f213fd072 100644
--- a/packages/core/src/storages/request_queue.ts
+++ b/packages/core/src/storages/request_queue.ts
@@ -860,6 +860,19 @@ export class RequestQueue implements IStorage, IRequestManager {
await this.backend.setExpectedRequestProcessingTimeSecs?.(secs);
}
+ /**
+ * @inheritdoc
+ * Unlike {@link RequestQueue.setExpectedRequestProcessingTimeSecs}, which sizes every future lock,
+ * this only touches the one request it is given.
+ */
+ async extendRequestProcessingTimeSecs(request: Request, secs: number): Promise<boolean> {
+ if (!request.id) {
+ return false;
+ }
+
+ return (await this.backend.extendRequestProcessingTimeSecs?.(request.id, secs)) ?? false;
+ }
+
/**
* Caches information about request to beware of unneeded addRequest() calls.
*/
diff --git a/packages/fs-storage/package.json b/packages/fs-storage/package.json
index f15178fba..17a5ca27c 100644
--- a/packages/fs-storage/package.json
+++ b/packages/fs-storage/package.json
@@ -42,7 +42,7 @@
"access": "public"
},
"dependencies": {
- "@crawlee/fs-storage-native": ">=0.2.1 <0.3",
+ "@crawlee/fs-storage-native": ">=0.2.2 <0.3",
"@crawlee/types": "workspace:*",
"@crawlee/utils": "workspace:*",
"zod": "catalog:"
diff --git a/packages/fs-storage/src/resource-clients/request-queue.ts b/packages/fs-storage/src/resource-clients/request-queue.ts
index afcd97ca8..b72ec36b0 100644
--- a/packages/fs-storage/src/resource-clients/request-queue.ts
+++ b/packages/fs-storage/src/resource-clients/request-queue.ts
@@ -75,6 +75,14 @@ export class RequestQueueBackend extends CachedIdClient implements storage.Reque
await this.#nativeBackend.setExpectedRequestProcessingTime(secs);
}
+ /**
+ * @inheritdoc
+ * The lock is owned and extended by the native client; `false` means it no longer holds the request.
+ */
+ async extendRequestProcessingTimeSecs(requestId: string, secs: number): Promise<boolean> {
+ return this.#nativeBackend.prolongRequestLock(requestId, secs);
+ }
+
async getMetadata(): Promise<storage.RequestQueueInfo> {
return this.#nativeBackend.getMetadata();
}
diff --git a/packages/fs-storage/test/request-queue/prolong-request-lock.test.ts b/packages/fs-storage/test/request-queue/prolong-request-lock.test.ts
new file mode 100644
index 000000000..44ef0717c
--- /dev/null
+++ b/packages/fs-storage/test/request-queue/prolong-request-lock.test.ts
@@ -0,0 +1,30 @@
+import { rm } from 'node:fs/promises';
+import { resolve } from 'node:path';
+
+import { FileSystemStorageBackend } from '@crawlee/fs-storage';
+
+describe('FileSystemStorageBackend extendRequestProcessingTimeSecs', () => {
+ const tmpLocation = resolve(import.meta.dirname, './tmp/prolong-request-lock');
+
+ afterEach(async () => {
+ await rm(tmpLocation, { force: true, recursive: true });
+ });
+
+ test('extends the lock on a fetched request, false for one never locked', async () => {
+ const storage = new FileSystemStorageBackend({
+ localDataDirectory: tmpLocation,
+ });
+ const queue = await storage.createRequestQueueBackend({
+ name: 'default',
+ });
+ await queue.addBatchOfRequests([{ url: 'http://example.com/1', uniqueKey: '1' }]);
+
+ const locked = await queue.fetchNextRequest();
+ expect(locked).toBeDefined();
+
+ expect(await queue.extendRequestProcessingTimeSecs(locked!.id!, 30)).toBe(true);
+ expect(await queue.extendRequestProcessingTimeSecs('no-such-request', 30)).toBe(false);
+
+ await queue.drop();
+ });
+});
diff --git a/packages/types/src/storages.ts b/packages/types/src/storages.ts
index d57ff19ef..49c337fa5 100644
--- a/packages/types/src/storages.ts
+++ b/packages/types/src/storages.ts
@@ -393,6 +393,16 @@ export interface RequestQueueBackend {
* still being processed. Clients that do not lock may ignore it.
*/
setExpectedRequestProcessingTimeSecs?(secs: number): Promise<void>;
+
+ /**
+ * Extends the lock on a request previously handed out by {@link fetchNextRequest}, for a consumer
+ * that needs more time than the sizing hint reserved.
+ *
+ * @returns `true` when prolonged, `false` when this client does not hold the request locked (or
+ * does not lock at all). `false` is information, not an error: the request may be processed twice.
+ * Non-locking backends leave this unimplemented.
+ */
+ extendRequestProcessingTimeSecs?(requestId: string, secs: number): Promise<boolean>;
}
/**
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 1bf31d3b6..d5e7d8090 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -835,8 +835,8 @@ importers:
packages/fs-storage:
dependencies:
'@crawlee/fs-storage-native':
- specifier: '>=0.2.1 <0.3'
- version: 0.2.1
+ specifier: '>=0.2.2 <0.3'
+ version: 0.2.2
'@crawlee/types':
specifier: workspace:*
version: link:../types
@@ -2233,60 +2233,60 @@ packages:
resolution: {integrity: sha512-TzlTVpKPjaqW6qOYjQcYUDuGsLCNsvFHVBXkYGTAnf5V37jCWrE5haKNXzz0WZUtVHjrpV76L1buANjwXMfT8w==}
engines: {node: '>=22'}
- '@crawlee/[email protected]':
- resolution: {integrity: sha512-vpjIyzhv2XB8oz5UbcXVMm6RwyfJgAE7xEiXG3pgxYjjtWOP2xGZX17dJiCw9fcOjH3bawfNZTZiKSDvxF0Q1A==}
+ '@crawlee/[email protected]':
+ resolution: {integrity: sha512-8R9fm6iyUVSxPdRA/m7XTgDawm6bjZIaDzmLmtQPwEIDTE97z6mk2MOyrlT53rr7B5aObN/nYJeCDENIGfcH1g==}
engines: {node: '>= 20'}
cpu: [arm64]
os: [darwin]
- '@crawlee/[email protected]':
- resolution: {integrity: sha512-os5MWPkUoNKM+JSLN5MxeVoRxlwqVlef2gMqXc+a359kSS3iqGjP+tUZQ8MTIU5s370igxIx6dnJl+xr6FplJA==}
+ '@crawlee/[email protected]':
+ resolution: {integrity: sha512-ETYUt8T9cNEpq4qfaDdIGlWrdXiDgYL7BzGkNRTYb+NCc3mN3SvY6kEKLSrezn8S6Hwmgud5tfPPe4HjlcxRoQ==}
engines: {node: '>= 20'}
cpu: [x64]
os: [darwin]
- '@crawlee/[email protected]':
- resolution: {integrity: sha512-7rO5NC6HKjl+rh8zK/bayKBP3MMQs/Ib5eyzZJi4U8Jb2Vke1pU9yta+XjKzVU2rQZB8ErntISTxczbUfZfhFQ==}
+ '@crawlee/[email protected]':
+ resolution: {integrity: sha512-rRyd22FOwlvuh+YVwwazlvxWRX2eUh8v2Wpp2So5oZjCEbfoY4Wp4R2Lbd8kjsMYWPBC9MIwv1RkiICpxVfqCA==}
engines: {node: '>= 20'}
cpu: [arm64]
os: [linux]
libc: [glibc]
- '@crawlee/[email protected]':
- resolution: {integrity: sha512-+oNToqNyXkcaT7oi8WBcKaQxJNZD4oXv1QWyUq+z69h5H6GfzcQe/nqslXALY5SNx9yTCzTtVS5AgrgqvGzyTg==}
+ '@crawlee/[email protected]':
+ resolution: {integrity: sha512-B0436STNN9aLMdlEeqEsfysarhcggQ0dS6p/9iffLUnqFC2OWGcXxHOVl+2pH597tPVyv0gsVfAC8BxJ5a1c4Q==}
engines: {node: '>= 20'}
cpu: [arm64]
os: [linux]
libc: [musl]
- '@crawlee/[email protected]':
- resolution: {integrity: sha512-qHQTBNFlBHajKfcwvzN8u68mLBWsEbaZQb7yNBmocUcBSYXm0U6JAFn1BcpNJyFb+CfpERR3QaCIUxBVdbRBDw==}
+ '@crawlee/[email protected]':
+ resolution: {integrity: sha512-OXB+0qlAzcZCbdncZ1koITkHotrzYCwm8r7p4jhcv9MUbhtx7h7EUJ/90aMARRvX0T00HxWyCzvGTiJQxNUidA==}
engines: {node: '>= 20'}
cpu: [x64]
os: [linux]
libc: [glibc]
- '@crawlee/[email protected]':
- resolution: {integrity: sha512-IjrfVP7mnyQqAgoRKTLxWmjYRyktVI/cBfXg+RIEtjss9a9OIt4jrHhjLhUo8cQr5IT6wyEXRto0AOxZth96SA==}
+ '@crawlee/[email protected]':
+ resolution: {integrity: sha512-uF92wHy7IExtblv3Jsi55z+05qtqnigucXtnsXmol0i1JFjwLGxiDCJC9cskZVph8bHVGyLQFitIlI4w5U1sUQ==}
engines: {node: '>= 20'}
cpu: [x64]
os: [linux]
libc: [musl]
- '@crawlee/[email protected]':
- resolution: {integrity: sha512-GsVD+XbiL5tTQlmIPtiThyGFp7B3vg1l8ERXyCHR6Gsz+85ZeMQ9jg3/ILFBKFY15I8v5FoDV0cQtdH7D5+FKg==}
+ '@crawlee/[email protected]':
+ resolution: {integrity: sha512-SKx5GIZJ1jPfnOmJCsHHwMwrBM+ZQUB+c1xcw+YijcD4un06MxuKRYS79nFSM2SvLzvMROBhsN+3LoIXOZjNow==}
engines: {node: '>= 20'}
cpu: [arm64]
os: [win32]
- '@crawlee/[email protected]':
- resolution: {integrity: sha512-H8uigGqYChXySUDpnDhwl9JrDbp/rZMFpjuoSDSssF1gr+83d7qRjISYKVC0TUlKiYAnT7SqYCBqNrsuSbJBUw==}
+ '@crawlee/[email protected]':
+ resolution: {integrity: sha512-d7oDXVLAnmpmDhQ9mkN/iTJTg+XJ60mwLyANxzPtyGqiapxWPT0Y2Yw7N+dUloSUFyuWDVl1R4A45eORuwVCYg==}
engines: {node: '>= 20'}
cpu: [x64]
os: [win32]
- '@crawlee/[email protected]':
- resolution: {integrity: sha512-x6umw+kVjuA6k+Qhx9RwJSjl/DRyeo1jlCiQH04ZKz5DP54NhQjMlZSQD1Y0v4SYuN0u+wqDVINQQQsarsFkeg==}
+ '@crawlee/[email protected]':
+ resolution: {integrity: sha512-WGKNyPoRq1LKqndeJ9LdEWigGNm9Z+3U8jJaSlEVgPq8CkmcqHANXxpZXkgGXl9jfEiWzFlJmWm4JUMqT/TxfA==}
engines: {node: '>= 20'}
'@crawlee/[email protected]':
@@ -14590,40 +14590,40 @@ snapshots:
'@conventional-changelog/[email protected]': {}
- '@crawlee/[email protected]':
+ '@crawlee/[email protected]':
optional: true
- '@crawlee/[email protected]':
+ '@crawlee/[email protected]':
optional: true
- '@crawlee/[email protected]':
+ '@crawlee/[email protected]':
optional: true
- '@crawlee/[email protected]':
+ '@crawlee/[email protected]':
optional: true
- '@crawlee/[email protected]':
+ '@crawlee/[email protected]':
optional: true
- '@crawlee/[email protected]':
+ '@crawlee/[email protected]':
optional: true
- '@crawlee/[email protected]':
+ '@crawlee/[email protected]':
optional: true
- '@crawlee/[email protected]':
+ '@crawlee/[email protected]':
optional: true
- '@crawlee/[email protected]':
+ '@crawlee/[email protected]':
optionalDependencies:
- '@crawlee/fs-storage-native-darwin-arm64': 0.2.1
- '@crawlee/fs-storage-native-darwin-x64': 0.2.1
- '@crawlee/fs-storage-native-linux-arm64-gnu': 0.2.1
- '@crawlee/fs-storage-native-linux-arm64-musl': 0.2.1
- '@crawlee/fs-storage-native-linux-x64-gnu': 0.2.1
- '@crawlee/fs-storage-native-linux-x64-musl': 0.2.1
- '@crawlee/fs-storage-native-win32-arm64-msvc': 0.2.1
- '@crawlee/fs-storage-native-win32-x64-msvc': 0.2.1
+ '@crawlee/fs-storage-native-darwin-arm64': 0.2.2
+ '@crawlee/fs-storage-native-darwin-x64': 0.2.2
+ '@crawlee/fs-storage-native-linux-arm64-gnu': 0.2.2
+ '@crawlee/fs-storage-native-linux-arm64-musl': 0.2.2
+ '@crawlee/fs-storage-native-linux-x64-gnu': 0.2.2
+ '@crawlee/fs-storage-native-linux-x64-musl': 0.2.2
+ '@crawlee/fs-storage-native-win32-arm64-msvc': 0.2.2
+ '@crawlee/fs-storage-native-win32-x64-msvc': 0.2.2
'@crawlee/[email protected]':
dependencies:
diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.ts
index 2b0d51a8b..35be620d3 100644
--- a/test/core/crawlers/basic_crawler.test.ts
+++ b/test/core/crawlers/basic_crawler.test.ts
@@ -1,4 +1,5 @@
import { readFile, rm } from 'node:fs/promises';
+import { resolve } from 'node:path';
import type { Server } from 'node:http';
import http from 'node:http';
import type { AddressInfo } from 'node:net';
@@ -45,6 +46,7 @@ import {
import type { StorageTransaction } from '@crawlee/core';
import { currentStorageTransaction, MemoryStorageBackend } from '@crawlee/core';
import { BaseHttpClient } from '@crawlee/http-client';
+import { FileSystemStorageBackend } from '@crawlee/fs-storage';
import type { Dictionary, ISession, ProxyInfo } from '@crawlee/types';
import { RobotsTxtFile, sleep } from '@crawlee/utils';
import express from 'express';
@@ -1914,6 +1916,42 @@ describe('BasicCrawler', () => {
expect(failed).toHaveLength(0);
});
+ test('context.extendTimeout prolongs the request lock on a locking request manager', async () => {
+ // The file-system backend is the real locking one: its queue takes actual locks,
+ // so the whole crawler -> queue -> backend -> native lock handoff runs.
+ const tmpLocation = resolve(import.meta.dirname, './tmp/extend-timeout-lock');
+ const requestQueue = await RequestQueue.open(null, {
+ storageBackend: new FileSystemStorageBackend({ localDataDirectory: tmpLocation }),
+ });
+ const extendSpy = vitest.spyOn(requestQueue.backend, 'extendRequestProcessingTimeSecs');
+
+ try {
+ let sawFailure = false;
+
+ const crawler = new BasicCrawler({
+ requestManager: requestQueue,
+ requestHandlerTimeoutSecs: 60,
+ maxRequestRetries: 0,
+ requestHandler: async ({ request, extendTimeout }) => {
+ await sleep(50);
+ extendTimeout(42);
+ expect(request.id).toBeTruthy();
+ },
+ failedRequestHandler: async () => {
+ sawFailure = true;
+ },
+ });
+
+ await crawler.run(['https://example.com']);
+
+ expect(sawFailure).toBe(false);
+ expect(extendSpy).toHaveBeenCalledExactlyOnceWith(expect.any(String), 42);
+ await expect(extendSpy.mock.results[0]!.value).resolves.toBe(true);
+ } finally {
+ await rm(tmpLocation, { force: true, recursive: true });
+ }
+ });
+
test('a route can override requestHandlerTimeoutSecs, other routes keep the default', async () => {
const requestList = await RequestList.open({
sources: [
diff --git a/test/core/request_manager_tandem.test.ts b/test/core/request_manager_tandem.test.ts
index d414f597a..547c91536 100644
--- a/test/core/request_manager_tandem.test.ts
+++ b/test/core/request_manager_tandem.test.ts
@@ -361,4 +361,20 @@ describe('RequestManagerTandem', () => {
await tandem.fetchNextRequest();
expect(hintSpy).toHaveBeenCalledWith(600);
});
+
+ test('extendRequestProcessingTimeSecs forwards to the writable request manager', async () => {
+ const requestList = await RequestList.open(null, []);
+ const requestQueue = await RequestQueue.open();
+ await requestQueue.addRequest({ url: 'https://example.com/1' });
+ const fetched = await requestQueue.fetchNextRequest();
+ expect(fetched).not.toBeNull();
+
+ const stub = vi.fn(async () => true);
+ (requestQueue.backend as any).extendRequestProcessingTimeSecs = stub;
+
+ const tandem = new RequestManagerTandem(requestList, requestQueue);
+
+ await expect(tandem.extendRequestProcessingTimeSecs!(fetched!, 30)).resolves.toBe(true);
+ expect(stub).toHaveBeenCalledExactlyOnceWith(fetched!.id, 30);
+ });
});
diff --git a/test/core/storages/request_queue.test.ts b/test/core/storages/request_queue.test.ts
index a75f0b07f..6ad7202ae 100644
--- a/test/core/storages/request_queue.test.ts
+++ b/test/core/storages/request_queue.test.ts
@@ -1,8 +1,11 @@
+import { rm } from 'node:fs/promises';
+import { resolve } from 'node:path';
/* eslint-disable dot-notation */
import { CrawlingRequest } from '@crawlee/basic';
import { MemoryStorageBackend, ProxyConfiguration, Request, RequestQueue, serviceLocator } from '@crawlee/core';
import { BaseHttpClient } from '@crawlee/http-client';
+import { FileSystemStorageBackend } from '@crawlee/fs-storage';
import { sleep } from '@crawlee/utils';
// `vitest.mockObject` clones the object and drops its prototype, so build the mock manually to
@@ -396,6 +399,38 @@ describe('RequestQueue remote', () => {
});
});
+ describe('extendRequestProcessingTimeSecs', () => {
+ test('forwards the request id and extension to the backend', async () => {
+ // The file-system backend actually locks fetched requests; the spy observes
+ // the real forwarding and the real lock extension.
+ const tmpLocation = resolve(import.meta.dirname, './tmp/extend-forwarding');
+ try {
+ const queue = await RequestQueue.open(null, {
+ storageBackend: new FileSystemStorageBackend({ localDataDirectory: tmpLocation }),
+ });
+ await queue.addRequest({ url: 'http://example.com/a' });
+ const fetched = await queue.fetchNextRequest();
+ expect(fetched).not.toBeNull();
+
+ const stub = vitest.spyOn(queue.backend, 'extendRequestProcessingTimeSecs');
+
+ await expect(queue.extendRequestProcessingTimeSecs(fetched!, 30)).resolves.toBe(true);
+ expect(stub).toHaveBeenCalledExactlyOnceWith(fetched!.id, 30);
+ } finally {
+ await rm(tmpLocation, { force: true, recursive: true });
+ }
+ });
+
+ test('resolves to false when the backend does not implement per-request locking', async () => {
+ const queue = await RequestQueue.open();
+ await queue.addRequest({ url: 'http://example.com/a' });
+ const fetched = await queue.fetchNextRequest();
+ expect(fetched).not.toBeNull();
+
+ await expect(queue.extendRequestProcessingTimeSecs(fetched!, 30)).resolves.toBe(false);
+ });
+ });
+
describe('stats', () => {
test('start at zero', async () => {
const queue = await RequestQueue.open();