Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix: disable storage access on `requestHandler` timeouts (#3788)

Jindřich Bär4f0c4c452ddc
diff --git a/packages/core/src/storages/access_checking.ts b/packages/core/src/storages/access_checking.ts
index c56612a2c..3ca0ff52b 100644
--- a/packages/core/src/storages/access_checking.ts
+++ b/packages/core/src/storages/access_checking.ts
@@ -1,13 +1,17 @@
import { AsyncLocalStorage } from 'node:async_hooks';
import type { Awaitable } from '../typedefs.js';
+import { tryCancel } from '@apify/timeout';
const storage = new AsyncLocalStorage<{ checkFunction: () => void }>();
/**
* Invoke a storage access checker function defined using {@link withCheckedStorageAccess} higher up in the call stack.
*/
-export const checkStorageAccess = () => storage.getStore()?.checkFunction();
+export const checkStorageAccess = () => {
+ tryCancel();
+ return storage.getStore()?.checkFunction();
+};
/**
* Define a storage access checker function that should be used by calls to {@link checkStorageAccess} in the callbacks.
diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.ts
index d7d3cb023..a55a7cfee 100644
--- a/test/core/crawlers/basic_crawler.test.ts
+++ b/test/core/crawlers/basic_crawler.test.ts
@@ -22,7 +22,7 @@ import {
serviceLocator,
SessionPool,
} from '@crawlee/basic';
-import { RequestState, SessionPool, Statistics } from '@crawlee/core';
+import { Dataset, RequestState, SessionPool, Statistics } from '@crawlee/core';
import { MemoryStorageClient } from '@crawlee/memory-storage';
import type { ISession, ProxyInfo } from '@crawlee/types';
import type { Dictionary } from '@crawlee/utils';
@@ -1313,6 +1313,34 @@ describe('BasicCrawler', () => {
results[0].errorMessages.forEach((msg) => expect(msg).toMatch('requestHandler timed out'));
});
+ test('timeouted request should not access storages', async () => {
+ const url = 'https://example.com';
+ const requestList = await RequestList.open({ sources: [{ url }] });
+
+ const results: Request[] = [];
+ const crawler = new BasicCrawler({
+ requestList,
+ requestHandlerTimeoutSecs: 0.01,
+ maxRequestRetries: 0,
+ requestHandler: async ({ pushData }) => {
+ await sleep(10);
+ await pushData({ foo: 'bar' });
+ },
+ failedRequestHandler: async ({ request }) => {
+ results.push(request);
+ await sleep(100);
+ },
+ });
+
+ await crawler.run();
+ expect(results).toHaveLength(1);
+ expect(results[0].url).toEqual(url);
+ results[0].errorMessages.forEach((msg) => expect(msg).toMatch('requestHandler timed out'));
+
+ const dataset = await crawler.getDataset();
+ expect((await dataset.getInfo()).itemCount).toBe(0);
+ });
+
test('limits requestHandlerTimeoutSecs and derived vars to a valid value', async () => {
const url = 'https://example.com';
const requestList = await RequestList.open({ sources: [{ url }] });