Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix: requestHandlerTimeout timing (#1660)

Jindřich Bär493ea0ce80e5
diff --git a/packages/browser-crawler/src/internals/browser-crawler.ts b/packages/browser-crawler/src/internals/browser-crawler.ts
index 5dbc50f27..d00f0aa75 100644
--- a/packages/browser-crawler/src/internals/browser-crawler.ts
+++ b/packages/browser-crawler/src/internals/browser-crawler.ts
@@ -305,6 +305,7 @@ export abstract class BrowserCrawler<
protected userProvidedRequestHandler!: BrowserRequestHandler<Context>;
protected navigationTimeoutMillis: number;
+ protected requestHandlerTimeoutInnerMillis: number;
protected preNavigationHooks: BrowserHook<Context>[];
protected postNavigationHooks: BrowserHook<Context>[];
protected persistCookiesPerSession: boolean;
@@ -387,6 +388,7 @@ export abstract class BrowserCrawler<
this.launchContext = launchContext;
this.navigationTimeoutMillis = navigationTimeoutSecs * 1000;
+ this.requestHandlerTimeoutInnerMillis = requestHandlerTimeoutSecs * 1000;
this.proxyConfiguration = proxyConfiguration;
this.preNavigationHooks = preNavigationHooks;
this.postNavigationHooks = postNavigationHooks;
@@ -491,8 +493,8 @@ export abstract class BrowserCrawler<
try {
await addTimeoutToPromise(
() => Promise.resolve(this.userProvidedRequestHandler(crawlingContext)),
- this.requestHandlerTimeoutMillis,
- `requestHandler timed out after ${this.requestHandlerTimeoutMillis / 1000} seconds.`,
+ this.requestHandlerTimeoutInnerMillis,
+ `requestHandler timed out after ${this.requestHandlerTimeoutInnerMillis / 1000} seconds.`,
);
request.state = RequestState.DONE;
diff --git a/test/core/crawlers/browser_crawler.test.ts b/test/core/crawlers/browser_crawler.test.ts
index c781000f3..a7f94b7ad 100644
--- a/test/core/crawlers/browser_crawler.test.ts
+++ b/test/core/crawlers/browser_crawler.test.ts
@@ -261,7 +261,7 @@ describe('BrowserCrawler', () => {
expect(result[0]).toBe(serverAddress);
});
- test.only('should correctly track request.state', async () => {
+ test('should correctly track request.state', async () => {
const sources = [
{ url: `${serverAddress}/?q=1` },
];
@@ -371,6 +371,34 @@ describe('BrowserCrawler', () => {
}
});
+ test('should respect the requestHandlerTimeoutSecs option', async () => {
+ const requestList = await RequestList.open({
+ sources: [
+ { url: `${serverAddress}/?q=1` },
+ ],
+ });
+
+ const callSpy = jest.fn();
+
+ const browserCrawler = new BrowserCrawlerTest({
+ browserPoolOptions: {
+ browserPlugins: [puppeteerPlugin],
+ },
+ requestList,
+ requestHandler: async () => {
+ setTimeout(() => callSpy('good'), 300);
+ setTimeout(() => callSpy('bad'), 1500);
+ await new Promise(() => {});
+ },
+ requestHandlerTimeoutSecs: 0.5,
+ maxRequestRetries: 0,
+ });
+ await browserCrawler.run();
+
+ expect(callSpy).toBeCalledTimes(1);
+ expect(callSpy).toBeCalledWith('good');
+ });
+
test('should not throw without SessionPool', async () => {
const requestList = await RequestList.open({
sources: [