Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Add handleRequestTimeout test

Ondra Urbanfdfd4379f9bb
diff --git a/test/basic_crawler.js b/test/basic_crawler.js
index 652166c14..3c6610af0 100644
--- a/test/basic_crawler.js
+++ b/test/basic_crawler.js
@@ -578,4 +578,24 @@ describe('BasicCrawler', () => {
expect(count).to.be.eql(7);
sinon.restore();
});
+
+ it('should timeout after handleRequestTimeoutSecs', async () => {
+ const url = 'https://example.com';
+ const requestList = new Apify.RequestList({ sources: [{ url }] });
+ await requestList.initialize();
+
+ const results = [];
+ const crawler = new Apify.BasicCrawler({
+ requestList,
+ handleRequestTimeoutSecs: 0.01,
+ maxRequestRetries: 1,
+ handleRequestFunction: () => delayPromise(1000),
+ handleFailedRequestFunction: ({ request }) => results.push(request),
+ });
+
+ await crawler.run();
+ expect(results).to.have.lengthOf(1);
+ expect(results[0].url).to.be.eql(url);
+ results[0].errorMessages.forEach(msg => expect(msg).to.include('handleRequestFunction timed out'));
+ });
});