Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Add handleRequestTimeout test
Ondra Urbanfdfd4379f9bb
diff --git a/test/basic_crawler.js b/test/basic_crawler.jsindex 652166c14..3c6610af0 100644--- a/test/basic_crawler.js+++ b/test/basic_crawler.js@@ -578,4 +578,24 @@ describe('BasicCrawler', () => {expect(count).to.be.eql(7);sinon.restore();});++ it('should timeout after handleRequestTimeoutSecs', async () => {+ const url = 'https://example.com';+ const requestList = new Apify.RequestList({ sources: [{ url }] });+ await requestList.initialize();++ const results = [];+ const crawler = new Apify.BasicCrawler({+ requestList,+ handleRequestTimeoutSecs: 0.01,+ maxRequestRetries: 1,+ handleRequestFunction: () => delayPromise(1000),+ handleFailedRequestFunction: ({ request }) => results.push(request),+ });++ await crawler.run();+ expect(results).to.have.lengthOf(1);+ expect(results[0].url).to.be.eql(url);+ results[0].errorMessages.forEach(msg => expect(msg).to.include('handleRequestFunction timed out'));+ });});