Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Increase version of httpRequest. Added the timeout error. Improved throwing.
Petr Patek5fc4954c017e
diff --git a/package.json b/package.jsonindex 53cb43d88..21ef13e12 100644--- a/package.json+++ b/package.json@@ -49,7 +49,7 @@"lint:fix": "./node_modules/.bin/eslint ./src ./test --ext .js,.jsx --fix"},"dependencies": {- "@apify/http-request": "^1.1.2",+ "@apify/http-request": "^1.1.3","@apify/ps-tree": "^1.1.3","apify-client": "^0.5.25","apify-shared": "^0.1.66",diff --git a/src/utils_request.js b/src/utils_request.jsindex 4d92760a4..e8b7ebe7b 100644--- a/src/utils_request.js+++ b/src/utils_request.js@@ -81,13 +81,17 @@ export const requestAsBrowser = async (options) => {};opts.headers = _.defaults({}, opts.headers, defaultHeaders);+try {return await httpRequest(opts);} catch (e) {if (e instanceof errors.TimeoutError) {throw new TimeoutError('Request Timed-out');}++ throw e;}+};/**