Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
chore: add over-ridden request queue timeout (#1288)
Andrey Bykovf418f2f98645
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex f1cbf48d5..f5a87dba1 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -1,6 +1,7 @@2.2.2 / 2022/01/xx====================- fix: ensure `request.headers` is set+- Lower RequestQueue timeout to 30 seconds- 2.2.1 / 2022/01/03====================diff --git a/package.json b/package.jsonindex 5c85ac4dc..d60688e75 100644--- a/package.json+++ b/package.json@@ -53,10 +53,10 @@"@apify/datastructures": "^1.0.1","@apify/log": "^1.1.1","@apify/ps-tree": "^1.1.4",- "@apify/storage-local": "^2.0.1",+ "@apify/storage-local": "^2.0.2-beta.0","@apify/timeout": "^0.2.1","@apify/utilities": "^1.1.2",- "apify-client": "^2.0.2",+ "apify-client": "^2.2.0-beta.2","browser-pool": "^3.1.0","cheerio": "1.0.0-rc.10","content-type": "^1.0.4",diff --git a/src/storages/request_queue.js b/src/storages/request_queue.jsindex 462881b2b..67fb40d88 100644--- a/src/storages/request_queue.js+++ b/src/storages/request_queue.js@@ -127,9 +127,11 @@ export class RequestQueue {this.id = options.id;this.name = options.name;this.isLocal = options.isLocal;+ this.timeoutSecs = 30;this.clientKey = cryptoRandomObjectId();this.client = options.client.requestQueue(this.id, {clientKey: this.clientKey,+ timeoutSecs: this.timeoutSecs,});this.log = log.child({ prefix: 'RequestQueue' });