Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix: reset `RequestQueue` state after 5 minutes of inactivity (#1324)
Martin Adámekfc6e4bf7566a
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 1fe64f46a..7ae8603c4 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -1,3 +1,36 @@+2.3.0 / 2022/04/07+====================+- feat: accept more social media patterns (#1286)+- feat: add multiple click support to `enqueueLinksByClickingElements` (#1295)+- feat: instance-scoped "global" configuration (#1315)+- feat: requestList accepts proxyConfiguration for requestsFromUrls (#1317)+- feat: update `playwright` to v1.20.2+- feat: update `puppeteer` to v13.5.2+- feat: stealth deprecation (#1314)+- fix: use correct apify-client instance for snapshotting (#1308)+- fix: automatically reset `RequestQueue` state after 5 minutes of inactivity, closes #997+- fix: improve guessing of chrome executable path on windows (#1294)+- fix: prune CPU snapshots locally (#1313)+- fix: improve browser launcher types (#1318)++### 0 concurrency mitigation++This release should resolve the 0 concurrency bug by automatically resetting the+internal `RequestQueue` state after 5 minutes of inactivity.++We now track last activity done on a `RequestQueue` instance:+- added new request+- started processing a request (added to `inProgress` cache)+- marked request as handled+- reclaimed request++If we don't detect one of those actions in last 5 minutes, and we have some+requests in the `inProgress` cache, we try to reset the state. We can override+this limit via `APIFY_INTERNAL_TIMEOUT` env var.++This should finally resolve the 0 concurrency bug, as it was always about+stuck requests in the `inProgress` cache.+2.2.2 / 2022/02/xx====================- fix: ensure `request.headers` is setdiff --git a/package.json b/package.jsonindex 9ef24de7c..a0e6fde98 100644--- a/package.json+++ b/package.json@@ -1,6 +1,6 @@{"name": "apify",- "version": "2.2.3",+ "version": "2.3.0","description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.","engines": {"node": ">=15.10.0"diff --git a/src/crawlers/basic_crawler.js b/src/crawlers/basic_crawler.jsindex e1aae1c02..904b38779 100644--- a/src/crawlers/basic_crawler.js+++ b/src/crawlers/basic_crawler.js@@ -262,7 +262,13 @@ export class BasicCrawler {this.userProvidedHandler = handleRequestFunction;this.failedContextHandler = handleFailedRequestFunction;this.handleRequestTimeoutMillis = handleRequestTimeoutSecs * 1000;- this.internalTimeoutMillis = Math.max(this.handleRequestTimeoutMillis, 300e3); // allow at least 5min for internal timeouts+ const tryEnv = (val) => (val == null ? val : +val);+ // allow at least 5min for internal timeouts+ this.internalTimeoutMillis = tryEnv(process.env.APIFY_INTERNAL_TIMEOUT) ?? Math.max(this.handleRequestTimeoutMillis, 300e3);+ // override the default internal timeout of request queue to respect `handleRequestTimeoutMillis`+ if (this.requestQueue) {+ this.requestQueue.internalTimeoutMillis = this.internalTimeoutMillis;+ }this.handleFailedRequestFunction = handleFailedRequestFunction;this.maxRequestRetries = maxRequestRetries;this.handledRequestsCount = 0;diff --git a/src/storages/request_queue.js b/src/storages/request_queue.jsindex c4d8e3080..5cabec522 100644--- a/src/storages/request_queue.js+++ b/src/storages/request_queue.js@@ -149,6 +149,12 @@ export class RequestQueue {// i.e. which were returned by fetchNextRequest() but not markRequestHandled()this.inProgress = new Set();+ // To track whether the queue gets stuck, and we need to reset it+ // `lastActivity` tracks the time when we either added, processed or reclaimed a request,+ // or when we add new request to in-progress cache+ this.lastActivity = new Date();+ this.internalTimeoutMillis = 5 * 60e3; // defaults to 5 minutes, will be overridden by BasicCrawler+// Contains a list of recently handled requests. It is used to avoid inconsistencies// caused by delays in the underlying DynamoDB storage.// Keys are request IDs, values are true.@@ -198,6 +204,7 @@ export class RequestQueue {forefront: ow.optional.boolean,}));+ this.lastActivity = new Date();const { forefront = false } = options;const request = requestLike instanceof Request@@ -296,6 +303,7 @@ export class RequestQueue {}this.inProgress.add(nextRequestId);+ this.lastActivity = new Date();let request;try {@@ -344,6 +352,7 @@ export class RequestQueue {* @return {Promise<QueueOperationInfo | null>}*/async markRequestHandled(request) {+ this.lastActivity = new Date();ow(request, ow.object.partialShape({id: ow.string,uniqueKey: ow.string,@@ -388,6 +397,7 @@ export class RequestQueue {* @return {Promise<QueueOperationInfo | null>}*/async reclaimRequest(request, options = {}) {+ this.lastActivity = new Date();ow(request, ow.object.partialShape({id: ow.string,uniqueKey: ow.string,@@ -448,12 +458,32 @@ export class RequestQueue {* @returns {Promise<boolean>}*/async isFinished() {+ if (this.inProgressCount() > 0 && (Date.now() - +this.lastActivity) > this.internalTimeoutMillis) {+ const message = `The request queue seems to be stuck for ${this.internalTimeoutMillis / 1e3}s, resetting internal state.`;+ this.log.warning(message, { inProgress: [...this.inProgress] });+ this._reset();+ }+if (this.queueHeadDict.length() > 0 || this.inProgressCount() > 0) return false;const isHeadConsistent = await this._ensureHeadIsNonEmpty(true);return isHeadConsistent && this.queueHeadDict.length() === 0 && this.inProgressCount() === 0;}+ /**+ * @internal+ */+ _reset() {+ this.queueHeadDict.clear();+ this.queryQueueHeadPromise = null;+ this.inProgress.clear();+ this.recentlyHandled.clear();+ this.assumedTotalCount = 0;+ this.assumedHandledCount = 0;+ this.requestsCache.clear();+ this.lastActivity = new Date();+ }+/*** Caches information about request to beware of unneeded addRequest() calls.* @param {string} cacheKeydiff --git a/test/request_queue_reset.test-e2e.js b/test/request_queue_reset.test-e2e.jsnew file mode 100644index 000000000..8c52ed38a--- /dev/null+++ b/test/request_queue_reset.test-e2e.js@@ -0,0 +1,26 @@+// $ npm run build && node test/request_queue_reset.test-e2e.js++const Apify = require('../build');++// RequestQueue auto-reset when stuck with requests in progress+Apify.main(async () => {+ process.env.APIFY_INTERNAL_TIMEOUT = '30000'; // 30s+ Apify.utils.log.setLevel(Apify.utils.log.LEVELS.DEBUG);+ await Apify.utils.purgeLocalStorage();+ const requestQueue = await Apify.openRequestQueue();+ await requestQueue.addRequest({ url: 'https://example.com/?q=1' });+ await requestQueue.addRequest({ url: 'https://example.com/?q=2' });+ const r3 = await requestQueue.addRequest({ url: 'https://example.com/?q=3' });++ // trigger 0 concurrency by marking one of the requests as already in progress+ requestQueue.inProgress.add(r3.requestId);++ const crawler = new Apify.CheerioCrawler({+ requestQueue,+ handlePageFunction: async (ctx) => {+ Apify.utils.log.info(ctx.request.id);+ },+ });++ await crawler.run();+});