Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Throttle API in AutoscaledPool on 2nd retry
Ondra Urban8a14a7db5ea4
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 501959acd..1e93d86a0 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -1,3 +1,7 @@+xxx+===================+- `Snapshotter` now marks Apify Client overloaded on the basis of 2nd retry errors.+0.13.0 / 2019-03-14===================- **BREAKING CHANGE:** Added `puppeteerOperationTimeoutSecs` option to `PuppeteerPool`.diff --git a/src/autoscaling/snapshotter.js b/src/autoscaling/snapshotter.jsindex a5be3e12a..bcf5d4be3 100644--- a/src/autoscaling/snapshotter.js+++ b/src/autoscaling/snapshotter.js@@ -13,7 +13,7 @@ const DEFAULT_OPTIONS = {clientSnapshotIntervalSecs: 1,maxUsedMemoryRatio: 0.7,snapshotHistorySecs: 30,- maxClientErrors: 1,+ maxClientErrors: 3,};const RESERVE_MEMORY_RATIO = 0.5;@@ -302,7 +302,13 @@ class Snapshotter {}/**- * Creates a snapshot of current API state.+ * Creates a snapshot of current API state by checking for+ * rate limit errors. Only errors produced by a 2nd retry+ * of the API call are considered for snapshotting since+ * earlier errors may just be caused by a random spike in+ * number of requests and do not necessarily signify API+ * overloading.+ ** @param intervalCallback* @return {number}* @private@@ -312,7 +318,8 @@ class Snapshotter {const now = new Date();this._pruneSnapshots(this.clientSnapshots, now);- const currentErrCount = apifyClient.stats.rateLimitErrors;+ const allErrorCounts = apifyClient.stats.rateLimitErrors;+ const currentErrCount = allErrorCounts[2]; // 2nd retry.// Handle empty snapshots arrayconst snapshot = {diff --git a/src/autoscaling/system_status.js b/src/autoscaling/system_status.jsindex adead9bc6..0918e68ef 100644--- a/src/autoscaling/system_status.js+++ b/src/autoscaling/system_status.js@@ -8,7 +8,7 @@ const DEFAULT_OPTIONS = {maxMemoryOverloadedRatio: 0.2,maxEventLoopOverloadedRatio: 0.4,maxCpuOverloadedRatio: 0.4,- maxClientOverloadedRatio: 0.2,+ maxClientOverloadedRatio: 0.3,};/**diff --git a/test/autoscaling/snapshotter.js b/test/autoscaling/snapshotter.jsindex babd68ded..1264e617b 100644--- a/test/autoscaling/snapshotter.js+++ b/test/autoscaling/snapshotter.js@@ -24,13 +24,13 @@ describe('Snapshotter', () => {// mock client dataconst oldStats = utils.apifyClient.stats;utils.apifyClient.stats = {};- utils.apifyClient.stats.rateLimitErrors = 0;+ utils.apifyClient.stats.rateLimitErrors = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0];const snapshotter = new Snapshotter();await snapshotter.start();await Apify.utils.sleep(625);- utils.apifyClient.stats.rateLimitErrors = 2;+ utils.apifyClient.stats.rateLimitErrors = [0, 0, 2, 0, 0, 0, 0, 0, 0, 0];await Apify.utils.sleep(625);await snapshotter.stop();@@ -218,15 +218,15 @@ describe('Snapshotter', () => {// mock client dataconst oldStats = utils.apifyClient.stats;utils.apifyClient.stats = {};- utils.apifyClient.stats.rateLimitErrors = 0;+ utils.apifyClient.stats.rateLimitErrors = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0];- const snapshotter = new Snapshotter();+ const snapshotter = new Snapshotter({ maxClientErrors: 1 });snapshotter._snapshotClient(noop);- utils.apifyClient.stats.rateLimitErrors = 1;+ utils.apifyClient.stats.rateLimitErrors = [1, 1, 1, 0, 0, 0, 0, 0, 0, 0];snapshotter._snapshotClient(noop);- utils.apifyClient.stats.rateLimitErrors = 2;+ utils.apifyClient.stats.rateLimitErrors = [10, 5, 2, 0, 0, 0, 0, 0, 0, 0];snapshotter._snapshotClient(noop);- utils.apifyClient.stats.rateLimitErrors = 4;+ utils.apifyClient.stats.rateLimitErrors = [100, 24, 4, 2, 0, 0, 0, 0, 0, 0];snapshotter._snapshotClient(noop);const clientSnapshots = snapshotter.getClientSample();