Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Throttle API in AutoscaledPool on 2nd retry

Ondra Urban8a14a7db5ea4
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 501959acd..1e93d86a0 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,7 @@
+xxx
+===================
+- `Snapshotter` now marks Apify Client overloaded on the basis of 2nd retry errors.
+
0.13.0 / 2019-03-14
===================
- **BREAKING CHANGE:** Added `puppeteerOperationTimeoutSecs` option to `PuppeteerPool`.
diff --git a/src/autoscaling/snapshotter.js b/src/autoscaling/snapshotter.js
index a5be3e12a..bcf5d4be3 100644
--- a/src/autoscaling/snapshotter.js
+++ b/src/autoscaling/snapshotter.js
@@ -13,7 +13,7 @@ const DEFAULT_OPTIONS = {
clientSnapshotIntervalSecs: 1,
maxUsedMemoryRatio: 0.7,
snapshotHistorySecs: 30,
- maxClientErrors: 1,
+ maxClientErrors: 3,
};
const RESERVE_MEMORY_RATIO = 0.5;
@@ -302,7 +302,13 @@ class Snapshotter {
}
/**
- * Creates a snapshot of current API state.
+ * Creates a snapshot of current API state by checking for
+ * rate limit errors. Only errors produced by a 2nd retry
+ * of the API call are considered for snapshotting since
+ * earlier errors may just be caused by a random spike in
+ * number of requests and do not necessarily signify API
+ * overloading.
+ *
* @param intervalCallback
* @return {number}
* @private
@@ -312,7 +318,8 @@ class Snapshotter {
const now = new Date();
this._pruneSnapshots(this.clientSnapshots, now);
- const currentErrCount = apifyClient.stats.rateLimitErrors;
+ const allErrorCounts = apifyClient.stats.rateLimitErrors;
+ const currentErrCount = allErrorCounts[2]; // 2nd retry.
// Handle empty snapshots array
const snapshot = {
diff --git a/src/autoscaling/system_status.js b/src/autoscaling/system_status.js
index adead9bc6..0918e68ef 100644
--- a/src/autoscaling/system_status.js
+++ b/src/autoscaling/system_status.js
@@ -8,7 +8,7 @@ const DEFAULT_OPTIONS = {
maxMemoryOverloadedRatio: 0.2,
maxEventLoopOverloadedRatio: 0.4,
maxCpuOverloadedRatio: 0.4,
- maxClientOverloadedRatio: 0.2,
+ maxClientOverloadedRatio: 0.3,
};
/**
diff --git a/test/autoscaling/snapshotter.js b/test/autoscaling/snapshotter.js
index babd68ded..1264e617b 100644
--- a/test/autoscaling/snapshotter.js
+++ b/test/autoscaling/snapshotter.js
@@ -24,13 +24,13 @@ describe('Snapshotter', () => {
// mock client data
const oldStats = utils.apifyClient.stats;
utils.apifyClient.stats = {};
- utils.apifyClient.stats.rateLimitErrors = 0;
+ utils.apifyClient.stats.rateLimitErrors = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0];
const snapshotter = new Snapshotter();
await snapshotter.start();
await Apify.utils.sleep(625);
- utils.apifyClient.stats.rateLimitErrors = 2;
+ utils.apifyClient.stats.rateLimitErrors = [0, 0, 2, 0, 0, 0, 0, 0, 0, 0];
await Apify.utils.sleep(625);
await snapshotter.stop();
@@ -218,15 +218,15 @@ describe('Snapshotter', () => {
// mock client data
const oldStats = utils.apifyClient.stats;
utils.apifyClient.stats = {};
- utils.apifyClient.stats.rateLimitErrors = 0;
+ utils.apifyClient.stats.rateLimitErrors = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0];
- const snapshotter = new Snapshotter();
+ const snapshotter = new Snapshotter({ maxClientErrors: 1 });
snapshotter._snapshotClient(noop);
- utils.apifyClient.stats.rateLimitErrors = 1;
+ utils.apifyClient.stats.rateLimitErrors = [1, 1, 1, 0, 0, 0, 0, 0, 0, 0];
snapshotter._snapshotClient(noop);
- utils.apifyClient.stats.rateLimitErrors = 2;
+ utils.apifyClient.stats.rateLimitErrors = [10, 5, 2, 0, 0, 0, 0, 0, 0, 0];
snapshotter._snapshotClient(noop);
- utils.apifyClient.stats.rateLimitErrors = 4;
+ utils.apifyClient.stats.rateLimitErrors = [100, 24, 4, 2, 0, 0, 0, 0, 0, 0];
snapshotter._snapshotClient(noop);
const clientSnapshots = snapshotter.getClientSample();