Skip to content

A platform for community discussion. Free, open, simple.

FIX: Retry rate limited backup upload chunks

Régis Hanoleb9667fcc77d
diff --git a/frontend/discourse/app/lib/uppy-chunked-upload.js b/frontend/discourse/app/lib/uppy-chunked-upload.js
index 64fcefb1bf9..e4163e6f73f 100644
--- a/frontend/discourse/app/lib/uppy-chunked-upload.js
+++ b/frontend/discourse/app/lib/uppy-chunked-upload.js
@@ -1,5 +1,10 @@
import { AbortController, createAbortError, delay } from "@uppy/utils";
import { Promise } from "rsvp";
+import {
+ isRateLimitError,
+ MAX_RATE_LIMIT_RETRY_SECONDS,
+ rateLimitWaitSeconds,
+} from "discourse/lib/ajax-error";
const MB = 1024 * 1024;
@@ -35,10 +40,8 @@ export default class UppyChunkedUpload {
};
this.file = file;
- if (!this.options.getChunkSize) {
- this.options.getChunkSize = defaultOptions.getChunkSize;
- this.chunkSize = this.options.getChunkSize(this.file);
- }
+ this.options.getChunkSize ??= defaultOptions.getChunkSize;
+ this.chunkSize = this.options.getChunkSize(this.file);
this.abortController = new AbortController();
this._initChunks();
@@ -149,7 +152,8 @@ export default class UppyChunkedUpload {
status === 0 ||
status === 409 ||
status === 423 ||
- (status >= 500 && status < 600)
+ (status >= 500 && status < 600) ||
+ isRateLimitError(err)
);
}
return false;
@@ -163,21 +167,33 @@ export default class UppyChunkedUpload {
before();
}
- const doAttempt = (retryAttempt) =>
+ const doAttempt = (retryAttempt, rateLimitWaited) =>
attempt().catch((err) => {
if (this._aborted()) {
throw createAbortError();
}
- if (this._shouldRetry(err) && retryAttempt < retryDelays.length) {
+ if (!this._shouldRetry(err) || retryAttempt >= retryDelays.length) {
+ throw err;
+ }
+
+ if (!isRateLimitError(err)) {
return delay(retryDelays[retryAttempt], { signal }).then(() =>
- doAttempt(retryAttempt + 1)
+ doAttempt(retryAttempt + 1, rateLimitWaited)
);
}
- throw err;
+
+ const waitSeconds = rateLimitWaitSeconds(err);
+ if (rateLimitWaited + waitSeconds > MAX_RATE_LIMIT_RETRY_SECONDS) {
+ throw err;
+ }
+
+ return delay(waitSeconds * 1000, { signal }).then(() =>
+ doAttempt(retryAttempt + 1, rateLimitWaited + waitSeconds)
+ );
});
- return doAttempt(0).then(
+ return doAttempt(0, 0).then(
(result) => {
if (after) {
after();
@@ -241,10 +257,11 @@ export default class UppyChunkedUpload {
return new Promise((resolve, reject) => {
const xhr = new XMLHttpRequest();
+ const onAbortSignal = () => xhr.abort();
function cleanup() {
- signal.removeEventListener("abort", () => xhr.abort());
+ signal.removeEventListener("abort", onAbortSignal);
}
- signal.addEventListener("abort", xhr.abort());
+ signal.addEventListener("abort", onAbortSignal);
xhr.open(this.options.method || "POST", url, true);
if (headers) {
@@ -263,22 +280,24 @@ export default class UppyChunkedUpload {
xhr.addEventListener("abort", () => {
cleanup();
- this.chunkState[index].busy = false;
reject(createAbortError());
});
xhr.addEventListener("load", (ev) => {
cleanup();
- this.chunkState[index].busy = false;
if (ev.target.status < 200 || ev.target.status >= 300) {
+ this._onChunkProgress(index, 0);
+
const error = new Error("Non 2xx");
error.source = ev.target;
reject(error);
return;
}
+ this.chunkState[index].busy = false;
+
// This avoids the net::ERR_OUT_OF_MEMORY in Chromium Browsers.
this.chunks[index] = null;
@@ -290,7 +309,7 @@ export default class UppyChunkedUpload {
xhr.addEventListener("error", (ev) => {
cleanup();
- this.chunkState[index].busy = false;
+ this._onChunkProgress(index, 0);
const error = new Error("Unknown error");
error.source = ev.target;
diff --git a/frontend/discourse/app/lib/uppy-chunked-uploader-plugin.js b/frontend/discourse/app/lib/uppy-chunked-uploader-plugin.js
index ff8868d579e..5d3fa84a718 100644
--- a/frontend/discourse/app/lib/uppy-chunked-uploader-plugin.js
+++ b/frontend/discourse/app/lib/uppy-chunked-uploader-plugin.js
@@ -91,9 +91,7 @@ export default class UppyChunkedUploader extends UploaderPlugin {
};
const upload = new UppyChunkedUpload(file, {
- getChunkSize: this.opts.getChunkSize
- ? this.opts.getChunkSize.bind(this)
- : null,
+ getChunkSize: this.opts.getChunkSize?.bind(this),
onStart,
onProgress,
@@ -127,20 +125,6 @@ export default class UppyChunkedUploader extends UploaderPlugin {
resolve(`upload ${file.id} was canceled`);
});
- this._onFilePause(file.id, (isPaused) => {
- if (isPaused) {
- upload.pause();
- } else {
- next(() => {
- upload.start();
- });
- }
- });
-
- this._onPauseAll(file.id, () => {
- upload.pause();
- });
-
this._onResumeAll(file.id, () => {
if (file.error) {
upload.abort();
@@ -165,23 +149,6 @@ export default class UppyChunkedUploader extends UploaderPlugin {
});
}
- _onFilePause(fileID, cb) {
- this.uploaderEvents[fileID].on("upload-pause", (targetFileID, isPaused) => {
- if (fileID === targetFileID) {
- cb(isPaused);
- }
- });
- }
-
- _onPauseAll(fileID, cb) {
- this.uploaderEvents[fileID].on("pause-all", () => {
- if (!this.uppy.getFile(fileID)) {
- return;
- }
- cb();
- });
- }
-
_onCancelAll(fileID, cb) {
this.uploaderEvents[fileID].on("cancel-all", () => {
if (!this.uppy.getFile(fileID)) {
diff --git a/frontend/discourse/tests/unit/lib/uppy-chunked-upload-test.js b/frontend/discourse/tests/unit/lib/uppy-chunked-upload-test.js
new file mode 100644
index 00000000000..08c94d71d3b
--- /dev/null
+++ b/frontend/discourse/tests/unit/lib/uppy-chunked-upload-test.js
@@ -0,0 +1,143 @@
+import { waitUntil } from "@ember/test-helpers";
+import { setupTest } from "ember-qunit";
+import { module, test } from "qunit";
+import sinon from "sinon";
+import UppyChunkedUpload from "discourse/lib/uppy-chunked-upload";
+import pretender, {
+ middlewareRateLimit,
+} from "discourse/tests/helpers/create-pretender";
+import { createFile } from "discourse/tests/helpers/qunit-helpers";
+
+const UPLOAD_URL = "/admin/backups/upload";
+const RETRY_AFTER_SECONDS = 1;
+const RATE_LIMITED_CHUNK = 1;
+const FILE_CONTENT = "abcdef";
+const MANUAL_PRETENDER_RESOLUTION = true;
+const OK = [200, {}, "{}"];
+const SERVICE_UNAVAILABLE = [503, {}, ""];
+
+function startUpload(respondWith, { options, timing } = {}) {
+ const requests = [];
+
+ pretender.post(
+ UPLOAD_URL,
+ (request) => {
+ const chunk = parseInt(
+ request.requestBody.get("resumableChunkNumber"),
+ 10
+ );
+ requests.push({ chunk, at: Date.now() });
+
+ return respondWith(chunk, requests);
+ },
+ timing
+ );
+
+ const outcome = { errors: [], successes: 0 };
+ const data = createFile("backup.tar.gz", "application/gzip", FILE_CONTENT);
+
+ const upload = new UppyChunkedUpload(
+ { data, size: data.size },
+ {
+ url: UPLOAD_URL,
+ method: "POST",
+ getChunkSize: null,
+ limit: 5,
+ retryDelays: [0, 0, 0, 0],
+ onSuccess: () => outcome.successes++,
+ onError: (err) => outcome.errors.push(err),
+ ...options,
+ }
+ );
+ upload.start();
+
+ return { upload, requests, outcome };
+}
+
+function attemptsFor(requests, chunk) {
+ return requests.filter((request) => request.chunk === chunk);
+}
+
+module("Unit | Lib | uppy-chunked-upload", function (hooks) {
+ setupTest(hooks);
+
+ test("a rate limited chunk is retried after the advertised Retry-After", async function (assert) {
+ const { requests, outcome } = startUpload(
+ (chunk, sent) =>
+ chunk === RATE_LIMITED_CHUNK && attemptsFor(sent, chunk).length === 1
+ ? middlewareRateLimit(RETRY_AFTER_SECONDS)
+ : OK,
+ { options: { getChunkSize: () => 1 } }
+ );
+
+ await waitUntil(() => outcome.successes > 0, {
+ timeout: (RETRY_AFTER_SECONDS + 4) * 1000,
+ });
+
+ const attempts = attemptsFor(requests, RATE_LIMITED_CHUNK);
+
+ assert.strictEqual(
+ attempts.length,
+ 2,
+ "the rate limited chunk is retried exactly once, with no concurrent duplicate retry"
+ );
+ assert.true(
+ attempts[1].at - attempts[0].at >= RETRY_AFTER_SECONDS * 1000,
+ "the retry waits for Retry-After rather than the fixed retry delay"
+ );
+ assert.strictEqual(
+ requests.length,
+ FILE_CONTENT.length + 1,
+ "every other chunk is uploaded exactly once"
+ );
+ assert.strictEqual(outcome.errors.length, 0, "no error is surfaced");
+ });
+
+ test("a 503 is retried for every retry delay", async function (assert) {
+ const { requests, outcome } = startUpload((chunk, sent) =>
+ sent.length > 4 ? OK : SERVICE_UNAVAILABLE
+ );
+
+ await waitUntil(() => outcome.successes > 0);
+
+ assert.strictEqual(
+ requests.length,
+ 5,
+ "the chunk is retried once per retry delay before succeeding"
+ );
+ assert.strictEqual(outcome.errors.length, 0, "no error is surfaced");
+ });
+
+ test("a supplied getChunkSize is used to split the file", async function (assert) {
+ const { requests, outcome } = startUpload(() => OK, {
+ options: { getChunkSize: () => 2 },
+ });
+
+ await waitUntil(() => outcome.successes > 0);
+
+ assert.deepEqual(
+ requests.map((request) => request.chunk),
+ [1, 2, 3],
+ "the file is split into 2 byte chunks"
+ );
+ });
+
+ test("aborting cancels the in-flight request", async function (assert) {
+ const abort = sinon.spy(XMLHttpRequest.prototype, "abort");
+ const { upload, requests } = startUpload(() => OK, {
+ timing: MANUAL_PRETENDER_RESOLUTION,
+ });
+
+ await waitUntil(() => requests.length > 0);
+
+ assert.strictEqual(
+ abort.callCount,
+ 0,
+ "nothing is aborted while the upload is running"
+ );
+
+ upload.abort({ really: true });
+
+ assert.strictEqual(abort.callCount, 1, "the in-flight request is aborted");
+ });
+});
diff --git a/spec/requests/admin/backups_controller_spec.rb b/spec/requests/admin/backups_controller_spec.rb
index 9172391468c..769adab451b 100644
--- a/spec/requests/admin/backups_controller_spec.rb
+++ b/spec/requests/admin/backups_controller_spec.rb
@@ -702,6 +702,56 @@ RSpec.describe Admin::BackupsController do
)
end
end
+
+ context "when the global request rate limiter trips" do
+ before do
+ described_class.any_instance.stubs(:has_enough_space_on_disk?).returns(true)
+
+ global_setting :max_reqs_per_ip_mode, "block"
+ global_setting :max_reqs_per_ip_per_10_seconds, 2
+ global_setting :max_reqs_rate_limit_on_private, true
+
+ RateLimiter.enable
+ RateLimiter.clear_all_global!
+ freeze_time_safe
+ end
+
+ def upload_chunk(number)
+ post "/admin/backups/upload.json",
+ params: {
+ resumableFilename: "test_Site-0123456789.tar.gz",
+ resumableTotalSize: 100.megabytes,
+ resumableIdentifier: "test",
+ resumableChunkNumber: number.to_s,
+ resumableChunkSize: "1",
+ resumableCurrentChunkSize: "1",
+ file: fixture_file_upload(Tempfile.new),
+ }
+ end
+
+ it "returns a 429 with a Retry-After header" do
+ @paths =
+ (1..3).map do
+ backup_path(File.join("tmp", "test", "test_Site-0123456789.tar.gz.part#{_1}"))
+ end
+
+ upload_chunk(1)
+ expect(response.status).to eq(200)
+
+ upload_chunk(2)
+ expect(response.status).to eq(200)
+
+ upload_chunk(3)
+ expect(response.status).to eq(429)
+ expect(response.headers["Retry-After"]).to eq("10")
+ expect(response.headers["Discourse-Rate-Limit-Error-Code"]).to eq("user_10_secs_limit")
+ expect(response.body).to eq(<<~MSG)
+ Slow down, you're making too many requests.
+ Please retry again in 10 seconds.
+ Error code: user_10_secs_limit.
+ MSG
+ end
+ end
end
shared_examples "uploading backup chunk not allowed" do