A platform for community discussion. Free, open, simple.
FIX: Retry rate limited backup upload chunks
Régis Hanoleb9667fcc77d
diff --git a/frontend/discourse/app/lib/uppy-chunked-upload.js b/frontend/discourse/app/lib/uppy-chunked-upload.jsindex 64fcefb1bf9..e4163e6f73f 100644--- a/frontend/discourse/app/lib/uppy-chunked-upload.js+++ b/frontend/discourse/app/lib/uppy-chunked-upload.js@@ -1,5 +1,10 @@import { AbortController, createAbortError, delay } from "@uppy/utils";import { Promise } from "rsvp";+import {+ isRateLimitError,+ MAX_RATE_LIMIT_RETRY_SECONDS,+ rateLimitWaitSeconds,+} from "discourse/lib/ajax-error";const MB = 1024 * 1024;@@ -35,10 +40,8 @@ export default class UppyChunkedUpload {};this.file = file;- if (!this.options.getChunkSize) {- this.options.getChunkSize = defaultOptions.getChunkSize;- this.chunkSize = this.options.getChunkSize(this.file);- }+ this.options.getChunkSize ??= defaultOptions.getChunkSize;+ this.chunkSize = this.options.getChunkSize(this.file);this.abortController = new AbortController();this._initChunks();@@ -149,7 +152,8 @@ export default class UppyChunkedUpload {status === 0 ||status === 409 ||status === 423 ||- (status >= 500 && status < 600)+ (status >= 500 && status < 600) ||+ isRateLimitError(err));}return false;@@ -163,21 +167,33 @@ export default class UppyChunkedUpload {before();}- const doAttempt = (retryAttempt) =>+ const doAttempt = (retryAttempt, rateLimitWaited) =>attempt().catch((err) => {if (this._aborted()) {throw createAbortError();}- if (this._shouldRetry(err) && retryAttempt < retryDelays.length) {+ if (!this._shouldRetry(err) || retryAttempt >= retryDelays.length) {+ throw err;+ }++ if (!isRateLimitError(err)) {return delay(retryDelays[retryAttempt], { signal }).then(() =>- doAttempt(retryAttempt + 1)+ doAttempt(retryAttempt + 1, rateLimitWaited));}- throw err;++ const waitSeconds = rateLimitWaitSeconds(err);+ if (rateLimitWaited + waitSeconds > MAX_RATE_LIMIT_RETRY_SECONDS) {+ throw err;+ }++ return delay(waitSeconds * 1000, { signal }).then(() =>+ doAttempt(retryAttempt + 1, rateLimitWaited + waitSeconds)+ );});- return doAttempt(0).then(+ return doAttempt(0, 0).then((result) => {if (after) {after();@@ -241,10 +257,11 @@ export default class UppyChunkedUpload {return new Promise((resolve, reject) => {const xhr = new XMLHttpRequest();+ const onAbortSignal = () => xhr.abort();function cleanup() {- signal.removeEventListener("abort", () => xhr.abort());+ signal.removeEventListener("abort", onAbortSignal);}- signal.addEventListener("abort", xhr.abort());+ signal.addEventListener("abort", onAbortSignal);xhr.open(this.options.method || "POST", url, true);if (headers) {@@ -263,22 +280,24 @@ export default class UppyChunkedUpload {xhr.addEventListener("abort", () => {cleanup();- this.chunkState[index].busy = false;reject(createAbortError());});xhr.addEventListener("load", (ev) => {cleanup();- this.chunkState[index].busy = false;if (ev.target.status < 200 || ev.target.status >= 300) {+ this._onChunkProgress(index, 0);+const error = new Error("Non 2xx");error.source = ev.target;reject(error);return;}+ this.chunkState[index].busy = false;+// This avoids the net::ERR_OUT_OF_MEMORY in Chromium Browsers.this.chunks[index] = null;@@ -290,7 +309,7 @@ export default class UppyChunkedUpload {xhr.addEventListener("error", (ev) => {cleanup();- this.chunkState[index].busy = false;+ this._onChunkProgress(index, 0);const error = new Error("Unknown error");error.source = ev.target;diff --git a/frontend/discourse/app/lib/uppy-chunked-uploader-plugin.js b/frontend/discourse/app/lib/uppy-chunked-uploader-plugin.jsindex ff8868d579e..5d3fa84a718 100644--- a/frontend/discourse/app/lib/uppy-chunked-uploader-plugin.js+++ b/frontend/discourse/app/lib/uppy-chunked-uploader-plugin.js@@ -91,9 +91,7 @@ export default class UppyChunkedUploader extends UploaderPlugin {};const upload = new UppyChunkedUpload(file, {- getChunkSize: this.opts.getChunkSize- ? this.opts.getChunkSize.bind(this)- : null,+ getChunkSize: this.opts.getChunkSize?.bind(this),onStart,onProgress,@@ -127,20 +125,6 @@ export default class UppyChunkedUploader extends UploaderPlugin {resolve(`upload ${file.id} was canceled`);});- this._onFilePause(file.id, (isPaused) => {- if (isPaused) {- upload.pause();- } else {- next(() => {- upload.start();- });- }- });-- this._onPauseAll(file.id, () => {- upload.pause();- });-this._onResumeAll(file.id, () => {if (file.error) {upload.abort();@@ -165,23 +149,6 @@ export default class UppyChunkedUploader extends UploaderPlugin {});}- _onFilePause(fileID, cb) {- this.uploaderEvents[fileID].on("upload-pause", (targetFileID, isPaused) => {- if (fileID === targetFileID) {- cb(isPaused);- }- });- }-- _onPauseAll(fileID, cb) {- this.uploaderEvents[fileID].on("pause-all", () => {- if (!this.uppy.getFile(fileID)) {- return;- }- cb();- });- }-_onCancelAll(fileID, cb) {this.uploaderEvents[fileID].on("cancel-all", () => {if (!this.uppy.getFile(fileID)) {diff --git a/frontend/discourse/tests/unit/lib/uppy-chunked-upload-test.js b/frontend/discourse/tests/unit/lib/uppy-chunked-upload-test.jsnew file mode 100644index 00000000000..08c94d71d3b--- /dev/null+++ b/frontend/discourse/tests/unit/lib/uppy-chunked-upload-test.js@@ -0,0 +1,143 @@+import { waitUntil } from "@ember/test-helpers";+import { setupTest } from "ember-qunit";+import { module, test } from "qunit";+import sinon from "sinon";+import UppyChunkedUpload from "discourse/lib/uppy-chunked-upload";+import pretender, {+ middlewareRateLimit,+} from "discourse/tests/helpers/create-pretender";+import { createFile } from "discourse/tests/helpers/qunit-helpers";++const UPLOAD_URL = "/admin/backups/upload";+const RETRY_AFTER_SECONDS = 1;+const RATE_LIMITED_CHUNK = 1;+const FILE_CONTENT = "abcdef";+const MANUAL_PRETENDER_RESOLUTION = true;+const OK = [200, {}, "{}"];+const SERVICE_UNAVAILABLE = [503, {}, ""];++function startUpload(respondWith, { options, timing } = {}) {+ const requests = [];++ pretender.post(+ UPLOAD_URL,+ (request) => {+ const chunk = parseInt(+ request.requestBody.get("resumableChunkNumber"),+ 10+ );+ requests.push({ chunk, at: Date.now() });++ return respondWith(chunk, requests);+ },+ timing+ );++ const outcome = { errors: [], successes: 0 };+ const data = createFile("backup.tar.gz", "application/gzip", FILE_CONTENT);++ const upload = new UppyChunkedUpload(+ { data, size: data.size },+ {+ url: UPLOAD_URL,+ method: "POST",+ getChunkSize: null,+ limit: 5,+ retryDelays: [0, 0, 0, 0],+ onSuccess: () => outcome.successes++,+ onError: (err) => outcome.errors.push(err),+ ...options,+ }+ );+ upload.start();++ return { upload, requests, outcome };+}++function attemptsFor(requests, chunk) {+ return requests.filter((request) => request.chunk === chunk);+}++module("Unit | Lib | uppy-chunked-upload", function (hooks) {+ setupTest(hooks);++ test("a rate limited chunk is retried after the advertised Retry-After", async function (assert) {+ const { requests, outcome } = startUpload(+ (chunk, sent) =>+ chunk === RATE_LIMITED_CHUNK && attemptsFor(sent, chunk).length === 1+ ? middlewareRateLimit(RETRY_AFTER_SECONDS)+ : OK,+ { options: { getChunkSize: () => 1 } }+ );++ await waitUntil(() => outcome.successes > 0, {+ timeout: (RETRY_AFTER_SECONDS + 4) * 1000,+ });++ const attempts = attemptsFor(requests, RATE_LIMITED_CHUNK);++ assert.strictEqual(+ attempts.length,+ 2,+ "the rate limited chunk is retried exactly once, with no concurrent duplicate retry"+ );+ assert.true(+ attempts[1].at - attempts[0].at >= RETRY_AFTER_SECONDS * 1000,+ "the retry waits for Retry-After rather than the fixed retry delay"+ );+ assert.strictEqual(+ requests.length,+ FILE_CONTENT.length + 1,+ "every other chunk is uploaded exactly once"+ );+ assert.strictEqual(outcome.errors.length, 0, "no error is surfaced");+ });++ test("a 503 is retried for every retry delay", async function (assert) {+ const { requests, outcome } = startUpload((chunk, sent) =>+ sent.length > 4 ? OK : SERVICE_UNAVAILABLE+ );++ await waitUntil(() => outcome.successes > 0);++ assert.strictEqual(+ requests.length,+ 5,+ "the chunk is retried once per retry delay before succeeding"+ );+ assert.strictEqual(outcome.errors.length, 0, "no error is surfaced");+ });++ test("a supplied getChunkSize is used to split the file", async function (assert) {+ const { requests, outcome } = startUpload(() => OK, {+ options: { getChunkSize: () => 2 },+ });++ await waitUntil(() => outcome.successes > 0);++ assert.deepEqual(+ requests.map((request) => request.chunk),+ [1, 2, 3],+ "the file is split into 2 byte chunks"+ );+ });++ test("aborting cancels the in-flight request", async function (assert) {+ const abort = sinon.spy(XMLHttpRequest.prototype, "abort");+ const { upload, requests } = startUpload(() => OK, {+ timing: MANUAL_PRETENDER_RESOLUTION,+ });++ await waitUntil(() => requests.length > 0);++ assert.strictEqual(+ abort.callCount,+ 0,+ "nothing is aborted while the upload is running"+ );++ upload.abort({ really: true });++ assert.strictEqual(abort.callCount, 1, "the in-flight request is aborted");+ });+});diff --git a/spec/requests/admin/backups_controller_spec.rb b/spec/requests/admin/backups_controller_spec.rbindex 9172391468c..769adab451b 100644--- a/spec/requests/admin/backups_controller_spec.rb+++ b/spec/requests/admin/backups_controller_spec.rb@@ -702,6 +702,56 @@ RSpec.describe Admin::BackupsController do)endend++ context "when the global request rate limiter trips" do+ before do+ described_class.any_instance.stubs(:has_enough_space_on_disk?).returns(true)++ global_setting :max_reqs_per_ip_mode, "block"+ global_setting :max_reqs_per_ip_per_10_seconds, 2+ global_setting :max_reqs_rate_limit_on_private, true++ RateLimiter.enable+ RateLimiter.clear_all_global!+ freeze_time_safe+ end++ def upload_chunk(number)+ post "/admin/backups/upload.json",+ params: {+ resumableFilename: "test_Site-0123456789.tar.gz",+ resumableTotalSize: 100.megabytes,+ resumableIdentifier: "test",+ resumableChunkNumber: number.to_s,+ resumableChunkSize: "1",+ resumableCurrentChunkSize: "1",+ file: fixture_file_upload(Tempfile.new),+ }+ end++ it "returns a 429 with a Retry-After header" do+ @paths =+ (1..3).map do+ backup_path(File.join("tmp", "test", "test_Site-0123456789.tar.gz.part#{_1}"))+ end++ upload_chunk(1)+ expect(response.status).to eq(200)++ upload_chunk(2)+ expect(response.status).to eq(200)++ upload_chunk(3)+ expect(response.status).to eq(429)+ expect(response.headers["Retry-After"]).to eq("10")+ expect(response.headers["Discourse-Rate-Limit-Error-Code"]).to eq("user_10_secs_limit")+ expect(response.body).to eq(<<~MSG)+ Slow down, you're making too many requests.+ Please retry again in 10 seconds.+ Error code: user_10_secs_limit.+ MSG+ end+ endendshared_examples "uploading backup chunk not allowed" do