Skip to content

armgit / kaneo

Public Imported from GitHub
0

⚡ Project management that doesn't become the project. Boards, Gantt charts, and GitHub integration. Self-host it or use Kaneo Cloud.

fix(api): retry upload checks on 403 and 404

Andrejb799520a19d2
diff --git a/apps/api/src/storage/head-uploaded-object.ts b/apps/api/src/storage/head-uploaded-object.ts
new file mode 100644
index 000000000..5d278edca
--- /dev/null
+++ b/apps/api/src/storage/head-uploaded-object.ts
@@ -0,0 +1,47 @@
+import {
+ HeadObjectCommand,
+ type HeadObjectCommandOutput,
+ type S3Client,
+} from "@aws-sdk/client-s3";
+
+export const UPLOAD_VISIBILITY_RETRY_DELAYS_MS = [250, 500, 1000, 2000, 4000];
+
+function storageStatus(error: unknown) {
+ return error instanceof Error && "$metadata" in error
+ ? (error.$metadata as { httpStatusCode?: number }).httpStatusCode
+ : undefined;
+}
+
+export function isMissingObjectError(error: unknown) {
+ return (
+ error instanceof Error &&
+ (error.name === "NotFound" ||
+ error.name === "NoSuchKey" ||
+ storageStatus(error) === 404)
+ );
+}
+
+function isNotVisibleYet(error: unknown) {
+ return isMissingObjectError(error) || storageStatus(error) === 403;
+}
+
+export async function headUploadedObject(
+ client: Pick<S3Client, "send">,
+ bucket: string,
+ key: string,
+ abortSignal: AbortSignal,
+): Promise<HeadObjectCommandOutput> {
+ const head = () =>
+ client.send(new HeadObjectCommand({ Bucket: bucket, Key: key }), {
+ abortSignal,
+ });
+ for (const delay of UPLOAD_VISIBILITY_RETRY_DELAYS_MS) {
+ try {
+ return await head();
+ } catch (error) {
+ if (!isNotVisibleYet(error) || abortSignal.aborted) throw error;
+ }
+ await new Promise((resolve) => setTimeout(resolve, delay));
+ }
+ return head();
+}
diff --git a/apps/api/src/storage/s3.ts b/apps/api/src/storage/s3.ts
index 2112b6110..b22191288 100644
--- a/apps/api/src/storage/s3.ts
+++ b/apps/api/src/storage/s3.ts
@@ -3,7 +3,6 @@ import {
CopyObjectCommand,
DeleteObjectCommand,
GetObjectCommand,
- HeadObjectCommand,
type HeadObjectCommandOutput,
PutObjectCommand,
S3Client,
@@ -13,6 +12,10 @@ import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
import { createId } from "@paralleldrive/cuid2";
import { config } from "dotenv-mono";
import { describeStorageError } from "./describe-storage-error";
+import {
+ headUploadedObject,
+ isMissingObjectError,
+} from "./head-uploaded-object";
config();
@@ -432,19 +435,9 @@ export async function verifyTaskAssetUpload(
const abortSignal = AbortSignal.timeout(10_000);
let object: HeadObjectCommandOutput;
try {
- object = await client.send(
- new HeadObjectCommand({ Bucket: config.bucket, Key: key }),
- { abortSignal },
- );
+ object = await headUploadedObject(client, config.bucket, key, abortSignal);
} catch (error) {
- if (
- error instanceof Error &&
- (error.name === "NotFound" ||
- error.name === "NoSuchKey" ||
- ("$metadata" in error &&
- (error.$metadata as { httpStatusCode?: number }).httpStatusCode ===
- 404))
- ) {
+ if (isMissingObjectError(error)) {
throw new InvalidUploadedAssetError("Uploaded object was not found.");
}
// Provider errors may include endpoints, signed URLs or other secrets.
diff --git a/tests/api/storage/head-uploaded-object.test.ts b/tests/api/storage/head-uploaded-object.test.ts
new file mode 100644
index 000000000..7145db51a
--- /dev/null
+++ b/tests/api/storage/head-uploaded-object.test.ts
@@ -0,0 +1,116 @@
+import {
+ afterEach,
+ beforeEach,
+ describe,
+ expect,
+ it,
+ vi,
+} from "vite-plus/test";
+import {
+ headUploadedObject,
+ UPLOAD_VISIBILITY_RETRY_DELAYS_MS,
+} from "../../../apps/api/src/storage/head-uploaded-object";
+
+function storageError(status: number, name = "Unknown") {
+ return Object.assign(new Error("UnknownError"), {
+ name,
+ $metadata: { httpStatusCode: status },
+ });
+}
+
+async function settle<T>(promise: Promise<T>) {
+ promise.catch(() => {});
+ await vi.runAllTimersAsync();
+ return promise;
+}
+
+describe("headUploadedObject", () => {
+ beforeEach(() => {
+ vi.useFakeTimers();
+ });
+
+ afterEach(() => {
+ vi.useRealTimers();
+ });
+
+ it("returns the object metadata on the first successful check", async () => {
+ const send = vi.fn().mockResolvedValue({ ContentLength: 12 });
+
+ await expect(
+ settle(
+ headUploadedObject(
+ { send },
+ "kaneo",
+ "a.png",
+ new AbortController().signal,
+ ),
+ ),
+ ).resolves.toEqual({ ContentLength: 12 });
+ expect(send).toHaveBeenCalledTimes(1);
+ });
+
+ it("retries while storage briefly refuses a fresh upload", async () => {
+ const send = vi
+ .fn()
+ .mockRejectedValueOnce(storageError(403))
+ .mockRejectedValueOnce(storageError(404, "NotFound"))
+ .mockResolvedValue({ ContentLength: 12 });
+
+ await expect(
+ settle(
+ headUploadedObject(
+ { send },
+ "kaneo",
+ "a.png",
+ new AbortController().signal,
+ ),
+ ),
+ ).resolves.toEqual({ ContentLength: 12 });
+ expect(send).toHaveBeenCalledTimes(3);
+ });
+
+ it("gives up with the last storage error after the retry budget", async () => {
+ const send = vi.fn().mockRejectedValue(storageError(403));
+
+ await expect(
+ settle(
+ headUploadedObject(
+ { send },
+ "kaneo",
+ "a.png",
+ new AbortController().signal,
+ ),
+ ),
+ ).rejects.toMatchObject({ $metadata: { httpStatusCode: 403 } });
+ expect(send).toHaveBeenCalledTimes(
+ UPLOAD_VISIBILITY_RETRY_DELAYS_MS.length + 1,
+ );
+ });
+
+ it("does not retry other storage failures", async () => {
+ const send = vi.fn().mockRejectedValue(storageError(500));
+
+ await expect(
+ settle(
+ headUploadedObject(
+ { send },
+ "kaneo",
+ "a.png",
+ new AbortController().signal,
+ ),
+ ),
+ ).rejects.toMatchObject({ $metadata: { httpStatusCode: 500 } });
+ expect(send).toHaveBeenCalledTimes(1);
+ });
+
+ it("stops retrying once the verification is aborted", async () => {
+ const controller = new AbortController();
+ controller.abort();
+ const send = vi.fn().mockRejectedValue(storageError(403));
+
+ await expect(
+ settle(headUploadedObject({ send }, "kaneo", "a.png", controller.signal)),
+ ).rejects.toMatchObject({ $metadata: { httpStatusCode: 403 } });
+ expect(send).toHaveBeenCalledTimes(1);
+ });
+});
diff --git a/tests/api/storage/s3.test.ts b/tests/api/storage/s3.test.ts
index 40b530910..7951b6383 100644
--- a/tests/api/storage/s3.test.ts
+++ b/tests/api/storage/s3.test.ts
@@ -386,7 +386,7 @@ describe("S3 helpers", () => {
vi.spyOn(S3Client.prototype, "send").mockRejectedValue(
Object.assign(
new Error("https://storage.example.test/kaneo?X-Amz-Signature=x"),
- { name: "Forbidden", $metadata: { httpStatusCode: 403 } },
+ { name: "InternalError", $metadata: { httpStatusCode: 500 } },
) as never,
);
const log = vi.spyOn(console, "error").mockImplementation(() => {});
@@ -399,12 +399,49 @@ describe("S3 helpers", () => {
).rejects.toThrow("Unable to verify uploaded object.");
expect(log).toHaveBeenCalledWith("Unable to verify uploaded object:", {
- name: "Forbidden",
- status: 403,
+ name: "InternalError",
+ status: 500,
});
expect(JSON.stringify(log.mock.calls)).not.toContain("X-Amz-Signature");
});
+ it("accepts an upload that storage refuses for a moment after the PUT", async () => {
+ process.env.S3_ENDPOINT = "https://storage.example.test";
+ process.env.S3_BUCKET = "kaneo";
+ process.env.S3_ACCESS_KEY_ID = "test-access-key";
+ process.env.S3_SECRET_ACCESS_KEY = "test-secret-key";
+
+ const send = vi
+ .spyOn(S3Client.prototype, "send")
+ .mockRejectedValueOnce(
+ Object.assign(new Error("UnknownError"), {
+ name: "Unknown",
+ $metadata: { httpStatusCode: 403 },
+ }) as never,
+ )
+ .mockResolvedValue({
+ ContentLength: 12,
+ ContentType: "image/png",
+ } as never);
+
+ vi.useFakeTimers();
+ try {
+ const verified = verifyTaskAssetUpload(
+ "workspace/w1/project/p1/task/t1/a.png",
+ { size: 12, contentType: "image/png" },
+ );
+ await vi.runAllTimersAsync();
+
+ await expect(verified).resolves.toEqual({
+ size: 12,
+ contentType: "image/png",
+ });
+ expect(send).toHaveBeenCalledTimes(2);
+ } finally {
+ vi.useRealTimers();
+ }
+ });
+
it("copies a stored object into the destination task prefix", async () => {
process.env.S3_ENDPOINT = "https://storage.example.test";
process.env.S3_BUCKET = "kaneo";