⚡ Project management that doesn't become the project. Boards, Gantt charts, and GitHub integration. Self-host it or use Kaneo Cloud.
TypeScript honoissue-managementissue-trackerjira-alternativekanbanlinear-alternativemcpproject-managementreactself-hostedtypescript
fix(api): retry upload checks on 403 and 404
Andrejb799520a19d2
diff --git a/apps/api/src/storage/head-uploaded-object.ts b/apps/api/src/storage/head-uploaded-object.tsnew file mode 100644index 000000000..5d278edca--- /dev/null+++ b/apps/api/src/storage/head-uploaded-object.ts@@ -0,0 +1,47 @@+import {+ HeadObjectCommand,+ type HeadObjectCommandOutput,+ type S3Client,+} from "@aws-sdk/client-s3";++export const UPLOAD_VISIBILITY_RETRY_DELAYS_MS = [250, 500, 1000, 2000, 4000];++function storageStatus(error: unknown) {+ return error instanceof Error && "$metadata" in error+ ? (error.$metadata as { httpStatusCode?: number }).httpStatusCode+ : undefined;+}++export function isMissingObjectError(error: unknown) {+ return (+ error instanceof Error &&+ (error.name === "NotFound" ||+ error.name === "NoSuchKey" ||+ storageStatus(error) === 404)+ );+}++function isNotVisibleYet(error: unknown) {+ return isMissingObjectError(error) || storageStatus(error) === 403;+}++export async function headUploadedObject(+ client: Pick<S3Client, "send">,+ bucket: string,+ key: string,+ abortSignal: AbortSignal,+): Promise<HeadObjectCommandOutput> {+ const head = () =>+ client.send(new HeadObjectCommand({ Bucket: bucket, Key: key }), {+ abortSignal,+ });+ for (const delay of UPLOAD_VISIBILITY_RETRY_DELAYS_MS) {+ try {+ return await head();+ } catch (error) {+ if (!isNotVisibleYet(error) || abortSignal.aborted) throw error;+ }+ await new Promise((resolve) => setTimeout(resolve, delay));+ }+ return head();+}diff --git a/apps/api/src/storage/s3.ts b/apps/api/src/storage/s3.tsindex 2112b6110..b22191288 100644--- a/apps/api/src/storage/s3.ts+++ b/apps/api/src/storage/s3.ts@@ -3,7 +3,6 @@ import {CopyObjectCommand,DeleteObjectCommand,GetObjectCommand,- HeadObjectCommand,type HeadObjectCommandOutput,PutObjectCommand,S3Client,@@ -13,6 +12,10 @@ import { getSignedUrl } from "@aws-sdk/s3-request-presigner";import { createId } from "@paralleldrive/cuid2";import { config } from "dotenv-mono";import { describeStorageError } from "./describe-storage-error";+import {+ headUploadedObject,+ isMissingObjectError,+} from "./head-uploaded-object";config();@@ -432,19 +435,9 @@ export async function verifyTaskAssetUpload(const abortSignal = AbortSignal.timeout(10_000);let object: HeadObjectCommandOutput;try {- object = await client.send(- new HeadObjectCommand({ Bucket: config.bucket, Key: key }),- { abortSignal },- );+ object = await headUploadedObject(client, config.bucket, key, abortSignal);} catch (error) {- if (- error instanceof Error &&- (error.name === "NotFound" ||- error.name === "NoSuchKey" ||- ("$metadata" in error &&- (error.$metadata as { httpStatusCode?: number }).httpStatusCode ===- 404))- ) {+ if (isMissingObjectError(error)) {throw new InvalidUploadedAssetError("Uploaded object was not found.");}// Provider errors may include endpoints, signed URLs or other secrets.diff --git a/tests/api/storage/head-uploaded-object.test.ts b/tests/api/storage/head-uploaded-object.test.tsnew file mode 100644index 000000000..7145db51a--- /dev/null+++ b/tests/api/storage/head-uploaded-object.test.ts@@ -0,0 +1,116 @@+import {+ afterEach,+ beforeEach,+ describe,+ expect,+ it,+ vi,+} from "vite-plus/test";+import {+ headUploadedObject,+ UPLOAD_VISIBILITY_RETRY_DELAYS_MS,+} from "../../../apps/api/src/storage/head-uploaded-object";++function storageError(status: number, name = "Unknown") {+ return Object.assign(new Error("UnknownError"), {+ name,+ $metadata: { httpStatusCode: status },+ });+}++async function settle<T>(promise: Promise<T>) {+ promise.catch(() => {});+ await vi.runAllTimersAsync();+ return promise;+}++describe("headUploadedObject", () => {+ beforeEach(() => {+ vi.useFakeTimers();+ });++ afterEach(() => {+ vi.useRealTimers();+ });++ it("returns the object metadata on the first successful check", async () => {+ const send = vi.fn().mockResolvedValue({ ContentLength: 12 });++ await expect(+ settle(+ headUploadedObject(+ { send },+ "kaneo",+ "a.png",+ new AbortController().signal,+ ),+ ),+ ).resolves.toEqual({ ContentLength: 12 });+ expect(send).toHaveBeenCalledTimes(1);+ });++ it("retries while storage briefly refuses a fresh upload", async () => {+ const send = vi+ .fn()+ .mockRejectedValueOnce(storageError(403))+ .mockRejectedValueOnce(storageError(404, "NotFound"))+ .mockResolvedValue({ ContentLength: 12 });++ await expect(+ settle(+ headUploadedObject(+ { send },+ "kaneo",+ "a.png",+ new AbortController().signal,+ ),+ ),+ ).resolves.toEqual({ ContentLength: 12 });+ expect(send).toHaveBeenCalledTimes(3);+ });++ it("gives up with the last storage error after the retry budget", async () => {+ const send = vi.fn().mockRejectedValue(storageError(403));++ await expect(+ settle(+ headUploadedObject(+ { send },+ "kaneo",+ "a.png",+ new AbortController().signal,+ ),+ ),+ ).rejects.toMatchObject({ $metadata: { httpStatusCode: 403 } });+ expect(send).toHaveBeenCalledTimes(+ UPLOAD_VISIBILITY_RETRY_DELAYS_MS.length + 1,+ );+ });++ it("does not retry other storage failures", async () => {+ const send = vi.fn().mockRejectedValue(storageError(500));++ await expect(+ settle(+ headUploadedObject(+ { send },+ "kaneo",+ "a.png",+ new AbortController().signal,+ ),+ ),+ ).rejects.toMatchObject({ $metadata: { httpStatusCode: 500 } });+ expect(send).toHaveBeenCalledTimes(1);+ });++ it("stops retrying once the verification is aborted", async () => {+ const controller = new AbortController();+ controller.abort();+ const send = vi.fn().mockRejectedValue(storageError(403));++ await expect(+ settle(headUploadedObject({ send }, "kaneo", "a.png", controller.signal)),+ ).rejects.toMatchObject({ $metadata: { httpStatusCode: 403 } });+ expect(send).toHaveBeenCalledTimes(1);+ });+});diff --git a/tests/api/storage/s3.test.ts b/tests/api/storage/s3.test.tsindex 40b530910..7951b6383 100644--- a/tests/api/storage/s3.test.ts+++ b/tests/api/storage/s3.test.ts@@ -386,7 +386,7 @@ describe("S3 helpers", () => {vi.spyOn(S3Client.prototype, "send").mockRejectedValue(Object.assign(new Error("https://storage.example.test/kaneo?X-Amz-Signature=x"),- { name: "Forbidden", $metadata: { httpStatusCode: 403 } },+ { name: "InternalError", $metadata: { httpStatusCode: 500 } },) as never,);const log = vi.spyOn(console, "error").mockImplementation(() => {});@@ -399,12 +399,49 @@ describe("S3 helpers", () => {).rejects.toThrow("Unable to verify uploaded object.");expect(log).toHaveBeenCalledWith("Unable to verify uploaded object:", {- name: "Forbidden",- status: 403,+ name: "InternalError",+ status: 500,});expect(JSON.stringify(log.mock.calls)).not.toContain("X-Amz-Signature");});+ it("accepts an upload that storage refuses for a moment after the PUT", async () => {+ process.env.S3_ENDPOINT = "https://storage.example.test";+ process.env.S3_BUCKET = "kaneo";+ process.env.S3_ACCESS_KEY_ID = "test-access-key";+ process.env.S3_SECRET_ACCESS_KEY = "test-secret-key";++ const send = vi+ .spyOn(S3Client.prototype, "send")+ .mockRejectedValueOnce(+ Object.assign(new Error("UnknownError"), {+ name: "Unknown",+ $metadata: { httpStatusCode: 403 },+ }) as never,+ )+ .mockResolvedValue({+ ContentLength: 12,+ ContentType: "image/png",+ } as never);++ vi.useFakeTimers();+ try {+ const verified = verifyTaskAssetUpload(+ "workspace/w1/project/p1/task/t1/a.png",+ { size: 12, contentType: "image/png" },+ );+ await vi.runAllTimersAsync();++ await expect(verified).resolves.toEqual({+ size: 12,+ contentType: "image/png",+ });+ expect(send).toHaveBeenCalledTimes(2);+ } finally {+ vi.useRealTimers();+ }+ });+it("copies a stored object into the destination task prefix", async () => {process.env.S3_ENDPOINT = "https://storage.example.test";process.env.S3_BUCKET = "kaneo";