Skip to content

armgit / hypit

Public Imported from GitHub
0

Clone any viral video with AI agents. Not just a script, the whole workflow: swap the face, the words, the B-roll, ship 100 variants in one command, and get your 100M views.

fix(hypihub): bound upload sessions and retry lifecycle operations

laurunshen6cfd46e8ebb9
diff --git a/packages/provider-hypihub/src/upload.ts b/packages/provider-hypihub/src/upload.ts
index 0db70fcc..df43bc2a 100644
--- a/packages/provider-hypihub/src/upload.ts
+++ b/packages/provider-hypihub/src/upload.ts
@@ -47,7 +47,7 @@ function apiBaseUrl(value: string): string {
}
class HypiHubHTTPError extends Error {
- constructor(readonly status: number, message: string) {
+ constructor(readonly status: number, message: string, readonly retryAfterMs?: number, readonly code?: string) {
super(message);
}
}
@@ -89,6 +89,22 @@ function assertHTTPS(value: string, subject: string): void {
assert(protocol === "https:", `${subject} must use HTTPS`);
}
+// A provider can upload references from several operations/uploader instances.
+// Bound whole file sessions, independently of each file's S3 part concurrency.
+const uploadGates = new Map<string, { active: number; waiting: (() => void)[] }>();
+async function acquireUploadSlot(origin: string, auth: UploadAuth): Promise<() => void> {
+ const key = `${origin}:${createHash("sha256").update(await uploadToken(auth)).digest("hex")}`;
+ let gate = uploadGates.get(key);
+ if (gate === undefined) { gate = { active: 0, waiting: [] }; uploadGates.set(key, gate); }
+ if (gate.active < 2) gate.active += 1;
+ else await new Promise<void>((resolve) => gate.waiting.push(resolve));
+ return () => {
+ const next = gate.waiting.shift();
+ if (next !== undefined) next();
+ else { gate.active -= 1; if (gate.active === 0) uploadGates.delete(key); }
+ };
+}
+
/** Uploads media through HypiHub's session-negotiated private regional S3 multipart flow. */
export class HypiHubUploader {
readonly baseUrl: string;
@@ -119,25 +135,61 @@ export class HypiHubUploader {
return message.replace(/https?:\/\/\S+/giu, "[redacted-url]").slice(0, 300);
}
- private async json(path: string, auth: UploadAuth, init: RequestInit = {}, retryAuth = true): Promise<Record<string, unknown>> {
+ private async json(path: string, auth: UploadAuth, init: RequestInit = {}): Promise<Record<string, unknown>> {
+ const existingSession = /^\/files\/uploads\/[^/]+(?:\/(?:parts|complete))?$/u.test(path);
+ const deadline = Date.now() + (existingSession ? Math.min(this.requestTimeout, 120_000) : this.requestTimeout);
+ for (let attempt = 0; ; attempt += 1) {
+ try {
+ return await this.jsonOnce(path, auth, init, Math.max(1, deadline - Date.now()));
+ } catch (error) {
+ const http = error instanceof HypiHubHTTPError ? error : undefined;
+ // Existing-session operations are idempotent. A new session may only
+ // be retried after an explicit 429 rejection, never an unknown result.
+ const retryable = http !== undefined
+ ? ([408, 429, 500, 502, 503, 504].includes(http.status)
+ || (http.status === 409 && http.code === "upload_incomplete"))
+ : existingSession;
+ const rejectedCreate = path === "/files/uploads" && http?.status === 429;
+ if (attempt >= 3 || !retryable || (!existingSession && !rejectedCreate)
+ || http?.code === "upload_daily_limit" || http?.code === "upload_storage_limit") throw error;
+ const delay = http?.retryAfterMs ?? 500 * 2 ** attempt;
+ // Respect a server's retry time; do not shorten it to fit our budget.
+ if (Date.now() + delay >= deadline) throw error;
+ this.log(`upload control retry method=${init.method ?? "GET"} path=${path} attempt=${attempt + 2} delay_ms=${delay}`);
+ await new Promise((resolve) => setTimeout(resolve, delay));
+ }
+ }
+ }
+
+ private async jsonOnce(path: string, auth: UploadAuth, init: RequestInit, timeoutMs: number, retryAuth = true): Promise<Record<string, unknown>> {
const controller = new AbortController();
- const timer = setTimeout(() => controller.abort(), this.requestTimeout);
+ const timer = setTimeout(() => controller.abort(), timeoutMs);
+ const deadline = Date.now() + timeoutMs;
try {
const response = await this.fetcher(`${this.baseUrl}${path}`, {
...init,
signal: controller.signal,
headers: { authorization: `Bearer ${await uploadToken(auth)}`, ...(init.headers ?? {}) },
});
+ const text = await response.text();
if (response.status === 401 && retryAuth && uploadCanRefresh(auth)) {
await (auth as HypiHubAuth).refresh();
- return await this.json(path, auth, init, false);
+ return await this.jsonOnce(path, auth, init, Math.max(1, deadline - Date.now()), false);
}
- const text = await response.text();
let body: unknown = {};
try { body = text.length === 0 ? {} : JSON.parse(text); }
- catch { throw new Error(`HypiHub returned invalid JSON (${response.status})`); }
+ catch {
+ if (response.ok) throw new Error(`HypiHub returned invalid JSON (${response.status})`);
+ }
if (!response.ok) {
- throw new HypiHubHTTPError(response.status, `HypiHub returned HTTP ${response.status}: ${text.slice(0, 300)}`);
+ const errorBody = body !== null && typeof body === "object" && "error" in body ? body.error : undefined;
+ const code = errorBody !== null && typeof errorBody === "object" && "code" in errorBody
+ && typeof errorBody.code === "string" ? errorBody.code : undefined;
+ const retryAfter = response.headers.get("retry-after");
+ const parsedRetry = retryAfter === null ? NaN : /^\d+$/u.test(retryAfter.trim())
+ ? Number(retryAfter) * 1000 : Date.parse(retryAfter) - Date.now();
+ throw new HypiHubHTTPError(response.status, `HypiHub returned HTTP ${response.status}: ${text.slice(0, 300)}`,
+ Number.isFinite(parsedRetry) ? Math.max(0, parsedRetry) : undefined, code);
}
return object(body, "HypiHub response");
} finally {
@@ -231,28 +283,29 @@ export class HypiHubUploader {
private async uploadDirect(bytes: Uint8Array, auth: UploadAuth, policy: Record<string, unknown>): Promise<string> {
const uploadId = requiredString(policy.upload_id, "HypiHub upload id");
- const partSize = requiredInteger(policy.part_size, "HypiHub upload part size");
- const partCount = requiredInteger(policy.part_count, "HypiHub upload part count");
- const requestedConcurrency = requiredInteger(policy.concurrency, "HypiHub upload concurrency");
- assert(requestedConcurrency <= 8, "HypiHub upload concurrency exceeds 8");
- assert(partCount <= 10_000, "HypiHub upload part count exceeds 10000");
- assert(partCount === Math.ceil(bytes.byteLength / partSize), "HypiHub upload part count differs from the file size");
- const declarations = Array.from({ length: partCount }, (_, index) => {
- const start = index * partSize;
- const part = bytes.subarray(start, Math.min(start + partSize, bytes.byteLength));
- return {
- part_number: index + 1,
- bytes: part.byteLength,
- checksum_sha256: createHash("sha256").update(part).digest("base64"),
- };
- });
- this.log(`multipart upload negotiated upload=${uploadId} bytes=${bytes.byteLength} part_size=${partSize} parts=${partCount} concurrency=${requestedConcurrency}`);
try {
+ const partSize = requiredInteger(policy.part_size, "HypiHub upload part size");
+ const partCount = requiredInteger(policy.part_count, "HypiHub upload part count");
+ const requestedConcurrency = requiredInteger(policy.concurrency, "HypiHub upload concurrency");
+ assert(requestedConcurrency <= 8, "HypiHub upload concurrency exceeds 8");
+ assert(partCount <= 10_000, "HypiHub upload part count exceeds 10000");
+ assert(partCount === Math.ceil(bytes.byteLength / partSize), "HypiHub upload part count differs from the file size");
+ const declarations = Array.from({ length: partCount }, (_, index) => {
+ const start = index * partSize;
+ const part = bytes.subarray(start, Math.min(start + partSize, bytes.byteLength));
+ return {
+ part_number: index + 1,
+ bytes: part.byteLength,
+ checksum_sha256: createHash("sha256").update(part).digest("base64"),
+ };
+ });
+ this.log(`multipart upload negotiated upload=${uploadId} bytes=${bytes.byteLength} part_size=${partSize} parts=${partCount} concurrency=${requestedConcurrency}`);
const signed = await this.signParts(uploadId, declarations, auth);
const completed = new Array<CompletedPart>(partCount);
let cursor = 0;
+ let workerFailed = false;
const worker = async (): Promise<void> => {
- while (cursor < partCount) {
+ while (!workerFailed && cursor < partCount) {
const index = cursor;
cursor += 1;
const declaration = declarations[index];
@@ -261,10 +314,15 @@ export class HypiHubUploader {
const body = bytes.subarray(start, Math.min(start + partSize, bytes.byteLength));
const capability = signed.get(declaration.part_number);
assert(capability !== undefined, `HypiHub upload part ${declaration.part_number} was not signed`);
- completed[index] = await this.putPart(uploadId, declaration, capability, body, auth);
+ try { completed[index] = await this.putPart(uploadId, declaration, capability, body, auth); }
+ catch (error) { workerFailed = true; throw error; }
}
};
- await Promise.all(Array.from({ length: Math.min(requestedConcurrency, partCount) }, worker));
+ // Let in-flight part requests settle before cancellation; otherwise a
+ // surviving worker can keep uploading into a session we just aborted.
+ const results = await Promise.allSettled(Array.from({ length: Math.min(requestedConcurrency, partCount) }, worker));
+ const failed = results.find((result) => result.status === "rejected");
+ if (failed?.status === "rejected") throw failed.reason;
const completeStartedAt = Date.now();
this.log(`multipart complete started upload=${uploadId} parts=${partCount}`);
const response = await this.json(`/files/uploads/${encodeURIComponent(uploadId)}/complete`, auth, {
@@ -279,12 +337,23 @@ export class HypiHubUploader {
} catch (error) {
this.log(`multipart upload failed upload=${uploadId} reason=${this.safeReason(error)}`);
try { await this.json(`/files/uploads/${encodeURIComponent(uploadId)}`, auth, { method: "DELETE" }); }
- catch { /* S3 Lifecycle is the final abort fallback. */ }
+ catch (cancelError) {
+ if (!(cancelError instanceof HypiHubHTTPError && (cancelError.status === 404
+ || (cancelError.status === 409 && cancelError.code === "upload_completed")))) {
+ this.log(`upload cancellation still pending upload=${uploadId} reason=${this.safeReason(cancelError)}`);
+ }
+ }
throw error;
}
}
async upload(input: HypiHubUploadInput, auth: UploadAuth): Promise<string> {
+ const release = await acquireUploadSlot(this.baseUrl, auth);
+ try { return await this.uploadWithSlot(input, auth); }
+ finally { release(); }
+ }
+
+ private async uploadWithSlot(input: HypiHubUploadInput, auth: UploadAuth): Promise<string> {
assert(input.bytes.byteLength > 0, "HypiHub reference artifact is empty");
const startedAt = Date.now();
const digest = createHash("sha256").update(input.bytes).digest("hex");
diff --git a/packages/provider-hypihub/test/upload.test.ts b/packages/provider-hypihub/test/upload.test.ts
new file mode 100644
index 00000000..714f20cc
--- /dev/null
+++ b/packages/provider-hypihub/test/upload.test.ts
@@ -0,0 +1,136 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+import { HypiHubUploader } from "../src/upload.js";
+
+const input = { bytes: new Uint8Array([1, 2, 3, 4]), mediaType: "video/mp4" };
+const retry = (status: number, code = "rate_limited") => Response.json({ error: { code } }, { status, headers: { "retry-after": "0" } });
+function fixture(hook: (url: string, init: RequestInit, calls: string[]) => Response | undefined | Promise<Response | undefined> = () => undefined) {
+ const calls: string[] = [];
+ const logs: string[] = [];
+ const fetcher: typeof fetch = async (resource, init = {}) => {
+ const url = String(resource);
+ calls.push(`${init.method} ${url}`);
+ const override = await hook(url, init, calls);
+ if (override !== undefined) return override;
+ if (url.endsWith("/files/uploads")) return Response.json({ upload_mode: "s3_multipart", upload_id: "up_test", part_size: 16, part_count: 1, concurrency: 4 });
+ if (url.endsWith("/parts")) {
+ const body = JSON.parse(String(init.body)) as { parts: { part_number: number; bytes: number; checksum_sha256: string }[] };
+ return Response.json({ parts: body.parts.map((p) => ({ part_number: p.part_number, url: `https://s3.test/${p.part_number}`, headers: { "content-length": String(p.bytes), "x-amz-checksum-sha256": p.checksum_sha256 } })) });
+ }
+ if (url.startsWith("https://s3.test/")) return new Response(null, { headers: { etag: "etag" } });
+ if (url.endsWith("/complete")) return Response.json({ url: "https://hub.test/files/result" });
+ if (init.method === "DELETE") return new Response(null, { status: 204 });
+ throw new Error(`unexpected request ${url}`);
+ };
+ const uploader = () => new HypiHubUploader({ baseUrl: "https://hub.test", requestTimeoutMs: 5000, uploadPartAttempts: 1, fetch: fetcher, logger: (message) => logs.push(message) });
+ return { uploader, calls, logs };
+}
+
+test("completion retries HTML 502 and 429 on the same session without creating another upload", async () => {
+ let completes = 0;
+ const f = fixture((url) => {
+ if (!url.endsWith("/complete")) return;
+ completes += 1;
+ if (completes === 1) return new Response("<html>gateway</html>", { status: 502, headers: { "retry-after": "0" } });
+ if (completes === 2) return retry(429);
+ });
+ assert.equal(await f.uploader().upload(input, "test-key"), "https://hub.test/files/result");
+ assert.equal(completes, 3);
+ assert.equal(f.calls.filter((c) => c.endsWith("/files/uploads")).length, 1);
+ assert.equal(f.calls.filter((c) => c.startsWith("DELETE")).length, 0);
+});
+
+test("part signing retries and cancellation survives a transient gateway failure", async () => {
+ let signs = 0;
+ let cancels = 0;
+ const f = fixture((url, init) => {
+ if (url.endsWith("/parts") && ++signs === 1) return retry(429);
+ if (url.startsWith("https://s3.test")) return new Response(null, { status: 400 });
+ if (init.method === "DELETE" && ++cancels === 1) return retry(502);
+ });
+ await assert.rejects(f.uploader().upload(input, "test-key"), /S3 upload part 1 failed/u);
+ assert.equal(signs, 2);
+ assert.equal(cancels, 2);
+ assert.equal(f.calls.filter((c) => c.endsWith("/files/uploads")).length, 1);
+});
+
+test("new sessions retry explicit temporary rejection, but never an unknown creation result", async () => {
+ for (const mode of ["network", "500", "daily", "storage", "429"] as const) {
+ let creates = 0;
+ const f = fixture((url) => {
+ if (!url.endsWith("/files/uploads")) return;
+ creates += 1;
+ if (mode === "network") throw new TypeError("fetch failed");
+ if (mode === "500") return retry(500);
+ if (mode === "daily") return retry(429, "upload_daily_limit");
+ if (mode === "storage") return retry(429, "upload_storage_limit");
+ if (creates === 1) return retry(429, "upload_concurrency_limit");
+ });
+ if (mode === "429") { await f.uploader().upload(input, "test-key"); assert.equal(creates, 2); }
+ else { await assert.rejects(f.uploader().upload(input, "test-key")); assert.equal(creates, 1); }
+ }
+});
+
+test("lifecycle retries are bounded and an unconfirmed cancellation retains its diagnostic id", async () => {
+ const f = fixture((url, init) => {
+ if (url.endsWith("/complete") || init.method === "DELETE") return retry(503);
+ });
+ await assert.rejects(f.uploader().upload(input, "test-key"), /503/u);
+ assert.equal(f.calls.filter((c) => c.endsWith("/complete")).length, 4);
+ assert.equal(f.calls.filter((c) => c.startsWith("DELETE")).length, 4);
+ assert(f.logs.some((line) => line.includes("cancellation still pending upload=up_test")));
+ assert(!f.logs.some((line) => line.includes("test-key")));
+});
+
+test("a Retry-After beyond the retry budget is not shortened or retried early", async () => {
+ const f = fixture((url) => url.endsWith("/complete")
+ ? new Response(null, { status: 429, headers: { "retry-after": "60" } }) : undefined);
+ await assert.rejects(f.uploader().upload(input, "test-key"), /429/u);
+ assert.equal(f.calls.filter((c) => c.endsWith("/complete")).length, 1);
+});
+
+test("file concurrency is shared across uploader instances for the same origin and credential", async () => {
+ let active = 0;
+ let maximum = 0;
+ let nextId = 0;
+ let release!: () => void;
+ const barrier = new Promise<void>((resolve) => { release = resolve; });
+ const f = fixture(async (url) => {
+ if (url.endsWith("/files/uploads")) {
+ active += 1; maximum = Math.max(maximum, active); nextId += 1;
+ return Response.json({ upload_mode: "s3_multipart", upload_id: `up_${nextId}`, part_size: 16, part_count: 1, concurrency: 4 });
+ }
+ if (url.endsWith("/complete")) {
+ await barrier; active -= 1;
+ return Response.json({ url: "https://hub.test/files/result" });
+ }
+ });
+ const uploads = Array.from({ length: 6 }, () => f.uploader().upload(input, "test-key"));
+ await new Promise<void>((resolve) => setImmediate(resolve));
+ assert.equal(nextId, 2);
+ release();
+ await Promise.all(uploads);
+ assert.equal(nextId, 6); assert.equal(maximum, 2); assert.equal(active, 0);
+});
+
+test("cancellation waits for other in-flight part workers to settle", async () => {
+ let release!: () => void;
+ const barrier = new Promise<void>((resolve) => { release = resolve; });
+ const f = fixture(async (url) => {
+ if (url.endsWith("/files/uploads")) return Response.json({ upload_mode: "s3_multipart", upload_id: "up_test", part_size: 2, part_count: 2, concurrency: 2 });
+ if (url === "https://s3.test/1") return new Response(null, { status: 400 });
+ if (url === "https://s3.test/2") { await barrier; return new Response(null, { headers: { etag: "etag" } }); }
+ });
+ const result = assert.rejects(f.uploader().upload(input, "test-key"), /S3 upload part 1 failed/u);
+ await new Promise<void>((resolve) => setImmediate(resolve));
+ assert.equal(f.calls.filter((c) => c.startsWith("DELETE")).length, 0);
+ release(); await result;
+ assert.equal(f.calls.filter((c) => c.startsWith("DELETE")).length, 1);
+});
+
+test("invalid negotiated policy still cancels its known upload session", async () => {
+ const f = fixture((url) => url.endsWith("/files/uploads")
+ ? Response.json({ upload_mode: "s3_multipart", upload_id: "up_test", part_size: 0 }) : undefined);
+ await assert.rejects(f.uploader().upload(input, "test-key"), /part size/u);
+ assert.equal(f.calls.filter((c) => c.startsWith("DELETE")).length, 1);
+});