Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
fix(media-server): drain jobs on shutdown, sweep orphaned temp dirs, heartbeat long steps and stop cancelled work
Richie McIlroy955c64738b4c
diff --git a/apps/media-server/Dockerfile b/apps/media-server/Dockerfileindex 4891e6452..5a1244bd8 100644--- a/apps/media-server/Dockerfile+++ b/apps/media-server/Dockerfile@@ -22,6 +22,8 @@ RUN bun test src/__tests__/lib/audio-quality-policy.test.ts src/__tests__/lib/au&& bun test src/__tests__/lib/media-transfer.test.ts src/__tests__/lib/capcodec-encoder.test.ts src/__tests__/lib/nvenc-encoder.test.ts \&& bun test src/__tests__/routes/recording-verification.test.ts \&& bun test src/__tests__/routes/video.test.ts \+ && bun test src/__tests__/lib/lifecycle.test.ts src/__tests__/lib/temp-files.test.ts \+ && bun test src/__tests__/routes/lifecycle.test.ts \&& bun test src/__tests__/lib/media-routes-real-world.integration.test.ts \&& bun test src/__tests__/lib/media-video.integration.test.ts@@ -37,6 +39,6 @@ ENV PORT=3456EXPOSE 3456HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \- CMD ["bun", "-e", "fetch(`http://127.0.0.1:${process.env.PORT || 3456}/health`).then((response) => process.exit(response.ok ? 0 : 1)).catch(() => process.exit(1))"]+ CMD ["bun", "-e", "fetch(`http://127.0.0.1:${process.env.PORT || 3456}/health`).then(async (response) => process.exit(response.ok || (await response.json().catch(() => ({}))).status === 'draining' ? 0 : 1)).catch(() => process.exit(1))"]CMD ["bun", "run", "src/index.ts"]diff --git a/apps/media-server/src/__tests__/lib/job-manager.test.ts b/apps/media-server/src/__tests__/lib/job-manager.test.tsindex ff2b43e69..5bada4e69 100644--- a/apps/media-server/src/__tests__/lib/job-manager.test.ts+++ b/apps/media-server/src/__tests__/lib/job-manager.test.ts@@ -6,7 +6,9 @@ import {cleanupExpiredJobs,createJob,deleteJob,+ extendJobLifetime,getJob,+ getJobTotals,type JobProgress,type RecordingWorkerAcknowledgement,sendWebhook,@@ -522,3 +524,83 @@ describe("job cleanup", () => {expect(currentJob?.error).toContain("maximum lifetime of 60 minutes");});});++describe("job lifetime", () => {+ test("extends the lifetime cap for long work without shortening it", () => {+ const job = createTrackedJob("job-extended-lifetime");+ const now = Date.now();+ job.phase = "processing";+ job.createdAt = now - 10 * 60 * 1000;++ expect(extendJobLifetime(job.jobId, 60_000)).toBe(+ job.createdAt + 60 * 60 * 1000,+ );+ const deadline = extendJobLifetime(job.jobId, 2 * 60 * 60 * 1000);+ expect(deadline).toBeGreaterThanOrEqual(now + 2 * 60 * 60 * 1000);+ expect(extendJobLifetime(job.jobId, 60_000)).toBe(deadline);++ job.createdAt = now - 90 * 60 * 1000;+ job.updatedAt = Date.now();+ expect(cleanupExpiredJobs()).toBe(0);+ expect(getJob(job.jobId)?.phase).toBe("processing");+ });++ test("keeps a hard upper bound on extended lifetimes", () => {+ const job = createTrackedJob("job-capped-lifetime");+ job.phase = "uploading";+ job.createdAt = Date.now() - 10 * 60 * 1000;++ expect(extendJobLifetime(job.jobId, 24 * 60 * 60 * 1000)).toBe(+ job.createdAt + 4 * 60 * 60 * 1000,+ );+ expect(extendJobLifetime(job.jobId, Number.NaN)).toBeUndefined();+ expect(extendJobLifetime(job.jobId, -1)).toBeUndefined();+ });++ test("expires a job past its extended deadline", () => {+ const job = createTrackedJob("job-past-extended-lifetime");+ const abortController = new AbortController();+ updateJob(job.jobId, { phase: "processing", abortController });+ job.createdAt = Date.now() - 10 * 60 * 1000;+ extendJobLifetime(job.jobId, 2 * 60 * 60 * 1000);+ job.createdAt -= 3 * 60 * 60 * 1000;+ job.lifetimeDeadlineAt = Date.now() - 1;+ job.updatedAt = Date.now();++ expect(cleanupExpiredJobs()).toBe(1);+ expect(job.phase).toBe("error");+ expect(job.error).toContain("maximum lifetime");+ expect(abortController.signal.aborted).toBe(true);+ });++ test("does not extend terminal jobs", () => {+ const job = createTrackedJob("job-terminal-lifetime");+ updateJob(job.jobId, { phase: "complete" });+ expect(extendJobLifetime(job.jobId, 2 * 60 * 60 * 1000)).toBeUndefined();+ expect(job.lifetimeDeadlineAt).toBeUndefined();+ });+});++describe("job totals", () => {+ test("counts created, terminal, and expired jobs", () => {+ const before = getJobTotals();+ const completed = createTrackedJob("job-total-complete");+ const errored = createTrackedJob("job-total-error");+ const cancelled = createTrackedJob("job-total-cancelled");+ const stale = createTrackedJob("job-total-stale");+ updateJob(completed.jobId, { phase: "complete" });+ updateJob(completed.jobId, { phase: "error" });+ updateJob(errored.jobId, { phase: "error" });+ updateJob(cancelled.jobId, { phase: "cancelled" });+ updateJob(stale.jobId, { phase: "processing" });+ stale.updatedAt = Date.now() - 16 * 60 * 1000;+ cleanupExpiredJobs();++ const after = getJobTotals();+ expect(after.created - before.created).toBe(4);+ expect(after.completed - before.completed).toBe(1);+ expect(after.errored - before.errored).toBe(2);+ expect(after.cancelled - before.cancelled).toBe(1);+ expect(after.expired - before.expired).toBe(1);+ });+});diff --git a/apps/media-server/src/__tests__/lib/lifecycle.test.ts b/apps/media-server/src/__tests__/lib/lifecycle.test.tsnew file mode 100644index 000000000..bd6f9454c--- /dev/null+++ b/apps/media-server/src/__tests__/lib/lifecycle.test.ts@@ -0,0 +1,174 @@+import { afterEach, describe, expect, test } from "bun:test";+import {+ beginRequest,+ createShutdownHandler,+ getDrainBlockers,+ getIdleTimeoutSeconds,+ getShutdownGraceMs,+ isDraining,+ isIdle,+ setDraining,+ setRequestIdleTimeout,+ trackBackgroundTask,+ waitForDrain,+} from "../../lib/lifecycle";++const originalGrace = process.env.MEDIA_SERVER_SHUTDOWN_GRACE_MS;+const originalIdle = process.env.MEDIA_SERVER_IDLE_TIMEOUT_SECONDS;++function deferred() {+ let resolve = () => {};+ const promise = new Promise<void>((done) => {+ resolve = done;+ });+ return { promise, resolve };+}++afterEach(() => {+ setDraining(false);+ if (originalGrace === undefined)+ delete process.env.MEDIA_SERVER_SHUTDOWN_GRACE_MS;+ else process.env.MEDIA_SERVER_SHUTDOWN_GRACE_MS = originalGrace;+ if (originalIdle === undefined)+ delete process.env.MEDIA_SERVER_IDLE_TIMEOUT_SECONDS;+ else process.env.MEDIA_SERVER_IDLE_TIMEOUT_SECONDS = originalIdle;+});++describe("shutdown configuration", () => {+ test("defaults to the immediate shutdown path", () => {+ delete process.env.MEDIA_SERVER_SHUTDOWN_GRACE_MS;+ expect(getShutdownGraceMs()).toBe(0);+ process.env.MEDIA_SERVER_SHUTDOWN_GRACE_MS = "not-a-number";+ expect(getShutdownGraceMs()).toBe(0);+ process.env.MEDIA_SERVER_SHUTDOWN_GRACE_MS = "120000";+ expect(getShutdownGraceMs()).toBe(120_000);+ });++ test("bounds the idle timeout to Bun's maximum", () => {+ delete process.env.MEDIA_SERVER_IDLE_TIMEOUT_SECONDS;+ expect(getIdleTimeoutSeconds()).toBe(255);+ process.env.MEDIA_SERVER_IDLE_TIMEOUT_SECONDS = "90";+ expect(getIdleTimeoutSeconds()).toBe(90);+ process.env.MEDIA_SERVER_IDLE_TIMEOUT_SECONDS = "10000";+ expect(getIdleTimeoutSeconds()).toBe(255);+ });++ test("ignores servers without per-request timeouts", () => {+ const calls: Array<[Request, number]> = [];+ const request = new Request("http://localhost/video/convert");+ setRequestIdleTimeout(undefined, request, 0);+ setRequestIdleTimeout(+ {+ timeout: (req: Request, seconds: number) => calls.push([req, seconds]),+ },+ request,+ 0,+ );+ expect(calls).toEqual([[request, 0]]);+ });+});++describe("drain", () => {+ test("waits for background work and in-flight requests", async () => {+ const work = deferred();+ const task = trackBackgroundTask(work.promise);+ const endRequest = beginRequest();+ expect(isIdle()).toBe(false);+ expect(getDrainBlockers().backgroundTasks).toBeGreaterThanOrEqual(1);++ const drained = waitForDrain(5_000);+ await Bun.sleep(50);+ work.resolve();+ await task;+ await Bun.sleep(50);+ endRequest();++ expect(await drained).toBe(true);+ expect(isIdle()).toBe(true);+ });++ test("gives up when the grace period elapses", async () => {+ const work = deferred();+ const task = trackBackgroundTask(work.promise);+ const startedAt = Date.now();++ expect(await waitForDrain(300)).toBe(false);+ expect(Date.now() - startedAt).toBeGreaterThanOrEqual(250);++ work.resolve();+ await task;+ });++ test("stops tracking rejected background work", async () => {+ const failure = trackBackgroundTask(Promise.reject(new Error("failed")));+ await failure.catch(() => {});+ await Bun.sleep(0);+ expect(await waitForDrain(1_000)).toBe(true);+ });+});++describe("shutdown handler", () => {+ test("keeps today's immediate shutdown when no grace period is set", async () => {+ const work = deferred();+ const task = trackBackgroundTask(work.promise);+ let finalized = 0;+ const shutdown = createShutdownHandler(+ async () => {+ finalized++;+ },+ () => 0,+ );++ await shutdown("SIGTERM");+ await shutdown("SIGTERM");++ expect(finalized).toBe(1);+ expect(isDraining()).toBe(false);+ work.resolve();+ await task;+ });++ test("drains before finalizing when a grace period is set", async () => {+ const work = deferred();+ const task = trackBackgroundTask(work.promise);+ let finalizedWhileBusy: boolean | undefined;+ const shutdown = createShutdownHandler(+ async () => {+ finalizedWhileBusy = !isIdle();+ },+ () => 10_000,+ );++ const shuttingDown = shutdown("SIGTERM");+ await Bun.sleep(50);+ expect(isDraining()).toBe(true);+ expect(finalizedWhileBusy).toBeUndefined();++ work.resolve();+ await task;+ await shuttingDown;+ expect(finalizedWhileBusy).toBe(false);+ });++ test("forces the immediate path on a second signal", async () => {+ const work = deferred();+ const task = trackBackgroundTask(work.promise);+ let finalized = 0;+ const shutdown = createShutdownHandler(+ async () => {+ finalized++;+ },+ () => 60_000,+ );++ const shuttingDown = shutdown("SIGTERM");+ await Bun.sleep(50);+ expect(finalized).toBe(0);+ await shutdown("SIGINT");+ await shuttingDown;+ expect(finalized).toBe(1);++ work.resolve();+ await task;+ });+});diff --git a/apps/media-server/src/__tests__/lib/temp-files.test.ts b/apps/media-server/src/__tests__/lib/temp-files.test.tsnew file mode 100644index 000000000..b90263a2f--- /dev/null+++ b/apps/media-server/src/__tests__/lib/temp-files.test.ts@@ -0,0 +1,177 @@+import { afterEach, describe, expect, test } from "bun:test";+import { randomUUID } from "node:crypto";+import { existsSync } from "node:fs";+import { mkdir, rm, utimes, writeFile } from "node:fs/promises";+import { tmpdir } from "node:os";+import { join } from "node:path";+import { spawn } from "bun";+import {+ cleanupMediaTransferCache,+ sweepMediaTransferCache,+} from "../../lib/media-transfer";+import {+ cleanupStaleTempFiles,+ createTempFile,+ ensureTempDir,+ getInstanceDir,+ getTempDir,+ getTempDirUsage,+ getTempRoot,+ removeOwnTempDir,+ retainTempPath,+ sweepOrphanedTempDirs,+} from "../../lib/temp-files";++const created: string[] = [];++afterEach(async () => {+ for (const path of created.splice(0)) {+ await rm(path, { recursive: true, force: true });+ }+});++async function makeTree(path: string, ageMs: number, lease = false) {+ created.push(path);+ await mkdir(join(path, "nested", "deeper"), { recursive: true });+ await writeFile(join(path, "nested", "deeper", "segment.m4s"), "segment");+ const past = new Date(Date.now() - ageMs);+ if (lease) {+ await writeFile(join(path, ".lease"), "");+ await utimes(join(path, ".lease"), past, past);+ }+ await utimes(path, past, past);+}++async function setAge(path: string, ageMs: number) {+ const past = new Date(Date.now() - ageMs);+ await utimes(path, past, past);+}++describe("startup temp sweep", () => {+ test("removes orphaned instance dirs and old legacy entries only", async () => {+ await ensureTempDir();+ const root = getTempRoot();+ const staleInstance = join(root, `instance-${randomUUID()}`);+ const liveInstance = join(root, `instance-${randomUUID()}`);+ const oldLegacyDir = join(root, `mux-job_${randomUUID()}`);+ const youngLegacyFile = join(root, `${randomUUID()}.mp4`);+ await makeTree(staleInstance, 10 * 60 * 1000, true);+ await makeTree(liveInstance, 10 * 60 * 1000);+ await writeFile(join(liveInstance, ".lease"), "");+ await makeTree(oldLegacyDir, 2 * 60 * 60 * 1000);+ created.push(youngLegacyFile);+ await writeFile(youngLegacyFile, "in use by an older server version");++ const removed = await sweepOrphanedTempDirs();++ expect(removed).toBeGreaterThanOrEqual(2);+ expect(existsSync(staleInstance)).toBe(false);+ expect(existsSync(oldLegacyDir)).toBe(false);+ expect(existsSync(liveInstance)).toBe(true);+ expect(existsSync(youngLegacyFile)).toBe(true);+ expect(existsSync(getTempDir())).toBe(true);+ expect(existsSync(join(getTempDir(), ".lease"))).toBe(true);+ });++ test("never treats the server's own instance dir as orphaned", async () => {+ await ensureTempDir();+ await setAge(join(getTempDir(), ".lease"), 24 * 60 * 60 * 1000);+ await setAge(getTempDir(), 24 * 60 * 60 * 1000);++ await sweepOrphanedTempDirs();++ expect(existsSync(getTempDir())).toBe(true);+ });+});++describe("stale temp cleanup", () => {+ test("removes stale directories recursively and keeps in-use entries", async () => {+ await ensureTempDir();+ const staleDir = join(getTempDir(), `stream-${randomUUID()}`);+ await makeTree(staleDir, 2 * 60 * 60 * 1000);+ const retainedDir = join(getTempDir(), `mux-job_${randomUUID()}`);+ await makeTree(retainedDir, 2 * 60 * 60 * 1000);+ const releaseRetainedDir = retainTempPath(retainedDir);+ const inUse = await createTempFile(".mp4");+ created.push(inUse.path);+ await writeFile(inUse.path, "input still read by ffmpeg");+ await setAge(inUse.path, 2 * 60 * 60 * 1000);+ const fresh = await createTempFile(".mp4");+ created.push(fresh.path);+ await writeFile(fresh.path, "fresh");++ try {+ expect(await cleanupStaleTempFiles()).toBeGreaterThanOrEqual(1);+ expect(existsSync(staleDir)).toBe(false);+ expect(existsSync(retainedDir)).toBe(true);+ expect(existsSync(inUse.path)).toBe(true);+ expect(existsSync(fresh.path)).toBe(true);++ releaseRetainedDir();+ await cleanupStaleTempFiles();+ expect(existsSync(retainedDir)).toBe(false);+ } finally {+ releaseRetainedDir();+ await inUse.cleanup();+ await fresh.cleanup();+ }+ });++ test("expires entries that were retained far beyond any job lifetime", async () => {+ await ensureTempDir();+ const leaked = await createTempFile(".mp4");+ created.push(leaked.path);+ await writeFile(leaked.path, "leaked handle");+ await setAge(leaked.path, 7 * 60 * 60 * 1000);++ await cleanupStaleTempFiles();++ expect(existsSync(leaked.path)).toBe(false);+ });++ test("reports temp usage including nested files", async () => {+ await ensureTempDir();+ const dir = join(getTempDir(), `usage-${randomUUID()}`);+ await makeTree(dir, 0);++ const usage = await getTempDirUsage();++ expect(usage.entries).toBeGreaterThanOrEqual(4);+ expect(usage.bytes).toBeGreaterThanOrEqual("segment".length);+ expect(usage.truncated).toBe(false);+ });++ test("removes the server's own dir on shutdown and recreates it on demand", async () => {+ const handle = await createTempFile(".mp4");+ await writeFile(handle.path, "shutdown");++ await removeOwnTempDir();+ expect(existsSync(getTempDir())).toBe(false);++ const next = await createTempFile(".mp4");+ expect(existsSync(join(getTempDir(), ".lease"))).toBe(true);+ await next.cleanup();+ });+});++describe("media transfer cache sweep", () => {+ test("removes orphaned instance and dead-pid cache dirs", async () => {+ const base = join(tmpdir(), "cap-media-download-cache");+ const staleInstance = join(base, `instance-${randomUUID()}`);+ const liveInstance = join(base, `instance-${randomUUID()}`);+ const exited = spawn({ cmd: ["true"] });+ await exited.exited;+ const deadPidDir = join(base, String(exited.pid));+ await makeTree(staleInstance, 10 * 60 * 1000, true);+ await makeTree(liveInstance, 0, true);+ await makeTree(deadPidDir, 0);++ await sweepMediaTransferCache();++ expect(existsSync(staleInstance)).toBe(false);+ expect(existsSync(deadPidDir)).toBe(false);+ expect(existsSync(liveInstance)).toBe(true);+ expect(existsSync(getInstanceDir(base))).toBe(false);+ await cleanupMediaTransferCache();+ });+});diff --git a/apps/media-server/src/__tests__/routes/lifecycle.test.ts b/apps/media-server/src/__tests__/routes/lifecycle.test.tsnew file mode 100644index 000000000..ae93d131a--- /dev/null+++ b/apps/media-server/src/__tests__/routes/lifecycle.test.ts@@ -0,0 +1,468 @@+import {+ afterEach,+ beforeEach,+ describe,+ expect,+ mock,+ spyOn,+ test,+} from "bun:test";+import { existsSync } from "node:fs";+import { writeFile } from "node:fs/promises";+import app from "../../app";+import * as jobManager from "../../lib/job-manager";+import {+ cleanupExpiredJobs,+ createJob,+ deleteJob,+ getActiveVideoProcessCount,+ getJob,+ type VideoMetadata,+} from "../../lib/job-manager";+import {+ DRAINING_DETAILS,+ setDraining,+ waitForDrain,+} from "../../lib/lifecycle";+import * as mediaEdit from "../../lib/media-edit";+import * as mediaProbe from "../../lib/media-probe";+import * as mediaVideo from "../../lib/media-video";+import { createTempFile, type TempFileHandle } from "../../lib/temp-files";+import { getActiveDirectVideoProcessCount } from "../../lib/video-capacity";++const MEDIA_SERVER_SECRET = "test-secret";+const AUTH_HEADERS = {+ "Content-Type": "application/json",+ "x-media-server-secret": MEDIA_SERVER_SECRET,+};+const originalPublicStatus = process.env.MEDIA_SERVER_PUBLIC_STATUS;++process.env.MEDIA_SERVER_WEBHOOK_SECRET = MEDIA_SERVER_SECRET;++const metadata: VideoMetadata = {+ duration: 3 * 60 * 60,+ width: 1920,+ height: 1080,+ fps: 30,+ videoCodec: "h264",+ audioCodec: "aac",+ audioChannels: 2,+ sampleRate: 48_000,+ bitrate: 5_000_000,+ fileSize: 1024 ** 3,+};++const createdJobs: string[] = [];++function post(path: string, body: unknown): Request {+ return new Request(`http://localhost${path}`, {+ method: "POST",+ headers: AUTH_HEADERS,+ body: JSON.stringify(body),+ });+}++function deferred<T = void>() {+ let resolve: (value: T) => void = () => {};+ const promise = new Promise<T>((done) => {+ resolve = done;+ });+ return { promise, resolve };+}++async function tempFileWith(extension: string, contents: string) {+ const handle = await createTempFile(extension);+ await writeFile(handle.path, contents);+ return handle;+}++async function waitFor(condition: () => boolean, timeoutMs = 2_000) {+ const deadline = Date.now() + timeoutMs;+ while (!condition()) {+ if (Date.now() > deadline) throw new Error("Timed out waiting");+ await Bun.sleep(10);+ }+}++beforeEach(() => {+ spyOn(jobManager, "canAcceptNewVideoProcess").mockReturnValue(true);+});++afterEach(async () => {+ mock.restore();+ setDraining(false);+ if (originalPublicStatus === undefined)+ delete process.env.MEDIA_SERVER_PUBLIC_STATUS;+ else process.env.MEDIA_SERVER_PUBLIC_STATUS = originalPublicStatus;+ await waitForDrain(2_000);+ for (const jobId of createdJobs.splice(0)) deleteJob(jobId);+});++describe("draining", () => {+ const url = "https://example.com/video.mp4";+ const admissions: Array<[string, unknown]> = [+ [+ "/video/process",+ { videoId: "v", userId: "u", videoUrl: url, outputPresignedUrl: url },+ ],+ [+ "/video/import",+ {+ videoId: "v",+ userId: "u",+ videoUrl: url,+ outputPresignedUrl: url,+ sourcePresignedUrl: url,+ },+ ],+ [+ "/video/edit",+ {+ videoId: "v",+ userId: "u",+ sourceUrl: url,+ outputPresignedUrl: url,+ keepRanges: [{ start: 0, end: 1 }],+ },+ ],+ [+ "/video/mux-segments",+ {+ videoId: "v",+ userId: "u",+ outputPresignedUrl: url,+ videoInitUrl: url,+ videoSegmentUrls: ["https://example.com/segment.m4s"],+ },+ ],+ [+ "/video/verify-recording",+ {+ videoId: "v",+ userId: "u",+ videoUrl: url,+ fileSize: 1,+ duration: 1,+ requiredAudio: false,+ objectIdentity: '"etag"',+ webhookUrl: "https://example.com/webhook",+ },+ ],+ ["/video/convert", { videoUrl: url }],+ ["/video/probe", { videoUrl: url }],+ ["/video/thumbnail", { videoUrl: url }],+ ["/audio/check", { videoUrl: url }],+ ["/audio/extract", { videoUrl: url }],+ ["/audio/convert", { audioUrl: url }],+ ];++ test.each(admissions)("rejects new work on %s", async (path, body) => {+ setDraining(true);+ const response = await app.fetch(post(path, body));++ expect(response.status).toBe(503);+ expect(response.headers.get("Retry-After")).toBe("15");+ const data = await response.json();+ expect(data.code).toBe("SERVER_BUSY");+ expect(data.details).toBe(DRAINING_DETAILS);+ });++ test("fails health checks but keeps status endpoints working", async () => {+ const job = createJob("job-draining-status", "video-id", "user-id");+ createdJobs.push(job.jobId);+ setDraining(true);++ const health = await app.fetch(new Request("http://localhost/health"));+ expect(health.status).toBe(503);+ expect((await health.json()).status).toBe("draining");++ const status = await app.fetch(+ new Request(`http://localhost/video/process/${job.jobId}/status`),+ );+ expect(status.status).toBe(200);+ expect((await status.json()).jobId).toBe(job.jobId);++ const serverStatus = await app.fetch(+ new Request("http://localhost/video/status"),+ );+ expect(serverStatus.status).toBe(200);+ });+});++describe("status access", () => {+ test("requires the secret when public status is disabled", async () => {+ process.env.MEDIA_SERVER_PUBLIC_STATUS = "false";+ for (const path of [+ "/video/status",+ "/audio/status",+ "/video/process/job-missing/status",+ ]) {+ const anonymous = await app.fetch(new Request(`http://localhost${path}`));+ expect(anonymous.status).toBe(401);+ const authenticated = await app.fetch(+ new Request(`http://localhost${path}`, { headers: AUTH_HEADERS }),+ );+ expect(authenticated.status).not.toBe(401);+ }+ });++ test("reports runtime counters only to authenticated callers", async () => {+ const job = createJob("job_runtime_report", "video-runtime", "user-id");+ createdJobs.push(job.jobId);+ const anonymous = await app.fetch(new Request("http://localhost/health"));+ expect((await anonymous.json()).runtime).toBeUndefined();++ for (const path of ["/health", "/video/status"]) {+ const response = await app.fetch(+ new Request(`http://localhost${path}`, { headers: AUTH_HEADERS }),+ );+ const { runtime } = await response.json();+ expect(runtime.activeJobs).toBeGreaterThanOrEqual(1);+ expect(runtime.jobTotals.created).toBeGreaterThanOrEqual(1);+ expect(typeof runtime.activeSubprocesses).toBe("number");+ expect(typeof runtime.tempDir.bytes).toBe("number");+ expect(typeof runtime.tempDir.entries).toBe("number");+ expect(typeof runtime.mediaTransferCache.bytes).toBe("number");+ expect(typeof runtime.slots.video.max).toBe("number");+ expect(typeof runtime.slots.directVideo.active).toBe("number");+ expect(typeof runtime.slots.audio.max).toBe("number");+ expect(typeof runtime.slots.probe.max).toBe("number");+ expect(JSON.stringify(runtime)).not.toContain(job.jobId);+ expect(JSON.stringify(runtime)).not.toContain(job.videoId);+ }+ });+});++describe("process jobs", () => {+ test("heartbeats a slow output upload and derives the lifetime from the input", async () => {+ const intervals: Array<{+ callback: () => void;+ delay?: number;+ handle: ReturnType<typeof setInterval>;+ }> = [];+ const cleared = new Set<unknown>();+ const originalSetInterval = globalThis.setInterval;+ const originalClearInterval = globalThis.clearInterval;+ spyOn(globalThis, "setInterval").mockImplementation(((+ callback: () => void,+ delay?: number,+ ) => {+ const handle = originalSetInterval(callback, delay);+ intervals.push({ callback, delay, handle });+ return handle;+ }) as unknown as typeof setInterval);+ spyOn(globalThis, "clearInterval").mockImplementation(((+ handle: ReturnType<typeof setInterval>,+ ) => {+ cleared.add(handle);+ originalClearInterval(handle);+ }) as unknown as typeof clearInterval);+ const files: TempFileHandle[] = [];+ spyOn(mediaVideo, "downloadVideoToTemp").mockImplementation(async () => {+ const input = await tempFileWith(".mp4", "input");+ files.push(input);+ return input;+ });+ spyOn(mediaProbe, "probeVideoFile").mockResolvedValue(metadata);+ spyOn(mediaVideo, "processVideo").mockImplementation(async () => {+ const output = await tempFileWith(".mp4", "output");+ files.push(output);+ return output;+ });+ const uploadStarted = deferred();+ const upload = deferred<mediaVideo.StorageUploadReceipt>();+ spyOn(mediaVideo, "uploadFileToS3").mockImplementation(async () => {+ uploadStarted.resolve();+ return await upload.promise;+ });++ const response = await app.fetch(+ post("/video/process", {+ videoId: "video-slow-upload",+ userId: "user-id",+ videoUrl: "https://example.com/input.mp4",+ outputPresignedUrl: "https://example.com/output.mp4",+ }),+ );+ const { jobId } = await response.json();+ createdJobs.push(jobId);+ await uploadStarted.promise;++ const job = getJob(jobId);+ expect(job?.phase).toBe("uploading");+ if (!job) throw new Error("Job missing");+ expect(job.lifetimeDeadlineAt).toBeGreaterThan(+ job.createdAt + 2 * 60 * 60 * 1000,+ );+ expect(job.lifetimeDeadlineAt).toBeLessThanOrEqual(+ job.createdAt + 4 * 60 * 60 * 1000,+ );++ job.updatedAt = Date.now() - 16 * 60 * 1000;+ const heartbeats = intervals.filter(+ ({ delay, handle }) => delay === 60_000 && !cleared.has(handle),+ );+ expect(heartbeats).toHaveLength(1);+ heartbeats[0]?.callback();+ expect(cleanupExpiredJobs()).toBe(0);+ expect(job.phase).toBe("uploading");++ upload.resolve({});+ await waitFor(() => job.phase === "complete");+ await waitForDrain(2_000);+ expect(files.every((handle) => !existsSync(handle.path))).toBe(true);+ });++ test("releases the job slot and temp files when processing fails", async () => {+ const before = getActiveVideoProcessCount();+ let input: TempFileHandle | undefined;+ spyOn(mediaVideo, "downloadVideoToTemp").mockImplementation(async () => {+ input = await tempFileWith(".mp4", "input");+ return input;+ });+ spyOn(mediaProbe, "probeVideoFile").mockResolvedValue({+ ...metadata,+ duration: 10,+ });+ spyOn(mediaVideo, "processVideo").mockRejectedValue(+ new Error("FFmpeg exited with code 1"),+ );++ const response = await app.fetch(+ post("/video/process", {+ videoId: "video-failed-process",+ userId: "user-id",+ videoUrl: "https://example.com/input.mp4",+ outputPresignedUrl: "https://example.com/output.mp4",+ }),+ );+ const { jobId } = await response.json();+ createdJobs.push(jobId);++ expect(await waitForDrain(2_000)).toBe(true);+ expect(getJob(jobId)?.phase).toBe("error");+ expect(getActiveVideoProcessCount()).toBe(before);+ expect(input && existsSync(input.path)).toBe(false);+ });+});++describe("edit jobs", () => {+ test("cancellation aborts the upload and skips preview work", async () => {+ const files: TempFileHandle[] = [];+ spyOn(mediaVideo, "downloadVideoToTemp").mockImplementation(async () => {+ const input = await tempFileWith(".mp4", "input");+ files.push(input);+ return input;+ });+ spyOn(mediaProbe, "probeVideoFile").mockResolvedValue({+ ...metadata,+ duration: 10,+ });+ spyOn(mediaEdit, "renderEditedVideo").mockImplementation(async () => {+ const output = await tempFileWith(".mp4", "edited");+ files.push(output);+ return output;+ });+ const thumbnail = spyOn(mediaVideo, "generateThumbnail");+ const uploadStarted = deferred();+ let uploadSignal: AbortSignal | undefined;+ spyOn(mediaVideo, "uploadFileToS3").mockImplementation(+ (_path, _url, _contentType, signal) => {+ uploadSignal = signal;+ uploadStarted.resolve();+ return new Promise((_resolve, reject) => {+ signal?.addEventListener("abort", () => reject(signal.reason), {+ once: true,+ });+ });+ },+ );++ const response = await app.fetch(+ post("/video/edit", {+ videoId: "video-cancelled-edit",+ userId: "user-id",+ sourceUrl: "https://example.com/source.mp4",+ outputPresignedUrl: "https://example.com/output.mp4",+ thumbnailPresignedUrl: "https://example.com/thumbnail.jpg",+ keepRanges: [{ start: 0, end: 5 }],+ }),+ );+ const { jobId } = await response.json();+ createdJobs.push(jobId);+ await uploadStarted.promise;++ const cancel = await app.fetch(post(`/video/process/${jobId}/cancel`, {}));+ expect(cancel.status).toBe(200);+ expect(uploadSignal?.aborted).toBe(true);++ expect(await waitForDrain(2_000)).toBe(true);+ expect(getJob(jobId)?.phase).toBe("cancelled");+ expect(thumbnail).not.toHaveBeenCalled();+ expect(files).toHaveLength(2);+ expect(files.every((handle) => !existsSync(handle.path))).toBe(true);+ });+});++describe("direct conversions", () => {+ test("release the slot when the download fails", async () => {+ spyOn(mediaVideo, "downloadVideoToTemp").mockRejectedValue(+ new Error("Failed to download video: 404"),+ );++ const response = await app.fetch(+ post("/video/convert", { videoUrl: "https://example.com/missing.mp4" }),+ );++ expect(response.status).toBe(500);+ expect(getActiveDirectVideoProcessCount()).toBe(0);+ });++ test("release the slot and temp files when the client cancels the body", async () => {+ const files: TempFileHandle[] = [];+ spyOn(mediaVideo, "downloadVideoToTemp").mockImplementation(async () => {+ const input = await tempFileWith(".webm", "input");+ files.push(input);+ return input;+ });+ spyOn(mediaProbe, "probeVideoFile").mockResolvedValue({+ ...metadata,+ duration: 10,+ });+ spyOn(mediaVideo, "processVideo").mockImplementation(async () => {+ const output = await tempFileWith(".mp4", "x".repeat(1024 * 1024));+ files.push(output);+ return output;+ });++ const response = await app.fetch(+ post("/video/convert", { videoUrl: "https://example.com/input.webm" }),+ );+ expect(response.status).toBe(200);+ expect(getActiveDirectVideoProcessCount()).toBe(1);++ await response.body?.cancel();+ await waitFor(() => getActiveDirectVideoProcessCount() === 0);+ await waitFor(() => files.every((handle) => !existsSync(handle.path)));+ });++ test("release the slot after the body is fully read", async () => {+ spyOn(mediaVideo, "downloadVideoToTemp").mockImplementation(() =>+ tempFileWith(".webm", "input"),+ );+ spyOn(mediaProbe, "probeVideoFile").mockResolvedValue({+ ...metadata,+ duration: 10,+ });+ spyOn(mediaVideo, "processVideo").mockImplementation(() =>+ tempFileWith(".mp4", "converted"),+ );++ const response = await app.fetch(+ post("/video/convert", { videoUrl: "https://example.com/input.webm" }),+ );++ expect(await response.text()).toBe("converted");+ await waitFor(() => getActiveDirectVideoProcessCount() === 0);+ });+});diff --git a/apps/media-server/src/app.ts b/apps/media-server/src/app.tsindex e50979955..162a7652d 100644--- a/apps/media-server/src/app.ts+++ b/apps/media-server/src/app.ts@@ -1,5 +1,6 @@import { Hono } from "hono";import { logger } from "hono/logger";+import { beginRequest } from "./lib/lifecycle";import audio from "./routes/audio";import health from "./routes/health";import video from "./routes/video";@@ -7,6 +8,15 @@ import video from "./routes/video";const app = new Hono();app.use("*", logger());+app.use("*", async (c, next) => {+ if (c.req.method === "GET" || c.req.method === "HEAD") return await next();+ const end = beginRequest();+ try {+ await next();+ } finally {+ end();+ }+});app.route("/health", health);app.route("/audio", audio);diff --git a/apps/media-server/src/index.ts b/apps/media-server/src/index.tsindex 7a5b2151d..05dbdac4f 100644--- a/apps/media-server/src/index.ts+++ b/apps/media-server/src/index.ts@@ -1,38 +1,52 @@import app from "./app";import { abortAllJobs } from "./lib/job-manager";+import { createShutdownHandler, getIdleTimeoutSeconds } from "./lib/lifecycle";import { cancelAllMediaOperations } from "./lib/media-operations";-import { cleanupMediaTransferCache } from "./lib/media-transfer";+import {+ cleanupMediaTransferCache,+ sweepMediaTransferCache,+} from "./lib/media-transfer";+import { removeOwnTempDir, startTempDirMaintenance } from "./lib/temp-files";const port = Number(process.env.PORT) || 3456;console.log(`[media-server] Starting on port ${port}`);-let shuttingDown = false;+void Promise.allSettled([+ startTempDirMaintenance(),+ sweepMediaTransferCache(),+]).then(([temp, cache]) => {+ const removed =+ (temp.status === "fulfilled" ? temp.value : 0) ++ (cache.status === "fulfilled" ? cache.value : 0);+ if (removed > 0) {+ console.log(`[media-server] Removed ${removed} orphaned temp entries`);+ }+});-const shutdown = async () => {- if (shuttingDown) return;- shuttingDown = true;- console.log("[media-server] Shutting down...");+const shutdown = createShutdownHandler(async () => {const abortedJobs = await abortAllJobs();if (abortedJobs > 0) {console.log(`[media-server] Aborted ${abortedJobs} active jobs`);}await cancelAllMediaOperations();await cleanupMediaTransferCache();+ await removeOwnTempDir();process.exit(0);-};+});process.on("SIGINT", () => {- void shutdown();+ void shutdown("SIGINT");});process.on("SIGTERM", () => {- void shutdown();+ void shutdown("SIGTERM");});process.on("SIGHUP", () => {- void shutdown();+ void shutdown("SIGHUP");});export default {port,+ idleTimeout: getIdleTimeoutSeconds(),fetch: app.fetch,};diff --git a/apps/media-server/src/lib/audio-quality.ts b/apps/media-server/src/lib/audio-quality.tsindex d51fe5b76..455824b25 100644--- a/apps/media-server/src/lib/audio-quality.ts+++ b/apps/media-server/src/lib/audio-quality.ts@@ -1,7 +1,6 @@import { createHash } from "node:crypto";import { createReadStream } from "node:fs";import { lstat, mkdtemp, rm } from "node:fs/promises";-import { tmpdir } from "node:os";import { isAbsolute, join } from "node:path";import { spawn } from "bun";import { z } from "zod";@@ -13,6 +12,7 @@ import {} from "./audio-quality-policy";import { proveRecordingPackets } from "./recording-packet-proof";import { registerSubprocess, unregisterSubprocess } from "./subprocess";+import { ensureTempDir, getTempDir } from "./temp-files";const streamSchema = z.object({codec_type: z.string(),@@ -307,7 +307,8 @@ export async function createAudioQualityCandidate(Math.abs(videoDuration - input.duration) > 0.1)return { status: "unchanged", reason: "source-duration-mismatch" };- directory = await mkdtemp(join(tmpdir(), "cap-audio-quality-"));+ await ensureTempDir();+ directory = await mkdtemp(join(getTempDir(), "audio-quality-"));let path = join(directory, "candidate.mp4");const renderArgs = ["ffmpeg",diff --git a/apps/media-server/src/lib/auth.ts b/apps/media-server/src/lib/auth.tsindex a8e322d28..9641d67c4 100644--- a/apps/media-server/src/lib/auth.ts+++ b/apps/media-server/src/lib/auth.ts@@ -12,3 +12,17 @@ export function validateMediaServerSecret(c: MediaServerContext): boolean {}return c.req.header("x-media-server-secret") === secret;}++export function hasMediaServerSecret(c: MediaServerContext): boolean {+ return (+ c.req.header("x-media-server-secret") !== undefined &&+ validateMediaServerSecret(c)+ );+}++export function canReadStatus(c: MediaServerContext): boolean {+ const publicStatus = process.env.MEDIA_SERVER_PUBLIC_STATUS?.toLowerCase();+ return publicStatus === "false" || publicStatus === "0"+ ? validateMediaServerSecret(c)+ : true;+}diff --git a/apps/media-server/src/lib/job-manager.ts b/apps/media-server/src/lib/job-manager.tsindex 74caa0dea..5c05238f2 100644--- a/apps/media-server/src/lib/job-manager.ts+++ b/apps/media-server/src/lib/job-manager.ts@@ -124,6 +124,7 @@ export interface Job {webhookLastAttemptAt?: number;recordingVerificationDeadlineAt?: number;recordingProcessingDeadlineAt?: number;+ lifetimeDeadlineAt?: number;recordingVerification?: RecordingVerificationProof;manifestSha256?: string;jobId: string;@@ -147,9 +148,17 @@ export interface Job {}const jobs = new Map<string, Job>();+const jobTotals = {+ created: 0,+ completed: 0,+ errored: 0,+ cancelled: 0,+ expired: 0,+};const JOB_TTL_MS = 60 * 60 * 1000;const STALE_JOB_MS = 15 * 60 * 1000;const MAX_JOB_LIFETIME_MS = 60 * 60 * 1000;+const MAX_EXTENDED_JOB_LIFETIME_MS = 4 * 60 * 60 * 1000;const MAX_RECORDING_PROCESSING_BUDGET_MS = 3 * 60 * 60 * 1000;const WEBHOOK_MAX_ATTEMPTS = 3;const WEBHOOK_RETRY_BASE_MS = 500;@@ -366,9 +375,14 @@ export function createJob(webhookSecret,};jobs.set(jobId, job);+ jobTotals.created++;return job;}+export function getJobTotals(): typeof jobTotals {+ return { ...jobTotals };+}+export function getJob(jobId: string): Job | undefined {return jobs.get(jobId);}@@ -410,12 +424,37 @@ export function updateJob(job.terminalAt ??= job.updatedAt;if (job.recordingWorkerLeaseTimer)clearTimeout(job.recordingWorkerLeaseTimer);+ if (job.phase === "complete") jobTotals.completed++;+ else if (job.phase === "error") jobTotals.errored++;+ else jobTotals.cancelled++;}if (!isActivePhase(job.phase) && job.webhookInFlight)job.webhookPending = true;return job;}+export function extendJobLifetime(+ jobId: string,+ remainingBudgetMs: number,+): number | undefined {+ const job = jobs.get(jobId);+ if (+ !job ||+ !isActivePhase(job.phase) ||+ !Number.isFinite(remainingBudgetMs) ||+ remainingBudgetMs <= 0+ )+ return undefined;+ job.lifetimeDeadlineAt = Math.min(+ job.createdAt + MAX_EXTENDED_JOB_LIFETIME_MS,+ Math.max(+ job.lifetimeDeadlineAt ?? job.createdAt + MAX_JOB_LIFETIME_MS,+ Date.now() + remainingBudgetMs,+ ),+ );+ return job.lifetimeDeadlineAt;+}+export function touchJob(jobId: string): Job | undefined {const job = jobs.get(jobId);if (!job) return undefined;@@ -556,11 +595,14 @@ export function cleanupExpiredJobs(): number {`[job-manager] Cleaning up expired job ${jobId} (phase=${job.phase}, age=${Math.round(age / 60000)}m)`,);job.abortController?.abort();- updateJob(jobId, {- phase: "error",- error: `Job expired: no progress update for ${Math.round(staleness / 60000)} minutes`,- message: "Processing failed (expired)",- });+ if (+ updateJob(jobId, {+ phase: "error",+ error: `Job expired: no progress update for ${Math.round(staleness / 60000)} minutes`,+ message: "Processing failed (expired)",+ })+ )+ jobTotals.expired++;void sendWebhook(job);}cleaned++;@@ -572,34 +614,40 @@ export function cleanupExpiredJobs(): number {`[job-manager] Marking stale job ${jobId} as error (phase=${job.phase}, no update for ${Math.round(staleness / 60000)}m)`,);job.abortController?.abort();- updateJob(jobId, {- phase: "error",- error: `Job stale: no progress update for ${Math.round(staleness / 60000)} minutes`,- message: "Processing failed (stale)",- });+ if (+ updateJob(jobId, {+ phase: "error",+ error: `Job stale: no progress update for ${Math.round(staleness / 60000)} minutes`,+ message: "Processing failed (stale)",+ })+ )+ jobTotals.expired++;void sendWebhook(job);cleaned++;continue;}+ const lifetimeDeadline =+ job.lifetimeDeadlineAt ?? job.createdAt + MAX_JOB_LIFETIME_MS;const deadline =job.recordingVerificationDeadlineAt ??job.recordingProcessingDeadlineAt ??- job.createdAt + MAX_JOB_LIFETIME_MS;+ lifetimeDeadline;if (isActivePhase(job.phase) && now > deadline) {console.warn(`[job-manager] Marking long-running job ${jobId} as error (phase=${job.phase}, age=${Math.round(age / 60000)}m)`,);job.abortController?.abort();- updateJob(jobId, {+ const expired = updateJob(jobId, {phase: "error",error:job.recordingVerificationDeadlineAt === undefined &&job.recordingProcessingDeadlineAt === undefined- ? `Job exceeded maximum lifetime of ${Math.round(MAX_JOB_LIFETIME_MS / 60000)} minutes`+ ? `Job exceeded maximum lifetime of ${Math.round((lifetimeDeadline - job.createdAt) / 60000)} minutes`: "Recording verification timed out",message: "Processing failed (timeout)",});+ if (expired) jobTotals.expired++;void sendWebhook(job);cleaned++;continue;diff --git a/apps/media-server/src/lib/lifecycle.ts b/apps/media-server/src/lib/lifecycle.tsnew file mode 100644index 000000000..d23ff92ff--- /dev/null+++ b/apps/media-server/src/lib/lifecycle.ts@@ -0,0 +1,172 @@+import { getActiveVideoProcessCount } from "./job-manager";+import {+ getActiveAudioOperationCount,+ getActiveProbeOperationCount,+} from "./media-operations";+import { getActiveSubprocessCount } from "./subprocess";++const DRAIN_POLL_INTERVAL_MS = 250;+const DRAIN_LOG_INTERVAL_MS = 30_000;+export const DRAIN_RETRY_AFTER_SECONDS = 15;+export const DRAINING_DETAILS =+ "Server is draining for shutdown, please retry later";++const MAX_IDLE_TIMEOUT_SECONDS = 255;++let draining = false;+let inFlightRequests = 0;+const backgroundTasks = new Set<Promise<unknown>>();++// Bun closes a connection after idleTimeout seconds without I/O, including+// while a handler is still working, and its 10 s default is shorter than the+// AWS ALB keep-alive idle timeout (60 s), which turns reused sockets into 502s.+export function getIdleTimeoutSeconds(): number {+ const value = Number.parseInt(+ process.env.MEDIA_SERVER_IDLE_TIMEOUT_SECONDS ?? "",+ 10,+ );+ return Number.isFinite(value) && value >= 0+ ? Math.min(MAX_IDLE_TIMEOUT_SECONDS, value)+ : MAX_IDLE_TIMEOUT_SECONDS;+}++type IdleTimeoutServer = {+ timeout?: (request: Request, seconds: number) => void;+};++export function setRequestIdleTimeout(+ server: unknown,+ request: Request,+ seconds: number,+): void {+ const timeout = (server as IdleTimeoutServer | undefined)?.timeout;+ if (typeof timeout !== "function") return;+ try {+ timeout.call(server, request, seconds);+ } catch {}+}++export function getShutdownGraceMs(): number {+ const value = Number.parseInt(+ process.env.MEDIA_SERVER_SHUTDOWN_GRACE_MS ?? "0",+ 10,+ );+ return Number.isFinite(value) && value > 0 ? value : 0;+}++export function isDraining(): boolean {+ return draining;+}++export function setDraining(value: boolean): void {+ draining = value;+}++export function trackBackgroundTask<T>(task: Promise<T>): Promise<T> {+ backgroundTasks.add(task);+ const settle = () => {+ backgroundTasks.delete(task);+ };+ task.then(settle, settle);+ return task;+}++export function beginRequest(): () => void {+ inFlightRequests++;+ let ended = false;+ return () => {+ if (ended) return;+ ended = true;+ inFlightRequests--;+ };+}++export function getDrainBlockers() {+ return {+ activeVideoProcesses: getActiveVideoProcessCount(),+ backgroundTasks: backgroundTasks.size,+ inFlightRequests,+ audioOperations: getActiveAudioOperationCount(),+ probeOperations: getActiveProbeOperationCount(),+ subprocesses: getActiveSubprocessCount(),+ };+}++export function isIdle(): boolean {+ return Object.values(getDrainBlockers()).every((count) => count === 0);+}++function wait(ms: number, signal?: AbortSignal): Promise<void> {+ return new Promise((resolve) => {+ const done = () => {+ clearTimeout(timer);+ signal?.removeEventListener("abort", done);+ resolve();+ };+ const timer = setTimeout(done, ms);+ signal?.addEventListener("abort", done, { once: true });+ });+}++export async function waitForDrain(+ graceMs: number,+ signal?: AbortSignal,+): Promise<boolean> {+ const startedAt = Date.now();+ const deadline = startedAt + graceMs;+ let lastLogAt = startedAt;+ while (!isIdle()) {+ const now = Date.now();+ if (signal?.aborted || now >= deadline) return false;+ if (now - lastLogAt >= DRAIN_LOG_INTERVAL_MS) {+ lastLogAt = now;+ console.log("[media-server] Draining", {+ remainingMs: deadline - now,+ ...getDrainBlockers(),+ });+ }+ await wait(Math.min(DRAIN_POLL_INTERVAL_MS, deadline - now), signal);+ }+ return true;+}++export function createShutdownHandler(+ finalize: () => Promise<void>,+ graceMs = getShutdownGraceMs,+): (signal: string) => Promise<void> {+ let shuttingDown = false;+ let forceDrain: AbortController | undefined;++ return async (signal) => {+ if (forceDrain) {+ console.log(+ `[media-server] Received ${signal} while draining, shutting down now`,+ );+ forceDrain.abort();+ return;+ }+ if (shuttingDown) return;+ shuttingDown = true;++ const drainMs = graceMs();+ if (drainMs > 0) {+ forceDrain = new AbortController();+ setDraining(true);+ console.log(+ `[media-server] Received ${signal}, draining for up to ${drainMs}ms`,+ getDrainBlockers(),+ );+ const drained = await waitForDrain(drainMs, forceDrain.signal);+ console.log(+ drained+ ? "[media-server] Drain complete"+ : "[media-server] Drain ended with work still active",+ getDrainBlockers(),+ );+ } else {+ console.log("[media-server] Shutting down...");+ }++ await finalize();+ };+}diff --git a/apps/media-server/src/lib/media-edit.ts b/apps/media-server/src/lib/media-edit.tsindex 406dd7034..91a9f0483 100644--- a/apps/media-server/src/lib/media-edit.ts+++ b/apps/media-server/src/lib/media-edit.ts@@ -46,7 +46,7 @@ function getTotalRangeDuration(ranges: EditRange[]) {return ranges.reduce((total, range) => total + getRangeDuration(range), 0);}-function getTimeoutMs(ranges: EditRange[]) {+export function getTimeoutMs(ranges: EditRange[]) {const durationMs = getTotalRangeDuration(ranges) * 20_000;return Math.min(MAX_TIMEOUT_MS, Math.max(DEFAULT_TIMEOUT_MS, durationMs));}diff --git a/apps/media-server/src/lib/media-operations.ts b/apps/media-server/src/lib/media-operations.tsindex 1c9901dd8..c1af41209 100644--- a/apps/media-server/src/lib/media-operations.ts+++ b/apps/media-server/src/lib/media-operations.ts@@ -30,6 +30,14 @@ export function getActiveAudioOperationCount(): number {return count;}+export function getMaxConcurrentAudioOperations(): number {+ return MAX_CONCURRENT_AUDIO_OPERATIONS;+}++export function getMaxConcurrentProbeOperations(): number {+ return MAX_CONCURRENT_PROBE_OPERATIONS;+}+export function canAcceptNewAudioOperation(): boolean {return (!hasCriticalMemoryPressure() &&diff --git a/apps/media-server/src/lib/media-transfer.ts b/apps/media-server/src/lib/media-transfer.tsindex 627207793..e78829a12 100644--- a/apps/media-server/src/lib/media-transfer.ts+++ b/apps/media-server/src/lib/media-transfer.ts@@ -1,9 +1,15 @@import { AsyncLocalStorage } from "node:async_hooks";import { createHash } from "node:crypto";-import { mkdir, open, readdir, rm, stat } from "node:fs/promises";+import { open, rm, stat } from "node:fs/promises";import { tmpdir } from "node:os";import { join } from "node:path";import { z } from "zod";+import {+ ensureInstanceDir,+ getInstanceDir,+ releaseInstanceDir,+ sweepOrphanedInstanceDirs,+} from "./temp-files";const targetSchema = z.object({version: z.literal(1),@@ -36,46 +42,58 @@ const transfers = new AsyncLocalStorage<TransferContext>();const cache = new Map<string, CachedFile>();const pendingTransfers = new Map<string, Promise<void>>();const cacheBase = join(tmpdir(), "cap-media-download-cache");-const cacheRoot = join(cacheBase, String(process.pid));+const cacheRoot = getInstanceDir(cacheBase);const CACHE_BYTES = 2 * 1024 ** 3;const CACHE_TTL_MS = 30 * 60_000;+const LEGACY_CACHE_ORPHAN_AGE_MS = 60 * 60_000;let cacheBytes = 0;let lastPruneAt = 0;-async function initializeCache() {- await mkdir(cacheRoot, { recursive: true, mode: 0o700 });- for (const entry of await readdir(cacheBase, { withFileTypes: true })) {- if (- !entry.isDirectory() ||- !/^\d+$/.test(entry.name) ||- entry.name === String(process.pid)- )- continue;- try {- process.kill(Number(entry.name), 0);- } catch (error) {- if (- typeof error === "object" &&- error !== null &&- "code" in error &&- error.code === "ESRCH"- )- await rm(join(cacheBase, entry.name), { recursive: true, force: true });- }+function isLegacyCacheOrphan(name: string, stats: { mtimeMs: number }) {+ if (!/^\d+$/.test(name)) return false;+ if (Date.now() - stats.mtimeMs > LEGACY_CACHE_ORPHAN_AGE_MS) return true;+ if (name === String(process.pid)) return false;+ try {+ process.kill(Number(name), 0);+ return false;+ } catch (error) {+ return (+ typeof error === "object" &&+ error !== null &&+ "code" in error &&+ error.code === "ESRCH"+ );}}++export function sweepMediaTransferCache(): Promise<number> {+ return sweepOrphanedInstanceDirs(cacheBase, isLegacyCacheOrphan);+}++async function initializeCache() {+ await ensureInstanceDir(cacheBase, 0o700);+ await sweepMediaTransferCache();+}let cacheReady: Promise<void> | undefined;const pruneTimer = setInterval(() => {void pruneCache().catch(() => {});}, 60_000);pruneTimer.unref?.();+const sweepTimer = setInterval(() => {+ void sweepMediaTransferCache().catch(() => {});+}, 5 * 60_000);+sweepTimer.unref?.();export async function cleanupMediaTransferCache() {- await rm(cacheRoot, { recursive: true, force: true }).catch(() => {});+ await releaseInstanceDir(cacheBase);cache.clear();cacheBytes = 0;cacheReady = undefined;}+export function getMediaTransferCacheUsage() {+ return { bytes: cacheBytes, entries: cache.size };+}+export function isInternalMediaUrl(input: string): boolean {try {const url = new URL(input);diff --git a/apps/media-server/src/lib/media-video.ts b/apps/media-server/src/lib/media-video.tsindex 3b74caa1c..feecf5830 100644--- a/apps/media-server/src/lib/media-video.ts+++ b/apps/media-server/src/lib/media-video.ts@@ -1297,15 +1297,18 @@ export async function downloadVideoToTemp(const reader = response.body.getReader();const writer = file(tempFile.path).writer();+ let writerEnded = false;try {while (true) {const { done, value } = await reader.read();if (done) break;writer.write(value);}+ writerEnded = true;await writer.end();} finally {reader.releaseLock();+ if (!writerEnded) await Promise.resolve(writer.end()).catch(() => {});}if (file(tempFile.path).size === 0) {@@ -1339,7 +1342,7 @@ function needsAudioTranscode(metadata: VideoMetadata): boolean {return Boolean(metadata.audioCodec && metadata.audioCodec !== "aac");}-function getProcessTimeoutMs(+export function getProcessTimeoutMs(durationSeconds: number,baseTimeoutMs: number,): number {diff --git a/apps/media-server/src/lib/runtime-report.ts b/apps/media-server/src/lib/runtime-report.tsnew file mode 100644index 000000000..bfe0bace9--- /dev/null+++ b/apps/media-server/src/lib/runtime-report.ts@@ -0,0 +1,60 @@+import {+ getActiveVideoProcessCount,+ getAllJobs,+ getJobTotals,+ getMaxConcurrentVideoProcesses,+ getSystemResources,+} from "./job-manager";+import { getDrainBlockers, getShutdownGraceMs, isDraining } from "./lifecycle";+import {+ getMaxConcurrentAudioOperations,+ getMaxConcurrentProbeOperations,+} from "./media-operations";+import { getMediaTransferCacheUsage } from "./media-transfer";+import { getTempDirUsage } from "./temp-files";+import {+ getActiveDirectVideoProcessCount,+ getMaxConcurrentDirectVideoProcesses,+} from "./video-capacity";++export async function getRuntimeReport() {+ const blockers = getDrainBlockers();+ const jobs = getAllJobs();+ return {+ draining: isDraining(),+ shutdownGraceMs: getShutdownGraceMs(),+ processUptimeSeconds: Math.round(process.uptime()),+ activeJobs: jobs.filter(+ (job) =>+ job.phase !== "complete" &&+ job.phase !== "error" &&+ job.phase !== "cancelled",+ ).length,+ jobsInMemory: jobs.length,+ jobTotals: getJobTotals(),+ activeSubprocesses: blockers.subprocesses,+ backgroundTasks: blockers.backgroundTasks,+ inFlightRequests: blockers.inFlightRequests,+ slots: {+ video: {+ active: getActiveVideoProcessCount(),+ max: getMaxConcurrentVideoProcesses(),+ effectiveMax: getSystemResources().effectiveMax,+ },+ directVideo: {+ active: getActiveDirectVideoProcessCount(),+ max: getMaxConcurrentDirectVideoProcesses(),+ },+ audio: {+ active: blockers.audioOperations,+ max: getMaxConcurrentAudioOperations(),+ },+ probe: {+ active: blockers.probeOperations,+ max: getMaxConcurrentProbeOperations(),+ },+ },+ tempDir: await getTempDirUsage(),+ mediaTransferCache: getMediaTransferCacheUsage(),+ };+}diff --git a/apps/media-server/src/lib/subprocess.ts b/apps/media-server/src/lib/subprocess.tsindex 31ac5ee4e..ffdf7c3f6 100644--- a/apps/media-server/src/lib/subprocess.ts+++ b/apps/media-server/src/lib/subprocess.ts@@ -19,6 +19,10 @@ export function unregisterSubprocess(proc: ManagedSubprocess): void {activeSubprocesses.delete(proc.pid);}+export function getActiveSubprocessCount(): number {+ return activeSubprocesses.size;+}+async function waitForProcessExit(proc: Pick<Subprocess, "exited" | "exitCode">,timeoutMs: number,diff --git a/apps/media-server/src/lib/temp-files.ts b/apps/media-server/src/lib/temp-files.tsindex 5141ad6c4..0c8ec1fdf 100644--- a/apps/media-server/src/lib/temp-files.ts+++ b/apps/media-server/src/lib/temp-files.ts@@ -1,18 +1,139 @@import { randomUUID } from "node:crypto";-import { mkdir, readdir, rm, stat, unlink } from "node:fs/promises";+import type { Dirent, Stats } from "node:fs";+import {+ lstat,+ mkdir,+ readdir,+ rm,+ unlink,+ utimes,+ writeFile,+} from "node:fs/promises";import { tmpdir } from "node:os";import { join } from "node:path";-const TEMP_DIR = join(tmpdir(), "cap-media-server");+const TEMP_ROOT = join(tmpdir(), "cap-media-server");+const INSTANCE_ID = randomUUID();+const INSTANCE_DIR_PREFIX = "instance-";+const INSTANCE_LEASE_FILE = ".lease";+const INSTANCE_LEASE_REFRESH_MS = 30_000;+const INSTANCE_LEASE_STALE_MS = 5 * 60 * 1000;const STALE_FILE_AGE_MS = 60 * 60 * 1000;+const MAX_ACTIVE_TEMP_AGE_MS = 6 * 60 * 60 * 1000;+const TEMP_MAINTENANCE_INTERVAL_MS = 5 * 60 * 1000;+const TEMP_USAGE_CACHE_MS = 5_000;+const TEMP_USAGE_MAX_ENTRIES = 20_000;++const leasedDirs = new Set<string>();+const activeTempPaths = new Set<string>();+let leaseTimer: ReturnType<typeof setInterval> | undefined;+let maintenanceTimer: ReturnType<typeof setInterval> | undefined;export interface TempFileHandle {path: string;cleanup: () => Promise<void>;}+export interface TempDirUsage {+ bytes: number;+ entries: number;+ truncated: boolean;+}++export function getInstanceDir(base: string): string {+ return join(base, `${INSTANCE_DIR_PREFIX}${INSTANCE_ID}`);+}++const TEMP_DIR = getInstanceDir(TEMP_ROOT);++async function refreshLeases(): Promise<void> {+ const now = new Date();+ await Promise.allSettled(+ Array.from(leasedDirs, async (dir) => {+ const lease = join(dir, INSTANCE_LEASE_FILE);+ try {+ await utimes(lease, now, now);+ } catch {+ await mkdir(dir, { recursive: true });+ await writeFile(lease, "");+ }+ }),+ );+}++export async function ensureInstanceDir(+ base: string,+ mode?: number,+): Promise<string> {+ const dir = getInstanceDir(base);+ await mkdir(dir, { recursive: true, mode });+ if (!leasedDirs.has(dir)) {+ await writeFile(join(dir, INSTANCE_LEASE_FILE), "");+ leasedDirs.add(dir);+ if (!leaseTimer) {+ leaseTimer = setInterval(() => {+ void refreshLeases();+ }, INSTANCE_LEASE_REFRESH_MS);+ leaseTimer.unref?.();+ }+ }+ return dir;+}++export async function releaseInstanceDir(base: string): Promise<void> {+ const dir = getInstanceDir(base);+ leasedDirs.delete(dir);+ await rm(dir, { recursive: true, force: true }).catch(() => {});+}++// Instance directories carry a lease so a host that shares one temp volume+// between an old (draining) and a new container never sweeps live work.+export async function sweepOrphanedInstanceDirs(+ base: string,+ isLegacyOrphan: (name: string, stats: Stats) => boolean,+): Promise<number> {+ let entries: Dirent[];+ try {+ entries = await readdir(base, { withFileTypes: true });+ } catch {+ return 0;+ }+ const ownDir = getInstanceDir(base);+ const now = Date.now();+ let removed = 0;++ for (const entry of entries) {+ const path = join(base, entry.name);+ if (path === ownDir) continue;+ try {+ const orphaned =+ entry.isDirectory() && entry.name.startsWith(INSTANCE_DIR_PREFIX)+ ? now -+ (+ await lstat(join(path, INSTANCE_LEASE_FILE)).catch(() =>+ lstat(path),+ )+ ).mtimeMs >+ INSTANCE_LEASE_STALE_MS+ : isLegacyOrphan(entry.name, await lstat(path));+ if (!orphaned) continue;+ await rm(path, { recursive: true, force: true });+ removed++;+ } catch {}+ }++ return removed;+}+export async function ensureTempDir(): Promise<void> {- await mkdir(TEMP_DIR, { recursive: true });+ await ensureInstanceDir(TEMP_ROOT);+}++export function retainTempPath(path: string): () => void {+ activeTempPaths.add(path);+ return () => {+ activeTempPaths.delete(path);+ };}export async function createTempFile(@@ -21,6 +142,7 @@ export async function createTempFile(await ensureTempDir();const filename = `${randomUUID()}${extension.startsWith(".") ? extension : `.${extension}`}`;const path = join(TEMP_DIR, filename);+ const release = retainTempPath(path);return {path,@@ -28,32 +150,116 @@ export async function createTempFile(try {await unlink(path);} catch {}+ release();},};}+export function sweepOrphanedTempDirs(): Promise<number> {+ return sweepOrphanedInstanceDirs(+ TEMP_ROOT,+ (_name, stats) => Date.now() - stats.mtimeMs > STALE_FILE_AGE_MS,+ );+}+export async function cleanupStaleTempFiles(): Promise<number> {+ let cleaned = await sweepOrphanedTempDirs();+ let entries: string[];try {- await ensureTempDir();- const files = await readdir(TEMP_DIR);- const now = Date.now();- let cleaned = 0;+ entries = await readdir(TEMP_DIR);+ } catch {+ return cleaned;+ }+ const now = Date.now();- for (const file of files) {- const filePath = join(TEMP_DIR, file);- try {- const fileStat = await stat(filePath);- if (now - fileStat.mtimeMs > STALE_FILE_AGE_MS) {- await rm(filePath, { force: true });- cleaned++;- }- } catch {}- }+ for (const entry of entries) {+ if (entry === INSTANCE_LEASE_FILE) continue;+ const path = join(TEMP_DIR, entry);+ try {+ const age = now - (await lstat(path)).mtimeMs;+ if (+ age <= STALE_FILE_AGE_MS ||+ (activeTempPaths.has(path) && age <= MAX_ACTIVE_TEMP_AGE_MS)+ )+ continue;+ await rm(path, { recursive: true, force: true });+ activeTempPaths.delete(path);+ cleaned++;+ } catch {}+ }- return cleaned;+ return cleaned;+}++export async function startTempDirMaintenance(): Promise<number> {+ if (!maintenanceTimer) {+ maintenanceTimer = setInterval(() => {+ void cleanupStaleTempFiles()+ .then((cleaned) => {+ if (cleaned > 0)+ console.log(`[temp-files] Removed ${cleaned} stale temp entries`);+ })+ .catch(() => {});+ }, TEMP_MAINTENANCE_INTERVAL_MS);+ maintenanceTimer.unref?.();+ }+ return await sweepOrphanedTempDirs();+}++export async function removeOwnTempDir(): Promise<void> {+ activeTempPaths.clear();+ await releaseInstanceDir(TEMP_ROOT);+}++async function measureDir(+ dir: string,+ usage: TempDirUsage,+): Promise<TempDirUsage> {+ let entries: Dirent[];+ try {+ entries = await readdir(dir, { withFileTypes: true });} catch {- return 0;+ return usage;+ }+ for (const entry of entries) {+ if (usage.entries >= TEMP_USAGE_MAX_ENTRIES) {+ usage.truncated = true;+ return usage;+ }+ usage.entries++;+ const path = join(dir, entry.name);+ if (entry.isDirectory()) {+ await measureDir(path, usage);+ } else {+ usage.bytes += await lstat(path).then(+ (stats) => stats.size,+ () => 0,+ );+ }}+ return usage;+}++let tempUsageCache:+ | { measuredAt: number; usage: Promise<TempDirUsage> }+ | undefined;++export function getTempDirUsage(): Promise<TempDirUsage> {+ const now = Date.now();+ if (+ !tempUsageCache ||+ now - tempUsageCache.measuredAt > TEMP_USAGE_CACHE_MS+ ) {+ tempUsageCache = {+ measuredAt: now,+ usage: measureDir(TEMP_ROOT, { bytes: 0, entries: 0, truncated: false }),+ };+ }+ return tempUsageCache.usage;+}++export function getTempRoot(): string {+ return TEMP_ROOT;}export function getTempDir(): string {diff --git a/apps/media-server/src/routes/audio.ts b/apps/media-server/src/routes/audio.tsindex d2f8cc71e..adf57bc54 100644--- a/apps/media-server/src/routes/audio.ts+++ b/apps/media-server/src/routes/audio.ts@@ -1,6 +1,12 @@-import { Hono } from "hono";+import { type Context, Hono } from "hono";import { z } from "zod";-import { validateMediaServerSecret } from "../lib/auth";+import { canReadStatus, validateMediaServerSecret } from "../lib/auth";+import {+ DRAIN_RETRY_AFTER_SECONDS,+ DRAINING_DETAILS,+ isDraining,+ setRequestIdleTimeout,+} from "../lib/lifecycle";import {canAcceptNewAudioOperation as canAcceptNewProcess,checkHasAudioTrack,@@ -35,7 +41,22 @@ function isTimeoutError(err: unknown): boolean {return err instanceof Error && err.message.includes("timed out");}+function drainingResponse(c: Context) {+ c.header("Retry-After", DRAIN_RETRY_AFTER_SECONDS.toString());+ return c.json(+ {+ error: "Server is busy",+ code: "SERVER_BUSY",+ details: DRAINING_DETAILS,+ },+ 503,+ );+}+audio.get("/status", (c) => {+ if (!canReadStatus(c)) {+ return c.json({ error: "Unauthorized" }, 401);+ }return c.json({activeProcesses: getActiveProcessCount(),canAcceptNewProcess: canAcceptNewProcess(),@@ -61,6 +82,8 @@ audio.post("/check", async (c) => {);}+ if (isDraining()) return drainingResponse(c);+try {const hasAudio = await checkHasAudioTrack(result.data.videoUrl);return c.json({ hasAudio });@@ -120,6 +143,8 @@ audio.post("/extract", async (c) => {}const { videoUrl, stream: useStreaming } = result.data;+ if (isDraining()) return drainingResponse(c);+ setRequestIdleTimeout(c.env, c.req.raw, 0);try {const hasAudio = await checkHasAudioTrack(videoUrl);@@ -209,6 +234,8 @@ audio.post("/convert", async (c) => {}const { audioUrl, bitrate } = result.data;+ if (isDraining()) return drainingResponse(c);+ setRequestIdleTimeout(c.env, c.req.raw, 0);try {const { stream, cleanup } = extractAudioStream(audioUrl, {diff --git a/apps/media-server/src/routes/health.ts b/apps/media-server/src/routes/health.tsindex 6a69c7ef2..eed6478b5 100644--- a/apps/media-server/src/routes/health.ts+++ b/apps/media-server/src/routes/health.ts@@ -1,12 +1,15 @@import os from "node:os";import { Hono } from "hono";+import { hasMediaServerSecret } from "../lib/auth";import { getSystemResources } from "../lib/job-manager";+import { isDraining } from "../lib/lifecycle";import { getMediaEngineStatus } from "../lib/media-engine";import { getVideoEncoderStatus } from "../lib/nvenc-encoder";+import { getRuntimeReport } from "../lib/runtime-report";const health = new Hono();-health.get("/", (c) => {+health.get("/", async (c) => {const mediaEngine = getMediaEngineStatus();const [, loadAvg5m, loadAvg15m] = os.loadavg();const totalMemoryBytes = os.totalmem();@@ -17,27 +20,35 @@ health.get("/", (c) => {(1 - freeMemoryBytes / totalMemoryBytes) * 100,);const resources = getSystemResources();+ const draining = isDraining();- return c.json({- status:- mediaEngine.available && resources.effectiveMax > 0 ? "ok" : "degraded",- mediaEngine,- ["ff" + "mpeg"]: {- available: mediaEngine.available,- version: mediaEngine.version,+ return c.json(+ {+ status: draining+ ? "draining"+ : mediaEngine.available && resources.effectiveMax > 0+ ? "ok"+ : "degraded",+ mediaEngine,+ ["ff" + "mpeg"]: {+ available: mediaEngine.available,+ version: mediaEngine.version,+ },+ videoEncoder: getVideoEncoderStatus(),+ system: {+ ...resources,+ loadAvg5m,+ loadAvg15m,+ totalMemoryMB,+ freeMemoryMB,+ memoryUsagePercent,+ uptimeSeconds: Math.round(os.uptime()),+ processUptimeSeconds: Math.round(process.uptime()),+ },+ ...(hasMediaServerSecret(c) && { runtime: await getRuntimeReport() }),},- videoEncoder: getVideoEncoderStatus(),- system: {- ...resources,- loadAvg5m,- loadAvg15m,- totalMemoryMB,- freeMemoryMB,- memoryUsagePercent,- uptimeSeconds: Math.round(os.uptime()),- processUptimeSeconds: Math.round(process.uptime()),- },- });+ draining ? 503 : 200,+ );});export default health;diff --git a/apps/media-server/src/routes/video.ts b/apps/media-server/src/routes/video.tsindex cd2b44e68..b194bc594 100644--- a/apps/media-server/src/routes/video.ts+++ b/apps/media-server/src/routes/video.ts@@ -4,7 +4,11 @@ import { file } from "bun";import { Hono } from "hono";import { z } from "zod";import { enhanceLocalRecording } from "../lib/audio-levels";-import { validateMediaServerSecret } from "../lib/auth";+import {+ canReadStatus,+ hasMediaServerSecret,+ validateMediaServerSecret,+} from "../lib/auth";import type {Job,RecordingErrorCode,@@ -17,6 +21,7 @@ import {claimRecordingWorker,createJob,deleteJob,+ extendJobLifetime,forceCleanupActiveJobs,generateJobId,getActiveVideoProcessCount,@@ -30,8 +35,18 @@ import {touchJob,updateJob,} from "../lib/job-manager";-import { PROCESS_TIMEOUT_MS } from "../lib/media-common";-import { renderEditedVideo } from "../lib/media-edit";+import {+ DRAINING_DETAILS,+ getIdleTimeoutSeconds,+ isDraining,+ setRequestIdleTimeout,+ trackBackgroundTask,+} from "../lib/lifecycle";+import { PROCESS_TIMEOUT_MS, UPLOAD_TIMEOUT_MS } from "../lib/media-common";+import {+ getTimeoutMs as getEditRenderTimeoutMs,+ renderEditedVideo,+} from "../lib/media-edit";import {canAcceptNewProbeOperation as canAcceptNewProbeProcess,getActiveProbeOperationCount as getActiveProbeProcessCount,@@ -54,6 +69,7 @@ import {downloadVideoToTemp,generatePreviewGif,generateThumbnail,+ getProcessTimeoutMs,muxMediaTracksToMp4,processVideo,repairContainer,@@ -69,6 +85,7 @@ import {verifyRemoteRecordingBytes,verifyRemuxedRecording,} from "../lib/recording-verification";+import { getRuntimeReport } from "../lib/runtime-report";import type { TempFileHandle } from "../lib/temp-files";import { cleanupStaleTempFiles } from "../lib/temp-files";import {@@ -93,6 +110,8 @@ const VIDEO_MEMORY_PRESSURE_ERROR ="SERVER_BUSY: container memory pressure is too high";const SEGMENT_DOWNLOAD_MAX_ATTEMPTS = 3;const SEGMENT_DOWNLOAD_RETRY_BASE_MS = 250;+const JOB_TRANSFER_ALLOWANCE_MS = 2 * UPLOAD_TIMEOUT_MS;+const JOB_TRANSFER_BYTES_PER_SECOND = 2 * 1024 ** 2;const probeSchema = z.object({videoUrl: z.string().url(),@@ -220,6 +239,12 @@ function getMuxBusyResponseBody(};}+const DRAINING_RESPONSE_BODY = {+ error: "Server is busy",+ code: "SERVER_BUSY",+ details: DRAINING_DETAILS,+};+function summarizeVideoInput(videoUrl: string, inputExtension?: string) {const extension = inputExtension?.toLowerCase() ?? null;try {@@ -286,25 +311,26 @@ async function createVideoDownloadResponse(onCleanup?.();};+ const reader = outputFile.stream().getReader();const stream = new ReadableStream<Uint8Array>({- async start(controller) {- const reader = outputFile.stream().getReader();-+ async pull(controller) {try {- while (true) {- const { done, value } = await reader.read();- if (done) break;- if (value) controller.enqueue(value);+ const { done, value } = await reader.read();+ if (done) {+ controller.close();+ reader.releaseLock();+ await cleanup();+ } else if (value) {+ controller.enqueue(value);}- controller.close();} catch (error) {controller.error(error);- } finally {reader.releaseLock();await cleanup();}},async cancel() {+ await reader.cancel().catch(() => {});await cleanup();},});@@ -345,6 +371,23 @@ async function withJobHeartbeat<T>(}}+function extendJobLifetimeForWork(+ jobId: string,+ workTimeoutMs: number,+ transferBytes: number,+): void {+ extendJobLifetime(+ jobId,+ workTimeoutMs ++ JOB_TRANSFER_ALLOWANCE_MS ++ POST_VERIFICATION_ASSET_BUDGET_MS ++ Math.ceil(+ (Math.max(0, transferBytes || 0) / JOB_TRANSFER_BYTES_PER_SECOND) *+ 1000,+ ),+ );+}+async function withMuxMemoryGuard<T>(abortController: AbortController,operation: () => Promise<T>,@@ -373,7 +416,10 @@ async function withMuxMemoryGuard<T>(}}-video.get("/status", (c) => {+video.get("/status", async (c) => {+ if (!canReadStatus(c)) {+ return c.json({ error: "Unauthorized" }, 401);+ }const jobs = getAllJobs();const resources = getSystemResources();const now = Date.now();@@ -401,6 +447,7 @@ video.get("/status", (c) => {stalenessMinutes: Math.round((now - j.updatedAt) / 60000),error: j.error,})),+ ...(hasMediaServerSecret(c) && { runtime: await getRuntimeReport() }),});});@@ -423,6 +470,11 @@ video.post("/probe", async (c) => {);}+ if (isDraining()) {+ c.header("Retry-After", VIDEO_BUSY_RETRY_AFTER_SECONDS.toString());+ return c.json(DRAINING_RESPONSE_BODY, 503);+ }+try {const metadata = await probeVideo(result.data.videoUrl);return c.json({ metadata });@@ -481,6 +533,11 @@ video.post("/thumbnail", async (c) => {);}+ if (isDraining()) {+ c.header("Retry-After", VIDEO_BUSY_RETRY_AFTER_SECONDS.toString());+ return c.json(DRAINING_RESPONSE_BODY, 503);+ }+try {const metadata = await probeVideo(result.data.videoUrl);@@ -562,6 +619,17 @@ video.post("/convert", async (c) => {const requestId = crypto.randomUUID();const startedAt = Date.now();+ if (isDraining()) {+ c.header("Retry-After", VIDEO_BUSY_RETRY_AFTER_SECONDS.toString());+ return c.json(+ {+ ...getBusyResponseBody(getVideoCapacitySnapshot()),+ details: DRAINING_DETAILS,+ },+ 503,+ );+ }+try {slot = tryAcquireDirectVideoProcessSlot(canAcceptNewVideoProcess);const capacity = getVideoCapacitySnapshot();@@ -583,6 +651,7 @@ video.post("/convert", async (c) => {),...capacity,});+ setRequestIdleTimeout(c.env, c.req.raw, 0);inputTempFile = await downloadVideoToTemp(result.data.videoUrl,@@ -618,12 +687,14 @@ video.post("/convert", async (c) => {...getVideoCapacitySnapshot(),});+ setRequestIdleTimeout(c.env, c.req.raw, getIdleTimeoutSeconds());return await createVideoDownloadResponse(outputTempFile,[inputTempFile, outputTempFile],slot.release,);} catch (err) {+ setRequestIdleTimeout(c.env, c.req.raw, getIdleTimeoutSeconds());slot?.release();await cleanupTempFiles([outputTempFile, inputTempFile]);logVideoEvent("video_convert_failed", {@@ -692,6 +763,13 @@ video.on("POST", ["/process", "/import"], async (c) => {}const capacity = getVideoCapacitySnapshot();+ if (isDraining()) {+ c.header("Retry-After", VIDEO_BUSY_RETRY_AFTER_SECONDS.toString());+ return c.json(+ { ...getBusyResponseBody(capacity), details: DRAINING_DETAILS },+ 503,+ );+ }const bulkCapacityReached =result.data.priority === "bulk" &&capacity.effectiveMaxVideoProcesses > 1 &&@@ -715,34 +793,36 @@ video.on("POST", ["/process", "/import"], async (c) => {const jobId = generateJobId();const job = createJob(jobId, videoId, userId, webhookUrl, webhookSecret);- withMediaTransfers(32 * 1024 ** 3, () =>- processVideoAsync(- job.jobId,- videoUrl,- outputPresignedUrl,- thumbnailPresignedUrl,- previewGifPresignedUrl,- result.data,- ),- ).catch((err) => {- console.error(- `[video/process] Async processing error for job ${jobId}:`,- err,- );- const currentJob = getJob(jobId);- if (- currentJob &&- currentJob.phase !== "error" &&- currentJob.phase !== "complete" &&- currentJob.phase !== "cancelled"- ) {- updateJob(jobId, {- phase: "error",- error: err instanceof Error ? err.message : String(err),- message: "Processing failed (unhandled)",- });- }- });+ void trackBackgroundTask(+ withMediaTransfers(32 * 1024 ** 3, () =>+ processVideoAsync(+ job.jobId,+ videoUrl,+ outputPresignedUrl,+ thumbnailPresignedUrl,+ previewGifPresignedUrl,+ result.data,+ ),+ ).catch((err) => {+ console.error(+ `[video/process] Async processing error for job ${jobId}:`,+ err,+ );+ const currentJob = getJob(jobId);+ if (+ currentJob &&+ currentJob.phase !== "error" &&+ currentJob.phase !== "complete" &&+ currentJob.phase !== "cancelled"+ ) {+ updateJob(jobId, {+ phase: "error",+ error: err instanceof Error ? err.message : String(err),+ message: "Processing failed (unhandled)",+ });+ }+ }),+ );return c.json({jobId,@@ -775,6 +855,17 @@ video.post("/edit", async (c) => {);}+ if (isDraining()) {+ c.header("Retry-After", VIDEO_BUSY_RETRY_AFTER_SECONDS.toString());+ return c.json(+ {+ ...getBusyResponseBody(getVideoCapacitySnapshot()),+ details: DRAINING_DETAILS,+ },+ 503,+ );+ }+if (!canAcceptNewVideoProcess()) {c.header("Retry-After", VIDEO_BUSY_RETRY_AFTER_SECONDS.toString());return c.json(getBusyResponseBody(getVideoCapacitySnapshot()), 503);@@ -794,31 +885,33 @@ video.post("/edit", async (c) => {const jobId = generateJobId();const job = createJob(jobId, videoId, userId, webhookUrl, webhookSecret);- withMediaTransfers(32 * 1024 ** 3, () =>- editVideoAsync(- job.jobId,- sourceUrl,- outputPresignedUrl,- thumbnailPresignedUrl,- previewGifPresignedUrl,- result.data,- ),- ).catch((err) => {- console.error(`[video/edit] Async edit error for job ${jobId}:`, err);- const currentJob = getJob(jobId);- if (- currentJob &&- currentJob.phase !== "error" &&- currentJob.phase !== "complete" &&- currentJob.phase !== "cancelled"- ) {- updateJob(jobId, {- phase: "error",- error: err instanceof Error ? err.message : String(err),- message: "Edit failed (unhandled)",- });- }- });+ void trackBackgroundTask(+ withMediaTransfers(32 * 1024 ** 3, () =>+ editVideoAsync(+ job.jobId,+ sourceUrl,+ outputPresignedUrl,+ thumbnailPresignedUrl,+ previewGifPresignedUrl,+ result.data,+ ),+ ).catch((err) => {+ console.error(`[video/edit] Async edit error for job ${jobId}:`, err);+ const currentJob = getJob(jobId);+ if (+ currentJob &&+ currentJob.phase !== "error" &&+ currentJob.phase !== "complete" &&+ currentJob.phase !== "cancelled"+ ) {+ updateJob(jobId, {+ phase: "error",+ error: err instanceof Error ? err.message : String(err),+ message: "Edit failed (unhandled)",+ });+ }+ }),+ );return c.json({jobId,@@ -963,6 +1056,11 @@ async function processWithResilientRetry(progress: 10,message: "Retrying with error recovery...",});+ extendJobLifetimeForWork(+ jobId,+ getProcessTimeoutMs(metadata.duration, PROCESS_TIMEOUT_MS),+ metadata.fileSize,+ );try {const outputFile = await processVideo(@@ -988,6 +1086,11 @@ async function processWithResilientRetry(progress: 10,message: "Attempting full repair...",});+ extendJobLifetimeForWork(+ jobId,+ getProcessTimeoutMs(metadata.duration, PROCESS_TIMEOUT_MS),+ metadata.fileSize,+ );let lastResortRepairFile: TempFileHandle | null = null;try {@@ -1075,21 +1178,25 @@ function isDurationClose(actual: number, expected: number) {);}-async function waitForVerificationRetry(attempt: number) {- await new Promise((resolve) =>- setTimeout(resolve, UPLOAD_VERIFICATION_RETRY_MS * (attempt + 1)),- );+async function waitForVerificationRetry(+ attempt: number,+ abortSignal?: AbortSignal,+) {+ await Bun.sleep(UPLOAD_VERIFICATION_RETRY_MS * (attempt + 1));+ abortSignal?.throwIfAborted();}async function verifyUploadedVideo(outputVerificationUrl: string | undefined,expectedMetadata: VideoMetadata,+ abortSignal?: AbortSignal,) {if (!outputVerificationUrl) return;let lastError: Error | undefined;for (let attempt = 0; attempt < UPLOAD_VERIFICATION_ATTEMPTS; attempt++) {+ abortSignal?.throwIfAborted();try {const actualMetadata = await probeVideo(outputVerificationUrl);if (isDurationClose(actualMetadata.duration, expectedMetadata.duration)) {@@ -1104,7 +1211,7 @@ async function verifyUploadedVideo(}if (attempt < UPLOAD_VERIFICATION_ATTEMPTS - 1) {- await waitForVerificationRetry(attempt);+ await waitForVerificationRetry(attempt, abortSignal);}}@@ -1124,7 +1231,10 @@ async function editVideoAsync(}const abortController = new AbortController();+ const signal = abortController.signal;updateJob(jobId, { abortController });+ let inputTempFile: TempFileHandle | null = null;+ let outputTempFile: TempFileHandle | null = null;try {updateJob(jobId, {@@ -1137,10 +1247,11 @@ async function editVideoAsync(await sendWebhook(downloadingJob);}- const inputTempFile = await withJobHeartbeat(jobId, () =>- downloadVideoToTemp(sourceUrl, ".mp4", abortController.signal),+ const input = await withJobHeartbeat(jobId, () =>+ downloadVideoToTemp(sourceUrl, ".mp4", signal),);- updateJob(jobId, { inputTempFile });+ inputTempFile = input;+ updateJob(jobId, { inputTempFile: input });updateJob(jobId, {phase: "probing",@@ -1152,7 +1263,15 @@ async function editVideoAsync(await sendWebhook(probingJob);}- const sourceMetadata = await probeVideoFile(inputTempFile.path);+ const sourceMetadata = await withJobHeartbeat(jobId, () =>+ probeVideoFile(input.path),+ );+ signal.throwIfAborted();+ extendJobLifetimeForWork(+ jobId,+ getEditRenderTimeoutMs(options.keepRanges),+ sourceMetadata.fileSize,+ );updateJob(jobId, {phase: "processing",@@ -1164,12 +1283,12 @@ async function editVideoAsync(await sendWebhook(processingJob);}- const outputTempFile = await withJobHeartbeat(jobId, () =>+ const output = await withJobHeartbeat(jobId, () =>renderEditedVideo({- inputPath: inputTempFile.path,+ inputPath: input.path,keepRanges: options.keepRanges,metadata: sourceMetadata,- abortSignal: abortController.signal,+ abortSignal: signal,onProgress: (progress, message) => {updateJob(jobId, {progress: Math.min(80, 10 + progress * 0.9),@@ -1187,9 +1306,13 @@ async function editVideoAsync(},}),);- updateJob(jobId, { outputTempFile });+ outputTempFile = output;+ updateJob(jobId, { outputTempFile: output });- const outputMetadata = await probeVideoFile(outputTempFile.path);+ const outputMetadata = await withJobHeartbeat(jobId, () =>+ probeVideoFile(output.path),+ );+ signal.throwIfAborted();updateJob(jobId, { metadata: outputMetadata });updateJob(jobId, {@@ -1202,8 +1325,19 @@ async function editVideoAsync(await sendWebhook(uploadingJob);}- await uploadFileToS3(outputTempFile.path, outputPresignedUrl, "video/mp4");- await verifyUploadedVideo(options.outputVerificationUrl, outputMetadata);+ await withJobHeartbeat(jobId, async () => {+ await uploadFileToS3(+ output.path,+ outputPresignedUrl,+ "video/mp4",+ signal,+ );+ await verifyUploadedVideo(+ options.outputVerificationUrl,+ outputMetadata,+ signal,+ );+ });if (thumbnailPresignedUrl || previewGifPresignedUrl) {updateJob(jobId, {@@ -1217,21 +1351,31 @@ async function editVideoAsync(}}- if (thumbnailPresignedUrl) {- const thumbnailData = await generateThumbnail(- outputTempFile.path,+ await withJobHeartbeat(jobId, async () => {+ if (thumbnailPresignedUrl) {+ const thumbnailData = await generateThumbnail(+ output.path,+ outputMetadata.duration,+ {},+ signal,+ );+ await uploadToS3(+ thumbnailData,+ thumbnailPresignedUrl,+ "image/jpeg",+ signal,+ );+ }++ await generateAndUploadPreviewGif(+ output.path,outputMetadata.duration,+ previewGifPresignedUrl,+ signal,+ "video/edit",);- await uploadToS3(thumbnailData, thumbnailPresignedUrl, "image/jpeg");- }-- await generateAndUploadPreviewGif(- outputTempFile.path,- outputMetadata.duration,- previewGifPresignedUrl,- abortController.signal,- "video/edit",- );+ });+ signal.throwIfAborted();updateJob(jobId, {phase: "complete",@@ -1243,12 +1387,11 @@ async function editVideoAsync(await sendWebhook(completedJob);}- await inputTempFile.cleanup();- await outputTempFile.cleanup();-setTimeout(() => deleteJob(jobId), 5 * 60 * 1000);} catch (err) {- console.error(`[video/edit] Error editing job ${jobId}:`, err);+ if (!signal.aborted) {+ console.error(`[video/edit] Error editing job ${jobId}:`, err);+ }const updatedJob = updateJob(jobId, {phase: "error",@@ -1261,16 +1404,10 @@ async function editVideoAsync(await sendWebhook(updatedJob);}} finally {- const currentJob = getJob(jobId);- if (currentJob) {- await Promise.allSettled([- currentJob.inputTempFile?.cleanup(),- currentJob.outputTempFile?.cleanup(),- ]);- }-setTimeout(() => deleteJob(jobId), 5 * 60 * 1000);}+ } finally {+ await cleanupTempFiles([inputTempFile, outputTempFile]);}}@@ -1288,8 +1425,11 @@ async function processVideoAsync(}const abortController = new AbortController();+ const signal = abortController.signal;updateJob(jobId, { abortController });+ let inputTempFile: TempFileHandle | null = null;+ let outputTempFile: TempFileHandle | null = null;let repairedTempFile: TempFileHandle | null = null;let lastResortRepairFile: TempFileHandle | null = null;@@ -1301,26 +1441,18 @@ async function processVideoAsync(});await sendWebhook(job);- const inputTempFile = await withJobHeartbeat(jobId, () =>- downloadVideoToTemp(- videoUrl,- options.inputExtension,- abortController.signal,- ),+ const input = await withJobHeartbeat(jobId, () =>+ downloadVideoToTemp(videoUrl, options.inputExtension, signal),);- updateJob(jobId, { inputTempFile });+ inputTempFile = input;+ updateJob(jobId, { inputTempFile: input });const sourcePresignedUrl = options.sourcePresignedUrl;if (sourcePresignedUrl) {updateJob(jobId, { message: "Saving original video..." });await sendWebhook(job);await withJobHeartbeat(jobId, () =>- uploadFileToS3(- inputTempFile.path,- sourcePresignedUrl,- "video/mp4",- abortController.signal,- ),+ uploadFileToS3(input.path, sourcePresignedUrl, "video/mp4", signal),);}@@ -1328,7 +1460,7 @@ async function processVideoAsync(if (isWebm) {updateJob(jobId, { message: "Checking the original recording..." });await withJobHeartbeat(jobId, () =>- validateVideoInput(inputTempFile.path, abortController.signal),+ validateVideoInput(input.path, signal),);}@@ -1339,17 +1471,19 @@ async function processVideoAsync(});await sendWebhook(job);- const { metadata, repairedFile } = await probeWithRepairFallback(- inputTempFile.path,- isWebm,- abortController.signal,+ const { metadata, repairedFile } = await withJobHeartbeat(jobId, () =>+ probeWithRepairFallback(input.path, isWebm, signal),);repairedTempFile = repairedFile;+ signal.throwIfAborted();updateJob(jobId, { metadata });+ extendJobLifetimeForWork(+ jobId,+ getProcessTimeoutMs(metadata.duration, PROCESS_TIMEOUT_MS),+ metadata.fileSize,+ );- const processingInputPath = repairedFile- ? repairedFile.path- : inputTempFile.path;+ const processingInputPath = repairedFile ? repairedFile.path : input.path;updateJob(jobId, {phase: "processing",@@ -1360,18 +1494,19 @@ async function processVideoAsync(});await sendWebhook(job);- const { outputFile: outputTempFile, lastResortRepairFile: lrrf } =- await processWithResilientRetry(- processingInputPath,- inputTempFile.path,- metadata,- options,- isWebm,- jobId,- abortController.signal,- );- lastResortRepairFile = lrrf;- updateJob(jobId, { outputTempFile });+ const processed = await processWithResilientRetry(+ processingInputPath,+ input.path,+ metadata,+ options,+ isWebm,+ jobId,+ signal,+ );+ outputTempFile = processed.outputFile;+ lastResortRepairFile = processed.lastResortRepairFile;+ const outputPath = processed.outputFile.path;+ updateJob(jobId, { outputTempFile: processed.outputFile });updateJob(jobId, {phase: "uploading",@@ -1380,11 +1515,8 @@ async function processVideoAsync(});await sendWebhook(job);- const uploadReceipt = await uploadFileToS3(- outputTempFile.path,- outputPresignedUrl,- "video/mp4",- abortController.signal,+ const uploadReceipt = await withJobHeartbeat(jobId, () =>+ uploadFileToS3(outputPath, outputPresignedUrl, "video/mp4", signal),);if (thumbnailPresignedUrl || previewGifPresignedUrl) {@@ -1396,38 +1528,40 @@ async function processVideoAsync(await sendWebhook(job);}- if (thumbnailPresignedUrl) {- // The uploaded MP4 has not been independently verified, so decode failures must still fail the job.- const thumbnailData = await generateThumbnail(- outputTempFile.path,- metadata.duration,- {},- abortController.signal,- );- try {- await uploadToS3(- thumbnailData,- thumbnailPresignedUrl,- "image/jpeg",- abortController.signal,- );- } catch (error) {- abortController.signal.throwIfAborted();- console.warn(- `[video/process] Thumbnail upload failed for ${jobId}:`,- error,+ await withJobHeartbeat(jobId, async () => {+ if (thumbnailPresignedUrl) {+ // The uploaded MP4 has not been independently verified, so decode failures must still fail the job.+ const thumbnailData = await generateThumbnail(+ outputPath,+ metadata.duration,+ {},+ signal,);+ try {+ await uploadToS3(+ thumbnailData,+ thumbnailPresignedUrl,+ "image/jpeg",+ signal,+ );+ } catch (error) {+ signal.throwIfAborted();+ console.warn(+ `[video/process] Thumbnail upload failed for ${jobId}:`,+ error,+ );+ }}- }- await generateAndUploadPreviewGif(- outputTempFile.path,- metadata.duration,- previewGifPresignedUrl,- abortController.signal,- "video/process",- );- abortController.signal.throwIfAborted();+ await generateAndUploadPreviewGif(+ outputPath,+ metadata.duration,+ previewGifPresignedUrl,+ signal,+ "video/process",+ );+ });+ signal.throwIfAborted();updateJob(jobId, {phase: "complete",@@ -1441,7 +1575,7 @@ async function processVideoAsync(if (options.audioLevels)await enhanceLocalRecording({- path: outputTempFile.path,+ path: outputPath,videoId: options.videoId,userId: options.userId,jobId,@@ -1452,14 +1586,9 @@ async function processVideoAsync(webhookSecret: options.webhookSecret,});- await inputTempFile.cleanup();- await outputTempFile.cleanup();- await repairedTempFile?.cleanup();- await lastResortRepairFile?.cleanup();-setTimeout(() => deleteJob(jobId), 5 * 60 * 1000);} catch (err) {- if (!abortController.signal.aborted) {+ if (!signal.aborted) {console.error(`[video/process] Error processing job ${jobId}:`, err);}const failedJob = getJob(jobId);@@ -1469,7 +1598,7 @@ async function processVideoAsync(failedJob.phase !== "cancelled" &&failedJob.phase !== "error") {- const cancelled = abortController.signal.aborted;+ const cancelled = signal.aborted;const updatedJob = updateJob(jobId, {phase: cancelled ? "cancelled" : "error",error: cancelled@@ -1483,18 +1612,20 @@ async function processVideoAsync(await sendWebhook(updatedJob);}}-- const currentJob = getJob(jobId);- if (currentJob) {- await currentJob.inputTempFile?.cleanup();- await currentJob.outputTempFile?.cleanup();- }- await repairedTempFile?.cleanup();- await lastResortRepairFile?.cleanup();+ } finally {+ await cleanupTempFiles([+ inputTempFile,+ outputTempFile,+ repairedTempFile,+ lastResortRepairFile,+ ]);}}video.get("/process/:jobId/status", async (c) => {+ if (!canReadStatus(c)) {+ return c.json({ error: "Unauthorized" }, 401);+ }const jobId = c.req.param("jobId");const job = getJob(jobId);@@ -1845,6 +1976,16 @@ video.post("/verify-recording", async (c) => {);return c.json(recordingWorkerResponse(existing, ownerJobId));}+ if (isDraining()) {+ c.header("Retry-After", VIDEO_BUSY_RETRY_AFTER_SECONDS.toString());+ return c.json(+ {+ ...getMuxBusyResponseBody(getVideoCapacitySnapshot()),+ details: DRAINING_DETAILS,+ },+ 503,+ );+ }if (!canAcceptNewVideoProcess(body.processingPriority)) {c.header("Retry-After", VIDEO_BUSY_RETRY_AFTER_SECONDS.toString());return c.json(getMuxBusyResponseBody(getVideoCapacitySnapshot()), 503);@@ -1877,19 +2018,24 @@ video.post("/verify-recording", async (c) => {deleteJob(jobId);return c.json(recordingWorkerResponse(job, ownerJobId));}- void withMediaTransfers(body.downloadBudgetBytes ?? body.fileSize * 3, () =>- verifyUploadedRecordingAsync(jobId, body),- ).catch((error: unknown) => {- const current = getJob(jobId);- if (!current || ["complete", "error", "cancelled"].includes(current.phase))- return;- updateJob(jobId, {- phase: "error",- error: "Recording transfer could not complete",- errorCode: classifySourceError(error),- });- sendCurrentJobWebhook(jobId);- });+ void trackBackgroundTask(+ withMediaTransfers(body.downloadBudgetBytes ?? body.fileSize * 3, () =>+ verifyUploadedRecordingAsync(jobId, body),+ ).catch((error: unknown) => {+ const current = getJob(jobId);+ if (+ !current ||+ ["complete", "error", "cancelled"].includes(current.phase)+ )+ return;+ updateJob(jobId, {+ phase: "error",+ error: "Recording transfer could not complete",+ errorCode: classifySourceError(error),+ });+ sendCurrentJobWebhook(jobId);+ }),+ );return c.json(recordingWorkerResponse(job, jobId));});@@ -2174,6 +2320,17 @@ video.post("/mux-segments", async (c) => {});}+ if (isDraining()) {+ c.header("Retry-After", VIDEO_BUSY_RETRY_AFTER_SECONDS.toString());+ return c.json(+ {+ ...getMuxBusyResponseBody(getVideoCapacitySnapshot()),+ details: DRAINING_DETAILS,+ },+ 503,+ );+ }+if (!canAcceptNewVideoProcess(body.data.processingPriority)) {c.header("Retry-After", VIDEO_BUSY_RETRY_AFTER_SECONDS.toString());return c.json(getMuxBusyResponseBody(getVideoCapacitySnapshot()), 503);@@ -2215,36 +2372,38 @@ video.post("/mux-segments", async (c) => {});}- withMediaTransfers(body.data.downloadBudgetBytes ?? 32 * 1024 ** 3, () =>- muxSegmentsAsync(- jobId,- videoId,- outputUpload,- thumbnailPresignedUrl,- previewGifPresignedUrl,- videoInitUrl,- videoSegUrls,- audioInitUrl ?? null,- audioSegUrls ?? null,- body.data.outputVerificationUrl,- body.data,- ),- ).catch((err) => {- console.error(`[mux-segments] Async mux error for job ${jobId}:`, err);- const currentJob = getJob(jobId);- if (- currentJob &&- currentJob.phase !== "error" &&- currentJob.phase !== "complete" &&- currentJob.phase !== "cancelled"- ) {- updateJob(jobId, {- phase: "error",- error: err instanceof Error ? err.message : String(err),- });- sendCurrentJobWebhook(jobId);- }- });+ void trackBackgroundTask(+ withMediaTransfers(body.data.downloadBudgetBytes ?? 32 * 1024 ** 3, () =>+ muxSegmentsAsync(+ jobId,+ videoId,+ outputUpload,+ thumbnailPresignedUrl,+ previewGifPresignedUrl,+ videoInitUrl,+ videoSegUrls,+ audioInitUrl ?? null,+ audioSegUrls ?? null,+ body.data.outputVerificationUrl,+ body.data,+ ),+ ).catch((err) => {+ console.error(`[mux-segments] Async mux error for job ${jobId}:`, err);+ const currentJob = getJob(jobId);+ if (+ currentJob &&+ currentJob.phase !== "error" &&+ currentJob.phase !== "complete" &&+ currentJob.phase !== "cancelled"+ ) {+ updateJob(jobId, {+ phase: "error",+ error: err instanceof Error ? err.message : String(err),+ });+ sendCurrentJobWebhook(jobId);+ }+ }),+ );return c.json({...recordingWorkerResponse(job, jobId),@@ -2568,15 +2727,14 @@ async function muxSegmentsAsync(outputVerificationUrl?: string,context: MuxContext = {},): Promise<void> {- const { ensureTempDir } = await import("../lib/temp-files");+ const { ensureTempDir, getTempDir, retainTempPath } = await import(+ "../lib/temp-files"+ );const { lstat, mkdir, rm } = await import("node:fs/promises");const { join } = await import("node:path");- const workDir = join(- (await import("node:os")).tmpdir(),- "cap-media-server",- `mux-${jobId}`,- );+ const workDir = join(getTempDir(), `mux-${jobId}`);+ const releaseWorkDir = retainTempPath(workDir);const abortController = new AbortController();updateJob(jobId, { abortController });let outputUploadStarted = false;@@ -2666,10 +2824,12 @@ async function muxSegmentsAsync(const combinedVideoPath = join(workDir, "combined_video.mp4");const videoInitPath = join(videoDir, "init.mp4");- await streamConcatFiles(- [videoInitPath, ...videoSegmentFiles],- combinedVideoPath,- abortController.signal,+ await withJobHeartbeat(jobId, () =>+ streamConcatFiles(+ [videoInitPath, ...videoSegmentFiles],+ combinedVideoPath,+ abortController.signal,+ ),);await rm(videoDir, { recursive: true, force: true });@@ -2679,10 +2839,13 @@ async function muxSegmentsAsync(combinedAudioPath = join(workDir, "combined_audio.mp4");const audioInitPath = join(audioDir, "init.mp4");- await streamConcatFiles(- [audioInitPath, ...audioSegmentFiles],- combinedAudioPath,- abortController.signal,+ const audioOutputPath = combinedAudioPath;+ await withJobHeartbeat(jobId, () =>+ streamConcatFiles(+ [audioInitPath, ...audioSegmentFiles],+ audioOutputPath,+ abortController.signal,+ ),);await rm(audioDir, { recursive: true, force: true });}@@ -2789,7 +2952,9 @@ async function muxSegmentsAsync(resources: getSystemResources(),});- let metadata = await probeVideoFile(resultPath);+ let metadata = await withJobHeartbeat(jobId, () =>+ probeVideoFile(resultPath),+ );if (!Number.isFinite(metadata.duration) ||metadata.duration <= 0 ||@@ -2903,37 +3068,39 @@ async function muxSegmentsAsync(sendCurrentJobWebhook(jobId);}- if (thumbnailPresignedUrl) {- try {- const duration = metadata?.duration ?? 0;- const thumbnailData = await generateThumbnail(- resultPath,- duration,- {},- abortController.signal,- );- await uploadToS3(- thumbnailData,- thumbnailPresignedUrl,- "image/jpeg",- abortController.signal,- );- } catch (thumbErr) {- abortController.signal.throwIfAborted();- console.warn(- `[mux-segments] Thumbnail generation failed for ${videoId}:`,- thumbErr,- );+ const assetDuration = metadata?.duration ?? 0;+ await withJobHeartbeat(jobId, async () => {+ if (thumbnailPresignedUrl) {+ try {+ const thumbnailData = await generateThumbnail(+ resultPath,+ assetDuration,+ {},+ abortController.signal,+ );+ await uploadToS3(+ thumbnailData,+ thumbnailPresignedUrl,+ "image/jpeg",+ abortController.signal,+ );+ } catch (thumbErr) {+ abortController.signal.throwIfAborted();+ console.warn(+ `[mux-segments] Thumbnail generation failed for ${videoId}:`,+ thumbErr,+ );+ }}- }- await generateAndUploadPreviewGif(- resultPath,- metadata?.duration ?? 0,- previewGifPresignedUrl,- abortController.signal,- "mux-segments",- );+ await generateAndUploadPreviewGif(+ resultPath,+ assetDuration,+ previewGifPresignedUrl,+ abortController.signal,+ "mux-segments",+ );+ });abortController.signal.throwIfAborted();updateJob(jobId, {@@ -3014,6 +3181,7 @@ async function muxSegmentsAsync(} finally {if (processingTimeout) clearTimeout(processingTimeout);await rm(workDir, { recursive: true, force: true }).catch(() => {});+ releaseWorkDir();}}