Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

fix(media-server): drain jobs on shutdown, sweep orphaned temp dirs, heartbeat long steps and stop cancelled work

Richie McIlroy955c64738b4c
diff --git a/apps/media-server/Dockerfile b/apps/media-server/Dockerfile
index 4891e6452..5a1244bd8 100644
--- a/apps/media-server/Dockerfile
+++ b/apps/media-server/Dockerfile
@@ -22,6 +22,8 @@ RUN bun test src/__tests__/lib/audio-quality-policy.test.ts src/__tests__/lib/au
&& bun test src/__tests__/lib/media-transfer.test.ts src/__tests__/lib/capcodec-encoder.test.ts src/__tests__/lib/nvenc-encoder.test.ts \
&& bun test src/__tests__/routes/recording-verification.test.ts \
&& bun test src/__tests__/routes/video.test.ts \
+ && bun test src/__tests__/lib/lifecycle.test.ts src/__tests__/lib/temp-files.test.ts \
+ && bun test src/__tests__/routes/lifecycle.test.ts \
&& bun test src/__tests__/lib/media-routes-real-world.integration.test.ts \
&& bun test src/__tests__/lib/media-video.integration.test.ts
@@ -37,6 +39,6 @@ ENV PORT=3456
EXPOSE 3456
HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
- CMD ["bun", "-e", "fetch(`http://127.0.0.1:${process.env.PORT || 3456}/health`).then((response) => process.exit(response.ok ? 0 : 1)).catch(() => process.exit(1))"]
+ CMD ["bun", "-e", "fetch(`http://127.0.0.1:${process.env.PORT || 3456}/health`).then(async (response) => process.exit(response.ok || (await response.json().catch(() => ({}))).status === 'draining' ? 0 : 1)).catch(() => process.exit(1))"]
CMD ["bun", "run", "src/index.ts"]
diff --git a/apps/media-server/src/__tests__/lib/job-manager.test.ts b/apps/media-server/src/__tests__/lib/job-manager.test.ts
index ff2b43e69..5bada4e69 100644
--- a/apps/media-server/src/__tests__/lib/job-manager.test.ts
+++ b/apps/media-server/src/__tests__/lib/job-manager.test.ts
@@ -6,7 +6,9 @@ import {
cleanupExpiredJobs,
createJob,
deleteJob,
+ extendJobLifetime,
getJob,
+ getJobTotals,
type JobProgress,
type RecordingWorkerAcknowledgement,
sendWebhook,
@@ -522,3 +524,83 @@ describe("job cleanup", () => {
expect(currentJob?.error).toContain("maximum lifetime of 60 minutes");
});
});
+
+describe("job lifetime", () => {
+ test("extends the lifetime cap for long work without shortening it", () => {
+ const job = createTrackedJob("job-extended-lifetime");
+ const now = Date.now();
+ job.phase = "processing";
+ job.createdAt = now - 10 * 60 * 1000;
+
+ expect(extendJobLifetime(job.jobId, 60_000)).toBe(
+ job.createdAt + 60 * 60 * 1000,
+ );
+ const deadline = extendJobLifetime(job.jobId, 2 * 60 * 60 * 1000);
+ expect(deadline).toBeGreaterThanOrEqual(now + 2 * 60 * 60 * 1000);
+ expect(extendJobLifetime(job.jobId, 60_000)).toBe(deadline);
+
+ job.createdAt = now - 90 * 60 * 1000;
+ job.updatedAt = Date.now();
+ expect(cleanupExpiredJobs()).toBe(0);
+ expect(getJob(job.jobId)?.phase).toBe("processing");
+ });
+
+ test("keeps a hard upper bound on extended lifetimes", () => {
+ const job = createTrackedJob("job-capped-lifetime");
+ job.phase = "uploading";
+ job.createdAt = Date.now() - 10 * 60 * 1000;
+
+ expect(extendJobLifetime(job.jobId, 24 * 60 * 60 * 1000)).toBe(
+ job.createdAt + 4 * 60 * 60 * 1000,
+ );
+ expect(extendJobLifetime(job.jobId, Number.NaN)).toBeUndefined();
+ expect(extendJobLifetime(job.jobId, -1)).toBeUndefined();
+ });
+
+ test("expires a job past its extended deadline", () => {
+ const job = createTrackedJob("job-past-extended-lifetime");
+ const abortController = new AbortController();
+ updateJob(job.jobId, { phase: "processing", abortController });
+ job.createdAt = Date.now() - 10 * 60 * 1000;
+ extendJobLifetime(job.jobId, 2 * 60 * 60 * 1000);
+ job.createdAt -= 3 * 60 * 60 * 1000;
+ job.lifetimeDeadlineAt = Date.now() - 1;
+ job.updatedAt = Date.now();
+
+ expect(cleanupExpiredJobs()).toBe(1);
+ expect(job.phase).toBe("error");
+ expect(job.error).toContain("maximum lifetime");
+ expect(abortController.signal.aborted).toBe(true);
+ });
+
+ test("does not extend terminal jobs", () => {
+ const job = createTrackedJob("job-terminal-lifetime");
+ updateJob(job.jobId, { phase: "complete" });
+ expect(extendJobLifetime(job.jobId, 2 * 60 * 60 * 1000)).toBeUndefined();
+ expect(job.lifetimeDeadlineAt).toBeUndefined();
+ });
+});
+
+describe("job totals", () => {
+ test("counts created, terminal, and expired jobs", () => {
+ const before = getJobTotals();
+ const completed = createTrackedJob("job-total-complete");
+ const errored = createTrackedJob("job-total-error");
+ const cancelled = createTrackedJob("job-total-cancelled");
+ const stale = createTrackedJob("job-total-stale");
+ updateJob(completed.jobId, { phase: "complete" });
+ updateJob(completed.jobId, { phase: "error" });
+ updateJob(errored.jobId, { phase: "error" });
+ updateJob(cancelled.jobId, { phase: "cancelled" });
+ updateJob(stale.jobId, { phase: "processing" });
+ stale.updatedAt = Date.now() - 16 * 60 * 1000;
+ cleanupExpiredJobs();
+
+ const after = getJobTotals();
+ expect(after.created - before.created).toBe(4);
+ expect(after.completed - before.completed).toBe(1);
+ expect(after.errored - before.errored).toBe(2);
+ expect(after.cancelled - before.cancelled).toBe(1);
+ expect(after.expired - before.expired).toBe(1);
+ });
+});
diff --git a/apps/media-server/src/__tests__/lib/lifecycle.test.ts b/apps/media-server/src/__tests__/lib/lifecycle.test.ts
new file mode 100644
index 000000000..bd6f9454c
--- /dev/null
+++ b/apps/media-server/src/__tests__/lib/lifecycle.test.ts
@@ -0,0 +1,174 @@
+import { afterEach, describe, expect, test } from "bun:test";
+import {
+ beginRequest,
+ createShutdownHandler,
+ getDrainBlockers,
+ getIdleTimeoutSeconds,
+ getShutdownGraceMs,
+ isDraining,
+ isIdle,
+ setDraining,
+ setRequestIdleTimeout,
+ trackBackgroundTask,
+ waitForDrain,
+} from "../../lib/lifecycle";
+
+const originalGrace = process.env.MEDIA_SERVER_SHUTDOWN_GRACE_MS;
+const originalIdle = process.env.MEDIA_SERVER_IDLE_TIMEOUT_SECONDS;
+
+function deferred() {
+ let resolve = () => {};
+ const promise = new Promise<void>((done) => {
+ resolve = done;
+ });
+ return { promise, resolve };
+}
+
+afterEach(() => {
+ setDraining(false);
+ if (originalGrace === undefined)
+ delete process.env.MEDIA_SERVER_SHUTDOWN_GRACE_MS;
+ else process.env.MEDIA_SERVER_SHUTDOWN_GRACE_MS = originalGrace;
+ if (originalIdle === undefined)
+ delete process.env.MEDIA_SERVER_IDLE_TIMEOUT_SECONDS;
+ else process.env.MEDIA_SERVER_IDLE_TIMEOUT_SECONDS = originalIdle;
+});
+
+describe("shutdown configuration", () => {
+ test("defaults to the immediate shutdown path", () => {
+ delete process.env.MEDIA_SERVER_SHUTDOWN_GRACE_MS;
+ expect(getShutdownGraceMs()).toBe(0);
+ process.env.MEDIA_SERVER_SHUTDOWN_GRACE_MS = "not-a-number";
+ expect(getShutdownGraceMs()).toBe(0);
+ process.env.MEDIA_SERVER_SHUTDOWN_GRACE_MS = "120000";
+ expect(getShutdownGraceMs()).toBe(120_000);
+ });
+
+ test("bounds the idle timeout to Bun's maximum", () => {
+ delete process.env.MEDIA_SERVER_IDLE_TIMEOUT_SECONDS;
+ expect(getIdleTimeoutSeconds()).toBe(255);
+ process.env.MEDIA_SERVER_IDLE_TIMEOUT_SECONDS = "90";
+ expect(getIdleTimeoutSeconds()).toBe(90);
+ process.env.MEDIA_SERVER_IDLE_TIMEOUT_SECONDS = "10000";
+ expect(getIdleTimeoutSeconds()).toBe(255);
+ });
+
+ test("ignores servers without per-request timeouts", () => {
+ const calls: Array<[Request, number]> = [];
+ const request = new Request("http://localhost/video/convert");
+ setRequestIdleTimeout(undefined, request, 0);
+ setRequestIdleTimeout(
+ {
+ timeout: (req: Request, seconds: number) => calls.push([req, seconds]),
+ },
+ request,
+ 0,
+ );
+ expect(calls).toEqual([[request, 0]]);
+ });
+});
+
+describe("drain", () => {
+ test("waits for background work and in-flight requests", async () => {
+ const work = deferred();
+ const task = trackBackgroundTask(work.promise);
+ const endRequest = beginRequest();
+ expect(isIdle()).toBe(false);
+ expect(getDrainBlockers().backgroundTasks).toBeGreaterThanOrEqual(1);
+
+ const drained = waitForDrain(5_000);
+ await Bun.sleep(50);
+ work.resolve();
+ await task;
+ await Bun.sleep(50);
+ endRequest();
+
+ expect(await drained).toBe(true);
+ expect(isIdle()).toBe(true);
+ });
+
+ test("gives up when the grace period elapses", async () => {
+ const work = deferred();
+ const task = trackBackgroundTask(work.promise);
+ const startedAt = Date.now();
+
+ expect(await waitForDrain(300)).toBe(false);
+ expect(Date.now() - startedAt).toBeGreaterThanOrEqual(250);
+
+ work.resolve();
+ await task;
+ });
+
+ test("stops tracking rejected background work", async () => {
+ const failure = trackBackgroundTask(Promise.reject(new Error("failed")));
+ await failure.catch(() => {});
+ await Bun.sleep(0);
+ expect(await waitForDrain(1_000)).toBe(true);
+ });
+});
+
+describe("shutdown handler", () => {
+ test("keeps today's immediate shutdown when no grace period is set", async () => {
+ const work = deferred();
+ const task = trackBackgroundTask(work.promise);
+ let finalized = 0;
+ const shutdown = createShutdownHandler(
+ async () => {
+ finalized++;
+ },
+ () => 0,
+ );
+
+ await shutdown("SIGTERM");
+ await shutdown("SIGTERM");
+
+ expect(finalized).toBe(1);
+ expect(isDraining()).toBe(false);
+ work.resolve();
+ await task;
+ });
+
+ test("drains before finalizing when a grace period is set", async () => {
+ const work = deferred();
+ const task = trackBackgroundTask(work.promise);
+ let finalizedWhileBusy: boolean | undefined;
+ const shutdown = createShutdownHandler(
+ async () => {
+ finalizedWhileBusy = !isIdle();
+ },
+ () => 10_000,
+ );
+
+ const shuttingDown = shutdown("SIGTERM");
+ await Bun.sleep(50);
+ expect(isDraining()).toBe(true);
+ expect(finalizedWhileBusy).toBeUndefined();
+
+ work.resolve();
+ await task;
+ await shuttingDown;
+ expect(finalizedWhileBusy).toBe(false);
+ });
+
+ test("forces the immediate path on a second signal", async () => {
+ const work = deferred();
+ const task = trackBackgroundTask(work.promise);
+ let finalized = 0;
+ const shutdown = createShutdownHandler(
+ async () => {
+ finalized++;
+ },
+ () => 60_000,
+ );
+
+ const shuttingDown = shutdown("SIGTERM");
+ await Bun.sleep(50);
+ expect(finalized).toBe(0);
+ await shutdown("SIGINT");
+ await shuttingDown;
+ expect(finalized).toBe(1);
+
+ work.resolve();
+ await task;
+ });
+});
diff --git a/apps/media-server/src/__tests__/lib/temp-files.test.ts b/apps/media-server/src/__tests__/lib/temp-files.test.ts
new file mode 100644
index 000000000..b90263a2f
--- /dev/null
+++ b/apps/media-server/src/__tests__/lib/temp-files.test.ts
@@ -0,0 +1,177 @@
+import { afterEach, describe, expect, test } from "bun:test";
+import { randomUUID } from "node:crypto";
+import { existsSync } from "node:fs";
+import { mkdir, rm, utimes, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { spawn } from "bun";
+import {
+ cleanupMediaTransferCache,
+ sweepMediaTransferCache,
+} from "../../lib/media-transfer";
+import {
+ cleanupStaleTempFiles,
+ createTempFile,
+ ensureTempDir,
+ getInstanceDir,
+ getTempDir,
+ getTempDirUsage,
+ getTempRoot,
+ removeOwnTempDir,
+ retainTempPath,
+ sweepOrphanedTempDirs,
+} from "../../lib/temp-files";
+
+const created: string[] = [];
+
+afterEach(async () => {
+ for (const path of created.splice(0)) {
+ await rm(path, { recursive: true, force: true });
+ }
+});
+
+async function makeTree(path: string, ageMs: number, lease = false) {
+ created.push(path);
+ await mkdir(join(path, "nested", "deeper"), { recursive: true });
+ await writeFile(join(path, "nested", "deeper", "segment.m4s"), "segment");
+ const past = new Date(Date.now() - ageMs);
+ if (lease) {
+ await writeFile(join(path, ".lease"), "");
+ await utimes(join(path, ".lease"), past, past);
+ }
+ await utimes(path, past, past);
+}
+
+async function setAge(path: string, ageMs: number) {
+ const past = new Date(Date.now() - ageMs);
+ await utimes(path, past, past);
+}
+
+describe("startup temp sweep", () => {
+ test("removes orphaned instance dirs and old legacy entries only", async () => {
+ await ensureTempDir();
+ const root = getTempRoot();
+ const staleInstance = join(root, `instance-${randomUUID()}`);
+ const liveInstance = join(root, `instance-${randomUUID()}`);
+ const oldLegacyDir = join(root, `mux-job_${randomUUID()}`);
+ const youngLegacyFile = join(root, `${randomUUID()}.mp4`);
+ await makeTree(staleInstance, 10 * 60 * 1000, true);
+ await makeTree(liveInstance, 10 * 60 * 1000);
+ await writeFile(join(liveInstance, ".lease"), "");
+ await makeTree(oldLegacyDir, 2 * 60 * 60 * 1000);
+ created.push(youngLegacyFile);
+ await writeFile(youngLegacyFile, "in use by an older server version");
+
+ const removed = await sweepOrphanedTempDirs();
+
+ expect(removed).toBeGreaterThanOrEqual(2);
+ expect(existsSync(staleInstance)).toBe(false);
+ expect(existsSync(oldLegacyDir)).toBe(false);
+ expect(existsSync(liveInstance)).toBe(true);
+ expect(existsSync(youngLegacyFile)).toBe(true);
+ expect(existsSync(getTempDir())).toBe(true);
+ expect(existsSync(join(getTempDir(), ".lease"))).toBe(true);
+ });
+
+ test("never treats the server's own instance dir as orphaned", async () => {
+ await ensureTempDir();
+ await setAge(join(getTempDir(), ".lease"), 24 * 60 * 60 * 1000);
+ await setAge(getTempDir(), 24 * 60 * 60 * 1000);
+
+ await sweepOrphanedTempDirs();
+
+ expect(existsSync(getTempDir())).toBe(true);
+ });
+});
+
+describe("stale temp cleanup", () => {
+ test("removes stale directories recursively and keeps in-use entries", async () => {
+ await ensureTempDir();
+ const staleDir = join(getTempDir(), `stream-${randomUUID()}`);
+ await makeTree(staleDir, 2 * 60 * 60 * 1000);
+ const retainedDir = join(getTempDir(), `mux-job_${randomUUID()}`);
+ await makeTree(retainedDir, 2 * 60 * 60 * 1000);
+ const releaseRetainedDir = retainTempPath(retainedDir);
+ const inUse = await createTempFile(".mp4");
+ created.push(inUse.path);
+ await writeFile(inUse.path, "input still read by ffmpeg");
+ await setAge(inUse.path, 2 * 60 * 60 * 1000);
+ const fresh = await createTempFile(".mp4");
+ created.push(fresh.path);
+ await writeFile(fresh.path, "fresh");
+
+ try {
+ expect(await cleanupStaleTempFiles()).toBeGreaterThanOrEqual(1);
+ expect(existsSync(staleDir)).toBe(false);
+ expect(existsSync(retainedDir)).toBe(true);
+ expect(existsSync(inUse.path)).toBe(true);
+ expect(existsSync(fresh.path)).toBe(true);
+
+ releaseRetainedDir();
+ await cleanupStaleTempFiles();
+ expect(existsSync(retainedDir)).toBe(false);
+ } finally {
+ releaseRetainedDir();
+ await inUse.cleanup();
+ await fresh.cleanup();
+ }
+ });
+
+ test("expires entries that were retained far beyond any job lifetime", async () => {
+ await ensureTempDir();
+ const leaked = await createTempFile(".mp4");
+ created.push(leaked.path);
+ await writeFile(leaked.path, "leaked handle");
+ await setAge(leaked.path, 7 * 60 * 60 * 1000);
+
+ await cleanupStaleTempFiles();
+
+ expect(existsSync(leaked.path)).toBe(false);
+ });
+
+ test("reports temp usage including nested files", async () => {
+ await ensureTempDir();
+ const dir = join(getTempDir(), `usage-${randomUUID()}`);
+ await makeTree(dir, 0);
+
+ const usage = await getTempDirUsage();
+
+ expect(usage.entries).toBeGreaterThanOrEqual(4);
+ expect(usage.bytes).toBeGreaterThanOrEqual("segment".length);
+ expect(usage.truncated).toBe(false);
+ });
+
+ test("removes the server's own dir on shutdown and recreates it on demand", async () => {
+ const handle = await createTempFile(".mp4");
+ await writeFile(handle.path, "shutdown");
+
+ await removeOwnTempDir();
+ expect(existsSync(getTempDir())).toBe(false);
+
+ const next = await createTempFile(".mp4");
+ expect(existsSync(join(getTempDir(), ".lease"))).toBe(true);
+ await next.cleanup();
+ });
+});
+
+describe("media transfer cache sweep", () => {
+ test("removes orphaned instance and dead-pid cache dirs", async () => {
+ const base = join(tmpdir(), "cap-media-download-cache");
+ const staleInstance = join(base, `instance-${randomUUID()}`);
+ const liveInstance = join(base, `instance-${randomUUID()}`);
+ const exited = spawn({ cmd: ["true"] });
+ await exited.exited;
+ const deadPidDir = join(base, String(exited.pid));
+ await makeTree(staleInstance, 10 * 60 * 1000, true);
+ await makeTree(liveInstance, 0, true);
+ await makeTree(deadPidDir, 0);
+
+ await sweepMediaTransferCache();
+
+ expect(existsSync(staleInstance)).toBe(false);
+ expect(existsSync(deadPidDir)).toBe(false);
+ expect(existsSync(liveInstance)).toBe(true);
+ expect(existsSync(getInstanceDir(base))).toBe(false);
+ await cleanupMediaTransferCache();
+ });
+});
diff --git a/apps/media-server/src/__tests__/routes/lifecycle.test.ts b/apps/media-server/src/__tests__/routes/lifecycle.test.ts
new file mode 100644
index 000000000..ae93d131a
--- /dev/null
+++ b/apps/media-server/src/__tests__/routes/lifecycle.test.ts
@@ -0,0 +1,468 @@
+import {
+ afterEach,
+ beforeEach,
+ describe,
+ expect,
+ mock,
+ spyOn,
+ test,
+} from "bun:test";
+import { existsSync } from "node:fs";
+import { writeFile } from "node:fs/promises";
+import app from "../../app";
+import * as jobManager from "../../lib/job-manager";
+import {
+ cleanupExpiredJobs,
+ createJob,
+ deleteJob,
+ getActiveVideoProcessCount,
+ getJob,
+ type VideoMetadata,
+} from "../../lib/job-manager";
+import {
+ DRAINING_DETAILS,
+ setDraining,
+ waitForDrain,
+} from "../../lib/lifecycle";
+import * as mediaEdit from "../../lib/media-edit";
+import * as mediaProbe from "../../lib/media-probe";
+import * as mediaVideo from "../../lib/media-video";
+import { createTempFile, type TempFileHandle } from "../../lib/temp-files";
+import { getActiveDirectVideoProcessCount } from "../../lib/video-capacity";
+
+const MEDIA_SERVER_SECRET = "test-secret";
+const AUTH_HEADERS = {
+ "Content-Type": "application/json",
+ "x-media-server-secret": MEDIA_SERVER_SECRET,
+};
+const originalPublicStatus = process.env.MEDIA_SERVER_PUBLIC_STATUS;
+
+process.env.MEDIA_SERVER_WEBHOOK_SECRET = MEDIA_SERVER_SECRET;
+
+const metadata: VideoMetadata = {
+ duration: 3 * 60 * 60,
+ width: 1920,
+ height: 1080,
+ fps: 30,
+ videoCodec: "h264",
+ audioCodec: "aac",
+ audioChannels: 2,
+ sampleRate: 48_000,
+ bitrate: 5_000_000,
+ fileSize: 1024 ** 3,
+};
+
+const createdJobs: string[] = [];
+
+function post(path: string, body: unknown): Request {
+ return new Request(`http://localhost${path}`, {
+ method: "POST",
+ headers: AUTH_HEADERS,
+ body: JSON.stringify(body),
+ });
+}
+
+function deferred<T = void>() {
+ let resolve: (value: T) => void = () => {};
+ const promise = new Promise<T>((done) => {
+ resolve = done;
+ });
+ return { promise, resolve };
+}
+
+async function tempFileWith(extension: string, contents: string) {
+ const handle = await createTempFile(extension);
+ await writeFile(handle.path, contents);
+ return handle;
+}
+
+async function waitFor(condition: () => boolean, timeoutMs = 2_000) {
+ const deadline = Date.now() + timeoutMs;
+ while (!condition()) {
+ if (Date.now() > deadline) throw new Error("Timed out waiting");
+ await Bun.sleep(10);
+ }
+}
+
+beforeEach(() => {
+ spyOn(jobManager, "canAcceptNewVideoProcess").mockReturnValue(true);
+});
+
+afterEach(async () => {
+ mock.restore();
+ setDraining(false);
+ if (originalPublicStatus === undefined)
+ delete process.env.MEDIA_SERVER_PUBLIC_STATUS;
+ else process.env.MEDIA_SERVER_PUBLIC_STATUS = originalPublicStatus;
+ await waitForDrain(2_000);
+ for (const jobId of createdJobs.splice(0)) deleteJob(jobId);
+});
+
+describe("draining", () => {
+ const url = "https://example.com/video.mp4";
+ const admissions: Array<[string, unknown]> = [
+ [
+ "/video/process",
+ { videoId: "v", userId: "u", videoUrl: url, outputPresignedUrl: url },
+ ],
+ [
+ "/video/import",
+ {
+ videoId: "v",
+ userId: "u",
+ videoUrl: url,
+ outputPresignedUrl: url,
+ sourcePresignedUrl: url,
+ },
+ ],
+ [
+ "/video/edit",
+ {
+ videoId: "v",
+ userId: "u",
+ sourceUrl: url,
+ outputPresignedUrl: url,
+ keepRanges: [{ start: 0, end: 1 }],
+ },
+ ],
+ [
+ "/video/mux-segments",
+ {
+ videoId: "v",
+ userId: "u",
+ outputPresignedUrl: url,
+ videoInitUrl: url,
+ videoSegmentUrls: ["https://example.com/segment.m4s"],
+ },
+ ],
+ [
+ "/video/verify-recording",
+ {
+ videoId: "v",
+ userId: "u",
+ videoUrl: url,
+ fileSize: 1,
+ duration: 1,
+ requiredAudio: false,
+ objectIdentity: '"etag"',
+ webhookUrl: "https://example.com/webhook",
+ },
+ ],
+ ["/video/convert", { videoUrl: url }],
+ ["/video/probe", { videoUrl: url }],
+ ["/video/thumbnail", { videoUrl: url }],
+ ["/audio/check", { videoUrl: url }],
+ ["/audio/extract", { videoUrl: url }],
+ ["/audio/convert", { audioUrl: url }],
+ ];
+
+ test.each(admissions)("rejects new work on %s", async (path, body) => {
+ setDraining(true);
+ const response = await app.fetch(post(path, body));
+
+ expect(response.status).toBe(503);
+ expect(response.headers.get("Retry-After")).toBe("15");
+ const data = await response.json();
+ expect(data.code).toBe("SERVER_BUSY");
+ expect(data.details).toBe(DRAINING_DETAILS);
+ });
+
+ test("fails health checks but keeps status endpoints working", async () => {
+ const job = createJob("job-draining-status", "video-id", "user-id");
+ createdJobs.push(job.jobId);
+ setDraining(true);
+
+ const health = await app.fetch(new Request("http://localhost/health"));
+ expect(health.status).toBe(503);
+ expect((await health.json()).status).toBe("draining");
+
+ const status = await app.fetch(
+ new Request(`http://localhost/video/process/${job.jobId}/status`),
+ );
+ expect(status.status).toBe(200);
+ expect((await status.json()).jobId).toBe(job.jobId);
+
+ const serverStatus = await app.fetch(
+ new Request("http://localhost/video/status"),
+ );
+ expect(serverStatus.status).toBe(200);
+ });
+});
+
+describe("status access", () => {
+ test("requires the secret when public status is disabled", async () => {
+ process.env.MEDIA_SERVER_PUBLIC_STATUS = "false";
+ for (const path of [
+ "/video/status",
+ "/audio/status",
+ "/video/process/job-missing/status",
+ ]) {
+ const anonymous = await app.fetch(new Request(`http://localhost${path}`));
+ expect(anonymous.status).toBe(401);
+ const authenticated = await app.fetch(
+ new Request(`http://localhost${path}`, { headers: AUTH_HEADERS }),
+ );
+ expect(authenticated.status).not.toBe(401);
+ }
+ });
+
+ test("reports runtime counters only to authenticated callers", async () => {
+ const job = createJob("job_runtime_report", "video-runtime", "user-id");
+ createdJobs.push(job.jobId);
+ const anonymous = await app.fetch(new Request("http://localhost/health"));
+ expect((await anonymous.json()).runtime).toBeUndefined();
+
+ for (const path of ["/health", "/video/status"]) {
+ const response = await app.fetch(
+ new Request(`http://localhost${path}`, { headers: AUTH_HEADERS }),
+ );
+ const { runtime } = await response.json();
+ expect(runtime.activeJobs).toBeGreaterThanOrEqual(1);
+ expect(runtime.jobTotals.created).toBeGreaterThanOrEqual(1);
+ expect(typeof runtime.activeSubprocesses).toBe("number");
+ expect(typeof runtime.tempDir.bytes).toBe("number");
+ expect(typeof runtime.tempDir.entries).toBe("number");
+ expect(typeof runtime.mediaTransferCache.bytes).toBe("number");
+ expect(typeof runtime.slots.video.max).toBe("number");
+ expect(typeof runtime.slots.directVideo.active).toBe("number");
+ expect(typeof runtime.slots.audio.max).toBe("number");
+ expect(typeof runtime.slots.probe.max).toBe("number");
+ expect(JSON.stringify(runtime)).not.toContain(job.jobId);
+ expect(JSON.stringify(runtime)).not.toContain(job.videoId);
+ }
+ });
+});
+
+describe("process jobs", () => {
+ test("heartbeats a slow output upload and derives the lifetime from the input", async () => {
+ const intervals: Array<{
+ callback: () => void;
+ delay?: number;
+ handle: ReturnType<typeof setInterval>;
+ }> = [];
+ const cleared = new Set<unknown>();
+ const originalSetInterval = globalThis.setInterval;
+ const originalClearInterval = globalThis.clearInterval;
+ spyOn(globalThis, "setInterval").mockImplementation(((
+ callback: () => void,
+ delay?: number,
+ ) => {
+ const handle = originalSetInterval(callback, delay);
+ intervals.push({ callback, delay, handle });
+ return handle;
+ }) as unknown as typeof setInterval);
+ spyOn(globalThis, "clearInterval").mockImplementation(((
+ handle: ReturnType<typeof setInterval>,
+ ) => {
+ cleared.add(handle);
+ originalClearInterval(handle);
+ }) as unknown as typeof clearInterval);
+ const files: TempFileHandle[] = [];
+ spyOn(mediaVideo, "downloadVideoToTemp").mockImplementation(async () => {
+ const input = await tempFileWith(".mp4", "input");
+ files.push(input);
+ return input;
+ });
+ spyOn(mediaProbe, "probeVideoFile").mockResolvedValue(metadata);
+ spyOn(mediaVideo, "processVideo").mockImplementation(async () => {
+ const output = await tempFileWith(".mp4", "output");
+ files.push(output);
+ return output;
+ });
+ const uploadStarted = deferred();
+ const upload = deferred<mediaVideo.StorageUploadReceipt>();
+ spyOn(mediaVideo, "uploadFileToS3").mockImplementation(async () => {
+ uploadStarted.resolve();
+ return await upload.promise;
+ });
+
+ const response = await app.fetch(
+ post("/video/process", {
+ videoId: "video-slow-upload",
+ userId: "user-id",
+ videoUrl: "https://example.com/input.mp4",
+ outputPresignedUrl: "https://example.com/output.mp4",
+ }),
+ );
+ const { jobId } = await response.json();
+ createdJobs.push(jobId);
+ await uploadStarted.promise;
+
+ const job = getJob(jobId);
+ expect(job?.phase).toBe("uploading");
+ if (!job) throw new Error("Job missing");
+ expect(job.lifetimeDeadlineAt).toBeGreaterThan(
+ job.createdAt + 2 * 60 * 60 * 1000,
+ );
+ expect(job.lifetimeDeadlineAt).toBeLessThanOrEqual(
+ job.createdAt + 4 * 60 * 60 * 1000,
+ );
+
+ job.updatedAt = Date.now() - 16 * 60 * 1000;
+ const heartbeats = intervals.filter(
+ ({ delay, handle }) => delay === 60_000 && !cleared.has(handle),
+ );
+ expect(heartbeats).toHaveLength(1);
+ heartbeats[0]?.callback();
+ expect(cleanupExpiredJobs()).toBe(0);
+ expect(job.phase).toBe("uploading");
+
+ upload.resolve({});
+ await waitFor(() => job.phase === "complete");
+ await waitForDrain(2_000);
+ expect(files.every((handle) => !existsSync(handle.path))).toBe(true);
+ });
+
+ test("releases the job slot and temp files when processing fails", async () => {
+ const before = getActiveVideoProcessCount();
+ let input: TempFileHandle | undefined;
+ spyOn(mediaVideo, "downloadVideoToTemp").mockImplementation(async () => {
+ input = await tempFileWith(".mp4", "input");
+ return input;
+ });
+ spyOn(mediaProbe, "probeVideoFile").mockResolvedValue({
+ ...metadata,
+ duration: 10,
+ });
+ spyOn(mediaVideo, "processVideo").mockRejectedValue(
+ new Error("FFmpeg exited with code 1"),
+ );
+
+ const response = await app.fetch(
+ post("/video/process", {
+ videoId: "video-failed-process",
+ userId: "user-id",
+ videoUrl: "https://example.com/input.mp4",
+ outputPresignedUrl: "https://example.com/output.mp4",
+ }),
+ );
+ const { jobId } = await response.json();
+ createdJobs.push(jobId);
+
+ expect(await waitForDrain(2_000)).toBe(true);
+ expect(getJob(jobId)?.phase).toBe("error");
+ expect(getActiveVideoProcessCount()).toBe(before);
+ expect(input && existsSync(input.path)).toBe(false);
+ });
+});
+
+describe("edit jobs", () => {
+ test("cancellation aborts the upload and skips preview work", async () => {
+ const files: TempFileHandle[] = [];
+ spyOn(mediaVideo, "downloadVideoToTemp").mockImplementation(async () => {
+ const input = await tempFileWith(".mp4", "input");
+ files.push(input);
+ return input;
+ });
+ spyOn(mediaProbe, "probeVideoFile").mockResolvedValue({
+ ...metadata,
+ duration: 10,
+ });
+ spyOn(mediaEdit, "renderEditedVideo").mockImplementation(async () => {
+ const output = await tempFileWith(".mp4", "edited");
+ files.push(output);
+ return output;
+ });
+ const thumbnail = spyOn(mediaVideo, "generateThumbnail");
+ const uploadStarted = deferred();
+ let uploadSignal: AbortSignal | undefined;
+ spyOn(mediaVideo, "uploadFileToS3").mockImplementation(
+ (_path, _url, _contentType, signal) => {
+ uploadSignal = signal;
+ uploadStarted.resolve();
+ return new Promise((_resolve, reject) => {
+ signal?.addEventListener("abort", () => reject(signal.reason), {
+ once: true,
+ });
+ });
+ },
+ );
+
+ const response = await app.fetch(
+ post("/video/edit", {
+ videoId: "video-cancelled-edit",
+ userId: "user-id",
+ sourceUrl: "https://example.com/source.mp4",
+ outputPresignedUrl: "https://example.com/output.mp4",
+ thumbnailPresignedUrl: "https://example.com/thumbnail.jpg",
+ keepRanges: [{ start: 0, end: 5 }],
+ }),
+ );
+ const { jobId } = await response.json();
+ createdJobs.push(jobId);
+ await uploadStarted.promise;
+
+ const cancel = await app.fetch(post(`/video/process/${jobId}/cancel`, {}));
+ expect(cancel.status).toBe(200);
+ expect(uploadSignal?.aborted).toBe(true);
+
+ expect(await waitForDrain(2_000)).toBe(true);
+ expect(getJob(jobId)?.phase).toBe("cancelled");
+ expect(thumbnail).not.toHaveBeenCalled();
+ expect(files).toHaveLength(2);
+ expect(files.every((handle) => !existsSync(handle.path))).toBe(true);
+ });
+});
+
+describe("direct conversions", () => {
+ test("release the slot when the download fails", async () => {
+ spyOn(mediaVideo, "downloadVideoToTemp").mockRejectedValue(
+ new Error("Failed to download video: 404"),
+ );
+
+ const response = await app.fetch(
+ post("/video/convert", { videoUrl: "https://example.com/missing.mp4" }),
+ );
+
+ expect(response.status).toBe(500);
+ expect(getActiveDirectVideoProcessCount()).toBe(0);
+ });
+
+ test("release the slot and temp files when the client cancels the body", async () => {
+ const files: TempFileHandle[] = [];
+ spyOn(mediaVideo, "downloadVideoToTemp").mockImplementation(async () => {
+ const input = await tempFileWith(".webm", "input");
+ files.push(input);
+ return input;
+ });
+ spyOn(mediaProbe, "probeVideoFile").mockResolvedValue({
+ ...metadata,
+ duration: 10,
+ });
+ spyOn(mediaVideo, "processVideo").mockImplementation(async () => {
+ const output = await tempFileWith(".mp4", "x".repeat(1024 * 1024));
+ files.push(output);
+ return output;
+ });
+
+ const response = await app.fetch(
+ post("/video/convert", { videoUrl: "https://example.com/input.webm" }),
+ );
+ expect(response.status).toBe(200);
+ expect(getActiveDirectVideoProcessCount()).toBe(1);
+
+ await response.body?.cancel();
+ await waitFor(() => getActiveDirectVideoProcessCount() === 0);
+ await waitFor(() => files.every((handle) => !existsSync(handle.path)));
+ });
+
+ test("release the slot after the body is fully read", async () => {
+ spyOn(mediaVideo, "downloadVideoToTemp").mockImplementation(() =>
+ tempFileWith(".webm", "input"),
+ );
+ spyOn(mediaProbe, "probeVideoFile").mockResolvedValue({
+ ...metadata,
+ duration: 10,
+ });
+ spyOn(mediaVideo, "processVideo").mockImplementation(() =>
+ tempFileWith(".mp4", "converted"),
+ );
+
+ const response = await app.fetch(
+ post("/video/convert", { videoUrl: "https://example.com/input.webm" }),
+ );
+
+ expect(await response.text()).toBe("converted");
+ await waitFor(() => getActiveDirectVideoProcessCount() === 0);
+ });
+});
diff --git a/apps/media-server/src/app.ts b/apps/media-server/src/app.ts
index e50979955..162a7652d 100644
--- a/apps/media-server/src/app.ts
+++ b/apps/media-server/src/app.ts
@@ -1,5 +1,6 @@
import { Hono } from "hono";
import { logger } from "hono/logger";
+import { beginRequest } from "./lib/lifecycle";
import audio from "./routes/audio";
import health from "./routes/health";
import video from "./routes/video";
@@ -7,6 +8,15 @@ import video from "./routes/video";
const app = new Hono();
app.use("*", logger());
+app.use("*", async (c, next) => {
+ if (c.req.method === "GET" || c.req.method === "HEAD") return await next();
+ const end = beginRequest();
+ try {
+ await next();
+ } finally {
+ end();
+ }
+});
app.route("/health", health);
app.route("/audio", audio);
diff --git a/apps/media-server/src/index.ts b/apps/media-server/src/index.ts
index 7a5b2151d..05dbdac4f 100644
--- a/apps/media-server/src/index.ts
+++ b/apps/media-server/src/index.ts
@@ -1,38 +1,52 @@
import app from "./app";
import { abortAllJobs } from "./lib/job-manager";
+import { createShutdownHandler, getIdleTimeoutSeconds } from "./lib/lifecycle";
import { cancelAllMediaOperations } from "./lib/media-operations";
-import { cleanupMediaTransferCache } from "./lib/media-transfer";
+import {
+ cleanupMediaTransferCache,
+ sweepMediaTransferCache,
+} from "./lib/media-transfer";
+import { removeOwnTempDir, startTempDirMaintenance } from "./lib/temp-files";
const port = Number(process.env.PORT) || 3456;
console.log(`[media-server] Starting on port ${port}`);
-let shuttingDown = false;
+void Promise.allSettled([
+ startTempDirMaintenance(),
+ sweepMediaTransferCache(),
+]).then(([temp, cache]) => {
+ const removed =
+ (temp.status === "fulfilled" ? temp.value : 0) +
+ (cache.status === "fulfilled" ? cache.value : 0);
+ if (removed > 0) {
+ console.log(`[media-server] Removed ${removed} orphaned temp entries`);
+ }
+});
-const shutdown = async () => {
- if (shuttingDown) return;
- shuttingDown = true;
- console.log("[media-server] Shutting down...");
+const shutdown = createShutdownHandler(async () => {
const abortedJobs = await abortAllJobs();
if (abortedJobs > 0) {
console.log(`[media-server] Aborted ${abortedJobs} active jobs`);
}
await cancelAllMediaOperations();
await cleanupMediaTransferCache();
+ await removeOwnTempDir();
process.exit(0);
-};
+});
process.on("SIGINT", () => {
- void shutdown();
+ void shutdown("SIGINT");
});
process.on("SIGTERM", () => {
- void shutdown();
+ void shutdown("SIGTERM");
});
process.on("SIGHUP", () => {
- void shutdown();
+ void shutdown("SIGHUP");
});
export default {
port,
+ idleTimeout: getIdleTimeoutSeconds(),
fetch: app.fetch,
};
diff --git a/apps/media-server/src/lib/audio-quality.ts b/apps/media-server/src/lib/audio-quality.ts
index d51fe5b76..455824b25 100644
--- a/apps/media-server/src/lib/audio-quality.ts
+++ b/apps/media-server/src/lib/audio-quality.ts
@@ -1,7 +1,6 @@
import { createHash } from "node:crypto";
import { createReadStream } from "node:fs";
import { lstat, mkdtemp, rm } from "node:fs/promises";
-import { tmpdir } from "node:os";
import { isAbsolute, join } from "node:path";
import { spawn } from "bun";
import { z } from "zod";
@@ -13,6 +12,7 @@ import {
} from "./audio-quality-policy";
import { proveRecordingPackets } from "./recording-packet-proof";
import { registerSubprocess, unregisterSubprocess } from "./subprocess";
+import { ensureTempDir, getTempDir } from "./temp-files";
const streamSchema = z.object({
codec_type: z.string(),
@@ -307,7 +307,8 @@ export async function createAudioQualityCandidate(
Math.abs(videoDuration - input.duration) > 0.1
)
return { status: "unchanged", reason: "source-duration-mismatch" };
- directory = await mkdtemp(join(tmpdir(), "cap-audio-quality-"));
+ await ensureTempDir();
+ directory = await mkdtemp(join(getTempDir(), "audio-quality-"));
let path = join(directory, "candidate.mp4");
const renderArgs = [
"ffmpeg",
diff --git a/apps/media-server/src/lib/auth.ts b/apps/media-server/src/lib/auth.ts
index a8e322d28..9641d67c4 100644
--- a/apps/media-server/src/lib/auth.ts
+++ b/apps/media-server/src/lib/auth.ts
@@ -12,3 +12,17 @@ export function validateMediaServerSecret(c: MediaServerContext): boolean {
}
return c.req.header("x-media-server-secret") === secret;
}
+
+export function hasMediaServerSecret(c: MediaServerContext): boolean {
+ return (
+ c.req.header("x-media-server-secret") !== undefined &&
+ validateMediaServerSecret(c)
+ );
+}
+
+export function canReadStatus(c: MediaServerContext): boolean {
+ const publicStatus = process.env.MEDIA_SERVER_PUBLIC_STATUS?.toLowerCase();
+ return publicStatus === "false" || publicStatus === "0"
+ ? validateMediaServerSecret(c)
+ : true;
+}
diff --git a/apps/media-server/src/lib/job-manager.ts b/apps/media-server/src/lib/job-manager.ts
index 74caa0dea..5c05238f2 100644
--- a/apps/media-server/src/lib/job-manager.ts
+++ b/apps/media-server/src/lib/job-manager.ts
@@ -124,6 +124,7 @@ export interface Job {
webhookLastAttemptAt?: number;
recordingVerificationDeadlineAt?: number;
recordingProcessingDeadlineAt?: number;
+ lifetimeDeadlineAt?: number;
recordingVerification?: RecordingVerificationProof;
manifestSha256?: string;
jobId: string;
@@ -147,9 +148,17 @@ export interface Job {
}
const jobs = new Map<string, Job>();
+const jobTotals = {
+ created: 0,
+ completed: 0,
+ errored: 0,
+ cancelled: 0,
+ expired: 0,
+};
const JOB_TTL_MS = 60 * 60 * 1000;
const STALE_JOB_MS = 15 * 60 * 1000;
const MAX_JOB_LIFETIME_MS = 60 * 60 * 1000;
+const MAX_EXTENDED_JOB_LIFETIME_MS = 4 * 60 * 60 * 1000;
const MAX_RECORDING_PROCESSING_BUDGET_MS = 3 * 60 * 60 * 1000;
const WEBHOOK_MAX_ATTEMPTS = 3;
const WEBHOOK_RETRY_BASE_MS = 500;
@@ -366,9 +375,14 @@ export function createJob(
webhookSecret,
};
jobs.set(jobId, job);
+ jobTotals.created++;
return job;
}
+export function getJobTotals(): typeof jobTotals {
+ return { ...jobTotals };
+}
+
export function getJob(jobId: string): Job | undefined {
return jobs.get(jobId);
}
@@ -410,12 +424,37 @@ export function updateJob(
job.terminalAt ??= job.updatedAt;
if (job.recordingWorkerLeaseTimer)
clearTimeout(job.recordingWorkerLeaseTimer);
+ if (job.phase === "complete") jobTotals.completed++;
+ else if (job.phase === "error") jobTotals.errored++;
+ else jobTotals.cancelled++;
}
if (!isActivePhase(job.phase) && job.webhookInFlight)
job.webhookPending = true;
return job;
}
+export function extendJobLifetime(
+ jobId: string,
+ remainingBudgetMs: number,
+): number | undefined {
+ const job = jobs.get(jobId);
+ if (
+ !job ||
+ !isActivePhase(job.phase) ||
+ !Number.isFinite(remainingBudgetMs) ||
+ remainingBudgetMs <= 0
+ )
+ return undefined;
+ job.lifetimeDeadlineAt = Math.min(
+ job.createdAt + MAX_EXTENDED_JOB_LIFETIME_MS,
+ Math.max(
+ job.lifetimeDeadlineAt ?? job.createdAt + MAX_JOB_LIFETIME_MS,
+ Date.now() + remainingBudgetMs,
+ ),
+ );
+ return job.lifetimeDeadlineAt;
+}
+
export function touchJob(jobId: string): Job | undefined {
const job = jobs.get(jobId);
if (!job) return undefined;
@@ -556,11 +595,14 @@ export function cleanupExpiredJobs(): number {
`[job-manager] Cleaning up expired job ${jobId} (phase=${job.phase}, age=${Math.round(age / 60000)}m)`,
);
job.abortController?.abort();
- updateJob(jobId, {
- phase: "error",
- error: `Job expired: no progress update for ${Math.round(staleness / 60000)} minutes`,
- message: "Processing failed (expired)",
- });
+ if (
+ updateJob(jobId, {
+ phase: "error",
+ error: `Job expired: no progress update for ${Math.round(staleness / 60000)} minutes`,
+ message: "Processing failed (expired)",
+ })
+ )
+ jobTotals.expired++;
void sendWebhook(job);
}
cleaned++;
@@ -572,34 +614,40 @@ export function cleanupExpiredJobs(): number {
`[job-manager] Marking stale job ${jobId} as error (phase=${job.phase}, no update for ${Math.round(staleness / 60000)}m)`,
);
job.abortController?.abort();
- updateJob(jobId, {
- phase: "error",
- error: `Job stale: no progress update for ${Math.round(staleness / 60000)} minutes`,
- message: "Processing failed (stale)",
- });
+ if (
+ updateJob(jobId, {
+ phase: "error",
+ error: `Job stale: no progress update for ${Math.round(staleness / 60000)} minutes`,
+ message: "Processing failed (stale)",
+ })
+ )
+ jobTotals.expired++;
void sendWebhook(job);
cleaned++;
continue;
}
+ const lifetimeDeadline =
+ job.lifetimeDeadlineAt ?? job.createdAt + MAX_JOB_LIFETIME_MS;
const deadline =
job.recordingVerificationDeadlineAt ??
job.recordingProcessingDeadlineAt ??
- job.createdAt + MAX_JOB_LIFETIME_MS;
+ lifetimeDeadline;
if (isActivePhase(job.phase) && now > deadline) {
console.warn(
`[job-manager] Marking long-running job ${jobId} as error (phase=${job.phase}, age=${Math.round(age / 60000)}m)`,
);
job.abortController?.abort();
- updateJob(jobId, {
+ const expired = updateJob(jobId, {
phase: "error",
error:
job.recordingVerificationDeadlineAt === undefined &&
job.recordingProcessingDeadlineAt === undefined
- ? `Job exceeded maximum lifetime of ${Math.round(MAX_JOB_LIFETIME_MS / 60000)} minutes`
+ ? `Job exceeded maximum lifetime of ${Math.round((lifetimeDeadline - job.createdAt) / 60000)} minutes`
: "Recording verification timed out",
message: "Processing failed (timeout)",
});
+ if (expired) jobTotals.expired++;
void sendWebhook(job);
cleaned++;
continue;
diff --git a/apps/media-server/src/lib/lifecycle.ts b/apps/media-server/src/lib/lifecycle.ts
new file mode 100644
index 000000000..d23ff92ff
--- /dev/null
+++ b/apps/media-server/src/lib/lifecycle.ts
@@ -0,0 +1,172 @@
+import { getActiveVideoProcessCount } from "./job-manager";
+import {
+ getActiveAudioOperationCount,
+ getActiveProbeOperationCount,
+} from "./media-operations";
+import { getActiveSubprocessCount } from "./subprocess";
+
+const DRAIN_POLL_INTERVAL_MS = 250;
+const DRAIN_LOG_INTERVAL_MS = 30_000;
+export const DRAIN_RETRY_AFTER_SECONDS = 15;
+export const DRAINING_DETAILS =
+ "Server is draining for shutdown, please retry later";
+
+const MAX_IDLE_TIMEOUT_SECONDS = 255;
+
+let draining = false;
+let inFlightRequests = 0;
+const backgroundTasks = new Set<Promise<unknown>>();
+
+// Bun closes a connection after idleTimeout seconds without I/O, including
+// while a handler is still working, and its 10 s default is shorter than the
+// AWS ALB keep-alive idle timeout (60 s), which turns reused sockets into 502s.
+export function getIdleTimeoutSeconds(): number {
+ const value = Number.parseInt(
+ process.env.MEDIA_SERVER_IDLE_TIMEOUT_SECONDS ?? "",
+ 10,
+ );
+ return Number.isFinite(value) && value >= 0
+ ? Math.min(MAX_IDLE_TIMEOUT_SECONDS, value)
+ : MAX_IDLE_TIMEOUT_SECONDS;
+}
+
+type IdleTimeoutServer = {
+ timeout?: (request: Request, seconds: number) => void;
+};
+
+export function setRequestIdleTimeout(
+ server: unknown,
+ request: Request,
+ seconds: number,
+): void {
+ const timeout = (server as IdleTimeoutServer | undefined)?.timeout;
+ if (typeof timeout !== "function") return;
+ try {
+ timeout.call(server, request, seconds);
+ } catch {}
+}
+
+export function getShutdownGraceMs(): number {
+ const value = Number.parseInt(
+ process.env.MEDIA_SERVER_SHUTDOWN_GRACE_MS ?? "0",
+ 10,
+ );
+ return Number.isFinite(value) && value > 0 ? value : 0;
+}
+
+export function isDraining(): boolean {
+ return draining;
+}
+
+export function setDraining(value: boolean): void {
+ draining = value;
+}
+
+export function trackBackgroundTask<T>(task: Promise<T>): Promise<T> {
+ backgroundTasks.add(task);
+ const settle = () => {
+ backgroundTasks.delete(task);
+ };
+ task.then(settle, settle);
+ return task;
+}
+
+export function beginRequest(): () => void {
+ inFlightRequests++;
+ let ended = false;
+ return () => {
+ if (ended) return;
+ ended = true;
+ inFlightRequests--;
+ };
+}
+
+export function getDrainBlockers() {
+ return {
+ activeVideoProcesses: getActiveVideoProcessCount(),
+ backgroundTasks: backgroundTasks.size,
+ inFlightRequests,
+ audioOperations: getActiveAudioOperationCount(),
+ probeOperations: getActiveProbeOperationCount(),
+ subprocesses: getActiveSubprocessCount(),
+ };
+}
+
+export function isIdle(): boolean {
+ return Object.values(getDrainBlockers()).every((count) => count === 0);
+}
+
+function wait(ms: number, signal?: AbortSignal): Promise<void> {
+ return new Promise((resolve) => {
+ const done = () => {
+ clearTimeout(timer);
+ signal?.removeEventListener("abort", done);
+ resolve();
+ };
+ const timer = setTimeout(done, ms);
+ signal?.addEventListener("abort", done, { once: true });
+ });
+}
+
+export async function waitForDrain(
+ graceMs: number,
+ signal?: AbortSignal,
+): Promise<boolean> {
+ const startedAt = Date.now();
+ const deadline = startedAt + graceMs;
+ let lastLogAt = startedAt;
+ while (!isIdle()) {
+ const now = Date.now();
+ if (signal?.aborted || now >= deadline) return false;
+ if (now - lastLogAt >= DRAIN_LOG_INTERVAL_MS) {
+ lastLogAt = now;
+ console.log("[media-server] Draining", {
+ remainingMs: deadline - now,
+ ...getDrainBlockers(),
+ });
+ }
+ await wait(Math.min(DRAIN_POLL_INTERVAL_MS, deadline - now), signal);
+ }
+ return true;
+}
+
+export function createShutdownHandler(
+ finalize: () => Promise<void>,
+ graceMs = getShutdownGraceMs,
+): (signal: string) => Promise<void> {
+ let shuttingDown = false;
+ let forceDrain: AbortController | undefined;
+
+ return async (signal) => {
+ if (forceDrain) {
+ console.log(
+ `[media-server] Received ${signal} while draining, shutting down now`,
+ );
+ forceDrain.abort();
+ return;
+ }
+ if (shuttingDown) return;
+ shuttingDown = true;
+
+ const drainMs = graceMs();
+ if (drainMs > 0) {
+ forceDrain = new AbortController();
+ setDraining(true);
+ console.log(
+ `[media-server] Received ${signal}, draining for up to ${drainMs}ms`,
+ getDrainBlockers(),
+ );
+ const drained = await waitForDrain(drainMs, forceDrain.signal);
+ console.log(
+ drained
+ ? "[media-server] Drain complete"
+ : "[media-server] Drain ended with work still active",
+ getDrainBlockers(),
+ );
+ } else {
+ console.log("[media-server] Shutting down...");
+ }
+
+ await finalize();
+ };
+}
diff --git a/apps/media-server/src/lib/media-edit.ts b/apps/media-server/src/lib/media-edit.ts
index 406dd7034..91a9f0483 100644
--- a/apps/media-server/src/lib/media-edit.ts
+++ b/apps/media-server/src/lib/media-edit.ts
@@ -46,7 +46,7 @@ function getTotalRangeDuration(ranges: EditRange[]) {
return ranges.reduce((total, range) => total + getRangeDuration(range), 0);
}
-function getTimeoutMs(ranges: EditRange[]) {
+export function getTimeoutMs(ranges: EditRange[]) {
const durationMs = getTotalRangeDuration(ranges) * 20_000;
return Math.min(MAX_TIMEOUT_MS, Math.max(DEFAULT_TIMEOUT_MS, durationMs));
}
diff --git a/apps/media-server/src/lib/media-operations.ts b/apps/media-server/src/lib/media-operations.ts
index 1c9901dd8..c1af41209 100644
--- a/apps/media-server/src/lib/media-operations.ts
+++ b/apps/media-server/src/lib/media-operations.ts
@@ -30,6 +30,14 @@ export function getActiveAudioOperationCount(): number {
return count;
}
+export function getMaxConcurrentAudioOperations(): number {
+ return MAX_CONCURRENT_AUDIO_OPERATIONS;
+}
+
+export function getMaxConcurrentProbeOperations(): number {
+ return MAX_CONCURRENT_PROBE_OPERATIONS;
+}
+
export function canAcceptNewAudioOperation(): boolean {
return (
!hasCriticalMemoryPressure() &&
diff --git a/apps/media-server/src/lib/media-transfer.ts b/apps/media-server/src/lib/media-transfer.ts
index 627207793..e78829a12 100644
--- a/apps/media-server/src/lib/media-transfer.ts
+++ b/apps/media-server/src/lib/media-transfer.ts
@@ -1,9 +1,15 @@
import { AsyncLocalStorage } from "node:async_hooks";
import { createHash } from "node:crypto";
-import { mkdir, open, readdir, rm, stat } from "node:fs/promises";
+import { open, rm, stat } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { z } from "zod";
+import {
+ ensureInstanceDir,
+ getInstanceDir,
+ releaseInstanceDir,
+ sweepOrphanedInstanceDirs,
+} from "./temp-files";
const targetSchema = z.object({
version: z.literal(1),
@@ -36,46 +42,58 @@ const transfers = new AsyncLocalStorage<TransferContext>();
const cache = new Map<string, CachedFile>();
const pendingTransfers = new Map<string, Promise<void>>();
const cacheBase = join(tmpdir(), "cap-media-download-cache");
-const cacheRoot = join(cacheBase, String(process.pid));
+const cacheRoot = getInstanceDir(cacheBase);
const CACHE_BYTES = 2 * 1024 ** 3;
const CACHE_TTL_MS = 30 * 60_000;
+const LEGACY_CACHE_ORPHAN_AGE_MS = 60 * 60_000;
let cacheBytes = 0;
let lastPruneAt = 0;
-async function initializeCache() {
- await mkdir(cacheRoot, { recursive: true, mode: 0o700 });
- for (const entry of await readdir(cacheBase, { withFileTypes: true })) {
- if (
- !entry.isDirectory() ||
- !/^\d+$/.test(entry.name) ||
- entry.name === String(process.pid)
- )
- continue;
- try {
- process.kill(Number(entry.name), 0);
- } catch (error) {
- if (
- typeof error === "object" &&
- error !== null &&
- "code" in error &&
- error.code === "ESRCH"
- )
- await rm(join(cacheBase, entry.name), { recursive: true, force: true });
- }
+function isLegacyCacheOrphan(name: string, stats: { mtimeMs: number }) {
+ if (!/^\d+$/.test(name)) return false;
+ if (Date.now() - stats.mtimeMs > LEGACY_CACHE_ORPHAN_AGE_MS) return true;
+ if (name === String(process.pid)) return false;
+ try {
+ process.kill(Number(name), 0);
+ return false;
+ } catch (error) {
+ return (
+ typeof error === "object" &&
+ error !== null &&
+ "code" in error &&
+ error.code === "ESRCH"
+ );
}
}
+
+export function sweepMediaTransferCache(): Promise<number> {
+ return sweepOrphanedInstanceDirs(cacheBase, isLegacyCacheOrphan);
+}
+
+async function initializeCache() {
+ await ensureInstanceDir(cacheBase, 0o700);
+ await sweepMediaTransferCache();
+}
let cacheReady: Promise<void> | undefined;
const pruneTimer = setInterval(() => {
void pruneCache().catch(() => {});
}, 60_000);
pruneTimer.unref?.();
+const sweepTimer = setInterval(() => {
+ void sweepMediaTransferCache().catch(() => {});
+}, 5 * 60_000);
+sweepTimer.unref?.();
export async function cleanupMediaTransferCache() {
- await rm(cacheRoot, { recursive: true, force: true }).catch(() => {});
+ await releaseInstanceDir(cacheBase);
cache.clear();
cacheBytes = 0;
cacheReady = undefined;
}
+export function getMediaTransferCacheUsage() {
+ return { bytes: cacheBytes, entries: cache.size };
+}
+
export function isInternalMediaUrl(input: string): boolean {
try {
const url = new URL(input);
diff --git a/apps/media-server/src/lib/media-video.ts b/apps/media-server/src/lib/media-video.ts
index 3b74caa1c..feecf5830 100644
--- a/apps/media-server/src/lib/media-video.ts
+++ b/apps/media-server/src/lib/media-video.ts
@@ -1297,15 +1297,18 @@ export async function downloadVideoToTemp(
const reader = response.body.getReader();
const writer = file(tempFile.path).writer();
+ let writerEnded = false;
try {
while (true) {
const { done, value } = await reader.read();
if (done) break;
writer.write(value);
}
+ writerEnded = true;
await writer.end();
} finally {
reader.releaseLock();
+ if (!writerEnded) await Promise.resolve(writer.end()).catch(() => {});
}
if (file(tempFile.path).size === 0) {
@@ -1339,7 +1342,7 @@ function needsAudioTranscode(metadata: VideoMetadata): boolean {
return Boolean(metadata.audioCodec && metadata.audioCodec !== "aac");
}
-function getProcessTimeoutMs(
+export function getProcessTimeoutMs(
durationSeconds: number,
baseTimeoutMs: number,
): number {
diff --git a/apps/media-server/src/lib/runtime-report.ts b/apps/media-server/src/lib/runtime-report.ts
new file mode 100644
index 000000000..bfe0bace9
--- /dev/null
+++ b/apps/media-server/src/lib/runtime-report.ts
@@ -0,0 +1,60 @@
+import {
+ getActiveVideoProcessCount,
+ getAllJobs,
+ getJobTotals,
+ getMaxConcurrentVideoProcesses,
+ getSystemResources,
+} from "./job-manager";
+import { getDrainBlockers, getShutdownGraceMs, isDraining } from "./lifecycle";
+import {
+ getMaxConcurrentAudioOperations,
+ getMaxConcurrentProbeOperations,
+} from "./media-operations";
+import { getMediaTransferCacheUsage } from "./media-transfer";
+import { getTempDirUsage } from "./temp-files";
+import {
+ getActiveDirectVideoProcessCount,
+ getMaxConcurrentDirectVideoProcesses,
+} from "./video-capacity";
+
+export async function getRuntimeReport() {
+ const blockers = getDrainBlockers();
+ const jobs = getAllJobs();
+ return {
+ draining: isDraining(),
+ shutdownGraceMs: getShutdownGraceMs(),
+ processUptimeSeconds: Math.round(process.uptime()),
+ activeJobs: jobs.filter(
+ (job) =>
+ job.phase !== "complete" &&
+ job.phase !== "error" &&
+ job.phase !== "cancelled",
+ ).length,
+ jobsInMemory: jobs.length,
+ jobTotals: getJobTotals(),
+ activeSubprocesses: blockers.subprocesses,
+ backgroundTasks: blockers.backgroundTasks,
+ inFlightRequests: blockers.inFlightRequests,
+ slots: {
+ video: {
+ active: getActiveVideoProcessCount(),
+ max: getMaxConcurrentVideoProcesses(),
+ effectiveMax: getSystemResources().effectiveMax,
+ },
+ directVideo: {
+ active: getActiveDirectVideoProcessCount(),
+ max: getMaxConcurrentDirectVideoProcesses(),
+ },
+ audio: {
+ active: blockers.audioOperations,
+ max: getMaxConcurrentAudioOperations(),
+ },
+ probe: {
+ active: blockers.probeOperations,
+ max: getMaxConcurrentProbeOperations(),
+ },
+ },
+ tempDir: await getTempDirUsage(),
+ mediaTransferCache: getMediaTransferCacheUsage(),
+ };
+}
diff --git a/apps/media-server/src/lib/subprocess.ts b/apps/media-server/src/lib/subprocess.ts
index 31ac5ee4e..ffdf7c3f6 100644
--- a/apps/media-server/src/lib/subprocess.ts
+++ b/apps/media-server/src/lib/subprocess.ts
@@ -19,6 +19,10 @@ export function unregisterSubprocess(proc: ManagedSubprocess): void {
activeSubprocesses.delete(proc.pid);
}
+export function getActiveSubprocessCount(): number {
+ return activeSubprocesses.size;
+}
+
async function waitForProcessExit(
proc: Pick<Subprocess, "exited" | "exitCode">,
timeoutMs: number,
diff --git a/apps/media-server/src/lib/temp-files.ts b/apps/media-server/src/lib/temp-files.ts
index 5141ad6c4..0c8ec1fdf 100644
--- a/apps/media-server/src/lib/temp-files.ts
+++ b/apps/media-server/src/lib/temp-files.ts
@@ -1,18 +1,139 @@
import { randomUUID } from "node:crypto";
-import { mkdir, readdir, rm, stat, unlink } from "node:fs/promises";
+import type { Dirent, Stats } from "node:fs";
+import {
+ lstat,
+ mkdir,
+ readdir,
+ rm,
+ unlink,
+ utimes,
+ writeFile,
+} from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
-const TEMP_DIR = join(tmpdir(), "cap-media-server");
+const TEMP_ROOT = join(tmpdir(), "cap-media-server");
+const INSTANCE_ID = randomUUID();
+const INSTANCE_DIR_PREFIX = "instance-";
+const INSTANCE_LEASE_FILE = ".lease";
+const INSTANCE_LEASE_REFRESH_MS = 30_000;
+const INSTANCE_LEASE_STALE_MS = 5 * 60 * 1000;
const STALE_FILE_AGE_MS = 60 * 60 * 1000;
+const MAX_ACTIVE_TEMP_AGE_MS = 6 * 60 * 60 * 1000;
+const TEMP_MAINTENANCE_INTERVAL_MS = 5 * 60 * 1000;
+const TEMP_USAGE_CACHE_MS = 5_000;
+const TEMP_USAGE_MAX_ENTRIES = 20_000;
+
+const leasedDirs = new Set<string>();
+const activeTempPaths = new Set<string>();
+let leaseTimer: ReturnType<typeof setInterval> | undefined;
+let maintenanceTimer: ReturnType<typeof setInterval> | undefined;
export interface TempFileHandle {
path: string;
cleanup: () => Promise<void>;
}
+export interface TempDirUsage {
+ bytes: number;
+ entries: number;
+ truncated: boolean;
+}
+
+export function getInstanceDir(base: string): string {
+ return join(base, `${INSTANCE_DIR_PREFIX}${INSTANCE_ID}`);
+}
+
+const TEMP_DIR = getInstanceDir(TEMP_ROOT);
+
+async function refreshLeases(): Promise<void> {
+ const now = new Date();
+ await Promise.allSettled(
+ Array.from(leasedDirs, async (dir) => {
+ const lease = join(dir, INSTANCE_LEASE_FILE);
+ try {
+ await utimes(lease, now, now);
+ } catch {
+ await mkdir(dir, { recursive: true });
+ await writeFile(lease, "");
+ }
+ }),
+ );
+}
+
+export async function ensureInstanceDir(
+ base: string,
+ mode?: number,
+): Promise<string> {
+ const dir = getInstanceDir(base);
+ await mkdir(dir, { recursive: true, mode });
+ if (!leasedDirs.has(dir)) {
+ await writeFile(join(dir, INSTANCE_LEASE_FILE), "");
+ leasedDirs.add(dir);
+ if (!leaseTimer) {
+ leaseTimer = setInterval(() => {
+ void refreshLeases();
+ }, INSTANCE_LEASE_REFRESH_MS);
+ leaseTimer.unref?.();
+ }
+ }
+ return dir;
+}
+
+export async function releaseInstanceDir(base: string): Promise<void> {
+ const dir = getInstanceDir(base);
+ leasedDirs.delete(dir);
+ await rm(dir, { recursive: true, force: true }).catch(() => {});
+}
+
+// Instance directories carry a lease so a host that shares one temp volume
+// between an old (draining) and a new container never sweeps live work.
+export async function sweepOrphanedInstanceDirs(
+ base: string,
+ isLegacyOrphan: (name: string, stats: Stats) => boolean,
+): Promise<number> {
+ let entries: Dirent[];
+ try {
+ entries = await readdir(base, { withFileTypes: true });
+ } catch {
+ return 0;
+ }
+ const ownDir = getInstanceDir(base);
+ const now = Date.now();
+ let removed = 0;
+
+ for (const entry of entries) {
+ const path = join(base, entry.name);
+ if (path === ownDir) continue;
+ try {
+ const orphaned =
+ entry.isDirectory() && entry.name.startsWith(INSTANCE_DIR_PREFIX)
+ ? now -
+ (
+ await lstat(join(path, INSTANCE_LEASE_FILE)).catch(() =>
+ lstat(path),
+ )
+ ).mtimeMs >
+ INSTANCE_LEASE_STALE_MS
+ : isLegacyOrphan(entry.name, await lstat(path));
+ if (!orphaned) continue;
+ await rm(path, { recursive: true, force: true });
+ removed++;
+ } catch {}
+ }
+
+ return removed;
+}
+
export async function ensureTempDir(): Promise<void> {
- await mkdir(TEMP_DIR, { recursive: true });
+ await ensureInstanceDir(TEMP_ROOT);
+}
+
+export function retainTempPath(path: string): () => void {
+ activeTempPaths.add(path);
+ return () => {
+ activeTempPaths.delete(path);
+ };
}
export async function createTempFile(
@@ -21,6 +142,7 @@ export async function createTempFile(
await ensureTempDir();
const filename = `${randomUUID()}${extension.startsWith(".") ? extension : `.${extension}`}`;
const path = join(TEMP_DIR, filename);
+ const release = retainTempPath(path);
return {
path,
@@ -28,32 +150,116 @@ export async function createTempFile(
try {
await unlink(path);
} catch {}
+ release();
},
};
}
+export function sweepOrphanedTempDirs(): Promise<number> {
+ return sweepOrphanedInstanceDirs(
+ TEMP_ROOT,
+ (_name, stats) => Date.now() - stats.mtimeMs > STALE_FILE_AGE_MS,
+ );
+}
+
export async function cleanupStaleTempFiles(): Promise<number> {
+ let cleaned = await sweepOrphanedTempDirs();
+ let entries: string[];
try {
- await ensureTempDir();
- const files = await readdir(TEMP_DIR);
- const now = Date.now();
- let cleaned = 0;
+ entries = await readdir(TEMP_DIR);
+ } catch {
+ return cleaned;
+ }
+ const now = Date.now();
- for (const file of files) {
- const filePath = join(TEMP_DIR, file);
- try {
- const fileStat = await stat(filePath);
- if (now - fileStat.mtimeMs > STALE_FILE_AGE_MS) {
- await rm(filePath, { force: true });
- cleaned++;
- }
- } catch {}
- }
+ for (const entry of entries) {
+ if (entry === INSTANCE_LEASE_FILE) continue;
+ const path = join(TEMP_DIR, entry);
+ try {
+ const age = now - (await lstat(path)).mtimeMs;
+ if (
+ age <= STALE_FILE_AGE_MS ||
+ (activeTempPaths.has(path) && age <= MAX_ACTIVE_TEMP_AGE_MS)
+ )
+ continue;
+ await rm(path, { recursive: true, force: true });
+ activeTempPaths.delete(path);
+ cleaned++;
+ } catch {}
+ }
- return cleaned;
+ return cleaned;
+}
+
+export async function startTempDirMaintenance(): Promise<number> {
+ if (!maintenanceTimer) {
+ maintenanceTimer = setInterval(() => {
+ void cleanupStaleTempFiles()
+ .then((cleaned) => {
+ if (cleaned > 0)
+ console.log(`[temp-files] Removed ${cleaned} stale temp entries`);
+ })
+ .catch(() => {});
+ }, TEMP_MAINTENANCE_INTERVAL_MS);
+ maintenanceTimer.unref?.();
+ }
+ return await sweepOrphanedTempDirs();
+}
+
+export async function removeOwnTempDir(): Promise<void> {
+ activeTempPaths.clear();
+ await releaseInstanceDir(TEMP_ROOT);
+}
+
+async function measureDir(
+ dir: string,
+ usage: TempDirUsage,
+): Promise<TempDirUsage> {
+ let entries: Dirent[];
+ try {
+ entries = await readdir(dir, { withFileTypes: true });
} catch {
- return 0;
+ return usage;
+ }
+ for (const entry of entries) {
+ if (usage.entries >= TEMP_USAGE_MAX_ENTRIES) {
+ usage.truncated = true;
+ return usage;
+ }
+ usage.entries++;
+ const path = join(dir, entry.name);
+ if (entry.isDirectory()) {
+ await measureDir(path, usage);
+ } else {
+ usage.bytes += await lstat(path).then(
+ (stats) => stats.size,
+ () => 0,
+ );
+ }
}
+ return usage;
+}
+
+let tempUsageCache:
+ | { measuredAt: number; usage: Promise<TempDirUsage> }
+ | undefined;
+
+export function getTempDirUsage(): Promise<TempDirUsage> {
+ const now = Date.now();
+ if (
+ !tempUsageCache ||
+ now - tempUsageCache.measuredAt > TEMP_USAGE_CACHE_MS
+ ) {
+ tempUsageCache = {
+ measuredAt: now,
+ usage: measureDir(TEMP_ROOT, { bytes: 0, entries: 0, truncated: false }),
+ };
+ }
+ return tempUsageCache.usage;
+}
+
+export function getTempRoot(): string {
+ return TEMP_ROOT;
}
export function getTempDir(): string {
diff --git a/apps/media-server/src/routes/audio.ts b/apps/media-server/src/routes/audio.ts
index d2f8cc71e..adf57bc54 100644
--- a/apps/media-server/src/routes/audio.ts
+++ b/apps/media-server/src/routes/audio.ts
@@ -1,6 +1,12 @@
-import { Hono } from "hono";
+import { type Context, Hono } from "hono";
import { z } from "zod";
-import { validateMediaServerSecret } from "../lib/auth";
+import { canReadStatus, validateMediaServerSecret } from "../lib/auth";
+import {
+ DRAIN_RETRY_AFTER_SECONDS,
+ DRAINING_DETAILS,
+ isDraining,
+ setRequestIdleTimeout,
+} from "../lib/lifecycle";
import {
canAcceptNewAudioOperation as canAcceptNewProcess,
checkHasAudioTrack,
@@ -35,7 +41,22 @@ function isTimeoutError(err: unknown): boolean {
return err instanceof Error && err.message.includes("timed out");
}
+function drainingResponse(c: Context) {
+ c.header("Retry-After", DRAIN_RETRY_AFTER_SECONDS.toString());
+ return c.json(
+ {
+ error: "Server is busy",
+ code: "SERVER_BUSY",
+ details: DRAINING_DETAILS,
+ },
+ 503,
+ );
+}
+
audio.get("/status", (c) => {
+ if (!canReadStatus(c)) {
+ return c.json({ error: "Unauthorized" }, 401);
+ }
return c.json({
activeProcesses: getActiveProcessCount(),
canAcceptNewProcess: canAcceptNewProcess(),
@@ -61,6 +82,8 @@ audio.post("/check", async (c) => {
);
}
+ if (isDraining()) return drainingResponse(c);
+
try {
const hasAudio = await checkHasAudioTrack(result.data.videoUrl);
return c.json({ hasAudio });
@@ -120,6 +143,8 @@ audio.post("/extract", async (c) => {
}
const { videoUrl, stream: useStreaming } = result.data;
+ if (isDraining()) return drainingResponse(c);
+ setRequestIdleTimeout(c.env, c.req.raw, 0);
try {
const hasAudio = await checkHasAudioTrack(videoUrl);
@@ -209,6 +234,8 @@ audio.post("/convert", async (c) => {
}
const { audioUrl, bitrate } = result.data;
+ if (isDraining()) return drainingResponse(c);
+ setRequestIdleTimeout(c.env, c.req.raw, 0);
try {
const { stream, cleanup } = extractAudioStream(audioUrl, {
diff --git a/apps/media-server/src/routes/health.ts b/apps/media-server/src/routes/health.ts
index 6a69c7ef2..eed6478b5 100644
--- a/apps/media-server/src/routes/health.ts
+++ b/apps/media-server/src/routes/health.ts
@@ -1,12 +1,15 @@
import os from "node:os";
import { Hono } from "hono";
+import { hasMediaServerSecret } from "../lib/auth";
import { getSystemResources } from "../lib/job-manager";
+import { isDraining } from "../lib/lifecycle";
import { getMediaEngineStatus } from "../lib/media-engine";
import { getVideoEncoderStatus } from "../lib/nvenc-encoder";
+import { getRuntimeReport } from "../lib/runtime-report";
const health = new Hono();
-health.get("/", (c) => {
+health.get("/", async (c) => {
const mediaEngine = getMediaEngineStatus();
const [, loadAvg5m, loadAvg15m] = os.loadavg();
const totalMemoryBytes = os.totalmem();
@@ -17,27 +20,35 @@ health.get("/", (c) => {
(1 - freeMemoryBytes / totalMemoryBytes) * 100,
);
const resources = getSystemResources();
+ const draining = isDraining();
- return c.json({
- status:
- mediaEngine.available && resources.effectiveMax > 0 ? "ok" : "degraded",
- mediaEngine,
- ["ff" + "mpeg"]: {
- available: mediaEngine.available,
- version: mediaEngine.version,
+ return c.json(
+ {
+ status: draining
+ ? "draining"
+ : mediaEngine.available && resources.effectiveMax > 0
+ ? "ok"
+ : "degraded",
+ mediaEngine,
+ ["ff" + "mpeg"]: {
+ available: mediaEngine.available,
+ version: mediaEngine.version,
+ },
+ videoEncoder: getVideoEncoderStatus(),
+ system: {
+ ...resources,
+ loadAvg5m,
+ loadAvg15m,
+ totalMemoryMB,
+ freeMemoryMB,
+ memoryUsagePercent,
+ uptimeSeconds: Math.round(os.uptime()),
+ processUptimeSeconds: Math.round(process.uptime()),
+ },
+ ...(hasMediaServerSecret(c) && { runtime: await getRuntimeReport() }),
},
- videoEncoder: getVideoEncoderStatus(),
- system: {
- ...resources,
- loadAvg5m,
- loadAvg15m,
- totalMemoryMB,
- freeMemoryMB,
- memoryUsagePercent,
- uptimeSeconds: Math.round(os.uptime()),
- processUptimeSeconds: Math.round(process.uptime()),
- },
- });
+ draining ? 503 : 200,
+ );
});
export default health;
diff --git a/apps/media-server/src/routes/video.ts b/apps/media-server/src/routes/video.ts
index cd2b44e68..b194bc594 100644
--- a/apps/media-server/src/routes/video.ts
+++ b/apps/media-server/src/routes/video.ts
@@ -4,7 +4,11 @@ import { file } from "bun";
import { Hono } from "hono";
import { z } from "zod";
import { enhanceLocalRecording } from "../lib/audio-levels";
-import { validateMediaServerSecret } from "../lib/auth";
+import {
+ canReadStatus,
+ hasMediaServerSecret,
+ validateMediaServerSecret,
+} from "../lib/auth";
import type {
Job,
RecordingErrorCode,
@@ -17,6 +21,7 @@ import {
claimRecordingWorker,
createJob,
deleteJob,
+ extendJobLifetime,
forceCleanupActiveJobs,
generateJobId,
getActiveVideoProcessCount,
@@ -30,8 +35,18 @@ import {
touchJob,
updateJob,
} from "../lib/job-manager";
-import { PROCESS_TIMEOUT_MS } from "../lib/media-common";
-import { renderEditedVideo } from "../lib/media-edit";
+import {
+ DRAINING_DETAILS,
+ getIdleTimeoutSeconds,
+ isDraining,
+ setRequestIdleTimeout,
+ trackBackgroundTask,
+} from "../lib/lifecycle";
+import { PROCESS_TIMEOUT_MS, UPLOAD_TIMEOUT_MS } from "../lib/media-common";
+import {
+ getTimeoutMs as getEditRenderTimeoutMs,
+ renderEditedVideo,
+} from "../lib/media-edit";
import {
canAcceptNewProbeOperation as canAcceptNewProbeProcess,
getActiveProbeOperationCount as getActiveProbeProcessCount,
@@ -54,6 +69,7 @@ import {
downloadVideoToTemp,
generatePreviewGif,
generateThumbnail,
+ getProcessTimeoutMs,
muxMediaTracksToMp4,
processVideo,
repairContainer,
@@ -69,6 +85,7 @@ import {
verifyRemoteRecordingBytes,
verifyRemuxedRecording,
} from "../lib/recording-verification";
+import { getRuntimeReport } from "../lib/runtime-report";
import type { TempFileHandle } from "../lib/temp-files";
import { cleanupStaleTempFiles } from "../lib/temp-files";
import {
@@ -93,6 +110,8 @@ const VIDEO_MEMORY_PRESSURE_ERROR =
"SERVER_BUSY: container memory pressure is too high";
const SEGMENT_DOWNLOAD_MAX_ATTEMPTS = 3;
const SEGMENT_DOWNLOAD_RETRY_BASE_MS = 250;
+const JOB_TRANSFER_ALLOWANCE_MS = 2 * UPLOAD_TIMEOUT_MS;
+const JOB_TRANSFER_BYTES_PER_SECOND = 2 * 1024 ** 2;
const probeSchema = z.object({
videoUrl: z.string().url(),
@@ -220,6 +239,12 @@ function getMuxBusyResponseBody(
};
}
+const DRAINING_RESPONSE_BODY = {
+ error: "Server is busy",
+ code: "SERVER_BUSY",
+ details: DRAINING_DETAILS,
+};
+
function summarizeVideoInput(videoUrl: string, inputExtension?: string) {
const extension = inputExtension?.toLowerCase() ?? null;
try {
@@ -286,25 +311,26 @@ async function createVideoDownloadResponse(
onCleanup?.();
};
+ const reader = outputFile.stream().getReader();
const stream = new ReadableStream<Uint8Array>({
- async start(controller) {
- const reader = outputFile.stream().getReader();
-
+ async pull(controller) {
try {
- while (true) {
- const { done, value } = await reader.read();
- if (done) break;
- if (value) controller.enqueue(value);
+ const { done, value } = await reader.read();
+ if (done) {
+ controller.close();
+ reader.releaseLock();
+ await cleanup();
+ } else if (value) {
+ controller.enqueue(value);
}
- controller.close();
} catch (error) {
controller.error(error);
- } finally {
reader.releaseLock();
await cleanup();
}
},
async cancel() {
+ await reader.cancel().catch(() => {});
await cleanup();
},
});
@@ -345,6 +371,23 @@ async function withJobHeartbeat<T>(
}
}
+function extendJobLifetimeForWork(
+ jobId: string,
+ workTimeoutMs: number,
+ transferBytes: number,
+): void {
+ extendJobLifetime(
+ jobId,
+ workTimeoutMs +
+ JOB_TRANSFER_ALLOWANCE_MS +
+ POST_VERIFICATION_ASSET_BUDGET_MS +
+ Math.ceil(
+ (Math.max(0, transferBytes || 0) / JOB_TRANSFER_BYTES_PER_SECOND) *
+ 1000,
+ ),
+ );
+}
+
async function withMuxMemoryGuard<T>(
abortController: AbortController,
operation: () => Promise<T>,
@@ -373,7 +416,10 @@ async function withMuxMemoryGuard<T>(
}
}
-video.get("/status", (c) => {
+video.get("/status", async (c) => {
+ if (!canReadStatus(c)) {
+ return c.json({ error: "Unauthorized" }, 401);
+ }
const jobs = getAllJobs();
const resources = getSystemResources();
const now = Date.now();
@@ -401,6 +447,7 @@ video.get("/status", (c) => {
stalenessMinutes: Math.round((now - j.updatedAt) / 60000),
error: j.error,
})),
+ ...(hasMediaServerSecret(c) && { runtime: await getRuntimeReport() }),
});
});
@@ -423,6 +470,11 @@ video.post("/probe", async (c) => {
);
}
+ if (isDraining()) {
+ c.header("Retry-After", VIDEO_BUSY_RETRY_AFTER_SECONDS.toString());
+ return c.json(DRAINING_RESPONSE_BODY, 503);
+ }
+
try {
const metadata = await probeVideo(result.data.videoUrl);
return c.json({ metadata });
@@ -481,6 +533,11 @@ video.post("/thumbnail", async (c) => {
);
}
+ if (isDraining()) {
+ c.header("Retry-After", VIDEO_BUSY_RETRY_AFTER_SECONDS.toString());
+ return c.json(DRAINING_RESPONSE_BODY, 503);
+ }
+
try {
const metadata = await probeVideo(result.data.videoUrl);
@@ -562,6 +619,17 @@ video.post("/convert", async (c) => {
const requestId = crypto.randomUUID();
const startedAt = Date.now();
+ if (isDraining()) {
+ c.header("Retry-After", VIDEO_BUSY_RETRY_AFTER_SECONDS.toString());
+ return c.json(
+ {
+ ...getBusyResponseBody(getVideoCapacitySnapshot()),
+ details: DRAINING_DETAILS,
+ },
+ 503,
+ );
+ }
+
try {
slot = tryAcquireDirectVideoProcessSlot(canAcceptNewVideoProcess);
const capacity = getVideoCapacitySnapshot();
@@ -583,6 +651,7 @@ video.post("/convert", async (c) => {
),
...capacity,
});
+ setRequestIdleTimeout(c.env, c.req.raw, 0);
inputTempFile = await downloadVideoToTemp(
result.data.videoUrl,
@@ -618,12 +687,14 @@ video.post("/convert", async (c) => {
...getVideoCapacitySnapshot(),
});
+ setRequestIdleTimeout(c.env, c.req.raw, getIdleTimeoutSeconds());
return await createVideoDownloadResponse(
outputTempFile,
[inputTempFile, outputTempFile],
slot.release,
);
} catch (err) {
+ setRequestIdleTimeout(c.env, c.req.raw, getIdleTimeoutSeconds());
slot?.release();
await cleanupTempFiles([outputTempFile, inputTempFile]);
logVideoEvent("video_convert_failed", {
@@ -692,6 +763,13 @@ video.on("POST", ["/process", "/import"], async (c) => {
}
const capacity = getVideoCapacitySnapshot();
+ if (isDraining()) {
+ c.header("Retry-After", VIDEO_BUSY_RETRY_AFTER_SECONDS.toString());
+ return c.json(
+ { ...getBusyResponseBody(capacity), details: DRAINING_DETAILS },
+ 503,
+ );
+ }
const bulkCapacityReached =
result.data.priority === "bulk" &&
capacity.effectiveMaxVideoProcesses > 1 &&
@@ -715,34 +793,36 @@ video.on("POST", ["/process", "/import"], async (c) => {
const jobId = generateJobId();
const job = createJob(jobId, videoId, userId, webhookUrl, webhookSecret);
- withMediaTransfers(32 * 1024 ** 3, () =>
- processVideoAsync(
- job.jobId,
- videoUrl,
- outputPresignedUrl,
- thumbnailPresignedUrl,
- previewGifPresignedUrl,
- result.data,
- ),
- ).catch((err) => {
- console.error(
- `[video/process] Async processing error for job ${jobId}:`,
- err,
- );
- const currentJob = getJob(jobId);
- if (
- currentJob &&
- currentJob.phase !== "error" &&
- currentJob.phase !== "complete" &&
- currentJob.phase !== "cancelled"
- ) {
- updateJob(jobId, {
- phase: "error",
- error: err instanceof Error ? err.message : String(err),
- message: "Processing failed (unhandled)",
- });
- }
- });
+ void trackBackgroundTask(
+ withMediaTransfers(32 * 1024 ** 3, () =>
+ processVideoAsync(
+ job.jobId,
+ videoUrl,
+ outputPresignedUrl,
+ thumbnailPresignedUrl,
+ previewGifPresignedUrl,
+ result.data,
+ ),
+ ).catch((err) => {
+ console.error(
+ `[video/process] Async processing error for job ${jobId}:`,
+ err,
+ );
+ const currentJob = getJob(jobId);
+ if (
+ currentJob &&
+ currentJob.phase !== "error" &&
+ currentJob.phase !== "complete" &&
+ currentJob.phase !== "cancelled"
+ ) {
+ updateJob(jobId, {
+ phase: "error",
+ error: err instanceof Error ? err.message : String(err),
+ message: "Processing failed (unhandled)",
+ });
+ }
+ }),
+ );
return c.json({
jobId,
@@ -775,6 +855,17 @@ video.post("/edit", async (c) => {
);
}
+ if (isDraining()) {
+ c.header("Retry-After", VIDEO_BUSY_RETRY_AFTER_SECONDS.toString());
+ return c.json(
+ {
+ ...getBusyResponseBody(getVideoCapacitySnapshot()),
+ details: DRAINING_DETAILS,
+ },
+ 503,
+ );
+ }
+
if (!canAcceptNewVideoProcess()) {
c.header("Retry-After", VIDEO_BUSY_RETRY_AFTER_SECONDS.toString());
return c.json(getBusyResponseBody(getVideoCapacitySnapshot()), 503);
@@ -794,31 +885,33 @@ video.post("/edit", async (c) => {
const jobId = generateJobId();
const job = createJob(jobId, videoId, userId, webhookUrl, webhookSecret);
- withMediaTransfers(32 * 1024 ** 3, () =>
- editVideoAsync(
- job.jobId,
- sourceUrl,
- outputPresignedUrl,
- thumbnailPresignedUrl,
- previewGifPresignedUrl,
- result.data,
- ),
- ).catch((err) => {
- console.error(`[video/edit] Async edit error for job ${jobId}:`, err);
- const currentJob = getJob(jobId);
- if (
- currentJob &&
- currentJob.phase !== "error" &&
- currentJob.phase !== "complete" &&
- currentJob.phase !== "cancelled"
- ) {
- updateJob(jobId, {
- phase: "error",
- error: err instanceof Error ? err.message : String(err),
- message: "Edit failed (unhandled)",
- });
- }
- });
+ void trackBackgroundTask(
+ withMediaTransfers(32 * 1024 ** 3, () =>
+ editVideoAsync(
+ job.jobId,
+ sourceUrl,
+ outputPresignedUrl,
+ thumbnailPresignedUrl,
+ previewGifPresignedUrl,
+ result.data,
+ ),
+ ).catch((err) => {
+ console.error(`[video/edit] Async edit error for job ${jobId}:`, err);
+ const currentJob = getJob(jobId);
+ if (
+ currentJob &&
+ currentJob.phase !== "error" &&
+ currentJob.phase !== "complete" &&
+ currentJob.phase !== "cancelled"
+ ) {
+ updateJob(jobId, {
+ phase: "error",
+ error: err instanceof Error ? err.message : String(err),
+ message: "Edit failed (unhandled)",
+ });
+ }
+ }),
+ );
return c.json({
jobId,
@@ -963,6 +1056,11 @@ async function processWithResilientRetry(
progress: 10,
message: "Retrying with error recovery...",
});
+ extendJobLifetimeForWork(
+ jobId,
+ getProcessTimeoutMs(metadata.duration, PROCESS_TIMEOUT_MS),
+ metadata.fileSize,
+ );
try {
const outputFile = await processVideo(
@@ -988,6 +1086,11 @@ async function processWithResilientRetry(
progress: 10,
message: "Attempting full repair...",
});
+ extendJobLifetimeForWork(
+ jobId,
+ getProcessTimeoutMs(metadata.duration, PROCESS_TIMEOUT_MS),
+ metadata.fileSize,
+ );
let lastResortRepairFile: TempFileHandle | null = null;
try {
@@ -1075,21 +1178,25 @@ function isDurationClose(actual: number, expected: number) {
);
}
-async function waitForVerificationRetry(attempt: number) {
- await new Promise((resolve) =>
- setTimeout(resolve, UPLOAD_VERIFICATION_RETRY_MS * (attempt + 1)),
- );
+async function waitForVerificationRetry(
+ attempt: number,
+ abortSignal?: AbortSignal,
+) {
+ await Bun.sleep(UPLOAD_VERIFICATION_RETRY_MS * (attempt + 1));
+ abortSignal?.throwIfAborted();
}
async function verifyUploadedVideo(
outputVerificationUrl: string | undefined,
expectedMetadata: VideoMetadata,
+ abortSignal?: AbortSignal,
) {
if (!outputVerificationUrl) return;
let lastError: Error | undefined;
for (let attempt = 0; attempt < UPLOAD_VERIFICATION_ATTEMPTS; attempt++) {
+ abortSignal?.throwIfAborted();
try {
const actualMetadata = await probeVideo(outputVerificationUrl);
if (isDurationClose(actualMetadata.duration, expectedMetadata.duration)) {
@@ -1104,7 +1211,7 @@ async function verifyUploadedVideo(
}
if (attempt < UPLOAD_VERIFICATION_ATTEMPTS - 1) {
- await waitForVerificationRetry(attempt);
+ await waitForVerificationRetry(attempt, abortSignal);
}
}
@@ -1124,7 +1231,10 @@ async function editVideoAsync(
}
const abortController = new AbortController();
+ const signal = abortController.signal;
updateJob(jobId, { abortController });
+ let inputTempFile: TempFileHandle | null = null;
+ let outputTempFile: TempFileHandle | null = null;
try {
updateJob(jobId, {
@@ -1137,10 +1247,11 @@ async function editVideoAsync(
await sendWebhook(downloadingJob);
}
- const inputTempFile = await withJobHeartbeat(jobId, () =>
- downloadVideoToTemp(sourceUrl, ".mp4", abortController.signal),
+ const input = await withJobHeartbeat(jobId, () =>
+ downloadVideoToTemp(sourceUrl, ".mp4", signal),
);
- updateJob(jobId, { inputTempFile });
+ inputTempFile = input;
+ updateJob(jobId, { inputTempFile: input });
updateJob(jobId, {
phase: "probing",
@@ -1152,7 +1263,15 @@ async function editVideoAsync(
await sendWebhook(probingJob);
}
- const sourceMetadata = await probeVideoFile(inputTempFile.path);
+ const sourceMetadata = await withJobHeartbeat(jobId, () =>
+ probeVideoFile(input.path),
+ );
+ signal.throwIfAborted();
+ extendJobLifetimeForWork(
+ jobId,
+ getEditRenderTimeoutMs(options.keepRanges),
+ sourceMetadata.fileSize,
+ );
updateJob(jobId, {
phase: "processing",
@@ -1164,12 +1283,12 @@ async function editVideoAsync(
await sendWebhook(processingJob);
}
- const outputTempFile = await withJobHeartbeat(jobId, () =>
+ const output = await withJobHeartbeat(jobId, () =>
renderEditedVideo({
- inputPath: inputTempFile.path,
+ inputPath: input.path,
keepRanges: options.keepRanges,
metadata: sourceMetadata,
- abortSignal: abortController.signal,
+ abortSignal: signal,
onProgress: (progress, message) => {
updateJob(jobId, {
progress: Math.min(80, 10 + progress * 0.9),
@@ -1187,9 +1306,13 @@ async function editVideoAsync(
},
}),
);
- updateJob(jobId, { outputTempFile });
+ outputTempFile = output;
+ updateJob(jobId, { outputTempFile: output });
- const outputMetadata = await probeVideoFile(outputTempFile.path);
+ const outputMetadata = await withJobHeartbeat(jobId, () =>
+ probeVideoFile(output.path),
+ );
+ signal.throwIfAborted();
updateJob(jobId, { metadata: outputMetadata });
updateJob(jobId, {
@@ -1202,8 +1325,19 @@ async function editVideoAsync(
await sendWebhook(uploadingJob);
}
- await uploadFileToS3(outputTempFile.path, outputPresignedUrl, "video/mp4");
- await verifyUploadedVideo(options.outputVerificationUrl, outputMetadata);
+ await withJobHeartbeat(jobId, async () => {
+ await uploadFileToS3(
+ output.path,
+ outputPresignedUrl,
+ "video/mp4",
+ signal,
+ );
+ await verifyUploadedVideo(
+ options.outputVerificationUrl,
+ outputMetadata,
+ signal,
+ );
+ });
if (thumbnailPresignedUrl || previewGifPresignedUrl) {
updateJob(jobId, {
@@ -1217,21 +1351,31 @@ async function editVideoAsync(
}
}
- if (thumbnailPresignedUrl) {
- const thumbnailData = await generateThumbnail(
- outputTempFile.path,
+ await withJobHeartbeat(jobId, async () => {
+ if (thumbnailPresignedUrl) {
+ const thumbnailData = await generateThumbnail(
+ output.path,
+ outputMetadata.duration,
+ {},
+ signal,
+ );
+ await uploadToS3(
+ thumbnailData,
+ thumbnailPresignedUrl,
+ "image/jpeg",
+ signal,
+ );
+ }
+
+ await generateAndUploadPreviewGif(
+ output.path,
outputMetadata.duration,
+ previewGifPresignedUrl,
+ signal,
+ "video/edit",
);
- await uploadToS3(thumbnailData, thumbnailPresignedUrl, "image/jpeg");
- }
-
- await generateAndUploadPreviewGif(
- outputTempFile.path,
- outputMetadata.duration,
- previewGifPresignedUrl,
- abortController.signal,
- "video/edit",
- );
+ });
+ signal.throwIfAborted();
updateJob(jobId, {
phase: "complete",
@@ -1243,12 +1387,11 @@ async function editVideoAsync(
await sendWebhook(completedJob);
}
- await inputTempFile.cleanup();
- await outputTempFile.cleanup();
-
setTimeout(() => deleteJob(jobId), 5 * 60 * 1000);
} catch (err) {
- console.error(`[video/edit] Error editing job ${jobId}:`, err);
+ if (!signal.aborted) {
+ console.error(`[video/edit] Error editing job ${jobId}:`, err);
+ }
const updatedJob = updateJob(jobId, {
phase: "error",
@@ -1261,16 +1404,10 @@ async function editVideoAsync(
await sendWebhook(updatedJob);
}
} finally {
- const currentJob = getJob(jobId);
- if (currentJob) {
- await Promise.allSettled([
- currentJob.inputTempFile?.cleanup(),
- currentJob.outputTempFile?.cleanup(),
- ]);
- }
-
setTimeout(() => deleteJob(jobId), 5 * 60 * 1000);
}
+ } finally {
+ await cleanupTempFiles([inputTempFile, outputTempFile]);
}
}
@@ -1288,8 +1425,11 @@ async function processVideoAsync(
}
const abortController = new AbortController();
+ const signal = abortController.signal;
updateJob(jobId, { abortController });
+ let inputTempFile: TempFileHandle | null = null;
+ let outputTempFile: TempFileHandle | null = null;
let repairedTempFile: TempFileHandle | null = null;
let lastResortRepairFile: TempFileHandle | null = null;
@@ -1301,26 +1441,18 @@ async function processVideoAsync(
});
await sendWebhook(job);
- const inputTempFile = await withJobHeartbeat(jobId, () =>
- downloadVideoToTemp(
- videoUrl,
- options.inputExtension,
- abortController.signal,
- ),
+ const input = await withJobHeartbeat(jobId, () =>
+ downloadVideoToTemp(videoUrl, options.inputExtension, signal),
);
- updateJob(jobId, { inputTempFile });
+ inputTempFile = input;
+ updateJob(jobId, { inputTempFile: input });
const sourcePresignedUrl = options.sourcePresignedUrl;
if (sourcePresignedUrl) {
updateJob(jobId, { message: "Saving original video..." });
await sendWebhook(job);
await withJobHeartbeat(jobId, () =>
- uploadFileToS3(
- inputTempFile.path,
- sourcePresignedUrl,
- "video/mp4",
- abortController.signal,
- ),
+ uploadFileToS3(input.path, sourcePresignedUrl, "video/mp4", signal),
);
}
@@ -1328,7 +1460,7 @@ async function processVideoAsync(
if (isWebm) {
updateJob(jobId, { message: "Checking the original recording..." });
await withJobHeartbeat(jobId, () =>
- validateVideoInput(inputTempFile.path, abortController.signal),
+ validateVideoInput(input.path, signal),
);
}
@@ -1339,17 +1471,19 @@ async function processVideoAsync(
});
await sendWebhook(job);
- const { metadata, repairedFile } = await probeWithRepairFallback(
- inputTempFile.path,
- isWebm,
- abortController.signal,
+ const { metadata, repairedFile } = await withJobHeartbeat(jobId, () =>
+ probeWithRepairFallback(input.path, isWebm, signal),
);
repairedTempFile = repairedFile;
+ signal.throwIfAborted();
updateJob(jobId, { metadata });
+ extendJobLifetimeForWork(
+ jobId,
+ getProcessTimeoutMs(metadata.duration, PROCESS_TIMEOUT_MS),
+ metadata.fileSize,
+ );
- const processingInputPath = repairedFile
- ? repairedFile.path
- : inputTempFile.path;
+ const processingInputPath = repairedFile ? repairedFile.path : input.path;
updateJob(jobId, {
phase: "processing",
@@ -1360,18 +1494,19 @@ async function processVideoAsync(
});
await sendWebhook(job);
- const { outputFile: outputTempFile, lastResortRepairFile: lrrf } =
- await processWithResilientRetry(
- processingInputPath,
- inputTempFile.path,
- metadata,
- options,
- isWebm,
- jobId,
- abortController.signal,
- );
- lastResortRepairFile = lrrf;
- updateJob(jobId, { outputTempFile });
+ const processed = await processWithResilientRetry(
+ processingInputPath,
+ input.path,
+ metadata,
+ options,
+ isWebm,
+ jobId,
+ signal,
+ );
+ outputTempFile = processed.outputFile;
+ lastResortRepairFile = processed.lastResortRepairFile;
+ const outputPath = processed.outputFile.path;
+ updateJob(jobId, { outputTempFile: processed.outputFile });
updateJob(jobId, {
phase: "uploading",
@@ -1380,11 +1515,8 @@ async function processVideoAsync(
});
await sendWebhook(job);
- const uploadReceipt = await uploadFileToS3(
- outputTempFile.path,
- outputPresignedUrl,
- "video/mp4",
- abortController.signal,
+ const uploadReceipt = await withJobHeartbeat(jobId, () =>
+ uploadFileToS3(outputPath, outputPresignedUrl, "video/mp4", signal),
);
if (thumbnailPresignedUrl || previewGifPresignedUrl) {
@@ -1396,38 +1528,40 @@ async function processVideoAsync(
await sendWebhook(job);
}
- if (thumbnailPresignedUrl) {
- // The uploaded MP4 has not been independently verified, so decode failures must still fail the job.
- const thumbnailData = await generateThumbnail(
- outputTempFile.path,
- metadata.duration,
- {},
- abortController.signal,
- );
- try {
- await uploadToS3(
- thumbnailData,
- thumbnailPresignedUrl,
- "image/jpeg",
- abortController.signal,
- );
- } catch (error) {
- abortController.signal.throwIfAborted();
- console.warn(
- `[video/process] Thumbnail upload failed for ${jobId}:`,
- error,
+ await withJobHeartbeat(jobId, async () => {
+ if (thumbnailPresignedUrl) {
+ // The uploaded MP4 has not been independently verified, so decode failures must still fail the job.
+ const thumbnailData = await generateThumbnail(
+ outputPath,
+ metadata.duration,
+ {},
+ signal,
);
+ try {
+ await uploadToS3(
+ thumbnailData,
+ thumbnailPresignedUrl,
+ "image/jpeg",
+ signal,
+ );
+ } catch (error) {
+ signal.throwIfAborted();
+ console.warn(
+ `[video/process] Thumbnail upload failed for ${jobId}:`,
+ error,
+ );
+ }
}
- }
- await generateAndUploadPreviewGif(
- outputTempFile.path,
- metadata.duration,
- previewGifPresignedUrl,
- abortController.signal,
- "video/process",
- );
- abortController.signal.throwIfAborted();
+ await generateAndUploadPreviewGif(
+ outputPath,
+ metadata.duration,
+ previewGifPresignedUrl,
+ signal,
+ "video/process",
+ );
+ });
+ signal.throwIfAborted();
updateJob(jobId, {
phase: "complete",
@@ -1441,7 +1575,7 @@ async function processVideoAsync(
if (options.audioLevels)
await enhanceLocalRecording({
- path: outputTempFile.path,
+ path: outputPath,
videoId: options.videoId,
userId: options.userId,
jobId,
@@ -1452,14 +1586,9 @@ async function processVideoAsync(
webhookSecret: options.webhookSecret,
});
- await inputTempFile.cleanup();
- await outputTempFile.cleanup();
- await repairedTempFile?.cleanup();
- await lastResortRepairFile?.cleanup();
-
setTimeout(() => deleteJob(jobId), 5 * 60 * 1000);
} catch (err) {
- if (!abortController.signal.aborted) {
+ if (!signal.aborted) {
console.error(`[video/process] Error processing job ${jobId}:`, err);
}
const failedJob = getJob(jobId);
@@ -1469,7 +1598,7 @@ async function processVideoAsync(
failedJob.phase !== "cancelled" &&
failedJob.phase !== "error"
) {
- const cancelled = abortController.signal.aborted;
+ const cancelled = signal.aborted;
const updatedJob = updateJob(jobId, {
phase: cancelled ? "cancelled" : "error",
error: cancelled
@@ -1483,18 +1612,20 @@ async function processVideoAsync(
await sendWebhook(updatedJob);
}
}
-
- const currentJob = getJob(jobId);
- if (currentJob) {
- await currentJob.inputTempFile?.cleanup();
- await currentJob.outputTempFile?.cleanup();
- }
- await repairedTempFile?.cleanup();
- await lastResortRepairFile?.cleanup();
+ } finally {
+ await cleanupTempFiles([
+ inputTempFile,
+ outputTempFile,
+ repairedTempFile,
+ lastResortRepairFile,
+ ]);
}
}
video.get("/process/:jobId/status", async (c) => {
+ if (!canReadStatus(c)) {
+ return c.json({ error: "Unauthorized" }, 401);
+ }
const jobId = c.req.param("jobId");
const job = getJob(jobId);
@@ -1845,6 +1976,16 @@ video.post("/verify-recording", async (c) => {
);
return c.json(recordingWorkerResponse(existing, ownerJobId));
}
+ if (isDraining()) {
+ c.header("Retry-After", VIDEO_BUSY_RETRY_AFTER_SECONDS.toString());
+ return c.json(
+ {
+ ...getMuxBusyResponseBody(getVideoCapacitySnapshot()),
+ details: DRAINING_DETAILS,
+ },
+ 503,
+ );
+ }
if (!canAcceptNewVideoProcess(body.processingPriority)) {
c.header("Retry-After", VIDEO_BUSY_RETRY_AFTER_SECONDS.toString());
return c.json(getMuxBusyResponseBody(getVideoCapacitySnapshot()), 503);
@@ -1877,19 +2018,24 @@ video.post("/verify-recording", async (c) => {
deleteJob(jobId);
return c.json(recordingWorkerResponse(job, ownerJobId));
}
- void withMediaTransfers(body.downloadBudgetBytes ?? body.fileSize * 3, () =>
- verifyUploadedRecordingAsync(jobId, body),
- ).catch((error: unknown) => {
- const current = getJob(jobId);
- if (!current || ["complete", "error", "cancelled"].includes(current.phase))
- return;
- updateJob(jobId, {
- phase: "error",
- error: "Recording transfer could not complete",
- errorCode: classifySourceError(error),
- });
- sendCurrentJobWebhook(jobId);
- });
+ void trackBackgroundTask(
+ withMediaTransfers(body.downloadBudgetBytes ?? body.fileSize * 3, () =>
+ verifyUploadedRecordingAsync(jobId, body),
+ ).catch((error: unknown) => {
+ const current = getJob(jobId);
+ if (
+ !current ||
+ ["complete", "error", "cancelled"].includes(current.phase)
+ )
+ return;
+ updateJob(jobId, {
+ phase: "error",
+ error: "Recording transfer could not complete",
+ errorCode: classifySourceError(error),
+ });
+ sendCurrentJobWebhook(jobId);
+ }),
+ );
return c.json(recordingWorkerResponse(job, jobId));
});
@@ -2174,6 +2320,17 @@ video.post("/mux-segments", async (c) => {
});
}
+ if (isDraining()) {
+ c.header("Retry-After", VIDEO_BUSY_RETRY_AFTER_SECONDS.toString());
+ return c.json(
+ {
+ ...getMuxBusyResponseBody(getVideoCapacitySnapshot()),
+ details: DRAINING_DETAILS,
+ },
+ 503,
+ );
+ }
+
if (!canAcceptNewVideoProcess(body.data.processingPriority)) {
c.header("Retry-After", VIDEO_BUSY_RETRY_AFTER_SECONDS.toString());
return c.json(getMuxBusyResponseBody(getVideoCapacitySnapshot()), 503);
@@ -2215,36 +2372,38 @@ video.post("/mux-segments", async (c) => {
});
}
- withMediaTransfers(body.data.downloadBudgetBytes ?? 32 * 1024 ** 3, () =>
- muxSegmentsAsync(
- jobId,
- videoId,
- outputUpload,
- thumbnailPresignedUrl,
- previewGifPresignedUrl,
- videoInitUrl,
- videoSegUrls,
- audioInitUrl ?? null,
- audioSegUrls ?? null,
- body.data.outputVerificationUrl,
- body.data,
- ),
- ).catch((err) => {
- console.error(`[mux-segments] Async mux error for job ${jobId}:`, err);
- const currentJob = getJob(jobId);
- if (
- currentJob &&
- currentJob.phase !== "error" &&
- currentJob.phase !== "complete" &&
- currentJob.phase !== "cancelled"
- ) {
- updateJob(jobId, {
- phase: "error",
- error: err instanceof Error ? err.message : String(err),
- });
- sendCurrentJobWebhook(jobId);
- }
- });
+ void trackBackgroundTask(
+ withMediaTransfers(body.data.downloadBudgetBytes ?? 32 * 1024 ** 3, () =>
+ muxSegmentsAsync(
+ jobId,
+ videoId,
+ outputUpload,
+ thumbnailPresignedUrl,
+ previewGifPresignedUrl,
+ videoInitUrl,
+ videoSegUrls,
+ audioInitUrl ?? null,
+ audioSegUrls ?? null,
+ body.data.outputVerificationUrl,
+ body.data,
+ ),
+ ).catch((err) => {
+ console.error(`[mux-segments] Async mux error for job ${jobId}:`, err);
+ const currentJob = getJob(jobId);
+ if (
+ currentJob &&
+ currentJob.phase !== "error" &&
+ currentJob.phase !== "complete" &&
+ currentJob.phase !== "cancelled"
+ ) {
+ updateJob(jobId, {
+ phase: "error",
+ error: err instanceof Error ? err.message : String(err),
+ });
+ sendCurrentJobWebhook(jobId);
+ }
+ }),
+ );
return c.json({
...recordingWorkerResponse(job, jobId),
@@ -2568,15 +2727,14 @@ async function muxSegmentsAsync(
outputVerificationUrl?: string,
context: MuxContext = {},
): Promise<void> {
- const { ensureTempDir } = await import("../lib/temp-files");
+ const { ensureTempDir, getTempDir, retainTempPath } = await import(
+ "../lib/temp-files"
+ );
const { lstat, mkdir, rm } = await import("node:fs/promises");
const { join } = await import("node:path");
- const workDir = join(
- (await import("node:os")).tmpdir(),
- "cap-media-server",
- `mux-${jobId}`,
- );
+ const workDir = join(getTempDir(), `mux-${jobId}`);
+ const releaseWorkDir = retainTempPath(workDir);
const abortController = new AbortController();
updateJob(jobId, { abortController });
let outputUploadStarted = false;
@@ -2666,10 +2824,12 @@ async function muxSegmentsAsync(
const combinedVideoPath = join(workDir, "combined_video.mp4");
const videoInitPath = join(videoDir, "init.mp4");
- await streamConcatFiles(
- [videoInitPath, ...videoSegmentFiles],
- combinedVideoPath,
- abortController.signal,
+ await withJobHeartbeat(jobId, () =>
+ streamConcatFiles(
+ [videoInitPath, ...videoSegmentFiles],
+ combinedVideoPath,
+ abortController.signal,
+ ),
);
await rm(videoDir, { recursive: true, force: true });
@@ -2679,10 +2839,13 @@ async function muxSegmentsAsync(
combinedAudioPath = join(workDir, "combined_audio.mp4");
const audioInitPath = join(audioDir, "init.mp4");
- await streamConcatFiles(
- [audioInitPath, ...audioSegmentFiles],
- combinedAudioPath,
- abortController.signal,
+ const audioOutputPath = combinedAudioPath;
+ await withJobHeartbeat(jobId, () =>
+ streamConcatFiles(
+ [audioInitPath, ...audioSegmentFiles],
+ audioOutputPath,
+ abortController.signal,
+ ),
);
await rm(audioDir, { recursive: true, force: true });
}
@@ -2789,7 +2952,9 @@ async function muxSegmentsAsync(
resources: getSystemResources(),
});
- let metadata = await probeVideoFile(resultPath);
+ let metadata = await withJobHeartbeat(jobId, () =>
+ probeVideoFile(resultPath),
+ );
if (
!Number.isFinite(metadata.duration) ||
metadata.duration <= 0 ||
@@ -2903,37 +3068,39 @@ async function muxSegmentsAsync(
sendCurrentJobWebhook(jobId);
}
- if (thumbnailPresignedUrl) {
- try {
- const duration = metadata?.duration ?? 0;
- const thumbnailData = await generateThumbnail(
- resultPath,
- duration,
- {},
- abortController.signal,
- );
- await uploadToS3(
- thumbnailData,
- thumbnailPresignedUrl,
- "image/jpeg",
- abortController.signal,
- );
- } catch (thumbErr) {
- abortController.signal.throwIfAborted();
- console.warn(
- `[mux-segments] Thumbnail generation failed for ${videoId}:`,
- thumbErr,
- );
+ const assetDuration = metadata?.duration ?? 0;
+ await withJobHeartbeat(jobId, async () => {
+ if (thumbnailPresignedUrl) {
+ try {
+ const thumbnailData = await generateThumbnail(
+ resultPath,
+ assetDuration,
+ {},
+ abortController.signal,
+ );
+ await uploadToS3(
+ thumbnailData,
+ thumbnailPresignedUrl,
+ "image/jpeg",
+ abortController.signal,
+ );
+ } catch (thumbErr) {
+ abortController.signal.throwIfAborted();
+ console.warn(
+ `[mux-segments] Thumbnail generation failed for ${videoId}:`,
+ thumbErr,
+ );
+ }
}
- }
- await generateAndUploadPreviewGif(
- resultPath,
- metadata?.duration ?? 0,
- previewGifPresignedUrl,
- abortController.signal,
- "mux-segments",
- );
+ await generateAndUploadPreviewGif(
+ resultPath,
+ assetDuration,
+ previewGifPresignedUrl,
+ abortController.signal,
+ "mux-segments",
+ );
+ });
abortController.signal.throwIfAborted();
updateJob(jobId, {
@@ -3014,6 +3181,7 @@ async function muxSegmentsAsync(
} finally {
if (processingTimeout) clearTimeout(processingTimeout);
await rm(workDir, { recursive: true, force: true }).catch(() => {});
+ releaseWorkDir();
}
}