Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
fix(media-server): find jobs on peer instances before answering not found
Richie McIlroya1d37c38c226
diff --git a/apps/media-server/Dockerfile b/apps/media-server/Dockerfileindex c4974284f..64b89c176 100644--- a/apps/media-server/Dockerfile+++ b/apps/media-server/Dockerfile@@ -25,6 +25,7 @@ RUN bun test src/__tests__/lib/audio-quality-policy.test.ts src/__tests__/lib/au&& bun test src/__tests__/routes/video.test.ts \&& bun test src/__tests__/lib/lifecycle.test.ts src/__tests__/lib/temp-files.test.ts \&& bun test src/__tests__/routes/lifecycle.test.ts \+ && bun test src/__tests__/routes/peer-jobs.test.ts \&& bun test src/__tests__/lib/media-routes-real-world.integration.test.ts \&& bun test src/__tests__/lib/media-video.integration.test.tsdiff --git a/apps/media-server/src/__tests__/routes/peer-jobs.test.ts b/apps/media-server/src/__tests__/routes/peer-jobs.test.tsnew file mode 100644index 000000000..4aa4ea77d--- /dev/null+++ b/apps/media-server/src/__tests__/routes/peer-jobs.test.ts@@ -0,0 +1,134 @@+import {+ afterAll,+ afterEach,+ beforeAll,+ describe,+ expect,+ test,+} from "bun:test";+import app from "../../app";+import { mediaServerPeers } from "../../lib/peer-jobs";++const SECRET = "peer-test-secret";+process.env.MEDIA_SERVER_WEBHOOK_SECRET = SECRET;++const peerRequests: {+ method: string;+ peerLookup: string | null;+ secret: string | null;+}[] = [];+let respond: (request: Request) => Response = () =>+ new Response("not found", { status: 404 });+let peer: ReturnType<typeof Bun.serve>;+let missingPeer: ReturnType<typeof Bun.serve>;++beforeAll(() => {+ peer = Bun.serve({+ hostname: "127.0.0.1",+ port: 0,+ fetch(request) {+ peerRequests.push({+ method: request.method,+ peerLookup: request.headers.get("x-cap-peer-lookup"),+ secret: request.headers.get("x-media-server-secret"),+ });+ return respond(request);+ },+ });+ missingPeer = Bun.serve({+ hostname: "127.0.0.1",+ port: 0,+ fetch: () => new Response("not found", { status: 404 }),+ });+ process.env.MEDIA_SERVER_PEERS = `http://127.0.0.1:${missingPeer.port}, http://127.0.0.1:${peer.port}/`;+});++afterEach(() => {+ peerRequests.length = 0;+ respond = () => new Response("not found", { status: 404 });+});++afterAll(async () => {+ delete process.env.MEDIA_SERVER_PEERS;+ await peer.stop(true);+ await missingPeer.stop(true);+});++function statusRequest(jobId: string, headers: Record<string, string> = {}) {+ return new Request(`http://localhost/video/process/${jobId}/status`, {+ headers: { "x-media-server-secret": SECRET, ...headers },+ });+}++describe("peer job lookup", () => {+ test("parses configured peers and drops invalid entries", () => {+ expect(+ mediaServerPeers({+ MEDIA_SERVER_PEERS:+ "http://10.0.0.1:3456/, nonsense, ftp://x, https://b",+ }),+ ).toEqual(["http://10.0.0.1:3456", "https://b"]);+ expect(mediaServerPeers({})).toEqual([]);+ });++ test("answers a status request with the peer that owns the job", async () => {+ respond = () =>+ Response.json({ jobId: "job-elsewhere", videoId: "v1", phase: "error" });+ const response = await app.fetch(statusRequest("job-elsewhere"));+ expect(response.status).toBe(200);+ expect(await response.json()).toEqual({+ jobId: "job-elsewhere",+ videoId: "v1",+ phase: "error",+ });+ expect(peerRequests).toHaveLength(1);+ expect(peerRequests[0].peerLookup).toBe("1");+ expect(peerRequests[0].secret).toBe(SECRET);+ });++ test("still answers not found when no peer has the job", async () => {+ const response = await app.fetch(statusRequest("job-nowhere"));+ expect(response.status).toBe(404);+ expect((await response.json()).code).toBe("NOT_FOUND");+ });++ test("does not forward a lookup that already came from a peer", async () => {+ respond = () => Response.json({ jobId: "job-loop", phase: "processing" });+ const response = await app.fetch(+ statusRequest("job-loop", { "x-cap-peer-lookup": "1" }),+ );+ expect(response.status).toBe(404);+ expect(peerRequests).toHaveLength(0);+ });++ test("does not forward unauthenticated status reads", async () => {+ respond = () => Response.json({ jobId: "job-public", phase: "processing" });+ const response = await app.fetch(+ new Request("http://localhost/video/process/job-public/status"),+ );+ expect(response.status).toBe(404);+ expect(peerRequests).toHaveLength(0);+ });++ test("ignores a peer that rejects the shared secret", async () => {+ respond = () => new Response("Unauthorized", { status: 401 });+ const response = await app.fetch(statusRequest("job-rejected"));+ expect(response.status).toBe(404);+ });++ test("cancels a job on the peer that owns it", async () => {+ respond = (request) =>+ request.method === "POST"+ ? Response.json({ success: true, jobId: "job-cancel" })+ : new Response("not found", { status: 404 });+ const response = await app.fetch(+ new Request("http://localhost/video/process/job-cancel/cancel", {+ method: "POST",+ headers: { "x-media-server-secret": SECRET },+ }),+ );+ expect(response.status).toBe(200);+ expect((await response.json()).success).toBe(true);+ expect(peerRequests.map((request) => request.method)).toEqual(["POST"]);+ });+});diff --git a/apps/media-server/src/lib/peer-jobs.ts b/apps/media-server/src/lib/peer-jobs.tsnew file mode 100644index 000000000..ae7865ab5--- /dev/null+++ b/apps/media-server/src/lib/peer-jobs.ts@@ -0,0 +1,70 @@+const PEER_LOOKUP_HEADER = "x-cap-peer-lookup";+const PEER_TIMEOUT_MS = 2_000;++// Jobs live in the memory of the instance that runs them. Behind a load+// balancer a status or cancel request can land on another instance, so it asks+// the configured peers once before answering that the job does not exist.+export function mediaServerPeers(+ env: Record<string, string | undefined> = process.env,+): string[] {+ return (env.MEDIA_SERVER_PEERS ?? "")+ .split(",")+ .map((value) => value.trim().replace(/\/+$/, ""))+ .filter((value) => {+ try {+ const url = new URL(value);+ return url.protocol === "http:" || url.protocol === "https:";+ } catch {+ return false;+ }+ });+}++export function isPeerLookup(request: Request): boolean {+ return request.headers.get(PEER_LOOKUP_HEADER) === "1";+}++export async function findJobOnPeer(+ path: string,+ method: "GET" | "POST",+): Promise<Response | null> {+ const secret = process.env.MEDIA_SERVER_WEBHOOK_SECRET;+ const peers = mediaServerPeers();+ if (!secret || peers.length === 0) return null;+ const controller = new AbortController();+ const timeout = setTimeout(() => controller.abort(), PEER_TIMEOUT_MS);+ try {+ return await Promise.any(+ peers.map(async (peer) => {+ const response = await fetch(`${peer}${path}`, {+ method,+ headers: {+ "x-media-server-secret": secret,+ [PEER_LOOKUP_HEADER]: "1",+ Accept: "application/json",+ },+ signal: controller.signal,+ redirect: "error",+ });+ // Only a peer that owns the job answers 200, or 400 for a finished job's cancel.+ if (response.status !== 200 && response.status !== 400) {+ await response.body?.cancel().catch(() => {});+ throw new Error("Job is not on this peer");+ }+ const body = await response.text();+ return new Response(body, {+ status: response.status,+ headers: {+ "Content-Type":+ response.headers.get("content-type") ?? "application/json",+ },+ });+ }),+ );+ } catch {+ return null;+ } finally {+ clearTimeout(timeout);+ controller.abort();+ }+}diff --git a/apps/media-server/src/routes/video.ts b/apps/media-server/src/routes/video.tsindex 30c23e421..b0163bcb1 100644--- a/apps/media-server/src/routes/video.ts+++ b/apps/media-server/src/routes/video.ts@@ -80,6 +80,7 @@ import {uploadToS3,} from "../lib/media-video";import { encodeAttemptsBudgetMs } from "../lib/nvenc-encoder";+import { findJobOnPeer, isPeerLookup } from "../lib/peer-jobs";import { RecordingTimingError } from "../lib/recording-timing";import {hashRecordingFile,@@ -1771,8 +1772,20 @@ video.get("/process/:jobId/status", async (c) => {}const jobId = c.req.param("jobId");const job = getJob(jobId);+ const accept = c.req.header("Accept");if (!job) {+ if (+ hasMediaServerSecret(c) &&+ !isPeerLookup(c.req.raw) &&+ !accept?.includes("text/event-stream")+ ) {+ const owner = await findJobOnPeer(+ `/video/process/${encodeURIComponent(jobId)}/status`,+ "GET",+ );+ if (owner) return owner;+ }return c.json({error: "Job not found",@@ -1784,8 +1797,6 @@ video.get("/process/:jobId/status", async (c) => {);}- const accept = c.req.header("Accept");-if (accept?.includes("text/event-stream")) {const stream = new ReadableStream({start(controller) {@@ -1855,6 +1866,13 @@ video.post("/process/:jobId/cancel", async (c) => {const job = getJob(jobId);if (!job) {+ if (!isPeerLookup(c.req.raw)) {+ const owner = await findJobOnPeer(+ `/video/process/${encodeURIComponent(jobId)}/cancel`,+ "POST",+ );+ if (owner) return owner;+ }return c.json({error: "Job not found",