Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

fix(media-server): find jobs on peer instances before answering not found

Richie McIlroya1d37c38c226
diff --git a/apps/media-server/Dockerfile b/apps/media-server/Dockerfile
index c4974284f..64b89c176 100644
--- a/apps/media-server/Dockerfile
+++ b/apps/media-server/Dockerfile
@@ -25,6 +25,7 @@ RUN bun test src/__tests__/lib/audio-quality-policy.test.ts src/__tests__/lib/au
&& bun test src/__tests__/routes/video.test.ts \
&& bun test src/__tests__/lib/lifecycle.test.ts src/__tests__/lib/temp-files.test.ts \
&& bun test src/__tests__/routes/lifecycle.test.ts \
+ && bun test src/__tests__/routes/peer-jobs.test.ts \
&& bun test src/__tests__/lib/media-routes-real-world.integration.test.ts \
&& bun test src/__tests__/lib/media-video.integration.test.ts
diff --git a/apps/media-server/src/__tests__/routes/peer-jobs.test.ts b/apps/media-server/src/__tests__/routes/peer-jobs.test.ts
new file mode 100644
index 000000000..4aa4ea77d
--- /dev/null
+++ b/apps/media-server/src/__tests__/routes/peer-jobs.test.ts
@@ -0,0 +1,134 @@
+import {
+ afterAll,
+ afterEach,
+ beforeAll,
+ describe,
+ expect,
+ test,
+} from "bun:test";
+import app from "../../app";
+import { mediaServerPeers } from "../../lib/peer-jobs";
+
+const SECRET = "peer-test-secret";
+process.env.MEDIA_SERVER_WEBHOOK_SECRET = SECRET;
+
+const peerRequests: {
+ method: string;
+ peerLookup: string | null;
+ secret: string | null;
+}[] = [];
+let respond: (request: Request) => Response = () =>
+ new Response("not found", { status: 404 });
+let peer: ReturnType<typeof Bun.serve>;
+let missingPeer: ReturnType<typeof Bun.serve>;
+
+beforeAll(() => {
+ peer = Bun.serve({
+ hostname: "127.0.0.1",
+ port: 0,
+ fetch(request) {
+ peerRequests.push({
+ method: request.method,
+ peerLookup: request.headers.get("x-cap-peer-lookup"),
+ secret: request.headers.get("x-media-server-secret"),
+ });
+ return respond(request);
+ },
+ });
+ missingPeer = Bun.serve({
+ hostname: "127.0.0.1",
+ port: 0,
+ fetch: () => new Response("not found", { status: 404 }),
+ });
+ process.env.MEDIA_SERVER_PEERS = `http://127.0.0.1:${missingPeer.port}, http://127.0.0.1:${peer.port}/`;
+});
+
+afterEach(() => {
+ peerRequests.length = 0;
+ respond = () => new Response("not found", { status: 404 });
+});
+
+afterAll(async () => {
+ delete process.env.MEDIA_SERVER_PEERS;
+ await peer.stop(true);
+ await missingPeer.stop(true);
+});
+
+function statusRequest(jobId: string, headers: Record<string, string> = {}) {
+ return new Request(`http://localhost/video/process/${jobId}/status`, {
+ headers: { "x-media-server-secret": SECRET, ...headers },
+ });
+}
+
+describe("peer job lookup", () => {
+ test("parses configured peers and drops invalid entries", () => {
+ expect(
+ mediaServerPeers({
+ MEDIA_SERVER_PEERS:
+ "http://10.0.0.1:3456/, nonsense, ftp://x, https://b",
+ }),
+ ).toEqual(["http://10.0.0.1:3456", "https://b"]);
+ expect(mediaServerPeers({})).toEqual([]);
+ });
+
+ test("answers a status request with the peer that owns the job", async () => {
+ respond = () =>
+ Response.json({ jobId: "job-elsewhere", videoId: "v1", phase: "error" });
+ const response = await app.fetch(statusRequest("job-elsewhere"));
+ expect(response.status).toBe(200);
+ expect(await response.json()).toEqual({
+ jobId: "job-elsewhere",
+ videoId: "v1",
+ phase: "error",
+ });
+ expect(peerRequests).toHaveLength(1);
+ expect(peerRequests[0].peerLookup).toBe("1");
+ expect(peerRequests[0].secret).toBe(SECRET);
+ });
+
+ test("still answers not found when no peer has the job", async () => {
+ const response = await app.fetch(statusRequest("job-nowhere"));
+ expect(response.status).toBe(404);
+ expect((await response.json()).code).toBe("NOT_FOUND");
+ });
+
+ test("does not forward a lookup that already came from a peer", async () => {
+ respond = () => Response.json({ jobId: "job-loop", phase: "processing" });
+ const response = await app.fetch(
+ statusRequest("job-loop", { "x-cap-peer-lookup": "1" }),
+ );
+ expect(response.status).toBe(404);
+ expect(peerRequests).toHaveLength(0);
+ });
+
+ test("does not forward unauthenticated status reads", async () => {
+ respond = () => Response.json({ jobId: "job-public", phase: "processing" });
+ const response = await app.fetch(
+ new Request("http://localhost/video/process/job-public/status"),
+ );
+ expect(response.status).toBe(404);
+ expect(peerRequests).toHaveLength(0);
+ });
+
+ test("ignores a peer that rejects the shared secret", async () => {
+ respond = () => new Response("Unauthorized", { status: 401 });
+ const response = await app.fetch(statusRequest("job-rejected"));
+ expect(response.status).toBe(404);
+ });
+
+ test("cancels a job on the peer that owns it", async () => {
+ respond = (request) =>
+ request.method === "POST"
+ ? Response.json({ success: true, jobId: "job-cancel" })
+ : new Response("not found", { status: 404 });
+ const response = await app.fetch(
+ new Request("http://localhost/video/process/job-cancel/cancel", {
+ method: "POST",
+ headers: { "x-media-server-secret": SECRET },
+ }),
+ );
+ expect(response.status).toBe(200);
+ expect((await response.json()).success).toBe(true);
+ expect(peerRequests.map((request) => request.method)).toEqual(["POST"]);
+ });
+});
diff --git a/apps/media-server/src/lib/peer-jobs.ts b/apps/media-server/src/lib/peer-jobs.ts
new file mode 100644
index 000000000..ae7865ab5
--- /dev/null
+++ b/apps/media-server/src/lib/peer-jobs.ts
@@ -0,0 +1,70 @@
+const PEER_LOOKUP_HEADER = "x-cap-peer-lookup";
+const PEER_TIMEOUT_MS = 2_000;
+
+// Jobs live in the memory of the instance that runs them. Behind a load
+// balancer a status or cancel request can land on another instance, so it asks
+// the configured peers once before answering that the job does not exist.
+export function mediaServerPeers(
+ env: Record<string, string | undefined> = process.env,
+): string[] {
+ return (env.MEDIA_SERVER_PEERS ?? "")
+ .split(",")
+ .map((value) => value.trim().replace(/\/+$/, ""))
+ .filter((value) => {
+ try {
+ const url = new URL(value);
+ return url.protocol === "http:" || url.protocol === "https:";
+ } catch {
+ return false;
+ }
+ });
+}
+
+export function isPeerLookup(request: Request): boolean {
+ return request.headers.get(PEER_LOOKUP_HEADER) === "1";
+}
+
+export async function findJobOnPeer(
+ path: string,
+ method: "GET" | "POST",
+): Promise<Response | null> {
+ const secret = process.env.MEDIA_SERVER_WEBHOOK_SECRET;
+ const peers = mediaServerPeers();
+ if (!secret || peers.length === 0) return null;
+ const controller = new AbortController();
+ const timeout = setTimeout(() => controller.abort(), PEER_TIMEOUT_MS);
+ try {
+ return await Promise.any(
+ peers.map(async (peer) => {
+ const response = await fetch(`${peer}${path}`, {
+ method,
+ headers: {
+ "x-media-server-secret": secret,
+ [PEER_LOOKUP_HEADER]: "1",
+ Accept: "application/json",
+ },
+ signal: controller.signal,
+ redirect: "error",
+ });
+ // Only a peer that owns the job answers 200, or 400 for a finished job's cancel.
+ if (response.status !== 200 && response.status !== 400) {
+ await response.body?.cancel().catch(() => {});
+ throw new Error("Job is not on this peer");
+ }
+ const body = await response.text();
+ return new Response(body, {
+ status: response.status,
+ headers: {
+ "Content-Type":
+ response.headers.get("content-type") ?? "application/json",
+ },
+ });
+ }),
+ );
+ } catch {
+ return null;
+ } finally {
+ clearTimeout(timeout);
+ controller.abort();
+ }
+}
diff --git a/apps/media-server/src/routes/video.ts b/apps/media-server/src/routes/video.ts
index 30c23e421..b0163bcb1 100644
--- a/apps/media-server/src/routes/video.ts
+++ b/apps/media-server/src/routes/video.ts
@@ -80,6 +80,7 @@ import {
uploadToS3,
} from "../lib/media-video";
import { encodeAttemptsBudgetMs } from "../lib/nvenc-encoder";
+import { findJobOnPeer, isPeerLookup } from "../lib/peer-jobs";
import { RecordingTimingError } from "../lib/recording-timing";
import {
hashRecordingFile,
@@ -1771,8 +1772,20 @@ video.get("/process/:jobId/status", async (c) => {
}
const jobId = c.req.param("jobId");
const job = getJob(jobId);
+ const accept = c.req.header("Accept");
if (!job) {
+ if (
+ hasMediaServerSecret(c) &&
+ !isPeerLookup(c.req.raw) &&
+ !accept?.includes("text/event-stream")
+ ) {
+ const owner = await findJobOnPeer(
+ `/video/process/${encodeURIComponent(jobId)}/status`,
+ "GET",
+ );
+ if (owner) return owner;
+ }
return c.json(
{
error: "Job not found",
@@ -1784,8 +1797,6 @@ video.get("/process/:jobId/status", async (c) => {
);
}
- const accept = c.req.header("Accept");
-
if (accept?.includes("text/event-stream")) {
const stream = new ReadableStream({
start(controller) {
@@ -1855,6 +1866,13 @@ video.post("/process/:jobId/cancel", async (c) => {
const job = getJob(jobId);
if (!job) {
+ if (!isPeerLookup(c.req.raw)) {
+ const owner = await findJobOnPeer(
+ `/video/process/${encodeURIComponent(jobId)}/cancel`,
+ "POST",
+ );
+ if (owner) return owner;
+ }
return c.json(
{
error: "Job not found",