Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

fix: check real Loom run and importer state before restarting imports

Richie McIlroy96b78e0a9b83
diff --git a/apps/web/__tests__/unit/loom-import-start.test.ts b/apps/web/__tests__/unit/loom-import-start.test.ts
index bf4127d8e..6c13975a3 100644
--- a/apps/web/__tests__/unit/loom-import-start.test.ts
+++ b/apps/web/__tests__/unit/loom-import-start.test.ts
@@ -1,6 +1,7 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
const mocks = vi.hoisted(() => ({
+ importerJobState: vi.fn(),
where: vi.fn(),
set: vi.fn(),
queue: vi.fn(),
@@ -37,6 +38,7 @@ vi.mock("workflow/api", () => ({ start: mocks.start, getRun: mocks.getRun }));
vi.mock("workflow/runtime", () => ({ getWorld: mocks.getWorld }));
vi.mock("@/lib/loom-media-dispatch", () => ({
isLoomMediaJobFailed: mocks.mediaJobFailed,
+ getLoomImporterJobState: mocks.importerJobState,
}));
vi.mock("@/workflows/import-loom-video", () => ({
importLoomVideoWorkflow: vi.fn(),
@@ -83,6 +85,73 @@ describe("Loom workflow startup receipts", () => {
);
});
+ it("treats a run that was never created as inactive", async () => {
+ const { WorkflowRunNotFoundError } = await import(
+ "workflow/internal/errors"
+ );
+ mocks.getRun.mockReturnValue({
+ status: Promise.reject(new WorkflowRunNotFoundError("run-1")),
+ });
+ expect(
+ await getLoomImportRunState({ runId: "run-1", dispatch: "uncertain" }),
+ ).toBe("inactive");
+ mocks.getRun.mockReturnValue({
+ status: Promise.reject(new Error("network")),
+ });
+ expect(
+ await getLoomImportRunState({ runId: "run-1", dispatch: "uncertain" }),
+ ).toBe("unconfirmed");
+ });
+
+ it.each([
+ ["running", "active"],
+ ["finished", "inactive"],
+ ["missing", "inactive"],
+ ["unknown", "unconfirmed"],
+ ] as const)(
+ "asks the importer about its own jobs: %s",
+ async (importerState, expected) => {
+ mocks.getRun.mockReturnValue({ status: Promise.resolve("failed") });
+ mocks.importerJobState.mockResolvedValue(importerState);
+ mocks.mediaJobFailed.mockResolvedValue(false);
+ expect(
+ await getLoomImportRunState({
+ runId: "run-1",
+ dispatch: "accepted",
+ mediaDispatch: {
+ attemptId: "run-1:0",
+ videoId: "video-1",
+ jobId: "importer-1",
+ state: "accepted",
+ worker: "importer",
+ },
+ }),
+ ).toBe(expected);
+ expect(mocks.importerJobState).toHaveBeenCalledWith({
+ jobId: "importer-1",
+ videoId: "video-1",
+ });
+ },
+ );
+
+ it("does not trust a missing importer job for media server dispatches", async () => {
+ mocks.getRun.mockReturnValue({ status: Promise.resolve("failed") });
+ mocks.importerJobState.mockResolvedValue("missing");
+ mocks.mediaJobFailed.mockResolvedValue(false);
+ expect(
+ await getLoomImportRunState({
+ runId: "run-1",
+ dispatch: "accepted",
+ mediaDispatch: {
+ attemptId: "run-1:0",
+ videoId: "video-1",
+ jobId: "worker-1",
+ state: "accepted",
+ },
+ }),
+ ).toBe("unconfirmed");
+ });
+
it.each([false, true])(
"requires a matching failed worker before releasing an accepted dispatch: %s",
async (failed) => {
diff --git a/apps/web/__tests__/unit/loom-importer-job-state.test.ts b/apps/web/__tests__/unit/loom-importer-job-state.test.ts
new file mode 100644
index 000000000..85126e9be
--- /dev/null
+++ b/apps/web/__tests__/unit/loom-importer-job-state.test.ts
@@ -0,0 +1,79 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+
+vi.mock("@cap/env", () => ({
+ serverEnv: () => ({
+ MEDIA_SERVER_URL: "https://worker.example.com",
+ LOOM_IMPORTER_URL: "https://media.example.com/loom-importer",
+ MEDIA_SERVER_WEBHOOK_SECRET: "fixture-secret",
+ }),
+}));
+vi.mock("@cap/database", () => ({ db: vi.fn() }));
+
+import { getLoomImporterJobState } from "@/lib/loom-media-dispatch";
+
+describe("getLoomImporterJobState", () => {
+ afterEach(() => vi.unstubAllGlobals());
+
+ it.each([
+ [{ jobId: "job", videoId: "video", phase: "downloading" }, "running"],
+ [{ jobId: "job", videoId: "video", phase: "queued" }, "running"],
+ [{ jobId: "job", videoId: "video", phase: "complete" }, "finished"],
+ [{ jobId: "job", videoId: "video", phase: "error" }, "finished"],
+ [{ jobId: "job", videoId: "video", phase: "cancelled" }, "finished"],
+ [{ jobId: "job", videoId: "other", phase: "error" }, "unknown"],
+ ])("reads the importer's job %j", async (body, expected) => {
+ const fetch = vi.fn().mockResolvedValue(Response.json(body));
+ vi.stubGlobal("fetch", fetch);
+ await expect(
+ getLoomImporterJobState({ jobId: "job", videoId: "video" }),
+ ).resolves.toBe(expected);
+ expect(fetch).toHaveBeenCalledWith(
+ "https://media.example.com/loom-importer/video/process/job/status",
+ expect.objectContaining({ redirect: "error" }),
+ );
+ });
+
+ it("knows a job the importer no longer has", async () => {
+ vi.stubGlobal(
+ "fetch",
+ vi.fn().mockResolvedValue(new Response("missing", { status: 404 })),
+ );
+ await expect(
+ getLoomImporterJobState({ jobId: "job", videoId: "video" }),
+ ).resolves.toBe("missing");
+ });
+
+ it.each([
+ [() => Promise.resolve(new Response("down", { status: 503 }))],
+ [() => Promise.reject(new Error("network"))],
+ ])("cannot tell when the importer does not answer: %#", async (respond) => {
+ vi.stubGlobal("fetch", vi.fn().mockImplementation(respond));
+ await expect(
+ getLoomImporterJobState({ jobId: "job", videoId: "video" }),
+ ).resolves.toBe("unknown");
+ });
+
+ it("looks a video up in the importer's job list when there is no job id", async () => {
+ const fetch = vi.fn().mockResolvedValue(
+ Response.json({
+ jobs: [{ jobId: "other", videoId: "other-video", phase: "queued" }],
+ }),
+ );
+ vi.stubGlobal("fetch", fetch);
+ await expect(getLoomImporterJobState({ videoId: "video" })).resolves.toBe(
+ "missing",
+ );
+ fetch.mockResolvedValue(
+ Response.json({
+ jobs: [{ jobId: "j", videoId: "video", phase: "queued" }],
+ }),
+ );
+ await expect(getLoomImporterJobState({ videoId: "video" })).resolves.toBe(
+ "running",
+ );
+ expect(fetch).toHaveBeenCalledWith(
+ "https://media.example.com/loom-importer/video/status",
+ expect.anything(),
+ );
+ });
+});
diff --git a/apps/web/__tests__/unit/loom-importer-redrive.test.ts b/apps/web/__tests__/unit/loom-importer-redrive.test.ts
index 3c0fcd242..2044d3185 100644
--- a/apps/web/__tests__/unit/loom-importer-redrive.test.ts
+++ b/apps/web/__tests__/unit/loom-importer-redrive.test.ts
@@ -149,7 +149,7 @@ describe("redriveLoomImport", () => {
[[], "not an unfinished Loom import"],
[[failedImport({ phase: "uploading" })], "phase uploading"],
[
- [failedImport({ phase: "processing", uploadUpdatedAt: hoursAgo(30) })],
+ [failedImport({ phase: "processing", uploadUpdatedAt: hoursAgo(0.1) })],
"phase processing",
],
])("skips imports that are not failed: %#", async (rows, reason) => {
@@ -192,14 +192,34 @@ describe("redriveLoomImport", () => {
expect(mocks.updates).toEqual([]);
});
+ it("restarts an import left processing when nothing is running it", async () => {
+ mocks.rows = [
+ failedImport({ phase: "processing", uploadUpdatedAt: hoursAgo(1) }),
+ ];
+ await expect(redriveLoomImport("video", now)).resolves.toMatchObject({
+ status: "restarted",
+ });
+ });
+
+ it("leaves an import processing while its run is active", async () => {
+ mocks.runState.mockResolvedValue("active");
+ mocks.rows = [
+ failedImport({ phase: "processing", uploadUpdatedAt: hoursAgo(1) }),
+ ];
+ await expect(redriveLoomImport("video", now)).resolves.toMatchObject({
+ status: "skipped",
+ });
+ expect(mocks.start).not.toHaveBeenCalled();
+ });
+
it("only restarts an unconfirmed worker once it can no longer be running", async () => {
mocks.runState.mockResolvedValue("unconfirmed");
- mocks.rows = [failedImport({ uploadUpdatedAt: hoursAgo(2) })];
+ mocks.rows = [failedImport({ uploadUpdatedAt: hoursAgo(0.5) })];
await expect(redriveLoomImport("video", now)).resolves.toMatchObject({
status: "skipped",
reason: "run unconfirmed",
});
- mocks.rows = [failedImport({ uploadUpdatedAt: hoursAgo(7) })];
+ mocks.rows = [failedImport({ uploadUpdatedAt: hoursAgo(2) })];
await expect(redriveLoomImport("video", now)).resolves.toMatchObject({
status: "restarted",
});
diff --git a/apps/web/lib/loom-import-redrive.ts b/apps/web/lib/loom-import-redrive.ts
index 5aa7125fc..1653c2029 100644
--- a/apps/web/lib/loom-import-redrive.ts
+++ b/apps/web/lib/loom-import-redrive.ts
@@ -9,7 +9,8 @@ import {
startLoomImportWorkflow,
} from "@/lib/loom-import-start";
-const FINISHED_WORKER_AGE_MS = 6 * 60 * 60 * 1000;
+const ABANDONED_PROCESSING_AGE_MS = 10 * 60 * 1000;
+const FINISHED_WORKER_AGE_MS = 60 * 60 * 1000;
const STALE_PROCESSING_AGE_MS = 7 * 24 * 60 * 60 * 1000;
export type LoomRedriveOutcome =
@@ -58,9 +59,13 @@ export async function redriveLoomImport(
const stalled =
(row.phase === "processing" || row.phase === "uploading") &&
age > STALE_PROCESSING_AGE_MS;
- if (row.phase !== "error" && !stalled)
- return { videoId, status: "skipped", reason: `phase ${row.phase}` };
const runState = await getLoomImportRunState(row.metadata?.loomImportRun);
+ const abandoned =
+ row.phase === "processing" &&
+ runState === "inactive" &&
+ age > ABANDONED_PROCESSING_AGE_MS;
+ if (row.phase !== "error" && !stalled && !abandoned)
+ return { videoId, status: "skipped", reason: `phase ${row.phase}` };
if (runState === "pending" || runState === "active")
return { videoId, status: "skipped", reason: `run ${runState}` };
if (runState === "unconfirmed" && age < FINISHED_WORKER_AGE_MS)
diff --git a/apps/web/lib/loom-import-start.ts b/apps/web/lib/loom-import-start.ts
index d85097379..dd2f819b1 100644
--- a/apps/web/lib/loom-import-start.ts
+++ b/apps/web/lib/loom-import-start.ts
@@ -4,8 +4,12 @@ import type { VideoMetadata } from "@cap/database/types";
import { User, Video } from "@cap/web-domain";
import { and, eq, sql } from "drizzle-orm";
import { getRun, start } from "workflow/api";
+import { WorkflowRunNotFoundError } from "workflow/internal/errors";
import { getWorld } from "workflow/runtime";
-import { isLoomMediaJobFailed } from "@/lib/loom-media-dispatch";
+import {
+ getLoomImporterJobState,
+ isLoomMediaJobFailed,
+} from "@/lib/loom-media-dispatch";
import { importLoomVideoWorkflow } from "@/workflows/import-loom-video";
type LoomImportRun = NonNullable<VideoMetadata["loomImportRun"]>;
@@ -57,32 +61,36 @@ export async function getLoomImportRunState(
receipt?: LoomImportRun,
): Promise<"inactive" | "pending" | "active" | "unconfirmed"> {
if (!receipt || receipt.dispatch === "rejected") return "inactive";
+ let status: string;
try {
- const status = await getRun(receipt.runId).status;
- if (
- status === "completed" ||
- status === "failed" ||
- status === "cancelled"
- ) {
- const mediaDispatch = receipt.mediaDispatch;
- if (mediaDispatch?.state === "accepted") {
- return mediaDispatch.jobId &&
- mediaDispatch.videoId &&
- (await isLoomMediaJobFailed(
- mediaDispatch.jobId,
- mediaDispatch.videoId,
- ))
- ? "inactive"
- : "unconfirmed";
- }
- return receipt.mediaDispatch?.state === "uncertain"
- ? "unconfirmed"
- : "inactive";
- }
+ status = await getRun(receipt.runId).status;
+ } catch (error) {
+ return WorkflowRunNotFoundError.is(error) &&
+ receipt.dispatch === "uncertain" &&
+ !receipt.mediaDispatch
+ ? "inactive"
+ : "unconfirmed";
+ }
+ if (status !== "completed" && status !== "failed" && status !== "cancelled")
return status === "pending" ? "pending" : "active";
- } catch {
- return "unconfirmed";
+ const mediaDispatch = receipt.mediaDispatch;
+ if (!mediaDispatch || mediaDispatch.state === "rejected") return "inactive";
+ if (mediaDispatch.worker !== "media" && mediaDispatch.videoId) {
+ const importerState = await getLoomImporterJobState({
+ jobId: mediaDispatch.jobId,
+ videoId: mediaDispatch.videoId,
+ });
+ if (importerState === "running") return "active";
+ if (importerState === "finished") return "inactive";
+ if (importerState === "missing" && mediaDispatch.worker === "importer")
+ return "inactive";
}
+ if (mediaDispatch.state === "accepted" && mediaDispatch.jobId)
+ return mediaDispatch.videoId &&
+ (await isLoomMediaJobFailed(mediaDispatch.jobId, mediaDispatch.videoId))
+ ? "inactive"
+ : "unconfirmed";
+ return "unconfirmed";
}
async function saveReceipt(payload: ImportPayload, receipt: LoomImportRun) {
diff --git a/apps/web/lib/loom-media-dispatch.ts b/apps/web/lib/loom-media-dispatch.ts
index de30c3fa7..2e8ce5f75 100644
--- a/apps/web/lib/loom-media-dispatch.ts
+++ b/apps/web/lib/loom-media-dispatch.ts
@@ -11,6 +11,7 @@ export async function recordLoomMediaDispatch({
attemptId,
state,
jobId,
+ worker,
}: {
videoId: string;
userId: string;
@@ -18,11 +19,18 @@ export async function recordLoomMediaDispatch({
attemptId: string;
state: "accepted" | "rejected" | "uncertain";
jobId?: string;
+ worker?: "importer" | "media";
}) {
const receipt = {
runId,
dispatch: "accepted",
- mediaDispatch: { attemptId, videoId, state, jobId: jobId ?? null },
+ mediaDispatch: {
+ attemptId,
+ videoId,
+ state,
+ jobId: jobId ?? null,
+ worker: worker ?? null,
+ },
};
const result = await db()
.update(videos)
@@ -94,3 +102,67 @@ async function isJobFailedOnWorker(
return false;
}
}
+
+export type LoomImporterJobState =
+ | "running"
+ | "finished"
+ | "missing"
+ | "unknown";
+
+type ImporterJob = { jobId?: unknown; videoId?: unknown; phase?: unknown };
+
+function importerHeaders() {
+ const secret = serverEnv().MEDIA_SERVER_WEBHOOK_SECRET;
+ return secret ? { "x-media-server-secret": secret } : undefined;
+}
+
+export async function getLoomImporterJobState({
+ jobId,
+ videoId,
+}: {
+ jobId?: string | null;
+ videoId: string;
+}): Promise<LoomImporterJobState> {
+ const importerUrl = serverEnv().LOOM_IMPORTER_URL;
+ const headers = importerHeaders();
+ if (!importerUrl || !headers) return "unknown";
+ try {
+ if (jobId) {
+ const response = await fetch(
+ `${importerUrl}/video/process/${encodeURIComponent(jobId)}/status`,
+ { headers, signal: AbortSignal.timeout(15_000), redirect: "error" },
+ );
+ if (response.status === 404) {
+ await response.body?.cancel();
+ return "missing";
+ }
+ if (!response.ok) {
+ await response.body?.cancel();
+ return "unknown";
+ }
+ const job = (await response.json()) as ImporterJob;
+ if (job.jobId !== jobId || job.videoId !== videoId) return "unknown";
+ return job.phase === "complete" ||
+ job.phase === "error" ||
+ job.phase === "cancelled"
+ ? "finished"
+ : "running";
+ }
+ const response = await fetch(`${importerUrl}/video/status`, {
+ headers,
+ signal: AbortSignal.timeout(15_000),
+ redirect: "error",
+ });
+ if (!response.ok) {
+ await response.body?.cancel();
+ return "unknown";
+ }
+ const status = (await response.json()) as { jobs?: ImporterJob[] };
+ if (!Array.isArray(status.jobs)) return "unknown";
+ return status.jobs.some((job) => job.videoId === videoId)
+ ? "running"
+ : "missing";
+ } catch {
+ return "unknown";
+ }
+}
diff --git a/apps/web/workflows/import-loom-video.ts b/apps/web/workflows/import-loom-video.ts
index c936f6517..b0644a108 100644
--- a/apps/web/workflows/import-loom-video.ts
+++ b/apps/web/workflows/import-loom-video.ts
@@ -703,6 +703,7 @@ async function processVideoOnMediaServer(
attemptId,
state,
jobId,
+ worker,
}),
worker === "importer",
);
diff --git a/packages/database/types/metadata.ts b/packages/database/types/metadata.ts
index 697001872..38351e422 100644
--- a/packages/database/types/metadata.ts
+++ b/packages/database/types/metadata.ts
@@ -14,6 +14,7 @@ export interface VideoMetadata {
videoId?: string;
state: "accepted" | "rejected" | "uncertain";
jobId?: string;
+ worker?: "importer" | "media" | null;
};
};
loomImportUpload?: {