Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
fix: check real Loom run and importer state before restarting imports
Richie McIlroy96b78e0a9b83
diff --git a/apps/web/__tests__/unit/loom-import-start.test.ts b/apps/web/__tests__/unit/loom-import-start.test.tsindex bf4127d8e..6c13975a3 100644--- a/apps/web/__tests__/unit/loom-import-start.test.ts+++ b/apps/web/__tests__/unit/loom-import-start.test.ts@@ -1,6 +1,7 @@import { beforeEach, describe, expect, it, vi } from "vitest";const mocks = vi.hoisted(() => ({+ importerJobState: vi.fn(),where: vi.fn(),set: vi.fn(),queue: vi.fn(),@@ -37,6 +38,7 @@ vi.mock("workflow/api", () => ({ start: mocks.start, getRun: mocks.getRun }));vi.mock("workflow/runtime", () => ({ getWorld: mocks.getWorld }));vi.mock("@/lib/loom-media-dispatch", () => ({isLoomMediaJobFailed: mocks.mediaJobFailed,+ getLoomImporterJobState: mocks.importerJobState,}));vi.mock("@/workflows/import-loom-video", () => ({importLoomVideoWorkflow: vi.fn(),@@ -83,6 +85,73 @@ describe("Loom workflow startup receipts", () => {);});+ it("treats a run that was never created as inactive", async () => {+ const { WorkflowRunNotFoundError } = await import(+ "workflow/internal/errors"+ );+ mocks.getRun.mockReturnValue({+ status: Promise.reject(new WorkflowRunNotFoundError("run-1")),+ });+ expect(+ await getLoomImportRunState({ runId: "run-1", dispatch: "uncertain" }),+ ).toBe("inactive");+ mocks.getRun.mockReturnValue({+ status: Promise.reject(new Error("network")),+ });+ expect(+ await getLoomImportRunState({ runId: "run-1", dispatch: "uncertain" }),+ ).toBe("unconfirmed");+ });++ it.each([+ ["running", "active"],+ ["finished", "inactive"],+ ["missing", "inactive"],+ ["unknown", "unconfirmed"],+ ] as const)(+ "asks the importer about its own jobs: %s",+ async (importerState, expected) => {+ mocks.getRun.mockReturnValue({ status: Promise.resolve("failed") });+ mocks.importerJobState.mockResolvedValue(importerState);+ mocks.mediaJobFailed.mockResolvedValue(false);+ expect(+ await getLoomImportRunState({+ runId: "run-1",+ dispatch: "accepted",+ mediaDispatch: {+ attemptId: "run-1:0",+ videoId: "video-1",+ jobId: "importer-1",+ state: "accepted",+ worker: "importer",+ },+ }),+ ).toBe(expected);+ expect(mocks.importerJobState).toHaveBeenCalledWith({+ jobId: "importer-1",+ videoId: "video-1",+ });+ },+ );++ it("does not trust a missing importer job for media server dispatches", async () => {+ mocks.getRun.mockReturnValue({ status: Promise.resolve("failed") });+ mocks.importerJobState.mockResolvedValue("missing");+ mocks.mediaJobFailed.mockResolvedValue(false);+ expect(+ await getLoomImportRunState({+ runId: "run-1",+ dispatch: "accepted",+ mediaDispatch: {+ attemptId: "run-1:0",+ videoId: "video-1",+ jobId: "worker-1",+ state: "accepted",+ },+ }),+ ).toBe("unconfirmed");+ });+it.each([false, true])("requires a matching failed worker before releasing an accepted dispatch: %s",async (failed) => {diff --git a/apps/web/__tests__/unit/loom-importer-job-state.test.ts b/apps/web/__tests__/unit/loom-importer-job-state.test.tsnew file mode 100644index 000000000..85126e9be--- /dev/null+++ b/apps/web/__tests__/unit/loom-importer-job-state.test.ts@@ -0,0 +1,79 @@+import { afterEach, describe, expect, it, vi } from "vitest";++vi.mock("@cap/env", () => ({+ serverEnv: () => ({+ MEDIA_SERVER_URL: "https://worker.example.com",+ LOOM_IMPORTER_URL: "https://media.example.com/loom-importer",+ MEDIA_SERVER_WEBHOOK_SECRET: "fixture-secret",+ }),+}));+vi.mock("@cap/database", () => ({ db: vi.fn() }));++import { getLoomImporterJobState } from "@/lib/loom-media-dispatch";++describe("getLoomImporterJobState", () => {+ afterEach(() => vi.unstubAllGlobals());++ it.each([+ [{ jobId: "job", videoId: "video", phase: "downloading" }, "running"],+ [{ jobId: "job", videoId: "video", phase: "queued" }, "running"],+ [{ jobId: "job", videoId: "video", phase: "complete" }, "finished"],+ [{ jobId: "job", videoId: "video", phase: "error" }, "finished"],+ [{ jobId: "job", videoId: "video", phase: "cancelled" }, "finished"],+ [{ jobId: "job", videoId: "other", phase: "error" }, "unknown"],+ ])("reads the importer's job %j", async (body, expected) => {+ const fetch = vi.fn().mockResolvedValue(Response.json(body));+ vi.stubGlobal("fetch", fetch);+ await expect(+ getLoomImporterJobState({ jobId: "job", videoId: "video" }),+ ).resolves.toBe(expected);+ expect(fetch).toHaveBeenCalledWith(+ "https://media.example.com/loom-importer/video/process/job/status",+ expect.objectContaining({ redirect: "error" }),+ );+ });++ it("knows a job the importer no longer has", async () => {+ vi.stubGlobal(+ "fetch",+ vi.fn().mockResolvedValue(new Response("missing", { status: 404 })),+ );+ await expect(+ getLoomImporterJobState({ jobId: "job", videoId: "video" }),+ ).resolves.toBe("missing");+ });++ it.each([+ [() => Promise.resolve(new Response("down", { status: 503 }))],+ [() => Promise.reject(new Error("network"))],+ ])("cannot tell when the importer does not answer: %#", async (respond) => {+ vi.stubGlobal("fetch", vi.fn().mockImplementation(respond));+ await expect(+ getLoomImporterJobState({ jobId: "job", videoId: "video" }),+ ).resolves.toBe("unknown");+ });++ it("looks a video up in the importer's job list when there is no job id", async () => {+ const fetch = vi.fn().mockResolvedValue(+ Response.json({+ jobs: [{ jobId: "other", videoId: "other-video", phase: "queued" }],+ }),+ );+ vi.stubGlobal("fetch", fetch);+ await expect(getLoomImporterJobState({ videoId: "video" })).resolves.toBe(+ "missing",+ );+ fetch.mockResolvedValue(+ Response.json({+ jobs: [{ jobId: "j", videoId: "video", phase: "queued" }],+ }),+ );+ await expect(getLoomImporterJobState({ videoId: "video" })).resolves.toBe(+ "running",+ );+ expect(fetch).toHaveBeenCalledWith(+ "https://media.example.com/loom-importer/video/status",+ expect.anything(),+ );+ });+});diff --git a/apps/web/__tests__/unit/loom-importer-redrive.test.ts b/apps/web/__tests__/unit/loom-importer-redrive.test.tsindex 3c0fcd242..2044d3185 100644--- a/apps/web/__tests__/unit/loom-importer-redrive.test.ts+++ b/apps/web/__tests__/unit/loom-importer-redrive.test.ts@@ -149,7 +149,7 @@ describe("redriveLoomImport", () => {[[], "not an unfinished Loom import"],[[failedImport({ phase: "uploading" })], "phase uploading"],[- [failedImport({ phase: "processing", uploadUpdatedAt: hoursAgo(30) })],+ [failedImport({ phase: "processing", uploadUpdatedAt: hoursAgo(0.1) })],"phase processing",],])("skips imports that are not failed: %#", async (rows, reason) => {@@ -192,14 +192,34 @@ describe("redriveLoomImport", () => {expect(mocks.updates).toEqual([]);});+ it("restarts an import left processing when nothing is running it", async () => {+ mocks.rows = [+ failedImport({ phase: "processing", uploadUpdatedAt: hoursAgo(1) }),+ ];+ await expect(redriveLoomImport("video", now)).resolves.toMatchObject({+ status: "restarted",+ });+ });++ it("leaves an import processing while its run is active", async () => {+ mocks.runState.mockResolvedValue("active");+ mocks.rows = [+ failedImport({ phase: "processing", uploadUpdatedAt: hoursAgo(1) }),+ ];+ await expect(redriveLoomImport("video", now)).resolves.toMatchObject({+ status: "skipped",+ });+ expect(mocks.start).not.toHaveBeenCalled();+ });+it("only restarts an unconfirmed worker once it can no longer be running", async () => {mocks.runState.mockResolvedValue("unconfirmed");- mocks.rows = [failedImport({ uploadUpdatedAt: hoursAgo(2) })];+ mocks.rows = [failedImport({ uploadUpdatedAt: hoursAgo(0.5) })];await expect(redriveLoomImport("video", now)).resolves.toMatchObject({status: "skipped",reason: "run unconfirmed",});- mocks.rows = [failedImport({ uploadUpdatedAt: hoursAgo(7) })];+ mocks.rows = [failedImport({ uploadUpdatedAt: hoursAgo(2) })];await expect(redriveLoomImport("video", now)).resolves.toMatchObject({status: "restarted",});diff --git a/apps/web/lib/loom-import-redrive.ts b/apps/web/lib/loom-import-redrive.tsindex 5aa7125fc..1653c2029 100644--- a/apps/web/lib/loom-import-redrive.ts+++ b/apps/web/lib/loom-import-redrive.ts@@ -9,7 +9,8 @@ import {startLoomImportWorkflow,} from "@/lib/loom-import-start";-const FINISHED_WORKER_AGE_MS = 6 * 60 * 60 * 1000;+const ABANDONED_PROCESSING_AGE_MS = 10 * 60 * 1000;+const FINISHED_WORKER_AGE_MS = 60 * 60 * 1000;const STALE_PROCESSING_AGE_MS = 7 * 24 * 60 * 60 * 1000;export type LoomRedriveOutcome =@@ -58,9 +59,13 @@ export async function redriveLoomImport(const stalled =(row.phase === "processing" || row.phase === "uploading") &&age > STALE_PROCESSING_AGE_MS;- if (row.phase !== "error" && !stalled)- return { videoId, status: "skipped", reason: `phase ${row.phase}` };const runState = await getLoomImportRunState(row.metadata?.loomImportRun);+ const abandoned =+ row.phase === "processing" &&+ runState === "inactive" &&+ age > ABANDONED_PROCESSING_AGE_MS;+ if (row.phase !== "error" && !stalled && !abandoned)+ return { videoId, status: "skipped", reason: `phase ${row.phase}` };if (runState === "pending" || runState === "active")return { videoId, status: "skipped", reason: `run ${runState}` };if (runState === "unconfirmed" && age < FINISHED_WORKER_AGE_MS)diff --git a/apps/web/lib/loom-import-start.ts b/apps/web/lib/loom-import-start.tsindex d85097379..dd2f819b1 100644--- a/apps/web/lib/loom-import-start.ts+++ b/apps/web/lib/loom-import-start.ts@@ -4,8 +4,12 @@ import type { VideoMetadata } from "@cap/database/types";import { User, Video } from "@cap/web-domain";import { and, eq, sql } from "drizzle-orm";import { getRun, start } from "workflow/api";+import { WorkflowRunNotFoundError } from "workflow/internal/errors";import { getWorld } from "workflow/runtime";-import { isLoomMediaJobFailed } from "@/lib/loom-media-dispatch";+import {+ getLoomImporterJobState,+ isLoomMediaJobFailed,+} from "@/lib/loom-media-dispatch";import { importLoomVideoWorkflow } from "@/workflows/import-loom-video";type LoomImportRun = NonNullable<VideoMetadata["loomImportRun"]>;@@ -57,32 +61,36 @@ export async function getLoomImportRunState(receipt?: LoomImportRun,): Promise<"inactive" | "pending" | "active" | "unconfirmed"> {if (!receipt || receipt.dispatch === "rejected") return "inactive";+ let status: string;try {- const status = await getRun(receipt.runId).status;- if (- status === "completed" ||- status === "failed" ||- status === "cancelled"- ) {- const mediaDispatch = receipt.mediaDispatch;- if (mediaDispatch?.state === "accepted") {- return mediaDispatch.jobId &&- mediaDispatch.videoId &&- (await isLoomMediaJobFailed(- mediaDispatch.jobId,- mediaDispatch.videoId,- ))- ? "inactive"- : "unconfirmed";- }- return receipt.mediaDispatch?.state === "uncertain"- ? "unconfirmed"- : "inactive";- }+ status = await getRun(receipt.runId).status;+ } catch (error) {+ return WorkflowRunNotFoundError.is(error) &&+ receipt.dispatch === "uncertain" &&+ !receipt.mediaDispatch+ ? "inactive"+ : "unconfirmed";+ }+ if (status !== "completed" && status !== "failed" && status !== "cancelled")return status === "pending" ? "pending" : "active";- } catch {- return "unconfirmed";+ const mediaDispatch = receipt.mediaDispatch;+ if (!mediaDispatch || mediaDispatch.state === "rejected") return "inactive";+ if (mediaDispatch.worker !== "media" && mediaDispatch.videoId) {+ const importerState = await getLoomImporterJobState({+ jobId: mediaDispatch.jobId,+ videoId: mediaDispatch.videoId,+ });+ if (importerState === "running") return "active";+ if (importerState === "finished") return "inactive";+ if (importerState === "missing" && mediaDispatch.worker === "importer")+ return "inactive";}+ if (mediaDispatch.state === "accepted" && mediaDispatch.jobId)+ return mediaDispatch.videoId &&+ (await isLoomMediaJobFailed(mediaDispatch.jobId, mediaDispatch.videoId))+ ? "inactive"+ : "unconfirmed";+ return "unconfirmed";}async function saveReceipt(payload: ImportPayload, receipt: LoomImportRun) {diff --git a/apps/web/lib/loom-media-dispatch.ts b/apps/web/lib/loom-media-dispatch.tsindex de30c3fa7..2e8ce5f75 100644--- a/apps/web/lib/loom-media-dispatch.ts+++ b/apps/web/lib/loom-media-dispatch.ts@@ -11,6 +11,7 @@ export async function recordLoomMediaDispatch({attemptId,state,jobId,+ worker,}: {videoId: string;userId: string;@@ -18,11 +19,18 @@ export async function recordLoomMediaDispatch({attemptId: string;state: "accepted" | "rejected" | "uncertain";jobId?: string;+ worker?: "importer" | "media";}) {const receipt = {runId,dispatch: "accepted",- mediaDispatch: { attemptId, videoId, state, jobId: jobId ?? null },+ mediaDispatch: {+ attemptId,+ videoId,+ state,+ jobId: jobId ?? null,+ worker: worker ?? null,+ },};const result = await db().update(videos)@@ -94,3 +102,67 @@ async function isJobFailedOnWorker(return false;}}++export type LoomImporterJobState =+ | "running"+ | "finished"+ | "missing"+ | "unknown";++type ImporterJob = { jobId?: unknown; videoId?: unknown; phase?: unknown };++function importerHeaders() {+ const secret = serverEnv().MEDIA_SERVER_WEBHOOK_SECRET;+ return secret ? { "x-media-server-secret": secret } : undefined;+}++export async function getLoomImporterJobState({+ jobId,+ videoId,+}: {+ jobId?: string | null;+ videoId: string;+}): Promise<LoomImporterJobState> {+ const importerUrl = serverEnv().LOOM_IMPORTER_URL;+ const headers = importerHeaders();+ if (!importerUrl || !headers) return "unknown";+ try {+ if (jobId) {+ const response = await fetch(+ `${importerUrl}/video/process/${encodeURIComponent(jobId)}/status`,+ { headers, signal: AbortSignal.timeout(15_000), redirect: "error" },+ );+ if (response.status === 404) {+ await response.body?.cancel();+ return "missing";+ }+ if (!response.ok) {+ await response.body?.cancel();+ return "unknown";+ }+ const job = (await response.json()) as ImporterJob;+ if (job.jobId !== jobId || job.videoId !== videoId) return "unknown";+ return job.phase === "complete" ||+ job.phase === "error" ||+ job.phase === "cancelled"+ ? "finished"+ : "running";+ }+ const response = await fetch(`${importerUrl}/video/status`, {+ headers,+ signal: AbortSignal.timeout(15_000),+ redirect: "error",+ });+ if (!response.ok) {+ await response.body?.cancel();+ return "unknown";+ }+ const status = (await response.json()) as { jobs?: ImporterJob[] };+ if (!Array.isArray(status.jobs)) return "unknown";+ return status.jobs.some((job) => job.videoId === videoId)+ ? "running"+ : "missing";+ } catch {+ return "unknown";+ }+}diff --git a/apps/web/workflows/import-loom-video.ts b/apps/web/workflows/import-loom-video.tsindex c936f6517..b0644a108 100644--- a/apps/web/workflows/import-loom-video.ts+++ b/apps/web/workflows/import-loom-video.ts@@ -703,6 +703,7 @@ async function processVideoOnMediaServer(attemptId,state,jobId,+ worker,}),worker === "importer",);diff --git a/packages/database/types/metadata.ts b/packages/database/types/metadata.tsindex 697001872..38351e422 100644--- a/packages/database/types/metadata.ts+++ b/packages/database/types/metadata.ts@@ -14,6 +14,7 @@ export interface VideoMetadata {videoId?: string;state: "accepted" | "rejected" | "uncertain";jobId?: string;+ worker?: "importer" | "media" | null;};};loomImportUpload?: {