Skip to content

Open source CapCut alternative.

TypeScript
16.0 KB
Raw Blame
import type { Prisma } from "@/generated/prisma/client";
import type { RenderHandle } from "../server/exportPipeline";
import { claimNextJob, prisma, type ClaimedJob } from "./db";
import { overlayKeysOf, runExportJob } from "./exportJob";
import { runHlsJob } from "./hlsJob";
import { runConvertJob } from "./convertJob";
import { runProxyJob } from "./proxyJob";
import { runImportUrlJob } from "./importUrlJob";
import { runTurnJob } from "./turnJob";
import { describeRuntime, installHeadlessRuntime } from "../lib/headless/runtime";
import { deleteObjects } from "./r2";
import { failedJobFate, InputFault, staleRunningFate, TAB_EXPORT_SILENCE_MS } from "./settle";
import { failureCode, UserError } from "../lib/userError";

// The cloud render worker: a headless loop that claims CutRenderJob rows the
// hosted API queues (export, preview, import_url, convert, proxy, agent_turn),
// executes them with the same pipeline code the local engine runs, and writes
// progress and results
// back to the row the client polls. Scale is horizontal: one job per process,
// several processes — the atomic claim keeps them apart.

// One render per replica: ffmpeg already threads across every core it is given,
// so a second concurrent job halves the first one's cores and doubles how long
// someone watches a progress bar. Parallelism comes from replicas instead
// (max_instances / REPLICAS, wrangler.jsonc + cf/worker.ts).
const MAX_RUNNING = 1;

// The failures the loop itself writes to a row.
const RAN_TOO_LONG = new UserError("job_timed_out", "The job ran too long and was stopped.");
// A bare network rejection reads as "fetch failed" and says nothing to
// whoever asked for the render. The row carries words instead.
const UNREACHABLE = new UserError("render_unreachable", "Could not reach the service — the render stopped. Try it again.");
const IDLE_POLL_MS = 2000;
const WATCH_MS = 1000; // progress write + cancellation check cadence
// Drain-and-exit: once the queue stays empty this long with nothing running,
// the process exits so the container stops billing; the hosted API wakes it
// again when a job is queued. Only successful empty polls count as idle — a
// failing claim (DB unreachable) must not read as a drained queue.
const IDLE_EXIT_MS = 60_000;
// Sustained claim failure: exit non-zero so the next wake retries with a
// fresh process instead of burning container time on a dead DB connection.
const FAIL_EXIT_MS = 5 * 60_000;
// The watcher's unconditional 1s row write doubles as a heartbeat (updatedAt).
// A "running" row this quiet lost its worker; sweep it back to the queue.
const STALE_RUNNING_MS = 60_000;
const SWEEP_EVERY_MS = 15_000;
// Cloudflare sends SIGTERM 15 minutes before SIGKILL and rolls replicas one at
// a time, so a deploy can let a render finish rather than throw it away. Drain
// inside that window, keeping enough of it back to requeue whatever is still
// going when it closes.
const DRAIN_MS = 13 * 60_000;
/** Set when the platform sends SIGTERM: a run whose process stops from here
 * on is the shutdown's doing (`failedJobFate`). */
let signaled = false;

/** The longest a job of each kind may run. Everything else about a job's cost
 * is bounded when it is queued; this bounds the one thing that isn't — a
 * render that keeps ffmpeg talking forever holds a billed container. */
const MAX_RUNTIME_MS: Record<string, number> = {
  export: 60 * 60_000,
  preview: 20 * 60_000,
  card: 10 * 60_000,
  hls: 3 * 60 * 60_000,
  import_url: 20 * 60_000,
  convert: 60 * 60_000,
  proxy: 60 * 60_000,
  agent_turn: 20 * 60_000,
};
const DEFAULT_MAX_RUNTIME_MS = 60 * 60_000;

interface ActiveJob {
  handle: RenderHandle;
  kind: string;
  startedAt: number;
  canceled: boolean;
  /** This job goes back on the queue rather than settling here — swept out
   * from under us after a heartbeat gap, or still rendering when a shutdown
   * drain ran out. Either way: stop working it, write nothing more, and keep
   * its overlay PNGs for whichever worker claims the retry. */
  retry: boolean;
}

const active = new Map<string, ActiveJob>();
let stopping = false;

const sleep = (ms: number) => new Promise<void>((r) => setTimeout(r, ms));

/** Requeue "running" rows whose heartbeat (updatedAt) has gone quiet: their
 * worker died without SIGTERM (OOM, host kill), so nothing else will ever
 * settle them. Own in-flight jobs are excluded by id. A tab render's claim is
 * the tab's, never queued: one whose tab went silent is dismissed instead
 * (`staleRunningFate`). */
async function sweepStaleRunning(): Promise<void> {
  const now = Date.now();
  const stale = await prisma.cutRenderJob.findMany({
    where: {
      state: "running",
      updatedAt: { lt: new Date(now - STALE_RUNNING_MS) },
      id: { notIn: [...active.keys()] },
      // A command batch belongs to the editor in the ChatGPT card; the hosted
      // API expires one whose editor went quiet.
      kind: { not: "commands" },
    },
    select: { id: true, spec: true, updatedAt: true },
  });
  const ids = (fate: ReturnType<typeof staleRunningFate>) =>
    stale.filter((row) => staleRunningFate(row, now) === fate).map((row) => row.id);
  const requeue = ids("requeue");
  const dismiss = ids("dismiss");
  // The state and heartbeat guards again, so a row that moved or beat since
  // the read is left alone.
  if (requeue.length > 0) {
    const { count } = await prisma.cutRenderJob.updateMany({
      where: { id: { in: requeue }, state: "running", updatedAt: { lt: new Date(now - STALE_RUNNING_MS) } },
      data: { state: "queued", claimedAt: null, progress: 0 },
    });
    if (count > 0) console.log(`[cut-worker] requeued ${count} stale running job(s)`);
  }
  if (dismiss.length > 0) {
    const { count } = await prisma.cutRenderJob.updateMany({
      where: { id: { in: dismiss }, state: "running", updatedAt: { lt: new Date(now - TAB_EXPORT_SILENCE_MS) } },
      data: { state: "dismissed" },
    });
    if (count > 0) console.log(`[cut-worker] dismissed ${count} tab export(s) whose tab went quiet`);
  }
}

/** Mirror one running job to its row every WATCH_MS: push the pipeline's
 * progress out, and pull cancellation in (a canceled — or deleted — row kills
 * the live ffmpeg; the run then settles without touching the row again). The
 * write happens every tick even when progress hasn't moved — it bumps
 * updatedAt, the heartbeat the stale-running sweep keys off. */
function watchJob(id: string, entry: ActiveJob): ReturnType<typeof setInterval> {
  return setInterval(() => {
    void (async () => {
      try {
        if (Date.now() - entry.startedAt > (MAX_RUNTIME_MS[entry.kind] ?? DEFAULT_MAX_RUNTIME_MS)) {
          entry.canceled = true; // suppress the run's own row writes
          entry.handle.error = "Ran too long.";
          entry.handle.proc?.kill("SIGKILL");
          await prisma.cutRenderJob.updateMany({
            where: { id, state: "running" },
            data: { state: "error", error: RAN_TOO_LONG.message, result: { errorCode: RAN_TOO_LONG.code } },
          });
          return;
        }
        const row = await prisma.cutRenderJob.findUnique({
          where: { id },
          select: { state: true },
        });
        if (!row || row.state === "canceled") {
          entry.canceled = true;
          entry.handle.error = "Canceled.";
          entry.handle.proc?.kill("SIGKILL");
          return;
        }
        if (row.state !== "running") {
          entry.retry = true;
          entry.canceled = true; // suppress the run's own row writes
          entry.handle.error = "Requeued.";
          entry.handle.proc?.kill("SIGKILL");
          return;
        }
        await prisma.cutRenderJob.updateMany({
          where: { id, state: "running" },
          data: { progress: entry.handle.progress },
        });
      } catch {
        // Transient DB hiccup: keep watching; the run itself is unaffected.
      }
    })();
  }, WATCH_MS);
}

async function runJob(job: ClaimedJob): Promise<void> {
  const handle: RenderHandle = { tmpDir: "", outPath: "", progress: 0, log: [] };
  const entry: ActiveJob = {
    handle,
    kind: job.kind,
    startedAt: Date.now(),
    canceled: false,
    retry: false,
  };
  active.set(job.id, entry);
  const watcher = watchJob(job.id, entry);
  try {
    if (job.kind === "export" || job.kind === "preview" || job.kind === "card") {
      const { outputKey, outName } = await runExportJob(job, handle);
      await prisma.cutRenderJob.updateMany({
        where: { id: job.id, state: "running" },
        data: { state: "done", progress: 1, outputKey, outName },
      });
    } else if (job.kind === "hls") {
      const { outputKey, outName } = await runHlsJob(job, handle);
      await prisma.cutRenderJob.updateMany({
        where: { id: job.id, state: "running" },
        data: { state: "done", progress: 1, outputKey, outName },
      });
    } else if (job.kind === "import_url") {
      const result = await runImportUrlJob(job, () => entry.canceled);
      await prisma.cutRenderJob.updateMany({
        where: { id: job.id, state: "running" },
        data: { state: "done", progress: 1, result: result as unknown as Prisma.InputJsonValue },
      });
    } else if (job.kind === "convert") {
      const result = await runConvertJob(job, handle, () => entry.canceled);
      await prisma.cutRenderJob.updateMany({
        where: { id: job.id, state: "running" },
        data: { state: "done", progress: 1, result: result as unknown as Prisma.InputJsonValue },
      });
    } else if (job.kind === "proxy") {
      const result = await runProxyJob(job, handle, () => entry.canceled);
      await prisma.cutRenderJob.updateMany({
        where: { id: job.id, state: "running" },
        data: { state: "done", progress: 1, result: result as unknown as Prisma.InputJsonValue },
      });
    } else if (job.kind === "agent_turn") {
      const result = await runTurnJob(job, handle, () => entry.canceled);
      await prisma.cutRenderJob.updateMany({
        where: { id: job.id, state: "running" },
        data: { state: "done", progress: 1, result: result as unknown as Prisma.InputJsonValue },
      });
    } else {
      throw new Error(`Unknown job kind: ${job.kind}`);
    }
    console.log(`[cut-worker] ${job.kind} ${job.id} done`);
  } catch (err) {
    const raw = err instanceof Error ? err.message : String(err);
    const failure = /^(fetch failed|network error|terminated)$/i.test(raw.trim()) ? UNREACHABLE : err;
    const message = failure === UNREACHABLE ? UNREACHABLE.message : raw;
    // A canceled job's row already says so; the running-state guard also
    // keeps this from resurrecting a row canceled between watcher ticks.
    const fate = failedJobFate({ canceled: entry.canceled, signaled, proc: entry.handle.proc });
    if (fate === "requeue") {
      // Its overlays stay for the replica that claims it next.
      entry.retry = true;
      await prisma.cutRenderJob
        .updateMany({
          where: { id: job.id, state: "running" },
          data: { state: "queued", claimedAt: null, progress: 0 },
        })
        .catch(() => {});
    } else if (fate === "error") {
      // A failure on the job's input records which, and a coded failure its
      // code, for the route to read.
      const coded = failureCode(failure);
      const result = {
        ...(err instanceof InputFault ? { fault: err.fault } : {}),
        ...(coded ? { errorCode: coded.code, errorParams: coded.params } : {}),
      };
      await prisma.cutRenderJob
        .updateMany({
          where: { id: job.id, state: "running" },
          data: { state: "error", error: message, ...(Object.keys(result).length ? { result } : {}) },
        })
        .catch(() => {});
    }
    console.error(`[cut-worker] ${job.kind} ${job.id} ${entry.canceled ? "canceled" : fate === "requeue" ? "requeued on shutdown" : "failed"}: ${raw}`);
    // A canceled turn retires this process once its cleanup settles: the
    // abort tears the agent loop down, and a fresh process guarantees the
    // next claim starts with clean per-user transport bindings even if some
    // in-flight tool call outlived the abort. The wake on the next queued
    // job restarts the container in seconds.
    if (job.kind === "agent_turn" && entry.canceled) stopping = true;
  } finally {
    clearInterval(watcher);
    // The overlay PNGs are single-use render inputs: delete them once the job
    // settles — including one that settles while the worker drains. Only a job
    // going back on the queue keeps its overlays, for the retry to read.
    // (deleteObjects swallows its own failures, so this always reaches the
    // delete below and the drain can never hang on it.)
    if (!entry.retry) await deleteObjects(overlayKeysOf(job));
    // Last, so a drain waiting on `active` waits for the cleanup too rather
    // than exiting the process out from under it.
    active.delete(job.id);
  }
}

/** On shutdown, stop claiming and let the renders in hand finish — a deploy
 * rolls one replica at a time, so a job most of the way through an encode is
 * worth the wait, and restarting it means someone watches the same progress bar
 * twice. Whatever is still going when the drain window closes is handed back:
 * killed and requeued, so the next worker picks it up instead of leaving the row
 * stranded in "running" forever. */
async function stop(): Promise<void> {
  if (stopping) return;
  stopping = true;
  if (active.size > 0) console.log(`[cut-worker] draining ${active.size} job(s)…`);
  const deadline = Date.now() + DRAIN_MS;
  // The per-job watcher keeps heartbeating throughout, so another replica's
  // stale sweep leaves a draining render alone.
  while (active.size > 0 && Date.now() < deadline) await sleep(1000);
  for (const [id, entry] of active) {
    entry.canceled = true; // suppress the run's own error write
    entry.retry = true; // and keep its overlays for whoever claims it next
    entry.handle.proc?.kill("SIGKILL");
    await prisma.cutRenderJob
      .updateMany({
        where: { id, state: "running" },
        data: { state: "queued", claimedAt: null, progress: 0 },
      })
      .catch(() => {});
  }
  process.exit(0);
}

async function main(): Promise<void> {
  process.on("SIGTERM", () => {
    signaled = true;
    void stop();
  });
  process.on("SIGINT", () => void stop());
  // Canvas, decoders, Web Audio, and fonts, standing in for the page a turn
  // would otherwise run in. A worker missing a native module loses that
  // capability and still runs every ffmpeg-only job.
  const runtime = await installHeadlessRuntime();
  console.log(
    `[cut-worker] polling for jobs (max ${MAX_RUNNING} concurrent), ${describeRuntime(runtime)}`
  );
  const inFlight = new Set<Promise<void>>();
  let idleSince: number | null = null;
  let failingSince: number | null = null;
  let lastSweep = 0;
  while (!stopping) {
    if (inFlight.size >= MAX_RUNNING) {
      await Promise.race(inFlight);
      continue;
    }
    let job: ClaimedJob | null = null;
    try {
      if (Date.now() - lastSweep >= SWEEP_EVERY_MS) {
        lastSweep = Date.now();
        await sweepStaleRunning();
      }
      job = await claimNextJob();
      failingSince = null;
    } catch (err) {
      console.error("[cut-worker] claim failed:", err instanceof Error ? err.message : err);
      // A failing claim says nothing about the queue: keep the drain timer
      // from ticking, and give up with an error exit once the failure holds.
      idleSince = null;
      failingSince ??= Date.now();
      if (inFlight.size === 0 && Date.now() - failingSince >= FAIL_EXIT_MS) {
        console.error("[cut-worker] claims failing persistently; exiting");
        process.exit(1);
      }
      await sleep(IDLE_POLL_MS);
      continue;
    }
    if (!job) {
      if (inFlight.size === 0) {
        idleSince ??= Date.now();
        if (Date.now() - idleSince >= IDLE_EXIT_MS) {
          console.log("[cut-worker] queue drained; exiting");
          process.exit(0);
        }
      } else {
        idleSince = null;
      }
      await sleep(IDLE_POLL_MS);
      continue;
    }
    idleSince = null;
    const run = runJob(job).finally(() => inFlight.delete(run));
    inFlight.add(run);
  }
}

void main();