Skip to content

Open-source AI podcast clipper. Generate vertical clips with face tracking and burned-in captions. CLI, MCP server, web app.

19.7 KB
Raw Blame
import { basename } from "path";
import { randomUUID } from "crypto";
import { z } from "zod";
import { PythonExecutor, type TaskExecutor } from "../services/python-executor.js";
import { TranscriptCache, needsDiarizationRetry } from "../services/transcript-cache.js";
import { resolveTranscribeEngine, engineCanDiarize } from "../services/engine-resolve.js";
import { webServerUrl } from "../config/server.js";
import type { TranscriptResult } from "../models/index.js";

type LocalJob = {
  status: "pending" | "running" | "done" | "error";
  progress: number;
  message: string;
  result?: unknown;
  error?: string;
};

const localJobs = new Map<string, LocalJob>();
const processExecutor = new PythonExecutor();

const executor = new PythonExecutor();
const cache = new TranscriptCache();

/** What handleTranscribe talks to. Tests pass stand-ins instead of the Python process and the disk cache. */
export interface TranscribeDeps {
  executor: TaskExecutor;
  cache: Pick<TranscriptCache, "get" | "set" | "getPackedMarkdown" | "getFileHashForEngine">;
  resolveEngine: typeof resolveTranscribeEngine;
}

const defaultDeps: TranscribeDeps = { executor, cache, resolveEngine: resolveTranscribeEngine };

export interface TranscribeInput {
  file_path: string;
  model_size?: "tiny" | "base" | "small" | "medium" | "large";
  engine?: "whisper-py" | "whispercpp" | "assemblyai" | "omnilingual";
  language?: string;
  enable_diarization?: boolean;
  num_speakers?: number;
  start_seconds?: number;
  duration_seconds?: number;
}

export const transcribeToolDef = {
  name: "transcribe_podcast",
  description:
    "STEP 1: Transcribe a podcast video/audio file. This is typically the first tool you call.\n\n" +
    "What it does: Uses Whisper AI for word-level timestamps, with speaker detection (who said what) on by " +
    "default. Speaker labels are what let a clip tell a question from an answer, so leave them on for any " +
    "interview. Pass enable_diarization=false to skip them and save time on a single-speaker recording.\n" +
    "Returns: lightweight metadata only, namely duration, language, word/segment counts, speaker summary, and " +
    "packed_ready flag. The actual transcript body is NOT returned here (it would be 500KB+ for a typical " +
    "episode). Read the content via get_ui_state(include_transcript: true) which returns a compact " +
    "phrase-grouped markdown view (~10x smaller than raw segments).\n" +
    "Caching: Results are cached by file hash, so the same file won't be re-transcribed.\n" +
    "Supported formats: MP4, MOV, WebM, MKV, MP3, WAV.\n\n" +
    "After transcription: call rank_clip_candidates(video_path) for a ~20-clip shortlist " +
    "(do not read the whole transcript unless you need to rewrite a range). " +
    "Or call process_episode(video_path) to transcribe, rank, and render in one job.",
  inputSchema: {
    type: "object" as const,
    properties: {
      file_path: {
        type: "string",
        description: "Absolute path to the podcast file",
      },
      model_size: {
        type: "string",
        enum: ["tiny", "base", "small", "medium", "large"],
        description:
          "Whisper model size. tiny=fastest, large=most accurate. Default: base",
        default: "base",
      },
      language: {
        type: "string",
        description: "ISO language code (e.g. 'en'). Leave empty for auto-detect.",
      },
      engine: {
        type: "string",
        enum: ["whisper-py", "whispercpp", "assemblyai", "omnilingual"],
        description: "Transcription engine. Default: whisper-py.",
      },
      enable_diarization: {
        type: "boolean",
        description:
          "Speaker labels (who is speaking). On by default, and required for any clip logic that needs " +
          "to tell a question from an answer. Set false only for a single-speaker recording. " +
          "Falls back to no labels with a warning where torch/pyannote.audio is unavailable.",
        default: true,
      },
      num_speakers: {
        type: "number",
        description:
          "Exact number of speakers if known (e.g. 2 for a two-person podcast). " +
          "Leave empty to auto-detect (2-5 speakers).",
      },
      start_seconds: {
        type: "number",
        description:
          "Sample mode: only transcribe a window starting here (seconds into the source), " +
          "instead of the whole file, e.g. to test a language on 40s before committing to " +
          "a full run. Pair with duration_seconds. The result is marked complete: false and " +
          "is not written to the main transcript cache.",
      },
      duration_seconds: {
        type: "number",
        description: "Sample mode window length in seconds. Defaults start_seconds to 0 if omitted.",
      },
    },
    required: ["file_path"],
  },
};

export async function handleTranscribe(
  input: TranscribeInput,
  { executor, cache, resolveEngine }: TranscribeDeps = defaultDeps,
): Promise<string> {
  const filePath = input.file_path;
  const modelSize = input.model_size ?? "base";
  const engine = input.engine;
  const language = input.language;
  const enableDiarization = input.enable_diarization !== false;
  const numSpeakers = input.num_speakers;
  const startSeconds = input.start_seconds;
  const durationSeconds = input.duration_seconds;
  // A sample is a positive window, not merely a present key. An explicit
  // null or a duration_seconds: 0 both mean "no sample", matching
  // backend/services/transcription.py and backend/main.py's handle_transcribe.
  const isSample = (durationSeconds ?? 0) > 0 || (startSeconds ?? 0) > 0;

  // Resolve before reading the cache: an unset engine is written under
  // whatever transcribe_file actually ran (e.g. "whispercpp" on a native
  // install), so reading with the raw unset request always misses.
  const resolvedEngine = await resolveEngine(executor, engine, modelSize);
  const cacheKey = { engine: resolvedEngine, model: modelSize, language };

  // A sample is a throwaway check on a slice of the file. It must never
  // serve (or pollute) the main transcript cache, which is keyed by the
  // whole file and assumed complete.
  const cachedRaw = isSample ? null : await cache.get(filePath, cacheKey);

  const cached = needsDiarizationRetry(cachedRaw, enableDiarization, engineCanDiarize(resolvedEngine))
    ? null
    : cachedRaw;

  if (cached) {
    // Same combo cache.get(filePath, cacheKey) just matched: the packed
    // view has to be keyed identically, or a backfill here would write it
    // under a key the next read for this same request won't find.
    const packedKey = { engine: cached.engine ?? resolvedEngine, model: modelSize, language };
    // Backfill packed view if this cache predates auto-packing.
    let packed = await cache.getPackedMarkdown(filePath, packedKey);

    if (!packed) {
      try {
        const cacheHash = await cache.getFileHashForEngine(filePath, packedKey);
        await executor.execute("pack_transcript", {
          transcript: cached,
          cache_hash: cacheHash,
          source_label: basename(filePath),
          file_path: filePath,
        });
        packed = await cache.getPackedMarkdown(filePath, packedKey);
      } catch {
        // Non-fatal: caller still gets metadata
      }
    }

    return JSON.stringify({ cached: true, packed_ready: !!packed, ...formatResult(cached) });
  }

  // Execute transcription + diarization (Python side auto-writes the packed view)
  const result = await executor.execute<TranscriptResult>("transcribe", {
    file_path: filePath,
    model_size: modelSize,
    engine,
    language,
    enable_diarization: enableDiarization,
    num_speakers: numSpeakers,
    start_seconds: startSeconds,
    duration_seconds: durationSeconds,
  });

  if (!result.data) {
    throw new Error("Transcription returned no data");
  }

  const data = result.data;
  const actualEngine = data.engine ?? resolvedEngine;

  if (isSample) {
    // Not cached and not packed: it's a slice of the file, not the whole
    // transcript the cache/packed-view keys assume.
    return JSON.stringify({ cached: false, packed_ready: false, ...formatResult(data) });
  }

  // Cache the raw result under what it actually ran with, not the prediction
  // above: resolveTranscribeEngine can't see a model-load failure that only
  // shows up once transcribe_file tries it for real.
  await cache.set(filePath, data, { engine: actualEngine, model: modelSize, language });
  const packed = await cache.getPackedMarkdown(filePath, { engine: actualEngine, model: modelSize, language });

  return JSON.stringify({ cached: false, packed_ready: !!packed, ...formatResult(data) });
}

/**
 * Returns lightweight metadata: NOT the transcript body. The full transcript
 * is available via get_ui_state(include_transcript: true), which serves the
 * packed markdown view (~10x smaller than raw segments/words). Raw segments
 * and words arrays stay in the on-disk cache for internal Python consumers.
 */
function formatResult(data: TranscriptResult) {
  return {
    duration: data.duration,
    language: data.language,
    // Exposed so callers that need to re-read the cache (e.g. the UI state
    // push in server.ts) key it the same way this handler just wrote it.
    engine: data.engine,
    word_count: (data.words ?? []).length,
    segment_count: (data.segments ?? []).length,
    speakers: data.speakers ?? { num_speakers: 0, speakers: {} },
    ...(data.complete === false && {
      complete: false,
      sample_offset_seconds: data.sample_offset_seconds ?? 0,
    }),
    next_step: "Read the transcript via get_ui_state(include_transcript: true), then suggest_clips.",
  };
}

// =============================================
// Async transcription via Web UI job queue
// =============================================

const jobStartSchema = z.object({
  job_id: z.string(),
  status: z.string(),
  cached: z.boolean().optional(),
});

const jobPollSchema = z.object({
  status: z.enum(["running", "done", "error"]),
  progress: z.number(),
  message: z.string(),
  result: z.unknown().optional(),
  error: z.string().optional(),
});

export const transcribeStartToolDef = {
  name: "transcribe_start",
  description:
    "Start transcription as a background job and return a job_id immediately. " +
    "Use this instead of transcribe_podcast for long files so you can narrate progress " +
    "to the user while it runs (a 60-min episode takes 15-25 min).\n\n" +
    "Flow: call transcribe_start(file_path) → emit status text to user → " +
    "call job_status(job_id, wait_seconds: 30) in a loop until done → " +
    "then read the packed transcript via get_ui_state(include_transcript: true).\n\n" +
    "Requires the Web UI to be running (npm run ui). Returns { job_id, cached, status, estimate_minutes }.",
  inputSchema: {
    type: "object" as const,
    properties: {
      file_path: { type: "string", description: "Absolute path to the podcast file" },
      model_size: {
        type: "string",
        enum: ["tiny", "base", "small", "medium", "large"],
        default: "base",
      },
      language: { type: "string" },
      engine: {
        type: "string",
        enum: ["whisper-py", "whispercpp", "assemblyai", "omnilingual"],
      },
      enable_diarization: { type: "boolean", default: true },
      num_speakers: { type: "number" },
      start_seconds: {
        type: "number",
        description:
          "Sample mode: only transcribe a window starting here (seconds into the source). " +
          "Pair with duration_seconds. Not written to the main transcript cache.",
      },
      duration_seconds: {
        type: "number",
        description: "Sample mode window length in seconds. Defaults start_seconds to 0 if omitted.",
      },
    },
    required: ["file_path"],
  },
};

export async function handleTranscribeStart(input: TranscribeInput): Promise<string> {
  try {
    const res = await fetch(`${webServerUrl}/api/transcribe`, {
      method: "POST",
      headers: { "Content-Type": "application/json" },
      body: JSON.stringify({
        file_path: input.file_path,
        model_size: input.model_size ?? "base",
        engine: input.engine,
        language: input.language,
        enable_diarization: input.enable_diarization !== false,
        num_speakers: input.num_speakers,
        start_seconds: input.start_seconds,
        duration_seconds: input.duration_seconds,
      }),
    });

    if (!res.ok) throw new Error(`HTTP ${res.status}: ${await res.text()}`);
    const data = jobStartSchema.parse(await res.json());

    // Rough estimate: Whisper base + diarization is roughly 1/3× audio duration on CPU.
    // We don't know duration here; ballpark from file size.
    let estimate = "10-30 min depending on episode length";

    try {
      const { statSync } = await import("fs");
      const mb = statSync(input.file_path).size / (1024 * 1024);
      const mins = Math.max(2, Math.round(mb / 50)); // very rough: ~50MB/min of processing
      estimate = data.cached ? "instant (cached)" : `~${mins}-${mins * 2} min`;
    } catch {
      // ignore
    }

    return JSON.stringify({
      job_id: data.job_id,
      cached: !!data.cached,
      status: data.status,
      estimate,
      next_step: data.cached
        ? "Cached: skip polling, read via get_ui_state(include_transcript: true)."
        : `Poll job_status("${data.job_id}", wait_seconds: 30) in a loop, emitting one terse progress line to the user between polls.`,
    });
  } catch (err: unknown) {
    const msg = err instanceof Error ? err.message : String(err);

    if (msg.includes("ECONNREFUSED") || msg.includes("fetch failed")) {
      return JSON.stringify({
        error: "Web UI not running. Start it with `npm run ui`, or fall back to transcribe_podcast for synchronous transcription.",
      });
    }

    throw err;
  }
}

export const jobStatusToolDef = {
  name: "job_status",
  description:
    "Poll the status of any background job (transcription, clip render, batch export, process_episode). " +
    "Supports long-polling: pass wait_seconds (1-60) to block until the job changes state " +
    "or the timeout elapses, whichever comes first. Paces Claude's polling naturally so " +
    "the spinner doesn't spam and the user sees steady progress text.\n\n" +
    "Returns { status: 'running'|'done'|'error', progress, message, done, result? }.\n" +
    "Use after transcribe_start, process_episode, or batch_create_clips(async_mode: true).",
  inputSchema: {
    type: "object" as const,
    properties: {
      job_id: { type: "string" },
      wait_seconds: {
        type: "number",
        default: 30,
        minimum: 0,
        maximum: 60,
        description: "How long to wait for a status change before returning. Default 30s.",
      },
    },
    required: ["job_id"],
  },
};

export async function handleJobStatus(input: {
  job_id: string;
  wait_seconds?: number;
  job_kind?: string;
}): Promise<string> {
  const wait = Math.max(0, Math.min(60, input.wait_seconds ?? 30));
  const deadline = Date.now() + wait * 1000;
  // Seeded from the first read, so the wait ends on a change after the caller
  // asked, not on the first look.
  let lastProgress: number | null = null;

  const local = localJobs.get(input.job_id);

  if (local) {
    while (true) {
      const job = localJobs.get(input.job_id);

      if (!job) break;
      const terminal = job.status === "done" || job.status === "error";

      if (lastProgress === null) lastProgress = job.progress;
      const progressChanged = job.progress !== lastProgress;

      if (terminal || progressChanged || Date.now() >= deadline) {
        return JSON.stringify({
          status: job.status,
          progress: job.progress,
          message: job.message,
          done: terminal,
          result: job.result,
          error: job.error,
          next_step:
            job.status === "done"
              ? "Job complete. The result field lists clip paths, durations, scores, titles, and captions."
              : job.status === "error"
                ? "Job failed: check the error. Retry process_episode if appropriate."
                : `Still running: call job_status("${input.job_id}", wait_seconds: 30) again.`,
        });
      }

      lastProgress = job.progress;
      await new Promise((r) => setTimeout(r, 2000));
    }
  }

  try {
    while (true) {
      const res = await fetch(`${webServerUrl}/api/job/${input.job_id}`);

      if (res.status === 404) {
        return JSON.stringify({ error: `Job ${input.job_id} not found` });
      }

      if (!res.ok) throw new Error(`HTTP ${res.status}`);

      const job = jobPollSchema.parse(await res.json());

      if (lastProgress === null) lastProgress = job.progress;
      const progressChanged = job.progress !== lastProgress;
      const terminal = job.status === "done" || job.status === "error";

      if (terminal || progressChanged || Date.now() >= deadline) {
        return JSON.stringify({
          status: job.status,
          progress: job.progress,
          message: job.message,
          done: terminal,
          error: job.error,
          next_step:
            job.status === "done"
              ? "Job complete. For transcription: get_ui_state(include_transcript: true). For renders: clips are on disk. The result field lists paths."
              : job.status === "error"
              ? "Job failed: check the error. Retry the original start call if appropriate."
              : `Still running: call job_status("${input.job_id}", wait_seconds: 30) again and emit one terse progress line to the user between polls.`,
        });
      }

      lastProgress = job.progress;
      await new Promise((r) => setTimeout(r, 2000));
    }
  } catch (err: unknown) {
    const msg = err instanceof Error ? err.message : String(err);

    if (msg.includes("ECONNREFUSED") || msg.includes("fetch failed")) {
      return JSON.stringify({ error: "Web UI not running." });
    }

    throw err;
  }
}

export const processEpisodeToolDef = {
  name: "process_episode",
  description:
    "One-shot episode job: transcribe, rank moments, render clips, write titles/descriptions/tags. " +
    "Returns { job_id } immediately. Poll with job_status until done; the result is the same JSON " +
    "`podcli process --json` prints (clip paths, durations, scores, titles, captions).\n\n" +
    "Does not require the web studio. For a shortlist without rendering, use rank_clip_candidates instead.",
};

export async function handleProcessEpisode(input: {
  video_path: string;
  transcript?: string;
  top?: number;
  output?: string;
  quality?: string;
  caption_style?: string;
  crop?: string;
  format?: string;
  language?: string;
  engine?: string;
  fast?: boolean;
}): Promise<string> {
  const jobId = randomUUID();
  localJobs.set(jobId, {
    status: "running",
    progress: 0,
    message: "Starting episode process",
  });
  void processExecutor
    .execute("process_episode", {
      video_path: input.video_path,
      transcript: input.transcript,
      top: input.top,
      output: input.output,
      quality: input.quality,
      caption_style: input.caption_style,
      crop: input.crop,
      format: input.format,
      language: input.language,
      engine: input.engine,
      fast: input.fast,
    }, (event) => {
      const job = localJobs.get(jobId);

      if (!job || job.status !== "running") return;
      job.progress = event.percent;
      job.message = event.message;
    })
    .then((result) => {
      localJobs.set(jobId, {
        status: "done",
        progress: 100,
        message: "Done",
        result: result.data,
      });
    })
    .catch((cause: unknown) => {
      localJobs.set(jobId, {
        status: "error",
        progress: 0,
        message: "Failed",
        error: cause instanceof Error ? cause.message : String(cause),
      });
    });

  return JSON.stringify({
    job_id: jobId,
    status: "running",
    next_step: `Poll job_status("${jobId}", wait_seconds: 30) until done. The result field lists clip paths.`,
  });
}