Open-source AI podcast clipper. Generate vertical clips with face tracking and burned-in captions. CLI, MCP server, web app.
Python ai-toolsauto-captionscaptionscliffmpegmcpmcp-serveropusclip-alternativepodcast-clipspodcastsshortform-videovertical-videowhisper
feat: pack transcripts into compact markdown for LLM reading
Nika Siradzee710a31bf89b
diff --git a/backend/main.py b/backend/main.pyindex e3f0069..363ed2f 100644--- a/backend/main.py+++ b/backend/main.py@@ -51,10 +51,12 @@ def handle_transcribe(task_id: str, params: dict):"""Transcribe a podcast video/audio file with speaker detection."""from services.transcription import transcribe_filefrom services.corrections import apply_corrections+ from services.transcript_packer import compute_cache_hash, write_packedemit_progress(task_id, "transcribing", 0, "Starting transcription...")+ file_path = params["file_path"]result = transcribe_file(- file_path=params["file_path"],+ file_path=file_path,model_size=params.get("model_size", "base"),language=params.get("language"),enable_diarization=params.get("enable_diarization", True),@@ -63,6 +65,31 @@ def handle_transcribe(task_id: str, params: dict):)# Apply word corrections (Whisper misheard proper nouns)apply_corrections(result.get("words", []), result.get("segments", []))++ # Auto-pack: emit compact LLM-readable markdown alongside raw JSON.+ # Pulls energy data so the packed view includes peak moments for clip reasoning.+ try:+ from services.audio_analyzer import extract_audio_energy++ cache_hash = compute_cache_hash(file_path)+ energy_data = None+ try:+ energy_data = extract_audio_energy(file_path)+ except Exception:+ pass # energy is a nice-to-have++ packed_path, packed_md = write_packed(+ result,+ cache_hash,+ source_label=os.path.basename(file_path),+ energy_data=energy_data,+ )+ result["packed_path"] = packed_path+ result["packed_size_bytes"] = len(packed_md.encode("utf-8"))+ except Exception as e:+ # Non-fatal — transcription result is still useful without the packed view+ emit_progress(task_id, "packing", 99, f"Packer skipped: {e}")+emit_result(task_id, "success", data=result)@@ -149,8 +176,10 @@ def handle_batch_clips(task_id: str, params: dict):def handle_parse_transcript(task_id: str, params: dict):"""Parse a speaker-labeled transcript into word-level timestamps."""+ import hashlibfrom services.transcript_parser import detect_and_parsefrom services.corrections import apply_corrections+ from services.transcript_packer import write_packedraw_text = params.get("raw_text", "")total_duration = params.get("total_duration")@@ -170,10 +199,53 @@ def handle_parse_transcript(task_id: str, params: dict):# Apply word corrections (proper nouns, brand names)apply_corrections(result.get("words", []), result.get("segments", []))+ # Auto-pack: key pasted transcripts by content hash since there's no source file.+ try:+ content_hash = hashlib.sha256(raw_text.encode("utf-8")).hexdigest()[:16]+ packed_path, packed_md = write_packed(+ result, content_hash, source_label="pasted-transcript"+ )+ result["packed_path"] = packed_path+ result["packed_size_bytes"] = len(packed_md.encode("utf-8"))+ result["content_hash"] = content_hash+ except Exception as e:+ emit_progress(task_id, "packing", 99, f"Packer skipped: {e}")+emit_progress(task_id, "parsing", 100, "Transcript parsed!")emit_result(task_id, "success", data=result)+def handle_pack_transcript(task_id: str, params: dict):+ """Pack a transcript dict into LLM-readable markdown. Used to backfill+ the packed view for caches that predate auto-packing."""+ from services.transcript_packer import write_packed++ transcript = params.get("transcript")+ cache_hash = params.get("cache_hash")+ if not transcript or not cache_hash:+ emit_result(task_id, "error", error="transcript and cache_hash are required")+ return++ energy_data = params.get("energy_data")+ if energy_data is None and params.get("file_path"):+ try:+ from services.audio_analyzer import extract_audio_energy+ energy_data = extract_audio_energy(params["file_path"])+ except Exception:+ pass++ path, md = write_packed(+ transcript,+ cache_hash,+ source_label=params.get("source_label"),+ energy_data=energy_data,+ )+ emit_result(task_id, "success", data={+ "packed_path": path,+ "size_bytes": len(md.encode("utf-8")),+ })++def handle_analyze_energy(task_id: str, params: dict):"""Analyze audio energy of a video to improve clip scoring."""from services.audio_analyzer import get_energy_profile@@ -312,6 +384,7 @@ TASK_HANDLERS = {"create_clip": handle_create_clip,"batch_clips": handle_batch_clips,"analyze_energy": handle_analyze_energy,+ "pack_transcript": handle_pack_transcript,"detect_encoder": handle_detect_encoder,"presets": handle_presets,"corrections": handle_corrections,diff --git a/backend/services/transcript_packer.py b/backend/services/transcript_packer.pynew file mode 100644index 0000000..280f0d3--- /dev/null+++ b/backend/services/transcript_packer.py@@ -0,0 +1,329 @@+"""+Pack a cached transcription into a compact markdown surface for LLM reading.++The goal: give an LLM everything it needs to reason about clip selection and+cut points without watching video. Words, speakers, silences, and optional+energy peaks fit into ~10-20KB of text.++Input: cached transcription JSON (.podcli/cache/<hash>.json)+Output: .podcli/packed/<hash>.md+"""++from __future__ import annotations++import hashlib+import json+import os+import sys+from typing import Optional++PROJECT_ROOT = os.path.abspath(+ os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "..")+)+CACHE_DIR = os.path.join(PROJECT_ROOT, ".podcli", "cache")+PACKED_DIR = os.path.join(PROJECT_ROOT, ".podcli", "packed")++# Phrase construction tuning+SILENCE_SPLIT_SEC = 0.5 # split a phrase on word gap >= this+SILENCE_GAP_REPORT_SEC = 0.6 # list gaps >= this in the silence section+PHRASE_MAX_SEC = 12.0+PHRASE_MAX_CHARS = 160+ENERGY_PEAKS_TO_REPORT = 20+++def compute_cache_hash(video_path: str) -> str:+ """Match src/services/transcript-cache.ts: sha256 of first 10MB + 'size:<N>', 16 hex chars."""+ size = os.path.getsize(video_path)+ h = hashlib.sha256()+ remaining = 10 * 1024 * 1024+ with open(video_path, "rb") as f:+ while remaining > 0:+ chunk = f.read(min(1 << 20, remaining))+ if not chunk:+ break+ h.update(chunk)+ remaining -= len(chunk)+ h.update(f"size:{size}".encode())+ return h.hexdigest()[:16]+++def _fmt_duration(seconds: float) -> str:+ s = int(round(seconds))+ h, rem = divmod(s, 3600)+ m, s = divmod(rem, 60)+ if h:+ return f"{h}h {m}m {s}s"+ return f"{m}m {s}s"+++def _fmt_ts(seconds: float) -> str:+ return f"{seconds:07.2f}"+++def _normalize_speakers(speakers_block) -> dict[str, dict]:+ """Return {raw_id: {total_time, label, segments}} regardless of input shape.++ Accepts two shapes:+ - transcribe output: {"num_speakers": N, "speakers": {SPEAKER_00: {...}}}+ - parse_transcript output: [{"id": SPEAKER_0, "label": ..., "total_time": ...}]+ """+ if isinstance(speakers_block, dict):+ inner = speakers_block.get("speakers", {})+ if isinstance(inner, dict):+ return inner+ speakers_block = inner # list fall-through+ if isinstance(speakers_block, list):+ return {+ entry.get("id", f"S{i}"): {+ "total_time": entry.get("total_time", 0),+ "label": entry.get("label", entry.get("id", f"S{i}")),+ "segments": entry.get("segments", 0),+ }+ for i, entry in enumerate(speakers_block)+ if isinstance(entry, dict)+ }+ return {}+++def _speaker_short_map(speakers_block) -> dict[str, str]:+ """Map speaker IDs AND labels to short codes (S0, S1, ...) sorted by talk time.++ Accepts either key — transcribe output uses SPEAKER_00 everywhere, but+ parse_transcript uses labels ("Host") in word.speaker/segment.speaker+ while keeping SPEAKER_0 as the canonical id. Mapping both avoids S? holes.+ """+ normalized = _normalize_speakers(speakers_block)+ ranked = sorted(+ normalized.items(),+ key=lambda kv: kv[1].get("total_time", 0),+ reverse=True,+ )+ short: dict[str, str] = {}+ for i, (raw_id, info) in enumerate(ranked):+ code = f"S{i}"+ short[raw_id] = code+ label = info.get("label")+ if label and label not in short:+ short[label] = code+ return short+++def _speaker_at(speaker_segments: list[dict], t: float, last_known: Optional[str]) -> Optional[str]:+ """Return the raw speaker ID whose segment contains time t, else last_known."""+ for seg in speaker_segments:+ if seg["start"] <= t <= seg["end"]:+ return seg["speaker"]+ return last_known+++def _build_phrases(+ words: list[dict],+ short: dict[str, str],+ speaker_segments: list[dict],+) -> list[dict]:+ """Group words into phrases by speaker + silence gaps.++ Uses speaker_segments as authoritative for speaker attribution — per-word+ speaker fields are often noisy at segment boundaries, causing single-word+ fragments that make the transcript unreadable.+ """+ phrases: list[dict] = []+ current: Optional[dict] = None+ last_known: Optional[str] = None++ for w in words:+ text = w.get("word", "")+ if not text:+ continue+ start = float(w.get("start", 0.0))+ end = float(w.get("end", start))+ mid = (start + end) / 2+ raw_spk = _speaker_at(speaker_segments, mid, last_known) or w.get("speaker") or last_known+ if raw_spk is None:+ raw_spk = "UNKNOWN"+ last_known = raw_spk+ spk = short.get(raw_spk, "S?")++ should_split = (+ current is None+ or current["speaker"] != spk+ or (start - current["end"]) >= SILENCE_SPLIT_SEC+ or (end - current["start"]) >= PHRASE_MAX_SEC+ or (len(current["text"]) + len(text) + 1) >= PHRASE_MAX_CHARS+ )++ if should_split:+ if current is not None:+ phrases.append(current)+ current = {"speaker": spk, "start": start, "end": end, "text": text.lstrip()}+ else:+ glue = "" if text.startswith(("'", ",", ".", "?", "!", ";", ":")) else " "+ current["text"] += f"{glue}{text}"+ current["end"] = end++ if current is not None:+ phrases.append(current)+ return phrases+++def _find_silence_gaps(words: list[dict], min_gap: float) -> list[dict]:+ gaps = []+ for prev, nxt in zip(words, words[1:]):+ gap = float(nxt.get("start", 0.0)) - float(prev.get("end", 0.0))+ if gap >= min_gap:+ gaps.append({+ "start": float(prev.get("end", 0.0)),+ "end": float(nxt.get("start", 0.0)),+ "duration": gap,+ })+ return gaps+++def _phrase_for_time(phrases: list[dict], t: float) -> Optional[dict]:+ for p in phrases:+ if p["start"] <= t <= p["end"]:+ return p+ # nearest+ if not phrases:+ return None+ return min(phrases, key=lambda p: min(abs(p["start"] - t), abs(p["end"] - t)))+++def pack_transcript(+ transcript: dict,+ source_label: str,+ energy_data: Optional[list[dict]] = None,+) -> str:+ """Render a packed markdown view of a transcription JSON."""+ duration = float(transcript.get("duration", 0.0))+ language = transcript.get("language", "?")+ words = transcript.get("words", [])+ speakers_block = transcript.get("speakers", {}) or {}+ speakers_map = _normalize_speakers(speakers_block)+ num_speakers = (+ speakers_block.get("num_speakers")+ if isinstance(speakers_block, dict)+ else None+ ) or len(speakers_map)++ short = _speaker_short_map(speakers_block)+ speaker_segments = transcript.get("speaker_segments", []) or []+ phrases = _build_phrases(words, short, speaker_segments)+ gaps = _find_silence_gaps(words, SILENCE_GAP_REPORT_SEC)++ lines: list[str] = []+ lines.append(f"# Episode: {source_label}")+ lines.append(+ f"duration: {_fmt_duration(duration)} · language: {language} · "+ f"speakers: {num_speakers} · words: {len(words)} · phrases: {len(phrases)}"+ )+ lines.append("")++ # Speakers summary+ lines.append("## Speakers")+ ranked = sorted(+ speakers_map.items(),+ key=lambda kv: kv[1].get("total_time", 0),+ reverse=True,+ )+ for raw_id, info in ranked:+ talk = float(info.get("total_time", 0))+ pct = (talk / duration * 100) if duration > 0 else 0+ label = info.get("label") or raw_id+ lines.append(+ f"- {short[raw_id]} ({label}): {_fmt_duration(talk)} · {pct:.0f}%"+ )+ lines.append("")++ # Transcript phrases+ lines.append("## Transcript")+ for p in phrases:+ lines.append(+ f"[{_fmt_ts(p['start'])}-{_fmt_ts(p['end'])}] {p['speaker']} {p['text']}"+ )+ lines.append("")++ # Silence gaps+ if gaps:+ lines.append(f"## Silence gaps (>={SILENCE_GAP_REPORT_SEC}s)")+ for g in gaps:+ lines.append(+ f"[{_fmt_ts(g['start'])}-{_fmt_ts(g['end'])}] {g['duration']:.2f}s"+ )+ lines.append("")++ # Energy peaks (optional)+ if energy_data:+ sorted_by_rms = sorted(energy_data, key=lambda e: e.get("rms_db", -999), reverse=True)+ peaks = sorted_by_rms[:ENERGY_PEAKS_TO_REPORT]+ peaks.sort(key=lambda e: e.get("time", 0))+ lines.append(f"## Energy peaks (top {len(peaks)})")+ for e in peaks:+ t = float(e.get("time", 0))+ rms = float(e.get("rms_db", 0))+ near = _phrase_for_time(phrases, t)+ snippet = ""+ if near:+ txt = near["text"]+ snippet = f' near: "{txt[:70]}{"…" if len(txt) > 70 else ""}"'+ lines.append(f"[{_fmt_ts(t)}] {rms:.1f}dB{snippet}")+ lines.append("")++ return "\n".join(lines)+++def load_cache(cache_hash: str) -> dict:+ path = os.path.join(CACHE_DIR, f"{cache_hash}.json")+ with open(path, "r") as f:+ return json.load(f)+++def packed_path_for(cache_hash: str) -> str:+ return os.path.join(PACKED_DIR, f"{cache_hash}.md")+++def write_packed(+ transcript: dict,+ cache_hash: str,+ source_label: Optional[str] = None,+ energy_data: Optional[list[dict]] = None,+) -> tuple[str, str]:+ """Pack a transcript dict and write to .podcli/packed/<hash>.md."""+ label = source_label or cache_hash+ md = pack_transcript(transcript, label, energy_data=energy_data)+ os.makedirs(PACKED_DIR, exist_ok=True)+ out_path = packed_path_for(cache_hash)+ with open(out_path, "w") as f:+ f.write(md)+ return out_path, md+++def pack_from_hash(cache_hash: str, source_label: Optional[str] = None) -> tuple[str, str]:+ """Pack a cached transcript by its hash. Returns (output_path, markdown)."""+ transcript = load_cache(cache_hash)+ return write_packed(transcript, cache_hash, source_label)+++def pack_from_video(video_path: str) -> tuple[str, str]:+ """Pack a transcript given the source video path."""+ cache_hash = compute_cache_hash(video_path)+ label = os.path.basename(video_path)+ return pack_from_hash(cache_hash, label)+++def _main(argv: list[str]) -> int:+ if len(argv) < 2:+ print("usage: python -m services.transcript_packer <video-path-or-cache-hash>", file=sys.stderr)+ return 2+ target = argv[1]+ if os.path.exists(target):+ out_path, md = pack_from_video(target)+ else:+ out_path, md = pack_from_hash(target)+ size_kb = len(md.encode("utf-8")) / 1024+ print(f"wrote {out_path} ({size_kb:.1f} KB)")+ return 0+++if __name__ == "__main__":+ sys.exit(_main(sys.argv))diff --git a/src/config/paths.ts b/src/config/paths.tsindex 30c884f..d41cfbb 100644--- a/src/config/paths.ts+++ b/src/config/paths.ts@@ -26,6 +26,7 @@ export const paths = {projectRoot,cache: join(dataDir, "cache"),transcripts: join(dataDir, "cache", "transcripts"),+ packed: join(home, "packed"),working: join(dataDir, "working"),output: join(dataDir, "output"),logs: join(dataDir, "logs"),diff --git a/src/handlers/transcribe.handler.ts b/src/handlers/transcribe.handler.tsindex 9e8809c..b5a6c29 100644--- a/src/handlers/transcribe.handler.ts+++ b/src/handlers/transcribe.handler.ts@@ -1,3 +1,4 @@+import { basename } from "path";import { PythonExecutor } from "../services/python-executor.js";import { TranscriptCache } from "../services/transcript-cache.js";import type { TranscriptResult } from "../models/index.js";@@ -18,7 +19,10 @@ export const transcribeToolDef = {description:"STEP 1 — Transcribe a podcast video/audio file. This is typically the first tool you call.\n\n" +"What it does: Uses Whisper AI for word-level timestamps + pyannote for speaker detection (who said what).\n" +- "Returns: Full transcript with word timing, speaker labels, and segments.\n" ++ "Returns: Lightweight metadata only — duration, language, word/segment counts, speaker summary, and " ++ "packed_ready flag. The actual transcript body is NOT returned here (it would be 500KB+ for a typical " ++ "episode). Read the content via get_ui_state(include_transcript: true) which returns a compact " ++ "phrase-grouped markdown view (~10x smaller than raw segments).\n" +"Caching: Results are cached by file hash — same file won't be re-transcribed.\n" +"Supported formats: MP4, MOV, WebM, MKV, MP3, WAV.\n\n" +"After transcription: call get_ui_state(include_transcript: true) to read the transcript, " +@@ -68,10 +72,26 @@ export async function handleTranscribe(input: TranscribeInput): Promise<string>// Check cache firstconst cached = await cache.get(filePath);if (cached) {- return JSON.stringify({ cached: true, ...formatResult(cached) });+ // Backfill packed view if this cache predates auto-packing.+ let packed = await cache.getPackedMarkdown(filePath);+ if (!packed) {+ try {+ const cacheHash = await cache.getFileHash(filePath);+ await executor.execute("pack_transcript", {+ transcript: cached,+ cache_hash: cacheHash,+ source_label: basename(filePath),+ file_path: filePath,+ });+ packed = await cache.getPackedMarkdown(filePath);+ } catch {+ // Non-fatal — caller still gets metadata+ }+ }+ return JSON.stringify({ cached: true, packed_ready: !!packed, ...formatResult(cached) });}- // Execute transcription + diarization+ // Execute transcription + diarization (Python side auto-writes the packed view)const result = await executor.execute<TranscriptResult>("transcribe", {file_path: filePath,model_size: modelSize,@@ -85,25 +105,26 @@ export async function handleTranscribe(input: TranscribeInput): Promise<string>}const data = result.data;- // Cache the result+ // Cache the raw resultawait cache.set(filePath, data);+ const packed = await cache.getPackedMarkdown(filePath);- return JSON.stringify({ cached: false, ...formatResult(data) });+ return JSON.stringify({ cached: false, packed_ready: !!packed, ...formatResult(data) });}+/**+ * Returns lightweight metadata — NOT the transcript body. The full transcript+ * is available via get_ui_state(include_transcript: true), which serves the+ * packed markdown view (~10x smaller than raw segments/words). Raw segments+ * and words arrays stay in the on-disk cache for internal Python consumers.+ */function formatResult(data: TranscriptResult) {- const words = data.words ?? [];- const segments = data.segments ?? [];-return {- transcript: data.transcript,- segments,- words,duration: data.duration,language: data.language,- word_count: words.length,- segment_count: segments.length,+ word_count: (data.words ?? []).length,+ segment_count: (data.segments ?? []).length,speakers: data.speakers ?? { num_speakers: 0, speakers: {} },- speaker_segments: data.speaker_segments ?? [],+ next_step: "Read the transcript via get_ui_state(include_transcript: true), then suggest_clips.",};}diff --git a/src/models/index.ts b/src/models/index.tsindex 343d255..8686214 100644--- a/src/models/index.ts+++ b/src/models/index.ts@@ -2,7 +2,7 @@export interface TaskRequest {task_id: string;- task_type: "transcribe" | "parse_transcript" | "create_clip" | "batch_clips" | "analyze_energy" | "detect_encoder" | "presets" | "ping" | "suggest_clips" | "generate_content" | "corrections";+ task_type: "transcribe" | "parse_transcript" | "create_clip" | "batch_clips" | "analyze_energy" | "pack_transcript" | "detect_encoder" | "presets" | "ping" | "suggest_clips" | "generate_content" | "corrections";params: Record<string, unknown>;}diff --git a/src/server.ts b/src/server.tsindex 802f663..373d794 100644--- a/src/server.ts+++ b/src/server.ts@@ -9,6 +9,7 @@ import { FileManager } from "./services/file-manager.js";import { KnowledgeBase } from "./services/knowledge-base.js";import { AssetManager } from "./services/asset-manager.js";import { ClipsHistory } from "./services/clips-history.js";+import { TranscriptCache } from "./services/transcript-cache.js";import { paths } from "./config/paths.js";import { childLogger } from "./utils/logger.js";import type { BatchClipsResult, UIState } from "./models/index.js";@@ -872,7 +873,9 @@ export function createServer(): McpServer {"IMPORTANT: Call this FIRST when starting a new conversation to understand the current state.\n" +"Clips are numbered #1, #2, etc. Use these numbers with create_clip(clip_number), " +"batch_create_clips(clip_numbers), modify_clip, and toggle_clip.\n\n" +- "Set include_transcript=true when you need to analyze transcript content (e.g. for suggesting clips).",+ "Set include_transcript=true to analyze transcript content. Returns a compact phrase-grouped " ++ "markdown view (~10x smaller than raw segments) with speaker attribution, silence gaps, and " ++ "optional energy peaks — the primary surface for reasoning about clip boundaries.",{include_transcript: z.boolean().optional().default(false).describe("Include full transcript segments in the response. Set true when analyzing content for clip suggestions."@@ -914,7 +917,21 @@ export function createServer(): McpServer {}if (include_transcript) {- if (state.transcript) {+ // Prefer packed markdown (10x smaller, phrase-grouped, LLM-readable).+ // Auto-generated during transcription — see backend/services/transcript_packer.py.+ const transcriptCache = new TranscriptCache();+ let packed: string | null = null;+ if (state.videoPath) {+ packed = await transcriptCache.getPackedMarkdown(state.videoPath);+ }+ if (!packed && state.rawTranscriptText) {+ packed = await transcriptCache.getPackedMarkdownFromText(state.rawTranscriptText);+ }+ if (packed) {+ lines.push("");+ lines.push("=== PACKED TRANSCRIPT ===");+ lines.push(packed);+ } else if (state.transcript) {const segments = state.transcript.segments || [];if (segments.length) {lines.push("");diff --git a/src/services/transcript-cache.ts b/src/services/transcript-cache.tsindex 41deb35..e8deb0c 100644--- a/src/services/transcript-cache.ts+++ b/src/services/transcript-cache.ts@@ -84,4 +84,41 @@ export class TranscriptCache {const cachePath = join(this.cacheDir, `${hash}.json`);await writeFile(cachePath, JSON.stringify(transcript), "utf-8");}++ /**+ * Return the packed markdown view (LLM-readable, ~10x smaller than raw JSON)+ * written by backend/services/transcript_packer.py as a side-effect of+ * transcription. Returns null if not yet generated.+ */+ async getPackedMarkdown(filePath: string): Promise<string | null> {+ try {+ const hash = await this.getFileHash(filePath);+ return await this.readPackedByHash(hash);+ } catch {+ return null;+ }+ }++ /**+ * Look up the packed view for a pasted transcript (no source file).+ * Mirrors backend/services handle_parse_transcript's content-hash keying:+ * sha256 of UTF-8 raw text, first 16 hex chars.+ */+ async getPackedMarkdownFromText(rawText: string): Promise<string | null> {+ try {+ const hash = createHash("sha256")+ .update(rawText, "utf-8")+ .digest("hex")+ .slice(0, 16);+ return await this.readPackedByHash(hash);+ } catch {+ return null;+ }+ }++ private async readPackedByHash(hash: string): Promise<string | null> {+ const packedPath = join(paths.packed, `${hash}.md`);+ if (!existsSync(packedPath)) return null;+ return await readFile(packedPath, "utf-8");+ }}