Open-source AI podcast clipper. Generate vertical clips with face tracking and burned-in captions. CLI, MCP server, web app.
Python ai-toolsauto-captionscaptionscliffmpegmcpmcp-serveropusclip-alternativepodcast-clipspodcastsshortform-videovertical-videowhisper
Decode stdin as UTF-8 and sanitize export filenames (#67)
Nika Siradze8d8877b6c503
diff --git a/backend/cli.py b/backend/cli.pyindex 79dea8f..8058d1a 100644--- a/backend/cli.py+++ b/backend/cli.py@@ -20,8 +20,8 @@ import textwrapimport timefrom version import VERSION-# Windows stdout/stderr default to cp1252, which can't encode chars like '→'; output is UTF-8.-for _stream in (sys.stdout, sys.stderr):+# Windows streams default to cp1252, which can't encode chars like '→'; podcli is UTF-8.+for _stream in (sys.stdin, sys.stdout, sys.stderr):if hasattr(_stream, "reconfigure"):try:_stream.reconfigure(encoding="utf-8", errors="replace")diff --git a/backend/main.py b/backend/main.pyindex 22744b2..af90301 100644--- a/backend/main.py+++ b/backend/main.py@@ -10,8 +10,11 @@ import jsonimport osimport sys-# Windows stdout/stderr default to cp1252, which can't encode chars like '→'; IPC is UTF-8.-for _stream in (sys.stdout, sys.stderr):+# Windows streams default to cp1252; IPC is UTF-8. stdin matters as much as the+# others: the task JSON arrives from Node, and a curly quote (U+201D) carries byte+# 0x9D, which cp1252 leaves undefined, so decoding it raised UnicodeDecodeError+# before the request was ever parsed.+for _stream in (sys.stdin, sys.stdout, sys.stderr):if hasattr(_stream, "reconfigure"):try:_stream.reconfigure(encoding="utf-8", errors="replace")diff --git a/backend/services/clip_generator.py b/backend/services/clip_generator.pyindex 64b1138..7512295 100644--- a/backend/services/clip_generator.py+++ b/backend/services/clip_generator.py@@ -14,6 +14,7 @@ import refrom typing import Optional, Callablefrom utils.proc import run as proc_run, ProcError+from utils.text import safe_filenamefrom config.paths import pathssys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))@@ -902,10 +903,7 @@ def generate_clip(if progress_callback:progress_callback(95, "Saving final clip...")- # Clean filename- safe_title = "".join(c if c.isalnum() or c in "-_ " else "" for c in title)- safe_title = safe_title.strip().replace(" ", "_")[:50]- output_filename = f"{safe_title}_short.mp4"+ output_filename = f"{safe_filename(title)}_short.mp4"if output_dir:os.makedirs(output_dir, exist_ok=True)diff --git a/backend/services/integrations/davinci_resolve/cli.py b/backend/services/integrations/davinci_resolve/cli.pyindex a0d3c24..b5a54bc 100644--- a/backend/services/integrations/davinci_resolve/cli.py+++ b/backend/services/integrations/davinci_resolve/cli.py@@ -18,6 +18,7 @@ if str(REPO_ROOT / "backend") not in sys.path:sys.path.insert(0, str(REPO_ROOT / "backend"))from services.integrations.davinci_resolve import emitter+from utils.text import safe_filenamefrom services.integrations._shared.media_probe import probe_mediafrom services.integrations._shared.timeline_ir import (CaptionLayer,@@ -68,7 +69,7 @@ def main() -> int:return 1out = args.out or (- REPO_ROOT / "data" / "export" / "davinci_resolve" / f"{args.title.replace(' ', '_')}.fcpxml"+ REPO_ROOT / "data" / "export" / "davinci_resolve" / f"{safe_filename(args.title)}.fcpxml")short = _build_short(args.title, args.source, args.captions)diff --git a/backend/utils/text.py b/backend/utils/text.pyindex 07d2d35..1f734ba 100644--- a/backend/utils/text.py+++ b/backend/utils/text.py@@ -1,9 +1,18 @@-"""Title helpers shared by the suggestion pipeline and the CLI."""+"""Title helpers shared by the suggestion pipeline, the CLI, and exporters."""TITLE_MAX = 55+FILENAME_MAX = 50_TRAILING_PUNCT = " ,;:-–—"+# Windows refuses these as a filename stem whatever the extension, so "CON.fcpxml"+# fails just as "CON" does.+_WINDOWS_RESERVED = frozenset(+ ["CON", "PRN", "AUX", "NUL"]+ + [f"COM{i}" for i in range(1, 10)]+ + [f"LPT{i}" for i in range(1, 10)]+)+def clean_title(text: str) -> str:"""Normalize whitespace, keeping the title intact.@@ -28,3 +37,22 @@ def truncate_title(text: str, limit: int = TITLE_MAX) -> str:if " " in head:head = head.rsplit(" ", 1)[0]return head.rstrip(_TRAILING_PUNCT) + "…"+++def safe_filename(text: str, limit: int = FILENAME_MAX, fallback: str = "clip") -> str:+ """Reduce a title to a filename stem: alphanumerics, hyphen, underscore.++ Interpolating a title straight into a path breaks on separators, on characters+ Windows rejects (``:*?"<>|``), and on titles long enough to blow the path limit.+ Punctuation is dropped rather than substituted, so "Q&A" stays "QA" instead of+ growing separators. Whitespace is re-collapsed afterwards, since dropping a+ padded em dash otherwise leaves a gap that becomes "__". A title made entirely+ of punctuation reduces to nothing, hence the fallback.+ """+ kept = "".join(c for c in clean_title(text) if c.isalnum() or c in "-_ ")+ stem = "_".join(kept.split())[:limit].strip("_")+ if not stem:+ return fallback+ if stem.upper() in _WINDOWS_RESERVED:+ return f"{stem}_{fallback}"+ return stemdiff --git a/cli/internal/engine/engine.go b/cli/internal/engine/engine.goindex 82cae60..722be67 100644--- a/cli/internal/engine/engine.go+++ b/cli/internal/engine/engine.go@@ -159,6 +159,9 @@ func nodeEnv() []string {if root, ok := BackendRoot(); ok {env = append(env, "PODCLI_BACKEND="+root)}+ // Node spawns the Python backend and passes its own env through, so these must be+ // set here too, not only in Run(). Titles cross that pipe as UTF-8 JSON.+ env = append(env, "PYTHONIOENCODING=utf-8", "PYTHONUTF8=1")env = append(env, "PYTHON_PATH="+Python())if ff := FFmpeg(); ff != "" {env = append(env, "FFMPEG_PATH="+ff)diff --git a/src/config/paths.ts b/src/config/paths.tsindex fc2eff0..dc2cf1e 100644--- a/src/config/paths.ts+++ b/src/config/paths.ts@@ -89,3 +89,17 @@ export const paths = {ffmpegPath: process.env.FFMPEG_PATH || "ffmpeg",ffprobePath: process.env.FFPROBE_PATH || "ffprobe",};++// Python falls back to the host locale encoding, which is cp1252 on most Windows+// installs. Titles carry em dashes and smart quotes, and U+201D encodes byte 0x9D,+// which cp1252 leaves undefined: reading it raises UnicodeDecodeError. Every spawn+// of the Python backend goes through here so no call site can forget.+export function pythonEnv(extra: Record<string, string> = {}): NodeJS.ProcessEnv {+ return {+ ...process.env,+ PYTHONUNBUFFERED: "1",+ PYTHONIOENCODING: "utf-8",+ PYTHONUTF8: "1",+ ...extra,+ };+}diff --git a/src/services/asset-manager.ts b/src/services/asset-manager.tsindex 4905187..c0a1a42 100644--- a/src/services/asset-manager.ts+++ b/src/services/asset-manager.ts@@ -4,7 +4,7 @@ import { join, extname } from "path";import { spawn } from "child_process";import { Readable } from "stream";import { pipeline } from "stream/promises";-import { paths } from "../config/paths.js";+import { paths, pythonEnv } from "../config/paths.js";import { ASSETS_SCHEMA_VERSION } from "../models/index.js";import type { Asset, AssetType, AssetRegistry } from "../models/index.js";@@ -282,7 +282,7 @@ function downloadWithYtDlp(url: string, destDir: string, name: string): Promise<url,];return new Promise((resolveP, rejectP) => {- const proc = spawn(paths.pythonPath, args, { env: { ...process.env, PYTHONUNBUFFERED: "1" } });+ const proc = spawn(paths.pythonPath, args, { env: pythonEnv() });let stdout = "";let stderr = "";let filePath = "";diff --git a/src/services/python-executor.ts b/src/services/python-executor.tsindex c4f984d..de7e527 100644--- a/src/services/python-executor.ts+++ b/src/services/python-executor.ts@@ -1,6 +1,6 @@import { spawn, type ChildProcess } from "child_process";import { v4 as uuidv4 } from "uuid";-import { paths } from "../config/paths.js";+import { paths, pythonEnv } from "../config/paths.js";import type { TaskRequest, TaskResult, ProgressEvent } from "../models/index.js";type ProgressCallback = (event: ProgressEvent) => void;@@ -67,12 +67,10 @@ export class PythonExecutor {const proc = spawn(paths.pythonPath, [paths.pythonBackend], {stdio: ["pipe", "pipe", "pipe"],detached: !isWindows,- env: {- ...process.env,- PYTHONUNBUFFERED: "1",+ env: pythonEnv({PODCLI_HOME: paths.home,PODCLI_DATA: paths.dataDir,- },+ }),});let stdout = "";diff --git a/src/ui/web-server.ts b/src/ui/web-server.tsindex ac85987..710752e 100644--- a/src/ui/web-server.ts+++ b/src/ui/web-server.ts@@ -36,7 +36,7 @@ import { FileManager } from "../services/file-manager.js";import { AssetManager, inferType, safeName } from "../services/asset-manager.js";import { ClipsHistory } from "../services/clips-history.js";import { KnowledgeBase } from "../services/knowledge-base.js";-import { paths } from "../config/paths.js";+import { paths, pythonEnv } from "../config/paths.js";import { DEMO_ASSETS_DIR } from "./demo-fixtures.js";import { registerConfigIntegrationRoutes } from "../handlers/integrations.routes.js";import { childLogger } from "../utils/logger.js";@@ -597,9 +597,7 @@ app.post("/api/download-video", async (req, res) => {"after_move:podcli-filepath:%(filepath)s",url,];- const proc = spawn(paths.pythonPath, args, {- env: { ...process.env, PYTHONUNBUFFERED: "1" },- });+ const proc = spawn(paths.pythonPath, args, { env: pythonEnv() });let stdout = "";let stderr = "";@@ -1900,7 +1898,7 @@ const stripAnsi = (s: string) => s.replace(/\x1b\[[0-9;]*m/g, "").trim();function runPy(scriptAndArgs: string[]): Promise<{ code: number; stdout: string; stderr: string }> {return new Promise((resolve) => {const proc = spawn(paths.pythonPath, scriptAndArgs, {- env: { ...process.env, PYTHONUNBUFFERED: "1", PODCLI_HOME: paths.home, PODCLI_DATA: paths.dataDir },+ env: pythonEnv({ PODCLI_HOME: paths.home, PODCLI_DATA: paths.dataDir }),});let stdout = "", stderr = "";proc.stdout.on("data", (d) => (stdout += d));diff --git a/tests/test_backend_stdin_encoding.py b/tests/test_backend_stdin_encoding.pynew file mode 100644index 0000000..0f3fa2e--- /dev/null+++ b/tests/test_backend_stdin_encoding.py@@ -0,0 +1,49 @@+"""The task JSON crosses a Node->Python pipe as UTF-8 and must survive a cp1252 host.++On Windows sys.stdin defaults to the locale encoding. A curly quote (U+201D) encodes+to bytes E2 80 9D, and cp1252 leaves 0x9D undefined, so reading the request raised+UnicodeDecodeError before it was ever parsed. Titles are stored unabbreviated, so+em dashes and smart quotes reach this pipe routinely.+"""++import json+import os+import subprocess+import sys+from pathlib import Path++BACKEND = Path(__file__).resolve().parent.parent / "backend"+MAIN = BACKEND / "main.py"++TITLE = "In the end — your “shit” has to work"+++def _run_with_locale_encoding(payload: str) -> subprocess.CompletedProcess:+ env = {+ **os.environ,+ "PYTHONIOENCODING": "cp1252",+ "PYTHONUTF8": "0", # a UTF-8-mode host would mask the bug this guards+ }+ return subprocess.run(+ [sys.executable, str(MAIN)],+ input=payload,+ capture_output=True,+ text=True,+ encoding="utf-8",+ env=env,+ cwd=str(BACKEND),+ timeout=120,+ )+++def test_non_ascii_title_survives_cp1252_stdin():+ payload = json.dumps(+ {"task_id": TITLE, "task_type": "nonexistent_task", "params": {}},+ ensure_ascii=False,+ )+ proc = _run_with_locale_encoding(payload + "\n")++ assert "UnicodeDecodeError" not in proc.stderr+ result = json.loads(proc.stdout.strip().splitlines()[-1])+ assert result["status"] == "error"+ assert result["task_id"] == TITLEdiff --git a/tests/test_text_utils.py b/tests/test_text_utils.pyindex 3a0075b..64a2187 100644--- a/tests/test_text_utils.py+++ b/tests/test_text_utils.py@@ -3,7 +3,7 @@ import syssys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "backend"))-from utils.text import clean_title, truncate_title # noqa: E402+from utils.text import clean_title, safe_filename, truncate_title # noqa: E402def test_clean_title_keeps_full_text():@@ -42,3 +42,48 @@ def test_truncate_title_strips_trailing_punctuation():def test_truncate_title_hard_cuts_a_single_long_word():out = truncate_title("x" * 80)assert out == "x" * 55 + "…"+++def test_safe_filename_replaces_spaces():+ assert safe_filename("My great clip") == "My_great_clip"+++def test_safe_filename_strips_path_separators():+ assert safe_filename("../../etc/passwd") == "etcpasswd"+ assert safe_filename("a/b\\c") == "abc"+++def test_safe_filename_strips_characters_windows_rejects():+ assert safe_filename('what: is "this"? <yes> | no*') == "what_is_this_yes_no"+++def test_safe_filename_drops_smart_punctuation():+ assert safe_filename("In the end — your “shit” has to work") == "In_the_end_your_shit_has_to_work"+++def test_safe_filename_caps_length():+ assert len(safe_filename("word " * 50)) <= 50+++def test_safe_filename_has_no_leading_or_trailing_underscore():+ out = safe_filename(" — hello — ")+ assert out == "hello"+++def test_safe_filename_falls_back_when_nothing_survives():+ assert safe_filename("———") == "clip"+ assert safe_filename("") == "clip"+ assert safe_filename(None) == "clip"+++# Windows rejects these stems whatever the extension, so "CON.fcpxml" fails too.+def test_safe_filename_escapes_windows_reserved_device_names():+ assert safe_filename("CON") == "CON_clip"+ assert safe_filename("nul") == "nul_clip"+ assert safe_filename("com1") == "com1_clip"+ assert safe_filename("LPT9") == "LPT9_clip"+++def test_safe_filename_leaves_names_merely_containing_reserved_words():+ assert safe_filename("Conference") == "Conference"+ assert safe_filename("CON artists") == "CON_artists"