Skip to content

Open-source AI podcast clipper. Generate vertical clips with face tracking and burned-in captions. CLI, MCP server, web app.

Decode stdin as UTF-8 and sanitize export filenames (#67)

Nika Siradze8d8877b6c503
diff --git a/backend/cli.py b/backend/cli.py
index 79dea8f..8058d1a 100644
--- a/backend/cli.py
+++ b/backend/cli.py
@@ -20,8 +20,8 @@ import textwrap
import time
from version import VERSION
-# Windows stdout/stderr default to cp1252, which can't encode chars like '→'; output is UTF-8.
-for _stream in (sys.stdout, sys.stderr):
+# Windows streams default to cp1252, which can't encode chars like '→'; podcli is UTF-8.
+for _stream in (sys.stdin, sys.stdout, sys.stderr):
if hasattr(_stream, "reconfigure"):
try:
_stream.reconfigure(encoding="utf-8", errors="replace")
diff --git a/backend/main.py b/backend/main.py
index 22744b2..af90301 100644
--- a/backend/main.py
+++ b/backend/main.py
@@ -10,8 +10,11 @@ import json
import os
import sys
-# Windows stdout/stderr default to cp1252, which can't encode chars like '→'; IPC is UTF-8.
-for _stream in (sys.stdout, sys.stderr):
+# Windows streams default to cp1252; IPC is UTF-8. stdin matters as much as the
+# others: the task JSON arrives from Node, and a curly quote (U+201D) carries byte
+# 0x9D, which cp1252 leaves undefined, so decoding it raised UnicodeDecodeError
+# before the request was ever parsed.
+for _stream in (sys.stdin, sys.stdout, sys.stderr):
if hasattr(_stream, "reconfigure"):
try:
_stream.reconfigure(encoding="utf-8", errors="replace")
diff --git a/backend/services/clip_generator.py b/backend/services/clip_generator.py
index 64b1138..7512295 100644
--- a/backend/services/clip_generator.py
+++ b/backend/services/clip_generator.py
@@ -14,6 +14,7 @@ import re
from typing import Optional, Callable
from utils.proc import run as proc_run, ProcError
+from utils.text import safe_filename
from config.paths import paths
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
@@ -902,10 +903,7 @@ def generate_clip(
if progress_callback:
progress_callback(95, "Saving final clip...")
- # Clean filename
- safe_title = "".join(c if c.isalnum() or c in "-_ " else "" for c in title)
- safe_title = safe_title.strip().replace(" ", "_")[:50]
- output_filename = f"{safe_title}_short.mp4"
+ output_filename = f"{safe_filename(title)}_short.mp4"
if output_dir:
os.makedirs(output_dir, exist_ok=True)
diff --git a/backend/services/integrations/davinci_resolve/cli.py b/backend/services/integrations/davinci_resolve/cli.py
index a0d3c24..b5a54bc 100644
--- a/backend/services/integrations/davinci_resolve/cli.py
+++ b/backend/services/integrations/davinci_resolve/cli.py
@@ -18,6 +18,7 @@ if str(REPO_ROOT / "backend") not in sys.path:
sys.path.insert(0, str(REPO_ROOT / "backend"))
from services.integrations.davinci_resolve import emitter
+from utils.text import safe_filename
from services.integrations._shared.media_probe import probe_media
from services.integrations._shared.timeline_ir import (
CaptionLayer,
@@ -68,7 +69,7 @@ def main() -> int:
return 1
out = args.out or (
- REPO_ROOT / "data" / "export" / "davinci_resolve" / f"{args.title.replace(' ', '_')}.fcpxml"
+ REPO_ROOT / "data" / "export" / "davinci_resolve" / f"{safe_filename(args.title)}.fcpxml"
)
short = _build_short(args.title, args.source, args.captions)
diff --git a/backend/utils/text.py b/backend/utils/text.py
index 07d2d35..1f734ba 100644
--- a/backend/utils/text.py
+++ b/backend/utils/text.py
@@ -1,9 +1,18 @@
-"""Title helpers shared by the suggestion pipeline and the CLI."""
+"""Title helpers shared by the suggestion pipeline, the CLI, and exporters."""
TITLE_MAX = 55
+FILENAME_MAX = 50
_TRAILING_PUNCT = " ,;:-–—"
+# Windows refuses these as a filename stem whatever the extension, so "CON.fcpxml"
+# fails just as "CON" does.
+_WINDOWS_RESERVED = frozenset(
+ ["CON", "PRN", "AUX", "NUL"]
+ + [f"COM{i}" for i in range(1, 10)]
+ + [f"LPT{i}" for i in range(1, 10)]
+)
+
def clean_title(text: str) -> str:
"""Normalize whitespace, keeping the title intact.
@@ -28,3 +37,22 @@ def truncate_title(text: str, limit: int = TITLE_MAX) -> str:
if " " in head:
head = head.rsplit(" ", 1)[0]
return head.rstrip(_TRAILING_PUNCT) + "…"
+
+
+def safe_filename(text: str, limit: int = FILENAME_MAX, fallback: str = "clip") -> str:
+ """Reduce a title to a filename stem: alphanumerics, hyphen, underscore.
+
+ Interpolating a title straight into a path breaks on separators, on characters
+ Windows rejects (``:*?"<>|``), and on titles long enough to blow the path limit.
+ Punctuation is dropped rather than substituted, so "Q&A" stays "QA" instead of
+ growing separators. Whitespace is re-collapsed afterwards, since dropping a
+ padded em dash otherwise leaves a gap that becomes "__". A title made entirely
+ of punctuation reduces to nothing, hence the fallback.
+ """
+ kept = "".join(c for c in clean_title(text) if c.isalnum() or c in "-_ ")
+ stem = "_".join(kept.split())[:limit].strip("_")
+ if not stem:
+ return fallback
+ if stem.upper() in _WINDOWS_RESERVED:
+ return f"{stem}_{fallback}"
+ return stem
diff --git a/cli/internal/engine/engine.go b/cli/internal/engine/engine.go
index 82cae60..722be67 100644
--- a/cli/internal/engine/engine.go
+++ b/cli/internal/engine/engine.go
@@ -159,6 +159,9 @@ func nodeEnv() []string {
if root, ok := BackendRoot(); ok {
env = append(env, "PODCLI_BACKEND="+root)
}
+ // Node spawns the Python backend and passes its own env through, so these must be
+ // set here too, not only in Run(). Titles cross that pipe as UTF-8 JSON.
+ env = append(env, "PYTHONIOENCODING=utf-8", "PYTHONUTF8=1")
env = append(env, "PYTHON_PATH="+Python())
if ff := FFmpeg(); ff != "" {
env = append(env, "FFMPEG_PATH="+ff)
diff --git a/src/config/paths.ts b/src/config/paths.ts
index fc2eff0..dc2cf1e 100644
--- a/src/config/paths.ts
+++ b/src/config/paths.ts
@@ -89,3 +89,17 @@ export const paths = {
ffmpegPath: process.env.FFMPEG_PATH || "ffmpeg",
ffprobePath: process.env.FFPROBE_PATH || "ffprobe",
};
+
+// Python falls back to the host locale encoding, which is cp1252 on most Windows
+// installs. Titles carry em dashes and smart quotes, and U+201D encodes byte 0x9D,
+// which cp1252 leaves undefined: reading it raises UnicodeDecodeError. Every spawn
+// of the Python backend goes through here so no call site can forget.
+export function pythonEnv(extra: Record<string, string> = {}): NodeJS.ProcessEnv {
+ return {
+ ...process.env,
+ PYTHONUNBUFFERED: "1",
+ PYTHONIOENCODING: "utf-8",
+ PYTHONUTF8: "1",
+ ...extra,
+ };
+}
diff --git a/src/services/asset-manager.ts b/src/services/asset-manager.ts
index 4905187..c0a1a42 100644
--- a/src/services/asset-manager.ts
+++ b/src/services/asset-manager.ts
@@ -4,7 +4,7 @@ import { join, extname } from "path";
import { spawn } from "child_process";
import { Readable } from "stream";
import { pipeline } from "stream/promises";
-import { paths } from "../config/paths.js";
+import { paths, pythonEnv } from "../config/paths.js";
import { ASSETS_SCHEMA_VERSION } from "../models/index.js";
import type { Asset, AssetType, AssetRegistry } from "../models/index.js";
@@ -282,7 +282,7 @@ function downloadWithYtDlp(url: string, destDir: string, name: string): Promise<
url,
];
return new Promise((resolveP, rejectP) => {
- const proc = spawn(paths.pythonPath, args, { env: { ...process.env, PYTHONUNBUFFERED: "1" } });
+ const proc = spawn(paths.pythonPath, args, { env: pythonEnv() });
let stdout = "";
let stderr = "";
let filePath = "";
diff --git a/src/services/python-executor.ts b/src/services/python-executor.ts
index c4f984d..de7e527 100644
--- a/src/services/python-executor.ts
+++ b/src/services/python-executor.ts
@@ -1,6 +1,6 @@
import { spawn, type ChildProcess } from "child_process";
import { v4 as uuidv4 } from "uuid";
-import { paths } from "../config/paths.js";
+import { paths, pythonEnv } from "../config/paths.js";
import type { TaskRequest, TaskResult, ProgressEvent } from "../models/index.js";
type ProgressCallback = (event: ProgressEvent) => void;
@@ -67,12 +67,10 @@ export class PythonExecutor {
const proc = spawn(paths.pythonPath, [paths.pythonBackend], {
stdio: ["pipe", "pipe", "pipe"],
detached: !isWindows,
- env: {
- ...process.env,
- PYTHONUNBUFFERED: "1",
+ env: pythonEnv({
PODCLI_HOME: paths.home,
PODCLI_DATA: paths.dataDir,
- },
+ }),
});
let stdout = "";
diff --git a/src/ui/web-server.ts b/src/ui/web-server.ts
index ac85987..710752e 100644
--- a/src/ui/web-server.ts
+++ b/src/ui/web-server.ts
@@ -36,7 +36,7 @@ import { FileManager } from "../services/file-manager.js";
import { AssetManager, inferType, safeName } from "../services/asset-manager.js";
import { ClipsHistory } from "../services/clips-history.js";
import { KnowledgeBase } from "../services/knowledge-base.js";
-import { paths } from "../config/paths.js";
+import { paths, pythonEnv } from "../config/paths.js";
import { DEMO_ASSETS_DIR } from "./demo-fixtures.js";
import { registerConfigIntegrationRoutes } from "../handlers/integrations.routes.js";
import { childLogger } from "../utils/logger.js";
@@ -597,9 +597,7 @@ app.post("/api/download-video", async (req, res) => {
"after_move:podcli-filepath:%(filepath)s",
url,
];
- const proc = spawn(paths.pythonPath, args, {
- env: { ...process.env, PYTHONUNBUFFERED: "1" },
- });
+ const proc = spawn(paths.pythonPath, args, { env: pythonEnv() });
let stdout = "";
let stderr = "";
@@ -1900,7 +1898,7 @@ const stripAnsi = (s: string) => s.replace(/\x1b\[[0-9;]*m/g, "").trim();
function runPy(scriptAndArgs: string[]): Promise<{ code: number; stdout: string; stderr: string }> {
return new Promise((resolve) => {
const proc = spawn(paths.pythonPath, scriptAndArgs, {
- env: { ...process.env, PYTHONUNBUFFERED: "1", PODCLI_HOME: paths.home, PODCLI_DATA: paths.dataDir },
+ env: pythonEnv({ PODCLI_HOME: paths.home, PODCLI_DATA: paths.dataDir }),
});
let stdout = "", stderr = "";
proc.stdout.on("data", (d) => (stdout += d));
diff --git a/tests/test_backend_stdin_encoding.py b/tests/test_backend_stdin_encoding.py
new file mode 100644
index 0000000..0f3fa2e
--- /dev/null
+++ b/tests/test_backend_stdin_encoding.py
@@ -0,0 +1,49 @@
+"""The task JSON crosses a Node->Python pipe as UTF-8 and must survive a cp1252 host.
+
+On Windows sys.stdin defaults to the locale encoding. A curly quote (U+201D) encodes
+to bytes E2 80 9D, and cp1252 leaves 0x9D undefined, so reading the request raised
+UnicodeDecodeError before it was ever parsed. Titles are stored unabbreviated, so
+em dashes and smart quotes reach this pipe routinely.
+"""
+
+import json
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+BACKEND = Path(__file__).resolve().parent.parent / "backend"
+MAIN = BACKEND / "main.py"
+
+TITLE = "In the end — your “shit” has to work"
+
+
+def _run_with_locale_encoding(payload: str) -> subprocess.CompletedProcess:
+ env = {
+ **os.environ,
+ "PYTHONIOENCODING": "cp1252",
+ "PYTHONUTF8": "0", # a UTF-8-mode host would mask the bug this guards
+ }
+ return subprocess.run(
+ [sys.executable, str(MAIN)],
+ input=payload,
+ capture_output=True,
+ text=True,
+ encoding="utf-8",
+ env=env,
+ cwd=str(BACKEND),
+ timeout=120,
+ )
+
+
+def test_non_ascii_title_survives_cp1252_stdin():
+ payload = json.dumps(
+ {"task_id": TITLE, "task_type": "nonexistent_task", "params": {}},
+ ensure_ascii=False,
+ )
+ proc = _run_with_locale_encoding(payload + "\n")
+
+ assert "UnicodeDecodeError" not in proc.stderr
+ result = json.loads(proc.stdout.strip().splitlines()[-1])
+ assert result["status"] == "error"
+ assert result["task_id"] == TITLE
diff --git a/tests/test_text_utils.py b/tests/test_text_utils.py
index 3a0075b..64a2187 100644
--- a/tests/test_text_utils.py
+++ b/tests/test_text_utils.py
@@ -3,7 +3,7 @@ import sys
sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "backend"))
-from utils.text import clean_title, truncate_title # noqa: E402
+from utils.text import clean_title, safe_filename, truncate_title # noqa: E402
def test_clean_title_keeps_full_text():
@@ -42,3 +42,48 @@ def test_truncate_title_strips_trailing_punctuation():
def test_truncate_title_hard_cuts_a_single_long_word():
out = truncate_title("x" * 80)
assert out == "x" * 55 + "…"
+
+
+def test_safe_filename_replaces_spaces():
+ assert safe_filename("My great clip") == "My_great_clip"
+
+
+def test_safe_filename_strips_path_separators():
+ assert safe_filename("../../etc/passwd") == "etcpasswd"
+ assert safe_filename("a/b\\c") == "abc"
+
+
+def test_safe_filename_strips_characters_windows_rejects():
+ assert safe_filename('what: is "this"? <yes> | no*') == "what_is_this_yes_no"
+
+
+def test_safe_filename_drops_smart_punctuation():
+ assert safe_filename("In the end — your “shit” has to work") == "In_the_end_your_shit_has_to_work"
+
+
+def test_safe_filename_caps_length():
+ assert len(safe_filename("word " * 50)) <= 50
+
+
+def test_safe_filename_has_no_leading_or_trailing_underscore():
+ out = safe_filename(" — hello — ")
+ assert out == "hello"
+
+
+def test_safe_filename_falls_back_when_nothing_survives():
+ assert safe_filename("———") == "clip"
+ assert safe_filename("") == "clip"
+ assert safe_filename(None) == "clip"
+
+
+# Windows rejects these stems whatever the extension, so "CON.fcpxml" fails too.
+def test_safe_filename_escapes_windows_reserved_device_names():
+ assert safe_filename("CON") == "CON_clip"
+ assert safe_filename("nul") == "nul_clip"
+ assert safe_filename("com1") == "com1_clip"
+ assert safe_filename("LPT9") == "LPT9_clip"
+
+
+def test_safe_filename_leaves_names_merely_containing_reserved_words():
+ assert safe_filename("Conference") == "Conference"
+ assert safe_filename("CON artists") == "CON_artists"