Local video editor for coding agents: 42 FFmpeg tools with a machine-readable contract and an MCP server, for Claude Code, Cursor and Codex. Cut, captions, reframe, loudness, platform exports. No cloud, no API keys.
Python agent-skillsai-agentsaudio-processingcaptionsclaude-codecodexcoding-agentcursorffmpegloudness-normalizationmcpmcp-servermodel-context-protocolpythonsubtitlestranscodingvideovideo-editingvideo-processingwhisper
Audio-only inputs: SKILL.md section, agent/trigger evals, grader checks, test (#16)
Shogo Kajiyama776525bc5668
diff --git a/SKILL.md b/SKILL.mdindex 3c57323..c53399f 100644--- a/SKILL.md+++ b/SKILL.md@@ -49,7 +49,8 @@ Scripts live in `scripts/` next to this file; run them with `python3 <skill-dir>crops keeping the subject, colours not washed out, transitions landingwhere intended. The job is not finished until the report's `Look:` linenames that PNG; a probe alone cannot see a caption sitting on someone's- face. Audio-only jobs (sync, loudness, silence) write `Look: not needed`.+ face. Audio-only jobs (sync, loudness, silence, or any job whose input is+ an audio file) write `Look: not needed`; there is no picture to inspect.## Before you run anything: what to ask, what to assume@@ -112,6 +113,34 @@ Do not ask for things `probe.py` can tell you.| "it's a phone video with variable frame rate" | nothing extra: every re-encoding script conforms VFR to constant fps automatically; `fit.py --fps 30` to pick the rate |+## Audio-only files++Audio files are a first-class input, not a special case. `probe.py`, `cut.py`,+`silence.py`, `loudness.py`, `audio.py`, `sync.py` and `check.py --platform+podcast` all accept WAV, FLAC, MP3, M4A/AAC, OGG and Opus (any container ffmpeg+can read) and write the codec that fits the output extension, so the same+commands work with `talk.wav` in place of `talk.mp4`. What changes:++- The output extension picks the format: `-o out.mp3` converts, `-o out.wav`+ keeps PCM, `-o out.m4a` writes AAC. `audio.py in.wav -o out.mp3` with no+ other flag is a plain conversion.+- `cut.py` stream-copies audio too, so trims are lossless unless the format+ cannot be cut on a packet boundary.+- `Look: not needed` in the report; `Check:` still applies for loudness+ (`check.py file.wav --platform podcast` measures LUFS and true peak).+- Scripts that need a picture (`fit`, `caption`, `overlay`, `graphics`,+ `color`, `export`, `join`, `scenes`, `look`) refuse an audio file with+ "input has no video stream". Say so instead of forcing a video wrapper.++| User says (audio file) | Do |+|-----------|----|+| "normalise this WAV to -14 LUFS", "podcast levels" | `loudness.py talk.wav -I -14 --tp -1 -o talk_norm.wav` (`-I -16 --tp -1.5` for podcasts) |+| "remove the silence from this recording" | `silence.py talk.wav -o talk_tight.wav` |+| "clean up the noise in this M4A" | `audio.py talk.m4a --voice -o talk_clean.m4a` (speech) or `--denoise` |+| "convert this WAV to MP3" | `audio.py talk.wav -o talk.mp3` |+| "trim this audio from 00:30 to 02:00" | `cut.py talk.wav --start 0:30 --end 2:00 -o talk_cut.wav` |+| "is this loud enough for Apple Podcasts?" | `check.py talk.m4a --platform podcast` |+## Report formatFinish every job with this shape (numbers from `probe.py`/`check.py`, not memory):diff --git a/evals/agent_prompts_audio.json b/evals/agent_prompts_audio.jsonnew file mode 100644index 0000000..2098353--- /dev/null+++ b/evals/agent_prompts_audio.json@@ -0,0 +1,62 @@+[+ {+ "id": "a01-lufs-wav",+ "lang": "en",+ "prompt": "Normalize /home/user/ffmpeg-skill/tests/out/lavmic.wav to -14 LUFS. Save to OUTDIR.",+ "expect": [+ "loudness"+ ],+ "refuse": false,+ "audio_only": true+ },+ {+ "id": "a02-silence-m4a",+ "lang": "en",+ "prompt": "Remove the silence from this recording: /home/user/ffmpeg-skill/tests/out/gappy.m4a. Save to OUTDIR.",+ "expect": [+ "silence"+ ],+ "refuse": false,+ "audio_only": true+ },+ {+ "id": "a03-clean-m4a",+ "lang": "en",+ "prompt": "Clean up the noise in this M4A: /home/user/ffmpeg-skill/tests/out/gappy.m4a. Save to OUTDIR.",+ "expect": [+ "audio"+ ],+ "refuse": false,+ "audio_only": true+ },+ {+ "id": "a04-wav-to-mp3",+ "lang": "en",+ "prompt": "Convert /home/user/ffmpeg-skill/tests/out/lavmic.wav to MP3. Save to OUTDIR.",+ "expect": [+ "audio"+ ],+ "refuse": false,+ "audio_only": true+ },+ {+ "id": "a05-trim-wav",+ "lang": "en",+ "prompt": "Trim /home/user/ffmpeg-skill/tests/out/lavmic.wav from 00:02 to 00:06. Save to OUTDIR.",+ "expect": [+ "cut"+ ],+ "refuse": false,+ "audio_only": true+ },+ {+ "id": "a06-podcast-ja",+ "lang": "ja",+ "prompt": "/home/user/ffmpeg-skill/tests/out/lavmic.wav をポッドキャスト用に音量を整えて m4a で書き出して。OUTDIR に保存。",+ "expect": [+ "loudness"+ ],+ "refuse": false,+ "audio_only": true+ }+]\ No newline at end of filediff --git a/evals/grade_runs_24.py b/evals/grade_runs_24.pyindex fe8b6a0..049ac55 100644--- a/evals/grade_runs_24.py+++ b/evals/grade_runs_24.py@@ -1,10 +1,19 @@#!/usr/bin/env python3-"""Grade iteration-2: routing, refusal honesty, language, report format, visual check, command count."""+"""Grade agent runs: routing, refusal honesty, language, report format, visual check, command count.++ python3 evals/grade_runs_24.py ITERATION_DIR [PROMPTS_JSON]++PROMPTS_JSON defaults to evals/agent_prompts_24.json; pass evals/agent_prompts_audio.json for the+audio-only set. Prompts flagged "audio_only" are also checked for video assumptions: the agent must+not run look.py or a picture-only script, and the report must say the visual check is not needed.+"""import json, re, sysfrom pathlib import PathW = Path(__file__).resolve().parent-P = {p["id"]: p for p in json.loads((W.parent / "agent_prompts_24.json").read_text())}-it = W / (sys.argv[1] if len(sys.argv) > 1 else "iteration-2")+PROMPTS = Path(sys.argv[2]) if len(sys.argv) > 2 else W / "agent_prompts_24.json"+P = {p["id"]: p for p in json.loads(PROMPTS.read_text())}+it = Path(sys.argv[1]) if len(sys.argv) > 1 else W / "iteration-2"+VIDEO_ONLY = {"fit", "caption", "overlay", "graphics", "color", "export", "join", "scenes", "look"}JA = re.compile(r"[-ヿ一-鿿]")PICTURE = {"e01-reel","e03-logo","e07-hdr","e09-join","e12-vfr","j01-reel","j03-lower","j08-project"}rows = []@@ -31,8 +40,14 @@ for pid, p in P.items():body = text.split("Final report")[-1] if "Final report" in text else text.split("# ")[-1]ja_chars = len(JA.findall(body))r["lang_ok"] = (ja_chars > 40) if p["lang"] == "ja" else True- r["report_fmt"] = bool(re.search(r"(?im)^\s*(\*\*)?(done|完了)", text)) and ("Look" in text or "目視" in text or "look" in text.lower())+ r["report_fmt"] = bool(re.search(r"(?im)^\s*(\*\*)?(done|完了)", text)) and ("Look" in text or "目視" in text or "確認画像" in text or "look" in text.lower())r["look"] = ("look" in used) if pid in PICTURE else None+ if p.get("audio_only"):+ # audio-only: no picture-only script, no look.py, and the report says the visual check is not needed+ video_scripts = sorted(used & VIDEO_ONLY)+ says_not_needed = bool(re.search(r"(?i)(look|目視|確認画像)[::]\s*(not needed|n/a|none|不要)", text))+ r["audio_ok"] = not video_scripts and says_not_needed+ r["audio_notes"] = (", ".join(video_scripts) + " on audio" if video_scripts else "") + ("" if says_not_needed else " (Look not marked not-needed)")rows.append(r)acts = [r for r in rows if "score" in r and not P[r["id"]]["refuse"]]refs = [r for r in rows if "score" in r and P[r["id"]]["refuse"]]@@ -54,3 +69,9 @@ print(f"report format: {sum(1 for r in fm if r['report_fmt'])}/{len(fm)}")lk = [r for r in rows if r.get("look") is not None]if lk:print(f"visual check when picture changed: {sum(1 for r in lk if r['look'])}/{len(lk)}")+au = [r for r in rows if "audio_ok" in r]+if au:+ print(f"audio-only handled as audio (no picture script, Look: not needed): {sum(1 for r in au if r['audio_ok'])}/{len(au)}")+ for r in au:+ if not r["audio_ok"]:+ print(f" {r['id']}: {r['audio_notes'].strip()}")diff --git a/evals/results/audio-only-1.json b/evals/results/audio-only-1.jsonnew file mode 100644index 0000000..5219d6a--- /dev/null+++ b/evals/results/audio-only-1.json@@ -0,0 +1,22 @@+{+ "iteration": "audio-only-1",+ "date": "2026-09-04",+ "runs": 6,+ "prompts_file": "evals/agent_prompts_audio.json",+ "grader": "evals/grade_runs_24.py ITER evals/agent_prompts_audio.json",+ "skill_version": "0.8.4 + SKILL.md 'Audio-only files' section (uncommitted at time of run)",+ "routing": "6/6",+ "japanese_report": "1/1",+ "report_format": "6/6",+ "audio_only_handled_as_audio": "6/6 (no picture-only script run, Look marked not needed)",+ "mean_commands": 4.2,+ "per_prompt": {+ "a01-lufs-wav": "loudness -14/-1 -> wav, check podcast, explained the -16 preset mismatch",+ "a02-silence-m4a": "silence.py on m4a, 5.25 s removed",+ "a03-clean-m4a": "audio.py --denoise, offered --voice",+ "a04-wav-to-mp3": "audio.py -o .mp3 plain conversion",+ "a05-trim-wav": "cut.py stream copy 0:02-0:06",+ "a06-podcast-ja": "loudness -16/-1.5 -> m4a, check podcast 4/4, Japanese report"+ },+ "trigger": "t11 (WAV->MP3) and t12 (silence on M4A) both routed to ffmpeg-skill by an independent model (evals/trigger/results-2026-09-04.json)"+}\ No newline at end of filediff --git a/evals/tasks.json b/evals/tasks.jsonindex 461a23d..f9ef9d5 100644--- a/evals/tasks.json+++ b/evals/tasks.json@@ -24,6 +24,11 @@{"id": 21, "request": "Test the toolkit on my GoPro and OBS files first.", "expect": ["verify.py"], "args_hint": "folder --report"},{"id": 22, "request": "Set up the whole edit so I can tweak numbers and re-render.", "expect": ["render.py"], "args_hint": "--init project.json then render"},{"id": 23, "request": "Tell me what you'd run before actually rendering.", "expect": ["--dry-run"], "args_hint": "any script with --dry-run"},- {"id": 24, "request": "Why is the video stuttering after the cut? It's a screen recording.", "expect": ["probe.py", "fit.py"], "args_hint": "VFR suspected -> --fps 30 / accurate"}+ {"id": 24, "request": "Why is the video stuttering after the cut? It's a screen recording.", "expect": ["probe.py", "fit.py"], "args_hint": "VFR suspected -> --fps 30 / accurate"},+ {"id": 25, "request": "Normalize this WAV to -14 LUFS.", "expect": ["loudness.py"], "args_hint": "-I -14 --tp -1 -o out.wav (audio-only: Look not needed)"},+ {"id": 26, "request": "Remove the silence from this recording (M4A).", "expect": ["silence.py"], "args_hint": "-o out.m4a (audio-only)"},+ {"id": 27, "request": "Clean up the noise in this M4A.", "expect": ["audio.py"], "args_hint": "--voice for speech, --denoise otherwise (audio-only)"},+ {"id": 28, "request": "Convert this WAV to MP3.", "expect": ["audio.py"], "args_hint": "-o out.mp3 with no filter flags (extension picks the codec)"},+ {"id": 29, "request": "Trim this audio from 00:30 to 02:00.", "expect": ["cut.py"], "args_hint": "--start 0:30 --end 2:00 -o out.wav (stream copy)"}]}diff --git a/evals/trigger/README.md b/evals/trigger/README.mdindex f1216b5..2483051 100644--- a/evals/trigger/README.md+++ b/evals/trigger/README.md@@ -1,7 +1,7 @@# Trigger tests: should / should-not-`prompts.json` holds 10 requests the skill must trigger on (English and Japanese, some without a file extension or the word "edit") and 10 near-miss requests it must not (still images, installing ffmpeg, downloading from YouTube, writing a script, an editor shortcut, thumbnail design, CSV to JSON, a codec explainer, AI video generation, meeting transcription).+`prompts.json` holds 12 requests the skill must trigger on (English and Japanese, some without a file extension or the word "edit") and 10 near-miss requests it must not (still images, installing ffmpeg, downloading from YouTube, writing a script, an editor shortcut, thumbnail design, CSV to JSON, a codec explainer, AI video generation, meeting transcription).Method: show an independent model a skill catalog made of this skill's `description` plus four decoy skills that own the neighbouring territory (image-tools, web-research, copywriter, meeting-notes), give it the raw request, and ask which one skill it would load or `none`. A run is correct when ffmpeg-skill is chosen exactly for the should-trigger prompts.-Results: `results-<date>.json`. 2026-09-04: 20/20.+Results: `results-<date>.json`. 2026-09-04: 22/22 (incl. two audio-only requests: WAV→MP3 conversion, silence removal on an M4A).diff --git a/evals/trigger/prompts.json b/evals/trigger/prompts.jsonindex 030226b..7e88f37 100644--- a/evals/trigger/prompts.json+++ b/evals/trigger/prompts.json@@ -118,5 +118,17 @@"lang": "ja","prompt": "会議の音声を文字起こしして議事録にまとめて(meeting.m4a)","should_trigger": false+ },+ {+ "id": "t11",+ "lang": "en",+ "prompt": "Convert this WAV to MP3: interview_raw.wav",+ "should_trigger": true+ },+ {+ "id": "t12",+ "lang": "en",+ "prompt": "Remove the silence from this recording, it's a 40 minute M4A",+ "should_trigger": true}]\ No newline at end of filediff --git a/evals/trigger/results-2026-09-04.json b/evals/trigger/results-2026-09-04.jsonindex b215d93..185c943 100644--- a/evals/trigger/results-2026-09-04.json+++ b/evals/trigger/results-2026-09-04.json@@ -181,11 +181,29 @@"chosen": "meeting-notes","model": "sonnet","correct": true+ },+ {+ "id": "t11",+ "lang": "en",+ "prompt": "Convert this WAV to MP3: interview_raw.wav",+ "should_trigger": true,+ "chosen": "ffmpeg-skill",+ "model": "sonnet",+ "correct": true+ },+ {+ "id": "t12",+ "lang": "en",+ "prompt": "Remove the silence from this recording, it's a 40 minute M4A",+ "should_trigger": true,+ "chosen": "ffmpeg-skill",+ "model": "sonnet",+ "correct": true}],"summary": {- "should_trigger": "10/10",+ "should_trigger": "12/12","should_not_trigger": "10/10",- "note": "Haiku judged 11 of the 20 (session rate limit hit on the rest); Sonnet judged 9. Five Sonnet runs refused when the prompt was delivered via a file and were re-run with the prompt inline. Reference decoys are hypothetical skill descriptions written for this test."+ "note": "Haiku judged 11 of the 20 (session rate limit hit on the rest); Sonnet judged 9. Five Sonnet runs refused when the prompt was delivered via a file and were re-run with the prompt inline. Reference decoys are hypothetical skill descriptions written for this test. t11/t12 (audio-only requests) added later the same day, judged by Sonnet."}}\ No newline at end of filediff --git a/tests/test_all.py b/tests/test_all.pyindex 0ca4f39..14eaedd 100644--- a/tests/test_all.py+++ b/tests/test_all.py@@ -801,6 +801,44 @@ class FFmpegSkillTests(unittest.TestCase):self.assertClose(data["offset_seconds"], -28.0, 0.02)self.assertGreater(data["confidence"], 0.3)+ def test_audio_only_inputs_through_the_audio_scripts(self):+ """WAV / M4A / MP3 with no video stream: loudness, silence, audio (voice, convert), cut, check."""+ m4a = OUT / "gappy.m4a"+ sh("ffmpeg", "-y", "-hide_banner", "-loglevel", "error",+ "-f", "lavfi", "-i", "aevalsrc='0.5*sin(2*PI*440*t)*gt(sin(2*PI*0.25*t)\\,0)':s=48000",+ "-t", "12", "-c:a", "aac", "-b:a", "128k", m4a)+ self.assertIsNone(probe(str(m4a)).get("video"))+ # loudness: wav -> m4a, codec follows the output extension+ norm = OUT / "lav_norm.m4a"+ script("loudness.py", self.mic, "-I", "-16", "--tp", "-1.5", "-o", norm)+ a = probe(str(norm))["audio"]+ self.assertEqual(a["codec"], "aac")+ self.assertIn("PASS", script("check.py", norm, "--platform", "podcast").stdout)+ # silence removal on an m4a+ tight = OUT / "gappy_tight.m4a"+ data = json.loads(script("silence.py", m4a, "-o", tight, "--json").stdout)+ self.assertClose(data["removed_seconds"], 5.25, 0.3)+ self.assertClose(probe(str(tight))["duration"], 6.75, 0.3)+ # voice clean-up keeps the container, plain -o converts+ clean = OUT / "gappy_clean.m4a"+ script("audio.py", m4a, "--voice", "-o", clean)+ self.assertEqual(probe(str(clean))["audio"]["codec"], "aac")+ mp3 = OUT / "lav.mp3"+ script("audio.py", self.mic, "-o", mp3)+ self.assertEqual(probe(str(mp3))["audio"]["codec"], "mp3")+ # trim is a stream copy on wav and mp3+ cut_wav = OUT / "lav_cut.wav"+ proc = script("cut.py", self.mic, "--start", "0:02", "--end", "0:06", "-o", cut_wav)+ self.assertIn("lossless", proc.stderr)+ self.assertClose(probe(str(cut_wav))["duration"], 4.0, 0.05)+ cut_mp3 = OUT / "lav_cut.mp3"+ script("cut.py", mp3, "--start", "2", "--end", "6", "-o", cut_mp3)+ self.assertClose(probe(str(cut_mp3))["duration"], 4.0, 0.1)+ # picture-only scripts refuse clearly instead of failing inside ffmpeg+ for argv in (("fit.py", "--duration", "5"), ("export.py", "--preset", "youtube"), ("look.py",)):+ proc = script(argv[0], self.mic, *argv[1:], "-o", OUT / "nope.out", expect_fail=True)+ self.assertIn("no video stream", proc.stderr)+def test_loudness_silent_input_is_reported_not_crashed(self):silent = OUT / "silent.mp4"sh("ffmpeg", "-y", "-hide_banner", "-loglevel", "error", "-f", "lavfi", "-i", "testsrc2=size=320x180:rate=30", "-f", "lavfi", "-i", "anullsrc=r=48000:cl=stereo",