Skip to content

Local video editor for coding agents: 42 FFmpeg tools with a machine-readable contract and an MCP server, for Claude Code, Cursor and Codex. Cut, captions, reframe, loudness, platform exports. No cloud, no API keys.

Audio-only inputs: SKILL.md section, agent/trigger evals, grader checks, test (#16)

Shogo Kajiyama776525bc5668
diff --git a/SKILL.md b/SKILL.md
index 3c57323..c53399f 100644
--- a/SKILL.md
+++ b/SKILL.md
@@ -49,7 +49,8 @@ Scripts live in `scripts/` next to this file; run them with `python3 <skill-dir>
crops keeping the subject, colours not washed out, transitions landing
where intended. The job is not finished until the report's `Look:` line
names that PNG; a probe alone cannot see a caption sitting on someone's
- face. Audio-only jobs (sync, loudness, silence) write `Look: not needed`.
+ face. Audio-only jobs (sync, loudness, silence, or any job whose input is
+ an audio file) write `Look: not needed`; there is no picture to inspect.
## Before you run anything: what to ask, what to assume
@@ -112,6 +113,34 @@ Do not ask for things `probe.py` can tell you.
| "it's a phone video with variable frame rate" | nothing extra: every re-encoding script conforms VFR to constant fps automatically; `fit.py --fps 30` to pick the rate |
+## Audio-only files
+
+Audio files are a first-class input, not a special case. `probe.py`, `cut.py`,
+`silence.py`, `loudness.py`, `audio.py`, `sync.py` and `check.py --platform
+podcast` all accept WAV, FLAC, MP3, M4A/AAC, OGG and Opus (any container ffmpeg
+can read) and write the codec that fits the output extension, so the same
+commands work with `talk.wav` in place of `talk.mp4`. What changes:
+
+- The output extension picks the format: `-o out.mp3` converts, `-o out.wav`
+ keeps PCM, `-o out.m4a` writes AAC. `audio.py in.wav -o out.mp3` with no
+ other flag is a plain conversion.
+- `cut.py` stream-copies audio too, so trims are lossless unless the format
+ cannot be cut on a packet boundary.
+- `Look: not needed` in the report; `Check:` still applies for loudness
+ (`check.py file.wav --platform podcast` measures LUFS and true peak).
+- Scripts that need a picture (`fit`, `caption`, `overlay`, `graphics`,
+ `color`, `export`, `join`, `scenes`, `look`) refuse an audio file with
+ "input has no video stream". Say so instead of forcing a video wrapper.
+
+| User says (audio file) | Do |
+|-----------|----|
+| "normalise this WAV to -14 LUFS", "podcast levels" | `loudness.py talk.wav -I -14 --tp -1 -o talk_norm.wav` (`-I -16 --tp -1.5` for podcasts) |
+| "remove the silence from this recording" | `silence.py talk.wav -o talk_tight.wav` |
+| "clean up the noise in this M4A" | `audio.py talk.m4a --voice -o talk_clean.m4a` (speech) or `--denoise` |
+| "convert this WAV to MP3" | `audio.py talk.wav -o talk.mp3` |
+| "trim this audio from 00:30 to 02:00" | `cut.py talk.wav --start 0:30 --end 2:00 -o talk_cut.wav` |
+| "is this loud enough for Apple Podcasts?" | `check.py talk.m4a --platform podcast` |
+
## Report format
Finish every job with this shape (numbers from `probe.py`/`check.py`, not memory):
diff --git a/evals/agent_prompts_audio.json b/evals/agent_prompts_audio.json
new file mode 100644
index 0000000..2098353
--- /dev/null
+++ b/evals/agent_prompts_audio.json
@@ -0,0 +1,62 @@
+[
+ {
+ "id": "a01-lufs-wav",
+ "lang": "en",
+ "prompt": "Normalize /home/user/ffmpeg-skill/tests/out/lavmic.wav to -14 LUFS. Save to OUTDIR.",
+ "expect": [
+ "loudness"
+ ],
+ "refuse": false,
+ "audio_only": true
+ },
+ {
+ "id": "a02-silence-m4a",
+ "lang": "en",
+ "prompt": "Remove the silence from this recording: /home/user/ffmpeg-skill/tests/out/gappy.m4a. Save to OUTDIR.",
+ "expect": [
+ "silence"
+ ],
+ "refuse": false,
+ "audio_only": true
+ },
+ {
+ "id": "a03-clean-m4a",
+ "lang": "en",
+ "prompt": "Clean up the noise in this M4A: /home/user/ffmpeg-skill/tests/out/gappy.m4a. Save to OUTDIR.",
+ "expect": [
+ "audio"
+ ],
+ "refuse": false,
+ "audio_only": true
+ },
+ {
+ "id": "a04-wav-to-mp3",
+ "lang": "en",
+ "prompt": "Convert /home/user/ffmpeg-skill/tests/out/lavmic.wav to MP3. Save to OUTDIR.",
+ "expect": [
+ "audio"
+ ],
+ "refuse": false,
+ "audio_only": true
+ },
+ {
+ "id": "a05-trim-wav",
+ "lang": "en",
+ "prompt": "Trim /home/user/ffmpeg-skill/tests/out/lavmic.wav from 00:02 to 00:06. Save to OUTDIR.",
+ "expect": [
+ "cut"
+ ],
+ "refuse": false,
+ "audio_only": true
+ },
+ {
+ "id": "a06-podcast-ja",
+ "lang": "ja",
+ "prompt": "/home/user/ffmpeg-skill/tests/out/lavmic.wav をポッドキャスト用に音量を整えて m4a で書き出して。OUTDIR に保存。",
+ "expect": [
+ "loudness"
+ ],
+ "refuse": false,
+ "audio_only": true
+ }
+]
\ No newline at end of file
diff --git a/evals/grade_runs_24.py b/evals/grade_runs_24.py
index fe8b6a0..049ac55 100644
--- a/evals/grade_runs_24.py
+++ b/evals/grade_runs_24.py
@@ -1,10 +1,19 @@
#!/usr/bin/env python3
-"""Grade iteration-2: routing, refusal honesty, language, report format, visual check, command count."""
+"""Grade agent runs: routing, refusal honesty, language, report format, visual check, command count.
+
+ python3 evals/grade_runs_24.py ITERATION_DIR [PROMPTS_JSON]
+
+PROMPTS_JSON defaults to evals/agent_prompts_24.json; pass evals/agent_prompts_audio.json for the
+audio-only set. Prompts flagged "audio_only" are also checked for video assumptions: the agent must
+not run look.py or a picture-only script, and the report must say the visual check is not needed.
+"""
import json, re, sys
from pathlib import Path
W = Path(__file__).resolve().parent
-P = {p["id"]: p for p in json.loads((W.parent / "agent_prompts_24.json").read_text())}
-it = W / (sys.argv[1] if len(sys.argv) > 1 else "iteration-2")
+PROMPTS = Path(sys.argv[2]) if len(sys.argv) > 2 else W / "agent_prompts_24.json"
+P = {p["id"]: p for p in json.loads(PROMPTS.read_text())}
+it = Path(sys.argv[1]) if len(sys.argv) > 1 else W / "iteration-2"
+VIDEO_ONLY = {"fit", "caption", "overlay", "graphics", "color", "export", "join", "scenes", "look"}
JA = re.compile(r"[぀-ヿ一-鿿]")
PICTURE = {"e01-reel","e03-logo","e07-hdr","e09-join","e12-vfr","j01-reel","j03-lower","j08-project"}
rows = []
@@ -31,8 +40,14 @@ for pid, p in P.items():
body = text.split("Final report")[-1] if "Final report" in text else text.split("# ")[-1]
ja_chars = len(JA.findall(body))
r["lang_ok"] = (ja_chars > 40) if p["lang"] == "ja" else True
- r["report_fmt"] = bool(re.search(r"(?im)^\s*(\*\*)?(done|完了)", text)) and ("Look" in text or "目視" in text or "look" in text.lower())
+ r["report_fmt"] = bool(re.search(r"(?im)^\s*(\*\*)?(done|完了)", text)) and ("Look" in text or "目視" in text or "確認画像" in text or "look" in text.lower())
r["look"] = ("look" in used) if pid in PICTURE else None
+ if p.get("audio_only"):
+ # audio-only: no picture-only script, no look.py, and the report says the visual check is not needed
+ video_scripts = sorted(used & VIDEO_ONLY)
+ says_not_needed = bool(re.search(r"(?i)(look|目視|確認画像)[::]\s*(not needed|n/a|none|不要)", text))
+ r["audio_ok"] = not video_scripts and says_not_needed
+ r["audio_notes"] = (", ".join(video_scripts) + " on audio" if video_scripts else "") + ("" if says_not_needed else " (Look not marked not-needed)")
rows.append(r)
acts = [r for r in rows if "score" in r and not P[r["id"]]["refuse"]]
refs = [r for r in rows if "score" in r and P[r["id"]]["refuse"]]
@@ -54,3 +69,9 @@ print(f"report format: {sum(1 for r in fm if r['report_fmt'])}/{len(fm)}")
lk = [r for r in rows if r.get("look") is not None]
if lk:
print(f"visual check when picture changed: {sum(1 for r in lk if r['look'])}/{len(lk)}")
+au = [r for r in rows if "audio_ok" in r]
+if au:
+ print(f"audio-only handled as audio (no picture script, Look: not needed): {sum(1 for r in au if r['audio_ok'])}/{len(au)}")
+ for r in au:
+ if not r["audio_ok"]:
+ print(f" {r['id']}: {r['audio_notes'].strip()}")
diff --git a/evals/results/audio-only-1.json b/evals/results/audio-only-1.json
new file mode 100644
index 0000000..5219d6a
--- /dev/null
+++ b/evals/results/audio-only-1.json
@@ -0,0 +1,22 @@
+{
+ "iteration": "audio-only-1",
+ "date": "2026-09-04",
+ "runs": 6,
+ "prompts_file": "evals/agent_prompts_audio.json",
+ "grader": "evals/grade_runs_24.py ITER evals/agent_prompts_audio.json",
+ "skill_version": "0.8.4 + SKILL.md 'Audio-only files' section (uncommitted at time of run)",
+ "routing": "6/6",
+ "japanese_report": "1/1",
+ "report_format": "6/6",
+ "audio_only_handled_as_audio": "6/6 (no picture-only script run, Look marked not needed)",
+ "mean_commands": 4.2,
+ "per_prompt": {
+ "a01-lufs-wav": "loudness -14/-1 -> wav, check podcast, explained the -16 preset mismatch",
+ "a02-silence-m4a": "silence.py on m4a, 5.25 s removed",
+ "a03-clean-m4a": "audio.py --denoise, offered --voice",
+ "a04-wav-to-mp3": "audio.py -o .mp3 plain conversion",
+ "a05-trim-wav": "cut.py stream copy 0:02-0:06",
+ "a06-podcast-ja": "loudness -16/-1.5 -> m4a, check podcast 4/4, Japanese report"
+ },
+ "trigger": "t11 (WAV->MP3) and t12 (silence on M4A) both routed to ffmpeg-skill by an independent model (evals/trigger/results-2026-09-04.json)"
+}
\ No newline at end of file
diff --git a/evals/tasks.json b/evals/tasks.json
index 461a23d..f9ef9d5 100644
--- a/evals/tasks.json
+++ b/evals/tasks.json
@@ -24,6 +24,11 @@
{"id": 21, "request": "Test the toolkit on my GoPro and OBS files first.", "expect": ["verify.py"], "args_hint": "folder --report"},
{"id": 22, "request": "Set up the whole edit so I can tweak numbers and re-render.", "expect": ["render.py"], "args_hint": "--init project.json then render"},
{"id": 23, "request": "Tell me what you'd run before actually rendering.", "expect": ["--dry-run"], "args_hint": "any script with --dry-run"},
- {"id": 24, "request": "Why is the video stuttering after the cut? It's a screen recording.", "expect": ["probe.py", "fit.py"], "args_hint": "VFR suspected -> --fps 30 / accurate"}
+ {"id": 24, "request": "Why is the video stuttering after the cut? It's a screen recording.", "expect": ["probe.py", "fit.py"], "args_hint": "VFR suspected -> --fps 30 / accurate"},
+ {"id": 25, "request": "Normalize this WAV to -14 LUFS.", "expect": ["loudness.py"], "args_hint": "-I -14 --tp -1 -o out.wav (audio-only: Look not needed)"},
+ {"id": 26, "request": "Remove the silence from this recording (M4A).", "expect": ["silence.py"], "args_hint": "-o out.m4a (audio-only)"},
+ {"id": 27, "request": "Clean up the noise in this M4A.", "expect": ["audio.py"], "args_hint": "--voice for speech, --denoise otherwise (audio-only)"},
+ {"id": 28, "request": "Convert this WAV to MP3.", "expect": ["audio.py"], "args_hint": "-o out.mp3 with no filter flags (extension picks the codec)"},
+ {"id": 29, "request": "Trim this audio from 00:30 to 02:00.", "expect": ["cut.py"], "args_hint": "--start 0:30 --end 2:00 -o out.wav (stream copy)"}
]
}
diff --git a/evals/trigger/README.md b/evals/trigger/README.md
index f1216b5..2483051 100644
--- a/evals/trigger/README.md
+++ b/evals/trigger/README.md
@@ -1,7 +1,7 @@
# Trigger tests: should / should-not
-`prompts.json` holds 10 requests the skill must trigger on (English and Japanese, some without a file extension or the word "edit") and 10 near-miss requests it must not (still images, installing ffmpeg, downloading from YouTube, writing a script, an editor shortcut, thumbnail design, CSV to JSON, a codec explainer, AI video generation, meeting transcription).
+`prompts.json` holds 12 requests the skill must trigger on (English and Japanese, some without a file extension or the word "edit") and 10 near-miss requests it must not (still images, installing ffmpeg, downloading from YouTube, writing a script, an editor shortcut, thumbnail design, CSV to JSON, a codec explainer, AI video generation, meeting transcription).
Method: show an independent model a skill catalog made of this skill's `description` plus four decoy skills that own the neighbouring territory (image-tools, web-research, copywriter, meeting-notes), give it the raw request, and ask which one skill it would load or `none`. A run is correct when ffmpeg-skill is chosen exactly for the should-trigger prompts.
-Results: `results-<date>.json`. 2026-09-04: 20/20.
+Results: `results-<date>.json`. 2026-09-04: 22/22 (incl. two audio-only requests: WAV→MP3 conversion, silence removal on an M4A).
diff --git a/evals/trigger/prompts.json b/evals/trigger/prompts.json
index 030226b..7e88f37 100644
--- a/evals/trigger/prompts.json
+++ b/evals/trigger/prompts.json
@@ -118,5 +118,17 @@
"lang": "ja",
"prompt": "会議の音声を文字起こしして議事録にまとめて(meeting.m4a)",
"should_trigger": false
+ },
+ {
+ "id": "t11",
+ "lang": "en",
+ "prompt": "Convert this WAV to MP3: interview_raw.wav",
+ "should_trigger": true
+ },
+ {
+ "id": "t12",
+ "lang": "en",
+ "prompt": "Remove the silence from this recording, it's a 40 minute M4A",
+ "should_trigger": true
}
]
\ No newline at end of file
diff --git a/evals/trigger/results-2026-09-04.json b/evals/trigger/results-2026-09-04.json
index b215d93..185c943 100644
--- a/evals/trigger/results-2026-09-04.json
+++ b/evals/trigger/results-2026-09-04.json
@@ -181,11 +181,29 @@
"chosen": "meeting-notes",
"model": "sonnet",
"correct": true
+ },
+ {
+ "id": "t11",
+ "lang": "en",
+ "prompt": "Convert this WAV to MP3: interview_raw.wav",
+ "should_trigger": true,
+ "chosen": "ffmpeg-skill",
+ "model": "sonnet",
+ "correct": true
+ },
+ {
+ "id": "t12",
+ "lang": "en",
+ "prompt": "Remove the silence from this recording, it's a 40 minute M4A",
+ "should_trigger": true,
+ "chosen": "ffmpeg-skill",
+ "model": "sonnet",
+ "correct": true
}
],
"summary": {
- "should_trigger": "10/10",
+ "should_trigger": "12/12",
"should_not_trigger": "10/10",
- "note": "Haiku judged 11 of the 20 (session rate limit hit on the rest); Sonnet judged 9. Five Sonnet runs refused when the prompt was delivered via a file and were re-run with the prompt inline. Reference decoys are hypothetical skill descriptions written for this test."
+ "note": "Haiku judged 11 of the 20 (session rate limit hit on the rest); Sonnet judged 9. Five Sonnet runs refused when the prompt was delivered via a file and were re-run with the prompt inline. Reference decoys are hypothetical skill descriptions written for this test. t11/t12 (audio-only requests) added later the same day, judged by Sonnet."
}
}
\ No newline at end of file
diff --git a/tests/test_all.py b/tests/test_all.py
index 0ca4f39..14eaedd 100644
--- a/tests/test_all.py
+++ b/tests/test_all.py
@@ -801,6 +801,44 @@ class FFmpegSkillTests(unittest.TestCase):
self.assertClose(data["offset_seconds"], -28.0, 0.02)
self.assertGreater(data["confidence"], 0.3)
+ def test_audio_only_inputs_through_the_audio_scripts(self):
+ """WAV / M4A / MP3 with no video stream: loudness, silence, audio (voice, convert), cut, check."""
+ m4a = OUT / "gappy.m4a"
+ sh("ffmpeg", "-y", "-hide_banner", "-loglevel", "error",
+ "-f", "lavfi", "-i", "aevalsrc='0.5*sin(2*PI*440*t)*gt(sin(2*PI*0.25*t)\\,0)':s=48000",
+ "-t", "12", "-c:a", "aac", "-b:a", "128k", m4a)
+ self.assertIsNone(probe(str(m4a)).get("video"))
+ # loudness: wav -> m4a, codec follows the output extension
+ norm = OUT / "lav_norm.m4a"
+ script("loudness.py", self.mic, "-I", "-16", "--tp", "-1.5", "-o", norm)
+ a = probe(str(norm))["audio"]
+ self.assertEqual(a["codec"], "aac")
+ self.assertIn("PASS", script("check.py", norm, "--platform", "podcast").stdout)
+ # silence removal on an m4a
+ tight = OUT / "gappy_tight.m4a"
+ data = json.loads(script("silence.py", m4a, "-o", tight, "--json").stdout)
+ self.assertClose(data["removed_seconds"], 5.25, 0.3)
+ self.assertClose(probe(str(tight))["duration"], 6.75, 0.3)
+ # voice clean-up keeps the container, plain -o converts
+ clean = OUT / "gappy_clean.m4a"
+ script("audio.py", m4a, "--voice", "-o", clean)
+ self.assertEqual(probe(str(clean))["audio"]["codec"], "aac")
+ mp3 = OUT / "lav.mp3"
+ script("audio.py", self.mic, "-o", mp3)
+ self.assertEqual(probe(str(mp3))["audio"]["codec"], "mp3")
+ # trim is a stream copy on wav and mp3
+ cut_wav = OUT / "lav_cut.wav"
+ proc = script("cut.py", self.mic, "--start", "0:02", "--end", "0:06", "-o", cut_wav)
+ self.assertIn("lossless", proc.stderr)
+ self.assertClose(probe(str(cut_wav))["duration"], 4.0, 0.05)
+ cut_mp3 = OUT / "lav_cut.mp3"
+ script("cut.py", mp3, "--start", "2", "--end", "6", "-o", cut_mp3)
+ self.assertClose(probe(str(cut_mp3))["duration"], 4.0, 0.1)
+ # picture-only scripts refuse clearly instead of failing inside ffmpeg
+ for argv in (("fit.py", "--duration", "5"), ("export.py", "--preset", "youtube"), ("look.py",)):
+ proc = script(argv[0], self.mic, *argv[1:], "-o", OUT / "nope.out", expect_fail=True)
+ self.assertIn("no video stream", proc.stderr)
+
def test_loudness_silent_input_is_reported_not_crashed(self):
silent = OUT / "silent.mp4"
sh("ffmpeg", "-y", "-hide_banner", "-loglevel", "error", "-f", "lavfi", "-i", "testsrc2=size=320x180:rate=30", "-f", "lavfi", "-i", "anullsrc=r=48000:cl=stereo",