The open creative studio on ComfyUI: images, video, 3D and audio on open weights, on your own GPU or the cloud. Camera motions, effects, apps, Cinema, Layers, Image to 3D and an AI agent.
TypeScript ai-videocomfyuicreative-toolsgenerative-aiideogramimage-generationimage-to-3dimage-to-videoltx-videominimax-h3nextjsopen-sourceopen-weightsqwen-imagetext-to-imagevideo-generationwan
{
"$schema": "./case.schema.json",
"suite": "separation",
"title": "Soundtrack separation · dialogue, music and effects",
"description": "Generated by scripts/eval/gen-cases.mjs from the research files; edit the generator, not this file. Synthetic 12 s mixes with known stems (python3 -I scripts/eval/fixtures/make-separation.py --speech <folder> --speech-licence \"<source, licence>\" writes them into local/separation/; the speech is yours to bring with its words and its licence, the music, effects, hum and hiss are synthesized there, CC0; the measured runs used LibriSpeech dev-clean speaker 1272, CC BY 4.0) and your own clip for the blind review. Scores each stem against its known part (SI-SDR, SDR), whether voice + bed add back up to the input, the bed's level, peak and stereo image against the known bed, where a steady hum or hiss goes, the dialogue's word error rate (Whisper base.en, pnpm eval:setup --asr; werInput is the same ASR on the input as it is, which is \"Clean the voice first\" off) and the speed from ComfyUI-DBS's report. Clean up dialogue (clean-dialogue: one file with the bed at −18 dB) is scored against the voice plus the known bed at that level (remixSiSdr), with the bed's level in it (bedLevel), the noise left in it and its WER. Licences (information): DBS-TIGER-DnR Apache-2.0 (JusperLee/DBS-TIGER-DnR@22da7d4c; trained on DnR v2 (CC BY 4.0), from scratch); DBS-BSRNN-DnR Apache-2.0 (JusperLee/DBS-BSRNN-DnR@5af343f5; trained on DnR v2 (CC BY 4.0), from scratch); ComfyUI-DBS MIT; MelBand RoFormer MIT; Demucs MIT; DeepFilterNet 3 MIT OR Apache-2.0; VEED Clean Audio a hosted paid API; the test mixes' sources are recorded in each mix's meta.json. SAM Audio isn't here: it is not offered on the cloud engine.",
"original": [
"oss-dbs-separate",
"oss-dbs-voice-separate"
],
"kind": "audio",
"decides": "Defaults change only on these numbers, in a reviewed code change: the open split's model per host (src/lib/av/split.ts) is the faster model unless the other wins dialogue SI-SDR by more than 1 dB on most mixes at a speed the host can live with (a minute of audio in under 5 minutes on a CPU); stereo stems (each channel) become the default only if the fast stereo bed loses its stereo image (bed stereo gap over 0.1) or its level (over 1 dB) on the centred mixes; the dub's voice split (src/lib/av/dub.ts DEFAULT_VOICE_SPLIT) moves to DBS when its dialogue WER is no worse than MelBand RoFormer's and its bed keeps the music and effects (background SI-SDR at least 3 dB higher).",
"targets": {
"open": {
"models": [
"oss-dbs-separate"
],
"op": "separate-audio"
},
"cloud": {
"models": [
"demucs"
],
"op": "separate-audio"
},
"demo": {
"models": [
"oss-dbs-separate"
],
"op": "separate-audio"
}
},
"defaults": {
"seeds": [
1001
],
"metrics": [
"separation",
"wer"
],
"variant": "bsrnn-mono",
"candidate": "bsrnn-mono",
"timeoutSeconds": 3600
},
"variants": {
"bsrnn-mono": {
"description": "DBS-BSRNN (fast), mono: stems from the channels' mean, the bed is the original minus the speech (the fast stereo bed)",
"extra": {
"separationModel": "DBS-BSRNN-DnR",
"channels": "mono"
},
"engines": [
"open",
"demo"
]
},
"bsrnn-stereo": {
"description": "DBS-BSRNN, each channel: every channel separated on its own (twice the time for stereo)",
"extra": {
"separationModel": "DBS-BSRNN-DnR",
"channels": "each channel"
},
"engines": [
"open"
]
},
"tiger-mono": {
"description": "DBS-TIGER (best), mono",
"extra": {
"separationModel": "DBS-TIGER-DnR",
"channels": "mono"
},
"engines": [
"open"
]
},
"tiger-stereo": {
"description": "DBS-TIGER, each channel",
"extra": {
"separationModel": "DBS-TIGER-DnR",
"channels": "each channel"
},
"engines": [
"open"
]
},
"dbs-dub": {
"description": "The dub's first step with DBS (graph B: DBS-BSRNN voice and bed, then Qwen3-ASR 0.6B on the voice)",
"models": {
"open": [
"oss-dbs-voice-separate"
]
},
"extra": {
"separationModel": "DBS-BSRNN-DnR",
"asrModel": "Qwen/Qwen3-ASR-0.6B-hf"
},
"engines": [
"open"
]
},
"melband": {
"description": "The dub's first step with MelBand RoFormer (a vocals model; voice and background only, then Qwen3-ASR 0.6B)",
"models": {
"open": [
"oss-voice-separate"
]
},
"extra": {
"asrModel": "Qwen/Qwen3-ASR-0.6B-hf"
},
"engines": [
"open"
]
},
"demucs": {
"description": "Demucs (htdemucs on fal, as today's cloud dub uses it): vocals against drums, bass and other, summed into the bed",
"models": {
"cloud": [
"demucs"
]
},
"engines": [
"cloud"
]
},
"clean-dialogue": {
"description": "Clean up dialogue, open (graph C: DBS-BSRNN dialogue with the music and effects at −18 dB, one file)",
"models": {
"open": [
"oss-dbs-remix"
]
},
"extra": {
"separationModel": "DBS-BSRNN-DnR",
"channels": "mono",
"voiceDb": 0,
"musicDb": -18,
"effectsDb": -18
},
"engines": [
"open"
]
},
"clean-veed": {
"description": "Clean up dialogue, cloud: VEED Clean Audio (the voice with nothing under it)",
"models": {
"cloud": [
"veed-clean-audio"
]
},
"op": "enhance-voice",
"engines": [
"cloud"
]
},
"clean-deepfilternet": {
"description": "Clean up dialogue, cloud: DeepFilterNet 3 on fal (audio only; a denoiser, so music stays)",
"models": {
"cloud": [
"deepfilternet3"
]
},
"op": "enhance-voice",
"engines": [
"cloud"
]
}
},
"cases": [
{
"id": "separation/centre",
"title": "Dialogue in the centre over music and effects",
"inputs": {
"audio": "local/separation/centre/mix.flac"
},
"demoInputs": {
"audio": "fixtures/voice.wav"
},
"references": "local/separation/centre",
"tags": [
"synthetic",
"audio-only"
],
"note": "the reference mix: every stem against its known part"
},
{
"id": "separation/off-centre",
"title": "Dialogue panned left over the same music and effects",
"inputs": {
"audio": "local/separation/off-centre/mix.flac"
},
"demoInputs": {
"audio": "fixtures/voice.wav"
},
"references": "local/separation/off-centre",
"tags": [
"synthetic",
"audio-only"
],
"note": "the fast stereo bed (mono speech subtracted from each channel) leaves part of off-centre speech in the bed"
},
{
"id": "separation/hum",
"title": "Dialogue with a steady 50 Hz mains hum",
"inputs": {
"audio": "local/separation/hum/mix.flac"
},
"demoInputs": {
"audio": "fixtures/voice.wav"
},
"references": "local/separation/hum",
"tags": [
"synthetic",
"audio-only"
],
"note": "where steady hum goes (noiseInVoice: the share left in the dialogue)"
},
{
"id": "separation/hiss",
"title": "Dialogue with steady tape hiss",
"inputs": {
"audio": "local/separation/hiss/mix.flac"
},
"demoInputs": {
"audio": "fixtures/voice.wav"
},
"references": "local/separation/hiss",
"tags": [
"synthetic",
"audio-only"
],
"note": "where steady hiss goes"
},
{
"id": "separation/loud",
"title": "A loud master that peaks at −0.1 dBFS",
"inputs": {
"audio": "local/separation/loud/mix.flac"
},
"demoInputs": {
"audio": "fixtures/voice.wav"
},
"references": "local/separation/loud",
"tags": [
"synthetic",
"audio-only"
],
"note": "a loud master whose peaks are the dialogue's: the bed stays well under full scale (bedLevel, bedPeak)"
},
{
"id": "separation/loud-bed",
"title": "A loud master whose music and effects reach full scale",
"inputs": {
"audio": "local/separation/loud-bed/mix.flac"
},
"demoInputs": {
"audio": "fixtures/voice.wav"
},
"references": "local/separation/loud-bed",
"tags": [
"synthetic",
"audio-only"
],
"note": "AudioMerge scales a bed that peaks over full scale down as a whole (bedScale, bedLevel, bedPeak; the known bed's own peak is in meta.json)"
},
{
"id": "separation/video",
"title": "The centre mix inside a video (H.264 + AAC)",
"inputs": {
"video": "local/separation/video/clip.mp4"
},
"demoInputs": {
"audio": "fixtures/voice.wav"
},
"references": "local/separation/video",
"tags": [
"synthetic",
"video"
],
"note": "a video source: the soundtrack read straight from the clip"
},
{
"id": "separation/real",
"title": "Your own clip (a film scene, a vlog, a podcast with music): for the blind A/B page",
"inputs": {
"video": "local/separation/real/clip.mp4"
},
"demoInputs": {
"audio": "fixtures/voice.wav"
},
"metrics": [
"separation"
],
"tags": [
"real",
"video"
],
"note": "No known stems: the scores are speed and whether the stems add up; listen on pnpm eval:review. Use a clip you own or may use."
}
]
}