Skip to content

armgit / fuobe

Public Imported from GitHub
0

The open creative studio on ComfyUI: images, video, 3D and audio on open weights, on your own GPU or the cloud. Camera motions, effects, apps, Cinema, Layers, Image to 3D and an AI agent.

9.8 KB
Raw Blame
{
  "$schema": "./case.schema.json",
  "suite": "separation",
  "title": "Soundtrack separation · dialogue, music and effects",
  "description": "Generated by scripts/eval/gen-cases.mjs from the research files; edit the generator, not this file. Synthetic 12 s mixes with known stems (python3 -I scripts/eval/fixtures/make-separation.py --speech <folder> --speech-licence \"<source, licence>\" writes them into local/separation/; the speech is yours to bring with its words and its licence, the music, effects, hum and hiss are synthesized there, CC0; the measured runs used LibriSpeech dev-clean speaker 1272, CC BY 4.0) and your own clip for the blind review. Scores each stem against its known part (SI-SDR, SDR), whether voice + bed add back up to the input, the bed's level, peak and stereo image against the known bed, where a steady hum or hiss goes, the dialogue's word error rate (Whisper base.en, pnpm eval:setup --asr; werInput is the same ASR on the input as it is, which is \"Clean the voice first\" off) and the speed from ComfyUI-DBS's report. Clean up dialogue (clean-dialogue: one file with the bed at −18 dB) is scored against the voice plus the known bed at that level (remixSiSdr), with the bed's level in it (bedLevel), the noise left in it and its WER. Licences (information): DBS-TIGER-DnR Apache-2.0 (JusperLee/DBS-TIGER-DnR@22da7d4c; trained on DnR v2 (CC BY 4.0), from scratch); DBS-BSRNN-DnR Apache-2.0 (JusperLee/DBS-BSRNN-DnR@5af343f5; trained on DnR v2 (CC BY 4.0), from scratch); ComfyUI-DBS MIT; MelBand RoFormer MIT; Demucs MIT; DeepFilterNet 3 MIT OR Apache-2.0; VEED Clean Audio a hosted paid API; the test mixes' sources are recorded in each mix's meta.json. SAM Audio isn't here: it is not offered on the cloud engine.",
  "original": [
    "oss-dbs-separate",
    "oss-dbs-voice-separate"
  ],
  "kind": "audio",
  "decides": "Defaults change only on these numbers, in a reviewed code change: the open split's model per host (src/lib/av/split.ts) is the faster model unless the other wins dialogue SI-SDR by more than 1 dB on most mixes at a speed the host can live with (a minute of audio in under 5 minutes on a CPU); stereo stems (each channel) become the default only if the fast stereo bed loses its stereo image (bed stereo gap over 0.1) or its level (over 1 dB) on the centred mixes; the dub's voice split (src/lib/av/dub.ts DEFAULT_VOICE_SPLIT) moves to DBS when its dialogue WER is no worse than MelBand RoFormer's and its bed keeps the music and effects (background SI-SDR at least 3 dB higher).",
  "targets": {
    "open": {
      "models": [
        "oss-dbs-separate"
      ],
      "op": "separate-audio"
    },
    "cloud": {
      "models": [
        "demucs"
      ],
      "op": "separate-audio"
    },
    "demo": {
      "models": [
        "oss-dbs-separate"
      ],
      "op": "separate-audio"
    }
  },
  "defaults": {
    "seeds": [
      1001
    ],
    "metrics": [
      "separation",
      "wer"
    ],
    "variant": "bsrnn-mono",
    "candidate": "bsrnn-mono",
    "timeoutSeconds": 3600
  },
  "variants": {
    "bsrnn-mono": {
      "description": "DBS-BSRNN (fast), mono: stems from the channels' mean, the bed is the original minus the speech (the fast stereo bed)",
      "extra": {
        "separationModel": "DBS-BSRNN-DnR",
        "channels": "mono"
      },
      "engines": [
        "open",
        "demo"
      ]
    },
    "bsrnn-stereo": {
      "description": "DBS-BSRNN, each channel: every channel separated on its own (twice the time for stereo)",
      "extra": {
        "separationModel": "DBS-BSRNN-DnR",
        "channels": "each channel"
      },
      "engines": [
        "open"
      ]
    },
    "tiger-mono": {
      "description": "DBS-TIGER (best), mono",
      "extra": {
        "separationModel": "DBS-TIGER-DnR",
        "channels": "mono"
      },
      "engines": [
        "open"
      ]
    },
    "tiger-stereo": {
      "description": "DBS-TIGER, each channel",
      "extra": {
        "separationModel": "DBS-TIGER-DnR",
        "channels": "each channel"
      },
      "engines": [
        "open"
      ]
    },
    "dbs-dub": {
      "description": "The dub's first step with DBS (graph B: DBS-BSRNN voice and bed, then Qwen3-ASR 0.6B on the voice)",
      "models": {
        "open": [
          "oss-dbs-voice-separate"
        ]
      },
      "extra": {
        "separationModel": "DBS-BSRNN-DnR",
        "asrModel": "Qwen/Qwen3-ASR-0.6B-hf"
      },
      "engines": [
        "open"
      ]
    },
    "melband": {
      "description": "The dub's first step with MelBand RoFormer (a vocals model; voice and background only, then Qwen3-ASR 0.6B)",
      "models": {
        "open": [
          "oss-voice-separate"
        ]
      },
      "extra": {
        "asrModel": "Qwen/Qwen3-ASR-0.6B-hf"
      },
      "engines": [
        "open"
      ]
    },
    "demucs": {
      "description": "Demucs (htdemucs on fal, as today's cloud dub uses it): vocals against drums, bass and other, summed into the bed",
      "models": {
        "cloud": [
          "demucs"
        ]
      },
      "engines": [
        "cloud"
      ]
    },
    "clean-dialogue": {
      "description": "Clean up dialogue, open (graph C: DBS-BSRNN dialogue with the music and effects at −18 dB, one file)",
      "models": {
        "open": [
          "oss-dbs-remix"
        ]
      },
      "extra": {
        "separationModel": "DBS-BSRNN-DnR",
        "channels": "mono",
        "voiceDb": 0,
        "musicDb": -18,
        "effectsDb": -18
      },
      "engines": [
        "open"
      ]
    },
    "clean-veed": {
      "description": "Clean up dialogue, cloud: VEED Clean Audio (the voice with nothing under it)",
      "models": {
        "cloud": [
          "veed-clean-audio"
        ]
      },
      "op": "enhance-voice",
      "engines": [
        "cloud"
      ]
    },
    "clean-deepfilternet": {
      "description": "Clean up dialogue, cloud: DeepFilterNet 3 on fal (audio only; a denoiser, so music stays)",
      "models": {
        "cloud": [
          "deepfilternet3"
        ]
      },
      "op": "enhance-voice",
      "engines": [
        "cloud"
      ]
    }
  },
  "cases": [
    {
      "id": "separation/centre",
      "title": "Dialogue in the centre over music and effects",
      "inputs": {
        "audio": "local/separation/centre/mix.flac"
      },
      "demoInputs": {
        "audio": "fixtures/voice.wav"
      },
      "references": "local/separation/centre",
      "tags": [
        "synthetic",
        "audio-only"
      ],
      "note": "the reference mix: every stem against its known part"
    },
    {
      "id": "separation/off-centre",
      "title": "Dialogue panned left over the same music and effects",
      "inputs": {
        "audio": "local/separation/off-centre/mix.flac"
      },
      "demoInputs": {
        "audio": "fixtures/voice.wav"
      },
      "references": "local/separation/off-centre",
      "tags": [
        "synthetic",
        "audio-only"
      ],
      "note": "the fast stereo bed (mono speech subtracted from each channel) leaves part of off-centre speech in the bed"
    },
    {
      "id": "separation/hum",
      "title": "Dialogue with a steady 50 Hz mains hum",
      "inputs": {
        "audio": "local/separation/hum/mix.flac"
      },
      "demoInputs": {
        "audio": "fixtures/voice.wav"
      },
      "references": "local/separation/hum",
      "tags": [
        "synthetic",
        "audio-only"
      ],
      "note": "where steady hum goes (noiseInVoice: the share left in the dialogue)"
    },
    {
      "id": "separation/hiss",
      "title": "Dialogue with steady tape hiss",
      "inputs": {
        "audio": "local/separation/hiss/mix.flac"
      },
      "demoInputs": {
        "audio": "fixtures/voice.wav"
      },
      "references": "local/separation/hiss",
      "tags": [
        "synthetic",
        "audio-only"
      ],
      "note": "where steady hiss goes"
    },
    {
      "id": "separation/loud",
      "title": "A loud master that peaks at −0.1 dBFS",
      "inputs": {
        "audio": "local/separation/loud/mix.flac"
      },
      "demoInputs": {
        "audio": "fixtures/voice.wav"
      },
      "references": "local/separation/loud",
      "tags": [
        "synthetic",
        "audio-only"
      ],
      "note": "a loud master whose peaks are the dialogue's: the bed stays well under full scale (bedLevel, bedPeak)"
    },
    {
      "id": "separation/loud-bed",
      "title": "A loud master whose music and effects reach full scale",
      "inputs": {
        "audio": "local/separation/loud-bed/mix.flac"
      },
      "demoInputs": {
        "audio": "fixtures/voice.wav"
      },
      "references": "local/separation/loud-bed",
      "tags": [
        "synthetic",
        "audio-only"
      ],
      "note": "AudioMerge scales a bed that peaks over full scale down as a whole (bedScale, bedLevel, bedPeak; the known bed's own peak is in meta.json)"
    },
    {
      "id": "separation/video",
      "title": "The centre mix inside a video (H.264 + AAC)",
      "inputs": {
        "video": "local/separation/video/clip.mp4"
      },
      "demoInputs": {
        "audio": "fixtures/voice.wav"
      },
      "references": "local/separation/video",
      "tags": [
        "synthetic",
        "video"
      ],
      "note": "a video source: the soundtrack read straight from the clip"
    },
    {
      "id": "separation/real",
      "title": "Your own clip (a film scene, a vlog, a podcast with music): for the blind A/B page",
      "inputs": {
        "video": "local/separation/real/clip.mp4"
      },
      "demoInputs": {
        "audio": "fixtures/voice.wav"
      },
      "metrics": [
        "separation"
      ],
      "tags": [
        "real",
        "video"
      ],
      "note": "No known stems: the scores are speed and whether the stems add up; listen on pnpm eval:review. Use a clip you own or may use."
    }
  ]
}