Ears for coding agents: let AI hear, review and make sound. CPU-only, open source.
"""Tuning set for the timeline event detector: synthetic mixtures of REAL recordings with exact ground truth.
.venv/Scripts/python.exe eval/timeline/mixes.py [--n 8] [--dur 30] [--out out/timeline/mixes]
Only recordings that the Clawd soundtrack does NOT use (unused files in the Clawd asset library + its _rejected
folder) go into these mixes, so detector thresholds tuned here are evaluated on the Clawd soundtrack without leakage.
Each mix: 2-4 beds (music or ambience, hard switches = section boundaries) and one-shot hits every 0.35-1.6 s at a
random level relative to the bed (seeded, deterministic). Ground truth per hit: attack time (first 1 ms within 20 dB
of the hit's peak, the Clawd mixer's own definition), peak-aligned time for whooshes, and the hit-to-bed SNR in dB.
Writes <out>/mix_<k>.wav, mix_<k>_bed.wav (the beds alone) + mix_<k>.json (media stays in out/, never in git).
"""
import argparse
import json
import os
import sys
import numpy as np
import soundfile as sf
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
from earshot import audio # noqa: E402
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) # eval/: local data locations
import evalpaths as EP # noqa: E402
SR = audio.SR
ASSETS = EP.ASSETS
MIX_REPORT = f'{EP.FILM}/out/qa/mix_report.json'
HITS = ('foley', 'garden', 'keyboard', 'magic', 'toy')
def unused(cat):
used = {os.path.basename(f) for f in json.load(open(MIX_REPORT))['files_used']}
out = []
for d in (os.path.join(ASSETS, cat), os.path.join(ASSETS, '_rejected', cat)):
if os.path.isdir(d):
out += [os.path.join(d, f) for f in sorted(os.listdir(d))
if os.path.splitext(f)[1].lower() in audio.AUDIO_EXTS and f not in used]
return out
def env1(x):
n = len(x) // 48
return np.abs(x[: n * 48]).reshape(n, 48).max(axis=1)
def attack_ms(x):
e = env1(x)
return int(np.argmax(e >= e.max() * 0.1)) if len(e) else 0
def one_shots(path, kmax=3):
"""single isolated hits inside a recording: [(array, attack_s)]"""
x = audio.load(path, mono=True)[: 60 * SR]
if len(x) < SR // 10:
return []
r = np.sqrt(np.convolve(x.astype(np.float64) ** 2, np.ones(240) / 240, 'same')[::240] + 1e-12)
db = 20 * np.log10(r)
out, used = [], np.zeros(len(db), bool)
for ip in np.argsort(-db):
if len(out) >= kmax or db[ip] < db.max() - 25:
break
if used[max(0, ip - 60): ip + 60].any():
continue
used[max(0, ip - 60): ip + 60] = True
pre, post = db[max(0, ip - 40): max(0, ip - 4)], db[ip + 12: ip + 100]
if len(pre) < 10 or pre.max() > db[ip] - 20 or (len(post) and post.max() > db[ip] - 12):
continue
a = max(0, (ip - 40) * 240)
seg = x[a: a + int(0.6 * SR)].copy()
fade = int(0.05 * SR)
seg[-fade:] *= np.linspace(1, 0, fade)
out.append((seg, attack_ms(seg) / 1000))
return out
def whooshes(path):
x = audio.load(path, mono=True)[: 4 * SR]
if len(x) < SR // 5:
return []
fade = int(0.03 * SR)
x = x.copy()
x[-fade:] *= np.linspace(1, 0, fade)
e = env1(x)
return [(x, (int(np.argmax(e)) + 0.5) / 1000)]
def bed(path, n, rng):
x = audio.load(path, mono=True)
if len(x) < n:
x = np.tile(x, int(np.ceil(n / max(1, len(x)))) + 1)
a = int(rng.uniform(0, max(1, len(x) - n - 1)))
seg = x[a: a + n].copy()
return seg / (np.sqrt(np.mean(seg ** 2)) + 1e-9)
def build(k, dur, pools, rng):
music, amb, hits, swoosh = pools
nb = int(rng.integers(2, 5))
cuts = np.sort(rng.uniform(4, dur - 4, nb - 1))
while len(cuts) > 1 and np.min(np.diff(cuts)) < 4:
cuts = np.sort(rng.uniform(4, dur - 4, nb - 1))
edges = [0.0] + [round(float(c), 2) for c in cuts] + [float(dur)]
y = np.zeros(int(dur * SR), np.float32)
beds = []
for j in range(nb):
src = music if (j % 2 == 0) != (rng.uniform() < 0.3) else amb
f = src[int(rng.integers(len(src)))]
lvl = rng.uniform(-24, -16) if src is music else rng.uniform(-40, -28)
a, b = int(edges[j] * SR), int(edges[j + 1] * SR)
y[a:b] = bed(f, b - a, rng) * 10 ** (lvl / 20)
beds.append({'start': edges[j], 'end': edges[j + 1], 'kind': 'music' if src is music else 'ambience',
'file': os.path.basename(f), 'rms_dbfs': round(lvl, 1)})
bed_only = y.copy()
gt, t = [], rng.uniform(0.3, 0.8)
while t < dur - 0.8:
is_w = rng.uniform() < 0.12
pool = swoosh if is_w else hits
sig, al, cat, name = pool[int(rng.integers(len(pool)))]
lo = int(max(0, (t - 0.05) * SR))
bed_db = 20 * np.log10(np.sqrt(np.mean(bed_only[lo: lo + int(0.3 * SR)] ** 2)) + 1e-9)
snr = rng.uniform(-14, 14)
pk = 20 * np.log10(np.sqrt(np.max(np.convolve(sig ** 2, np.ones(480) / 480, 'same'))) + 1e-9)
g = 10 ** ((bed_db + snr - pk) / 20)
s0 = int(round(t * SR)) - int(round(al * SR))
a, b = max(0, s0), min(len(y), s0 + len(sig))
y[a:b] += (sig[a - s0: b - s0] * g).astype(np.float32)
gt.append({'t': round(float(t), 4), 'align': 'peak' if is_w else 'attack', 'snr_db': round(float(snr), 1),
'cat': cat, 'file': name})
t += rng.uniform(0.35, 1.6) + (1.0 if is_w else 0.0)
pk = np.abs(y).max()
if pk > 0.89:
y *= 0.89 / pk
bed_only *= 0.89 / pk
return y, bed_only, {'beds': beds, 'boundaries': edges[1:-1], 'events': gt, 'duration': dur}
def main():
ap = argparse.ArgumentParser()
ap.add_argument('--n', type=int, default=8)
ap.add_argument('--dur', type=float, default=30.0)
ap.add_argument('--out', default='out/timeline/mixes')
a = ap.parse_args()
os.makedirs(a.out, exist_ok=True)
rng = np.random.default_rng(20261014)
music = unused('music')
amb = unused('ambience')
hits = [(x, al, c, os.path.basename(f)) for c in HITS for f in unused(c) for x, al in one_shots(f)]
sw = [(x, al, 'motion', os.path.basename(f)) for f in unused('motion') for x, al in whooshes(f)]
print(f'pools: {len(music)} music, {len(amb)} ambience, {len(hits)} one-shots, {len(sw)} whooshes')
for k in range(a.n):
y, b, meta = build(k, a.dur, (music, amb, hits, sw), rng)
sf.write(os.path.join(a.out, f'mix_{k}.wav'), y, SR, subtype='FLOAT')
sf.write(os.path.join(a.out, f'mix_{k}_bed.wav'), b, SR, subtype='FLOAT')
json.dump(meta, open(os.path.join(a.out, f'mix_{k}.json'), 'w'), indent=1)
print(f'mix_{k}: {len(meta["events"])} events, boundaries {meta["boundaries"]}')
if __name__ == '__main__':
main()