Skip to content

Ears for coding agents: let AI hear, review and make sound. CPU-only, open source.

11.9 KB
Raw Blame
"""Measure, export and summarise the curated pro motion-SFX events (eval/forge/pro/events.py).

    python eval/forge/pro/study.py                 # measure all, export trims, manifest, registry, stats, sheets
    python eval/forge/pro/study.py --no-export     # measurements + stats only
    python eval/forge/pro/study.py --sheets-only   # only the per-category spectrogram crop sheets

Outputs (local only: the films are third-party):
  out/refs_pro/<category>/{forge,bench}/<ref>_<ms>.wav   trims (trim + fade only, original gain, stereo 48 kHz 24-bit)
  out/refs_pro/manifest.json    every event: film id, times, category, cut, bed level, measurements, earshot evidence
  out/refs_pro/registry.json    earshot.forge.refsets format: {'version', 'categories': {cat: {forge, bench, note}}}
  out/refs_pro/stats.json       per category: count, medians / quartiles of the measured numbers
  out/refs_pro/_work/cat/<category>.png   spectrogram crops of the category's events (look at these)
The split is by film: a film is 'forge' or 'bench' for every category (SPLIT below), so a forge tuned on the forge
side is benchmarked on films it never heard.
"""
import argparse
import json
import os
import sys

import numpy as np
import soundfile as sf

HERE = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, HERE)
from videos import OUT, VIDEOS, WORK, wav  # noqa: E402
from measure import measure  # noqa: E402
from events import EVENTS, SPLIT, NOTES  # noqa: E402

SR = 48000


def load(ref, _c={}):
    if ref not in _c:
        x, sr = sf.read(wav(ref), dtype='float64', always_2d=True)
        assert sr == SR
        _c[ref] = x
    return _c[ref]


def tl(ref, _c={}):
    if ref not in _c:
        p = f'{WORK}/tl/{ref}.json'
        _c[ref] = json.load(open(p, encoding='utf-8')) if os.path.exists(p) else None
    return _c[ref]


def evidence(ref, t0, t1):
    r = tl(ref)
    if not r:
        return {}
    evs = []
    for e in r['events']:
        if t0 - 0.02 <= e['t'] <= t1:
            evs.append({'t': e['t'], 'kind': e.get('kind'), 'label': e.get('label'),
                        'tag': (e.get('tags') or [{}])[0].get('tag'), 'prominence_db': e.get('prominence_db'),
                        'bed_db': e.get('bed_db'), 'contrast_db': e.get('contrast_db'), 'band': e.get('band'),
                        'exposed': e.get('exposed'), 'length_ms': e.get('length_ms')})
    cuts = [c for c in (r.get('cuts') or []) if t0 - 0.5 <= c <= t1 + 0.5]
    return {'timeline_events': evs[:12], 'cuts_near': cuts}


def norm(e):
    d = {'ref': e[0], 't0': e[1], 't1': e[2], 'cat': e[3], 'cut': e[4] if len(e) > 4 else None,
         'note': e[5] if len(e) > 5 else '', 'bed': e[6] if len(e) > 6 else None}
    d['export'] = not d['note'].startswith('noexp')
    return d


def export(d, x, split):
    pre = 0.005
    a, b = max(0, int((d['t0'] - pre) * SR)), min(len(x), int(d['t1'] * SR))
    y = x[a:b].copy()
    fi = int(0.002 * SR)
    fo = int(min(0.03 + 0.1 * (d['t1'] - d['t0']), 0.25) * SR)
    fo = min(fo, len(y) // 3)
    y[:fi] *= np.linspace(0, 1, fi)[:, None]
    y[-fo:] *= np.linspace(1, 0, fo)[:, None] ** 2
    folder = f"{OUT}/{d['cat']}/{split}"
    os.makedirs(folder, exist_ok=True)
    f = f"{folder}/{d['ref']}_{int(round(d['t0'] * 1000)):06d}.wav"
    sf.write(f, y.astype(np.float32), SR, subtype='PCM_24')
    return f, round(a / SR, 4), round(b / SR, 4), round(fo / SR, 3)


NUMS = ['dur_s', 'over_bed_db', 'snr_db', 'peak_over_bed_db', 'bed_lufs', 'ev_lufs', 'attack_ms', 'rise_ms',
        'peak_frac', 't20_ms', 't40_ms', 'decay_db_s', 'tilt_db_oct', 'centroid_hz', 'peak_vs_cut_ms',
        'attack_vs_cut_ms', 'end_vs_cut_ms']


def stats(rows):
    out = {}
    cats = sorted({r['cat'] for r in rows})
    for c in cats:
        rr = [r for r in rows if r['cat'] == c]
        s = {'n': len(rr), 'films': sorted({r['ref'] for r in rr})}
        for k in NUMS:
            v = [r['m'].get(k) for r in rr if r['m'].get(k) is not None]
            if v:
                s[k] = {'med': round(float(np.median(v)), 2), 'q1': round(float(np.percentile(v, 25)), 2),
                        'q3': round(float(np.percentile(v, 75)), 2), 'min': round(float(min(v)), 2),
                        'max': round(float(max(v)), 2), 'n': len(v)}
        for k in ('sweep_oct', 'to_peak_oct', 'from_peak_oct', 'start', 'peak', 'end'):
            v = [r['m'].get('centroid', {}).get(k) for r in rr if r['m'].get('centroid', {}).get(k) is not None]
            if v:
                s['centroid_' + k] = {'med': round(float(np.median(v)), 2), 'q1': round(float(np.percentile(v, 25)), 2),
                                      'q3': round(float(np.percentile(v, 75)), 2), 'n': len(v)}
        for k in ('side_mid_db', 'side_mid_raw_db', 'corr'):
            v = [r['m']['stereo'].get(k) for r in rr]
            s['stereo_' + k] = {'med': round(float(np.median(v)), 2), 'q1': round(float(np.percentile(v, 25)), 2),
                                'q3': round(float(np.percentile(v, 75)), 2)}
        v = [r['m']['stereo'].get('pan_db', {}).get('travel') for r in rr if r['m']['stereo'].get('pan_db')]
        if v:
            s['pan_travel_db'] = {'med': round(float(np.median(v)), 2), 'max': round(float(max(v)), 2)}
        for k in ('start_hz', 'end_hz', 'min_hz', 'max_hz', 'peak_hz', 'sweep_oct'):
            v = [r['m']['ridge'][k] for r in rr if r['m'].get('ridge')]
            if v:
                s['ridge_' + k] = {'med': round(float(np.median(v)), 2), 'min': round(float(min(v)), 2),
                                   'max': round(float(max(v)), 2), 'n': len(v)}
        v = [r['m'].get('am_rate_hz') for r in rr if r['m'].get('am_rate_hz')]
        if v:
            s['am_rate_hz'] = {'med': round(float(np.median(v)), 1), 'min': min(v), 'max': max(v)}
        f0 = [r['m']['f0_track'] for r in rr if r['m'].get('f0_track')]
        if f0:
            s['f0_start'] = round(float(np.median([q['start'] for q in f0])), 1)
            s['f0_end'] = round(float(np.median([q['end'] for q in f0])), 1)
            s['f0_oct'] = round(float(np.median([q['oct'] for q in f0])), 2)
        # bed-subtracted 1/3-octave balance: median over events of each band (dB re total), unmasked bands only
        ev = np.array([r['m']['third_oct']['ev_db'] for r in rr])
        mk = np.array([r['m']['third_oct']['masked'] for r in rr])
        med = []
        for j in range(ev.shape[1]):
            vv = ev[~mk[:, j], j]
            med.append(round(float(np.median(vv)), 1) if len(vv) else None)
        s['third_oct_med_db'] = med
        s['third_oct_centres'] = rr[0]['m']['third_oct']['centres']
        sh = np.array([r['m']['shares'] for r in rr])
        s['shares_med_db'] = [round(float(v), 1) for v in np.median(sh, 0)]
        shapes = [r['m'].get('decay_shape') for r in rr if r['m'].get('decay_shape')]
        s['decay_shapes'] = {k: shapes.count(k) for k in sorted(set(shapes))}
        rshapes = [r['m'].get('rise_shape') for r in rr if r['m'].get('rise_shape')]
        s['rise_shapes'] = {k: rshapes.count(k) for k in sorted(set(rshapes))}
        out[c] = s
    return out


def cat_sheets(rows, max_per=12):
    import matplotlib
    matplotlib.use('Agg')
    import matplotlib.pyplot as plt
    import viewer
    os.makedirs(f'{WORK}/cat', exist_ok=True)
    outs = []
    for c in sorted({r['cat'] for r in rows}):
        rr = [r for r in rows if r['cat'] == c][:max_per]
        cols = 3
        nrow = int(np.ceil(len(rr) / cols))
        fig = plt.figure(figsize=(5.6 * cols, 3.3 * nrow), dpi=90)
        gs = fig.add_gridspec(2 * nrow, cols, height_ratios=[3, 1.2] * nrow, hspace=0.3, wspace=0.18)
        for i, r in enumerate(rr):
            x = load(r['ref']).astype(np.float32)
            cuts = (tl(r['ref']) or {}).get('cuts') or []
            evs = (tl(r['ref']) or {}).get('events') or []
            pad = max(0.15, 0.25 * (r['t1'] - r['t0']))
            q, k = divmod(i, cols)
            m = r['m']
            lab = (f"{c} {r['t0']:.2f}-{r['t1']:.2f} +{m['over_bed_db']}dB atk {m['attack_ms']:.0f}ms "
                   f"sw {m.get('centroid', {}).get('sweep_oct', '-')}oct")
            ax, ax2 = viewer.lane(fig, gs[2 * q, k], gs[2 * q + 1, k], x, r['t0'] - pad, r['t1'] + pad, cuts, evs,
                                  title=f"{r['ref']} {lab}")
            ax.axvline(r['t0'], color='w', lw=0.6, ls=':')
            ax.axvline(r['t1'], color='w', lw=0.6, ls=':')
            ax2.tick_params(labelsize=6)
        out = f'{WORK}/cat/{c}.png'
        fig.savefig(out, bbox_inches='tight')
        plt.close(fig)
        outs.append(out)
    return outs


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument('--no-export', action='store_true')
    ap.add_argument('--sheets-only', action='store_true')
    ap.add_argument('--no-sheets', action='store_true')
    ap.add_argument('--cats', nargs='*')
    a = ap.parse_args()
    rows = []
    for e in EVENTS:
        d = norm(e)
        if a.cats and d['cat'] not in a.cats:
            continue
        x = load(d['ref'])
        d['m'] = measure(x, d['t0'], d['t1'], bed=d['bed'], cut=d['cut'])
        rows.append(d)
    if a.sheets_only:
        print('\n'.join(cat_sheets(rows)))
        return
    man = []
    reg = {'version': 1, 'source': 'pro motion-design launch films (third-party, local only); see '
                                   'docs/motion-sfx-study.md', 'categories': {}}
    if not a.no_export:
        import shutil
        for c in sorted({r['cat'] for r in rows}):
            if os.path.isdir(f'{OUT}/{c}'):
                shutil.rmtree(f'{OUT}/{c}')
    for d in rows:
        split = SPLIT[d['ref']]
        ent = {'film': d['ref'], 'source_file': VIDEOS[d['ref']], 'category': d['cat'], 't0': d['t0'], 't1': d['t1'],
               'cut': d['cut'], 'split': split, 'note': d['note'], 'bed_lufs': d['m']['bed_lufs'],
               'over_bed_db': d['m']['over_bed_db'], 'snr_db': d['m']['snr_db'], 'measured': d['m'],
               'earshot': evidence(d['ref'], d['t0'], d['t1'])}
        if not a.no_export and d['export']:
            f, ta, tb, fo = export(d, load(d['ref']), split)
            ent.update({'file': f, 'trim_s': [ta, tb], 'fade_in_s': 0.002, 'fade_out_s': fo, 'gain_db': 0.0})
            c = reg['categories'].setdefault(d['cat'], {'forge': [], 'bench': [], 'note': NOTES.get(d['cat'], '')})
            c[split].append(os.path.relpath(f, OUT).replace(chr(92), '/'))
        man.append(ent)
    st = stats(rows)
    with open(f'{OUT}/stats.json', 'w', encoding='utf-8') as fh:
        json.dump(st, fh, indent=1)
    if not a.no_export:
        with open(f'{OUT}/manifest.json', 'w', encoding='utf-8') as fh:
            json.dump(man, fh, indent=1)
        with open(f'{OUT}/registry.json', 'w', encoding='utf-8') as fh:
            json.dump(reg, fh, indent=1)
        # sanity: the split never leaks a film
        sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(HERE))))
        from earshot.forge import refsets
        srcs = {f"{OUT}/{p}": p.split('/')[-1].split('_')[0] + '_' + p.split('/')[-1].split('_')[1]
                for cc in reg['categories'].values() for s in ('forge', 'bench') for p in cc[s]}
        reg_abs = refsets.load(f'{OUT}/registry.json')
        srcs = {}
        for cc in reg_abs['categories'].values():
            for s in ('forge', 'bench'):
                for p in cc[s]:
                    srcs[p] = os.path.basename(p).rsplit('_', 1)[0]
        bad = refsets.check_split(reg_abs, srcs)
        print('split check:', bad or 'ok')
    for c, s in st.items():
        def g(k, f='med'):
            return s.get(k, {}).get(f, '-') if isinstance(s.get(k), dict) else '-'
        print(f"{c:12s} n={s['n']:2d} films={','.join(f[4:] for f in s['films'])} dur {g('dur_s')} "
              f"over_bed {g('over_bed_db')} atk {g('attack_ms')} t20 {g('t20_ms')} tilt {g('tilt_db_oct')} "
              f"cen {g('centroid_hz')} sweep {g('centroid_sweep_oct')} S/M {g('stereo_side_mid_db')} "
              f"pk-cut {g('peak_vs_cut_ms')}")
    if not a.no_sheets:
        print('\n'.join(cat_sheets(rows)))


if __name__ == '__main__':
    main()