Skip to content

Ears for coding agents: let AI hear, review and make sound. CPU-only, open source.

15.0 KB
Raw Blame
"""Measurements of one sound-design event inside a finished film mix (music bed underneath), bed-subtracted.

    from measure import measure
    m = measure(x, t0, t1, bed=(b0, b1), cut=6.38)      # x = stereo float array at 48 kHz, times in seconds

The bed is the mix just before the event (default: the quietest 150 ms in the 0.6 s before t0, quiet_bed()). Everything that can be is measured
on (event power - bed power), so the music under the event mostly drops out; where the event does not rise over the bed
in a band, that band is 'masked' (floored) and says nothing about the event.

Returned (all plain floats / lists, JSON-ready):
  dur_s, bed_lufs (K-weighted mean level of the bed, ~LUFS; bed_silent when < -60), ev_lufs (same over the event's loudest 100 ms),
  over_bed_db (ev_lufs - bed_lufs, a silent bed counted as -70: 'how far it sticks out of the music'),
  snr_db (bed-subtracted event power / bed power over the whole event window, broadband),
  third_oct: {centres, ev_db (event power per band, dB re its total), bed_db (same scale), excess_db (ev - bed in each
             band), masked (bands where the event adds < 1 dB)}, tilt_db_oct (slope of the bed-subtracted balance,
             100 Hz - 10 kHz, least squares), shares (5 broad regions, dB of total: <150, 150-1k, 1-4k, 4-12k, >12k),
  attack_ms (from the first ms within 20 dB of the peak to the peak, 1 ms bed-subtracted envelope), rise_ms (10 -> 90 %
             amplitude), t_peak (s, absolute), peak_frac (where in [t0, t1] the peak sits),
  t20_ms / t40_ms (peak -> first fall 20 / 40 dB under it; None when the bed comes first), decay_db_s (fit over the
             first 20 dB of the fall), decay_shape ('exp' = straight in dB, 'convex' = holds then drops (gated / faded),
             'concave' = drops fast then a long tail (reverb)),
  centroid: start / peak / end (Hz, bed-subtracted, frames >= -20 dB re peak), sweep_oct (end - start in octaves),
            sweep_oct_s, the 20 ms track (t, Hz)
  f0_track (dominant peak of the 20-250 Hz band per 20 ms frame, for sub drops / booms: start, end, octaves),
  stereo: side_mid_db (bed-subtracted side / mid power), corr (L/R correlation of the event window), pan_db range
          (L-R level difference over 20 ms frames: min, max, travel), width words
  cut_ms: t_peak - cut and t_attack_start - cut (ms) when a cut is given
"""
import numpy as np
from scipy import signal

SR = 48000
CENTRES = [1000 * 2 ** (k / 3) for k in range(-16, 13)]          # 25 Hz ... 16 kHz (29 bands)
KW = [((1.53512485958697, -2.69169618940638, 1.19839281085285), (1.0, -1.69065929318241, 0.73248077421585)),
      ((1.0, -2.0, 1.0), (1.0, -1.99004745483398, 0.99007225036621))]
REGIONS = [(0, 150), (150, 1000), (1000, 4000), (4000, 12000), (12000, 24000)]


def kweight(m):
    y = m
    for b, a in KW:
        y = signal.lfilter(b, a, y)
    return y


def _seg(x, a, b):
    return x[max(0, int(round(a * SR))):max(0, int(round(b * SR)))]


def _psd(m, nfft=None):
    """mean power per sample spectrum (Hann, zero-padded), returns f, P such that sum(P) ~ mean(m**2)"""
    n = len(m)
    nfft = nfft or max(8192, 1 << int(np.ceil(np.log2(max(n, 2)))))
    w = np.hanning(n)
    X = np.fft.rfft(m * w, nfft)
    P = np.abs(X) ** 2 / (np.sum(w ** 2) * n) * n / 1.0
    P[1:-1] *= 2
    P /= n
    f = np.fft.rfftfreq(nfft, 1 / SR)
    return f, P


def _bands(f, P):
    out = []
    for c in CENTRES:
        lo, hi = c * 2 ** (-1 / 6), c * 2 ** (1 / 6)
        k = (f >= lo) & (f < hi)
        out.append(P[k].sum() if k.any() else P[np.argmin(np.abs(f - c))])
    return np.array(out)


def _env_db(m, ms=1.0):
    n = max(1, int(SR * ms / 1000))
    k = len(m) // n
    return 10 * np.log10((m[:k * n].reshape(k, n) ** 2).mean(1) + 1e-14)


def _frames(m, ms=20.0):
    n = int(SR * ms / 1000)
    k = len(m) // n
    return m[:k * n].reshape(k, n), n


def quiet_bed(x, t0, search=0.6, win=0.15, gap=0.015):
    """the bed under an event: the quietest `win` s (K-weighted power, 10 ms hop) in the `search` s before it (so a
    run of clicks or a previous hit does not count as bed); shorter at the start of a file"""
    a, b = max(0.0, t0 - search), max(0.0, t0 - gap)
    if b - a <= win:
        return (a, b) if b - a >= 0.03 else (a, max(b, a + 0.03))
    k = kweight(_seg(x, a, b).mean(1)) ** 2
    n, h = int(win * SR), int(0.01 * SR)
    c = np.cumsum(np.concatenate([[0.0], k]))
    starts = np.arange(0, len(k) - n + 1, h)
    p = (c[starts + n] - c[starts]) / n
    i = int(starts[int(np.argmin(p))])
    return (round(a + i / SR, 4), round(a + (i + n) / SR, 4))


def measure(x, t0, t1, bed=None, cut=None):
    x = np.asarray(x, np.float64)
    if x.ndim == 1:
        x = np.stack([x, x], 1)
    if bed is None:
        bed = quiet_bed(x, t0)
    ev = _seg(x, t0, t1)
    bd = _seg(x, *bed)
    if len(bd) < int(0.02 * SR):
        bd = np.zeros((int(0.05 * SR), 2)) + 1e-7
    mev, mbd = ev.mean(1), bd.mean(1)
    r = {'t0': round(t0, 3), 't1': round(t1, 3), 'dur_s': round(t1 - t0, 3), 'bed': [round(bed[0], 3), round(bed[1], 3)]}

    # levels (K-weighted)
    pre = _seg(x, max(0, bed[0] - 0.5), t1).mean(1)
    kw = kweight(pre)
    off = len(pre) - len(mev)
    kb = kw[int(round((bed[0] - max(0, bed[0] - 0.5)) * SR)):][:len(mbd)]
    ke = kw[off:]
    bed_p = np.mean(kb ** 2) + 1e-14
    win = int(0.1 * SR)
    if len(ke) > win:
        c = np.convolve(ke ** 2, np.ones(win) / win, 'valid')
        ev_p = c.max()
    else:
        ev_p = np.mean(ke ** 2)
    r['bed_lufs'] = round(-0.691 + 10 * np.log10(bed_p), 1)
    r['ev_lufs'] = round(-0.691 + 10 * np.log10(ev_p + 1e-14), 1)
    r['bed_silent'] = bool(r['bed_lufs'] < -60)
    r['over_bed_db'] = round(r['ev_lufs'] - max(r['bed_lufs'], -70.0), 1)   # a silent bed counts as -70
    pe, pb = np.mean(mev ** 2), np.mean(mbd ** 2) + 1e-14
    r['snr_db'] = round(10 * np.log10(max(pe - pb, 1e-14) / pb), 1)

    # third-octave balance, bed-subtracted
    f, Pe = _psd(mev)
    # bed spectrum: average of bed chunks of the event's length (or the whole bed when shorter)
    L = min(len(mbd), len(mev))
    chunks = [mbd[i:i + L] for i in range(0, len(mbd) - L + 1, max(1, L // 2))] or [mbd]
    Pb = np.mean([_psd(c, len(f) * 2 - 2)[1] for c in chunks], 0)
    Be, Bb = _bands(f, Pe), _bands(f, Pb)
    D = np.maximum(Be - Bb, 1e-14)
    masked = (10 * np.log10(Be + 1e-14) - 10 * np.log10(Bb + 1e-14)) < 1.0
    D[masked] = np.minimum(D[masked], Be[masked] * 0.05)
    tot = D.sum() + 1e-14
    evdb = 10 * np.log10(D / tot)
    r['third_oct'] = {'centres': [round(c, 1) for c in CENTRES],
                      'ev_db': [round(max(v, -60), 1) for v in evdb],
                      'bed_db': [round(max(v, -60), 1) for v in 10 * np.log10(Bb / tot + 1e-14)],
                      'excess_db': [round(v, 1) for v in 10 * np.log10(Be + 1e-14) - 10 * np.log10(Bb + 1e-14)],
                      'masked': [bool(v) for v in masked]}
    lc = np.log2(np.array(CENTRES))
    k = (np.array(CENTRES) >= 100) & (np.array(CENTRES) <= 10000) & ~masked
    if k.sum() >= 4:
        r['tilt_db_oct'] = round(float(np.polyfit(lc[k] - lc[k].mean(), evdb[k], 1)[0]), 2)
    else:
        r['tilt_db_oct'] = None
    sh = []
    for lo, hi in REGIONS:
        kk = (f >= lo) & (f < hi)
        sh.append(round(10 * np.log10(max(Pe[kk].sum() - Pb[kk].sum(), 1e-14) / max(Pe.sum() - Pb.sum(), 1e-14)), 1))
    r['shares'] = sh
    # spectral centroid of the bed-subtracted spectrum
    Dp = np.maximum(Pe - Pb, 0)
    r['centroid_hz'] = round(float((f * Dp).sum() / (Dp.sum() + 1e-14)), 0)

    # envelope (1 ms, bed-subtracted power)
    e = 10 ** (_env_db(mev) / 10) - pb
    e = np.maximum(e, pb * 1e-3)
    edb = 10 * np.log10(e)
    ip = int(np.argmax(edb))
    pk = edb[ip]
    i20 = int(np.argmax(edb[:ip + 1] >= pk - 20))
    r['attack_ms'] = float(ip - i20)
    amp = np.sqrt(e)
    i10 = int(np.argmax(amp[:ip + 1] >= 0.1 * amp[ip]))
    i90 = int(np.argmax(amp[:ip + 1] >= 0.9 * amp[ip]))
    r['rise_ms'] = float(i90 - i10)
    r['t_peak'] = round(t0 + ip / 1000, 4)
    r['t_attack'] = round(t0 + i20 / 1000, 4)
    r['peak_frac'] = round(ip / max(1, len(edb)), 3)
    bedlvl = 10 * np.log10(pb)
    # smoothed envelope for decay (10 ms)
    es = np.convolve(e, np.ones(10) / 10, 'same')
    esdb = 10 * np.log10(es + 1e-14)
    after = esdb[ip:]
    pk_s = esdb[ip]

    def fall(db):
        j = np.nonzero(after <= pk_s - db)[0]
        return float(j[0]) if len(j) else None
    r['t20_ms'], r['t40_ms'] = fall(20), fall(40)
    r['peak_over_bed_db'] = round(float(min(pk_s - bedlvl, pk_s - 10 * np.log10(1e-7))), 1)
    j20 = r['t20_ms']
    if j20 and j20 >= 8:
        seg = after[:int(j20) + 1]
        tt = np.arange(len(seg)) / 1000
        p1 = np.polyfit(tt, seg, 1)
        r['decay_db_s'] = round(float(p1[0]), 1)
        mid = seg[len(seg) // 2] - (seg[0] + seg[-1]) / 2
        r['decay_shape'] = 'convex' if mid > 2.5 else ('concave' if mid < -2.5 else 'exp')
        r['decay_mid_dev_db'] = round(float(mid), 1)
    else:
        r['decay_db_s'] = None
        r['decay_shape'] = None
    # rise shape (onset -> peak) for swells / whooshes: mid-point deviation from a straight dB ramp
    if ip - i20 >= 20:
        seg = esdb[i20:ip + 1]
        mid = seg[len(seg) // 2] - (seg[0] + seg[-1]) / 2
        r['rise_shape'] = 'convex' if mid > 2.5 else ('concave' if mid < -2.5 else 'exp')
        r['rise_mid_dev_db'] = round(float(mid), 1)

    # centroid track (20 ms frames, bed-subtracted)
    Fm, n = _frames(mev, 20)
    if len(Fm) >= 2:
        w = np.hanning(n)
        S = np.abs(np.fft.rfft(Fm * w, 2048)) ** 2
        ff = np.fft.rfftfreq(2048, 1 / SR)
        Bs = np.abs(np.fft.rfft(chunks[0][:n] * w if len(chunks[0]) >= n else np.zeros(n), 2048)) ** 2
        if len(mbd) >= n:
            Bf, _ = _frames(mbd, 20)
            Bs = (np.abs(np.fft.rfft(Bf * w, 2048)) ** 2).mean(0)
        Sd = np.maximum(S - Bs, 0)
        en = Sd.sum(1)
        ok = en >= en.max() * 0.01
        cen = (Sd * ff).sum(1) / (en + 1e-14)
        idx = np.nonzero(ok)[0]
        tr = [(round(t0 + (i + 0.5) * 0.02, 3), round(float(cen[i]), 0)) for i in idx]
        r['centroid'] = {'start': tr[0][1], 'peak': round(float(cen[int(np.argmax(en))]), 0), 'end': tr[-1][1],
                         'track': tr}
        if len(tr) >= 3 and tr[0][1] > 0 and tr[-1][1] > 0:
            # robust: medians of first / last quarter of active frames
            q = max(1, len(tr) // 4)
            c0 = np.median([v for _, v in tr[:q]])
            c1 = np.median([v for _, v in tr[-q:]])
            r['centroid']['sweep_oct'] = round(float(np.log2(c1 / c0)), 2)
            span = tr[-1][0] - tr[0][0]
            r['centroid']['sweep_oct_s'] = round(float(np.log2(c1 / c0) / max(span, 0.02)), 2)
            # rise-to-peak and peak-to-end sweeps
            ipk = int(np.argmax([en[i] for i in idx]))
            cp = np.median([v for _, v in tr[max(0, ipk - 1):ipk + 2]])
            r['centroid']['to_peak_oct'] = round(float(np.log2(cp / c0)), 2)
            r['centroid']['from_peak_oct'] = round(float(np.log2(c1 / cp)), 2)
        # low f0 track (20-250 Hz peak) on 80 ms frames for sub content
        F8, n8 = _frames(mev, 80)
        if len(F8) >= 2:
            S8 = np.abs(np.fft.rfft(F8 * np.hanning(n8), 16384)) ** 2
            f8 = np.fft.rfftfreq(16384, 1 / SR)
            kb = (f8 >= 20) & (f8 <= 250)
            pk8 = f8[kb][np.argmax(S8[:, kb], 1)]
            lv8 = 10 * np.log10(S8[:, kb].max(1) + 1e-14)
            okk = lv8 >= lv8.max() - 25
            ii = np.nonzero(okk)[0]
            r['f0_track'] = {'track': [(round(t0 + (i + 0.5) * 0.08, 2), round(float(pk8[i]), 1)) for i in ii],
                             'start': round(float(pk8[ii[0]]), 1), 'end': round(float(pk8[ii[-1]]), 1),
                             'oct': round(float(np.log2(pk8[ii[-1]] / max(pk8[ii[0]], 1))), 2)}

    # ridge: the dominant (bed-subtracted) peak frequency per 10 ms frame, 80 Hz - 16 kHz (tonal sweeps: zips, pops,
    # stings, the tonal layer of a riser); frames within 20 dB of the loudest frame
    Fr, nr = _frames(mev, 10)
    if len(Fr) >= 3:
        wr = np.hanning(nr)
        Sr = np.abs(np.fft.rfft(Fr * wr, 8192)) ** 2
        fr = np.fft.rfftfreq(8192, 1 / SR)
        if len(mbd) >= nr:
            Bfr, _ = _frames(mbd, 10)
            Sr = np.maximum(Sr - (np.abs(np.fft.rfft(Bfr * wr, 8192)) ** 2).mean(0) * 1.5, 0)
        kb = (fr >= 80) & (fr <= 16000)
        pkf = fr[kb][np.argmax(Sr[:, kb], 1)]
        pe_ = Sr[:, kb].max(1)
        ok = pe_ >= pe_.max() * 0.01
        ii = np.nonzero(ok)[0]
        tr = [(round(t0 + (i + 0.5) * 0.01, 3), round(float(pkf[i]), 0)) for i in ii]
        q = max(1, len(tr) // 5)
        f0_, f1_ = float(np.median([v for _, v in tr[:q]])), float(np.median([v for _, v in tr[-q:]]))
        r['ridge'] = {'start_hz': round(f0_, 0), 'end_hz': round(f1_, 0),
                      'min_hz': round(float(min(v for _, v in tr)), 0), 'max_hz': round(float(max(v for _, v in tr)), 0),
                      'peak_hz': round(float(pkf[int(np.argmax(pe_))]), 0),
                      'sweep_oct': round(float(np.log2(max(f1_, 1) / max(f0_, 1))), 2), 'track': tr}
    # amplitude modulation of the 1 ms envelope (stutter / gate / typing rate): strongest 4-60 Hz component
    if len(edb) >= 200:
        env = np.sqrt(e)
        trend = np.convolve(env, np.ones(101) / 101, 'same')
        mod = env - trend
        Fm_ = np.abs(np.fft.rfft(mod * np.hanning(len(mod)), 1 << 15))
        fm = np.fft.rfftfreq(1 << 15, 1 / 1000)
        km = (fm >= 4) & (fm <= 60)
        if km.any():
            j = int(np.argmax(Fm_[km]))
            r['am_rate_hz'] = round(float(fm[km][j]), 1)
            r['am_depth'] = round(float(np.std(mod) / (np.mean(env) + 1e-12)), 2)

    # stereo
    Lc, Rc = ev[:, 0], ev[:, 1]
    M, Sd_ = (Lc + Rc) / 2, (Lc - Rc) / 2
    Mb, Sb = mbd, (bd[:, 0] - bd[:, 1]) / 2
    pm = max(np.mean(M ** 2) - np.mean(Mb ** 2), 1e-14)
    ps = max(np.mean(Sd_ ** 2) - np.mean(Sb ** 2), 1e-14)
    r['stereo'] = {'side_mid_db': round(max(10 * np.log10(ps / pm), -40.0), 1),
                   'side_mid_raw_db': round(10 * np.log10((np.mean(Sd_ ** 2) + 1e-14) / (np.mean(M ** 2) + 1e-14)), 1),
                   'bed_side_mid_db': round(10 * np.log10((np.mean(Sb ** 2) + 1e-14) / (np.mean(Mb ** 2) + 1e-14)), 1),
                   'corr': round(float(np.corrcoef(Lc, Rc)[0, 1]) if np.std(Lc) > 0 and np.std(Rc) > 0 else 1.0, 3)}
    FL, _ = _frames(Lc, 20)
    FR, _ = _frames(Rc, 20)
    if len(FL):
        el, er = (FL ** 2).mean(1), (FR ** 2).mean(1)
        act = (el + er) >= (el + er).max() * 0.05
        pan = 10 * np.log10((el[act] + 1e-14) / (er[act] + 1e-14))
        r['stereo']['pan_db'] = {'min': round(float(pan.min()), 1), 'max': round(float(pan.max()), 1),
                                 'travel': round(float(pan.max() - pan.min()), 1),
                                 'start': round(float(pan[0]), 1), 'end': round(float(pan[-1]), 1)}
    if cut is not None:
        r['cut'] = round(cut, 3)
        r['peak_vs_cut_ms'] = round((r['t_peak'] - cut) * 1000, 0)
        r['attack_vs_cut_ms'] = round((r['t_attack'] - cut) * 1000, 0)
        r['end_vs_cut_ms'] = round((t1 - cut) * 1000, 0)
    return r