Skip to content

Ears for coding agents: let AI hear, review and make sound. CPU-only, open source.

7.5 KB
Raw Blame
"""Choose how timeline tags events: excerpt treatment x vocabulary, scored on the synthetic mixes (mixes.py; every
one-shot carries its library category). Never uses the Clawd mix.

    EARSHOT_THREADS=2 .venv/Scripts/python.exe eval/timeline/tag_tune.py --vocab compact --variants delta,repeat,zero,iso

vocab     json = all event categories of earshot/data/vocab.json (kind from the category), compact = timeline's own
          EVENT_VOCAB (kind = its key)
variants  zero / repeat = CLAP on the event excerpt, zero- or repeat-padded to 10 s (repeat: what timeline ships);
          delta = zero-padded cosine minus the same for the bed just before the event (the first build);
          deltarep = the same with repeat padding;
          win = minus the 3 s section window around the event; debed = excerpt after subtracting the bed's average
          spectrum; iso = the one-shot alone (an upper bound: no bed at all)
--center  subtract this share of each phrase's mean cosine over the file's events (hub-phrase correction)
A tag is plausible when the kind of the top phrase fits the one-shot's library category (FIT below).
Scores here are the first build's (z within each event); timeline now also damps hub phrases (tag_bg.py).
"""
import argparse
import glob
import json
import os
import sys
from collections import Counter

import numpy as np
from scipy import signal

HERE = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, os.path.dirname(os.path.dirname(HERE)))
sys.path.insert(0, HERE)
from earshot import audio, timeline as tl  # noqa: E402
from earshot.models import clap  # noqa: E402
import mixes  # noqa: E402

FIT = {'keyboard': {'KEY'}, 'motion': {'WHOOSH', 'RISER'}, 'garden': {'STEP', 'FOLEY'},
       'toy': {'FOLEY', 'UI'}, 'magic': {'NOTE', 'UI'}, 'foley': {'FOLEY', 'STEP', 'UI'}}
JSON_CATS = ('transitions', 'impacts', 'ui', 'foley', 'instruments')


def json_kind(term, cat):
    """kind of a vocab.json phrase, for comparing against the compact vocabulary"""
    if cat == 'transitions':
        return 'RISER' if any(w in term for w in ('riser', 'uplifter', 'swell')) else 'WHOOSH'
    if cat == 'impacts':
        return 'HIT'
    if cat == 'ui':
        return 'KEY' if any(w in term for w in ('keyboard', 'key press', 'spacebar', 'typing')) else 'UI'
    if cat == 'foley':
        return 'STEP' if 'footsteps' in term else 'FOLEY'
    return 'NOTE'


def vocabulary(c, which):
    if which == 'compact':
        terms, kinds, E = tl._event_matrix(c)
        return terms, [str(k) for k in kinds], E
    terms, cats, E = tl._vocab_matrix(c)
    keep = [i for i, k in enumerate(cats) if k in JSON_CATS]
    return [terms[i] for i in keep], [json_kind(terms[i], cats[i]) for i in keep], E[keep]


def debed(x, a, b, pre_s=0.5):
    """excerpt with the bed's average magnitude spectrum (pre_s before it) subtracted, -20 dB floor, original phase"""
    seg = tl._seg(x, a, b).astype(np.float64)
    pre = tl._seg(x, max(0.0, a - pre_s), max(0.0, a - 0.01)).astype(np.float64) if a > 0.1 else np.zeros(0)
    if len(pre) < 2048 or len(seg) < 1024:
        return seg.astype(np.float32)
    _, _, P = signal.stft(pre, nperseg=1024, noverlap=768)
    _, _, S = signal.stft(seg, nperseg=1024, noverlap=768)
    mag = np.abs(S)
    y = signal.istft(np.maximum(mag - np.abs(P).mean(axis=1, keepdims=True), 0.1 * mag) * np.exp(1j * np.angle(S)),
                     nperseg=1024, noverlap=768)[1]
    return y[: len(seg)].astype(np.float32)


def iso(g):
    for d in (os.path.join(mixes.ASSETS, g['cat']), os.path.join(mixes.ASSETS, '_rejected', g['cat'])):
        f = os.path.join(d, g['file'])
        if os.path.exists(f):
            got = mixes.whooshes(f) if g['cat'] == 'motion' else mixes.one_shots(f)
            return got[0][0] if got else np.zeros(4800, np.float32)
    return np.zeros(4800, np.float32)


def cosines(v, c, E, x, fid, spans, picks):
    def emb(kind, sp, make, pad):
        return tl._cached(c, fid, sp, kind, lambda q: c.embed_audio([make(p, r) for p, r in q], pad=pad))

    def seg(p, r):
        return tl._seg(x, p, r)
    pre = [(max(0.0, p - (q - p) - 0.01), max(0.01, p - 0.01)) for p, q in spans]
    if v == 'iso':
        return c.embed_audio([iso(g) for g, _ in picks], pad='zero') @ E.T
    if v == 'delta':
        return emb('clap-zero', spans, seg, 'zero') @ E.T - emb('clap-zero', pre, seg, 'zero') @ E.T
    if v == 'deltarep':
        return emb('clap-repeat', spans, seg, 'repeatpad') @ E.T - emb('clap-repeat', pre, seg, 'repeatpad') @ E.T
    if v == 'win':
        dur = len(x) / tl.SR
        starts = list(np.arange(0.0, max(dur - 3.0, 0.0) + 1e-6, 2.0))
        if starts[-1] + 3.0 < dur - 0.5:
            starts.append(max(0.0, dur - 3.0))
        wsp = [(round(a, 3), round(min(dur, a + 3.0), 3)) for a in starts]
        W = emb('clap-repeat', wsp, seg, 'repeatpad')
        mids = np.array([(a + b) / 2 for a, b in wsp])
        ref = np.stack([W[int(np.argmin(np.abs(mids - (p + q) / 2)))] for p, q in spans])
        return emb('clap-repeat', spans, seg, 'repeatpad') @ E.T - ref @ E.T
    if v == 'debed':
        return emb('clap-debed', spans, lambda p, r: debed(x, p, r), 'zero') @ E.T
    return emb(f'clap-{v}', spans, seg, 'repeatpad' if v == 'repeat' else 'zero') @ E.T


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument('--dir', default='out/timeline/mixes')
    ap.add_argument('--vocab', default='compact', choices=('compact', 'json'))
    ap.add_argument('--variants', default='delta,repeat,zero')
    ap.add_argument('--center', type=float, default=0.0)
    a = ap.parse_args()
    c = clap.get()
    terms, kinds, E = vocabulary(c, a.vocab)
    rows = {v: [] for v in a.variants.split(',')}
    for f in sorted(glob.glob(os.path.join(a.dir, 'mix_*.json'))):
        meta = json.load(open(f))
        x = audio.load(f[:-5] + '.wav')
        ev = tl.detect_events(x)
        det = np.array([e['t'] for e in ev])
        picks = [(g, ev[int(np.argmin(np.abs(det - g['t'])))]) for g in meta['events']
                 if np.min(np.abs(det - g['t'])) <= (0.12 if g['align'] == 'peak' else 0.03)]
        spans = [tl._event_span(e, len(x) / tl.SR) for _, e in picks]
        for v in rows:
            cos = cosines(v, c, E, x, audio.file_id(f[:-5] + '.wav'), spans, picks)
            if a.center:
                cos = cos - a.center * cos.mean(axis=0, keepdims=True)
            z = (cos - cos.mean(axis=1, keepdims=True)) / (cos.std(axis=1, keepdims=True) + 1e-9)
            for j, (g, _) in enumerate(picks):
                idx = np.argsort(-z[j])[:3]
                rows[v].append({'cat': g['cat'], 'snr': g['snr_db'], 'top1': kinds[idx[0]] in FIT[g['cat']],
                                'top3': any(kinds[i] in FIT[g['cat']] for i in idx), 'tag': terms[idx[0]]})
    for v, r in rows.items():
        t1, t3 = np.array([q['top1'] for q in r]), np.array([q['top3'] for q in r])
        snr = np.array([q['snr'] for q in r])
        print(v, json.dumps({'vocab': a.vocab, 'center': a.center, 'n': len(r), 'top1': round(float(t1.mean()), 3),
                             'top3': round(float(t3.mean()), 3), 'top1_snr_ge0': round(float(t1[snr >= 0].mean()), 3),
                             'top1_snr_lt0': round(float(t1[snr < 0].mean()), 3),
                             'per_category': {k: round(float(np.mean([q['top1'] for q in r if q['cat'] == k])), 2)
                                              for k in sorted({q['cat'] for q in r})},
                             'most_common_tags': Counter(q['tag'] for q in r).most_common(4)}), flush=True)


if __name__ == '__main__':
    main()