Skip to content

Ears for coding agents: let AI hear, review and make sound. CPU-only, open source.

10.2 KB
Raw Blame
"""Shared training code for the realness detector: dataset loading, the two logistic heads, the fusion layer and
grouped cross-validation. Used by train.py (final model) and run_eval.py (held-out evaluation).

Model: head_clap = standardised L2 logistic regression on the 512-d CLAP embedding; head_signal = the same on the
signal measurements of earshot.realness.signal_features; fusion = logistic regression on the two head logits, fitted on
inner out-of-group head predictions (so its weights reflect how well each head generalises to unseen groups).
Sample weights: each class weighs 1 in total, split equally over its CV groups, equally over the clips of a group.
Layered clips (a real bed + one code-synthesized layer, label 0) belong to two groups: the layer's and the bed's. They
are never test clips of the clip-level schemes, and a model never trains on one whose layer group or bed group is held
out.
"""
import json
import os
import sys

import numpy as np
from scipy.optimize import minimize

HERE = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, os.path.dirname(os.path.dirname(HERE)))
OUT = os.path.join(os.path.dirname(os.path.dirname(HERE)), 'out', 'realness')
FEATS = os.path.join(OUT, 'feats')
SIG_INPUTS = ['silence_frac', 'floor_db', 'spec_floor_db', 'log_flatness', 'peakiness_db', 'line_share', 'bandwidth_khz',
              'quiet_flat', 'tonal_frames', 'pitch_wobble', 'env_resid_db', 'texture_kurt', 'crest_db']
# physical direction of each measurement (+1: larger reads as a recording): the signal head's coefficients are
# constrained to these signs, so a measurement can never vote against its physics because of a dataset quirk
SIGNS = {'silence_frac': -1, 'floor_db': 1, 'spec_floor_db': 1, 'log_flatness': 1, 'peakiness_db': -1, 'line_share': -1,
         'bandwidth_khz': 1, 'quiet_flat': 1, 'tonal_frames': -1, 'pitch_wobble': 1, 'env_resid_db': 1, 'texture_kurt': 1,
         'crest_db': 1}
C_CLAP = (0.001, 0.003, 0.01)
C_SIGNAL = 0.1
C_AST = 0.01
AUG_WEIGHT = 0.35


class Data:
    """clips with features, aligned arrays; .sub(mask) for a subset"""

    def __init__(self, clips, clap, sig, ast=None):
        self.clips = clips
        self.ids = np.array([c['id'] for c in clips])
        self.y = np.array([c['label'] for c in clips])
        self.family = np.array([c['family'] for c in clips])
        self.group = np.array([c['group'] for c in clips])
        self.type = np.array([c['type'] for c in clips])
        self.src = np.array([c['src'] for c in clips])
        self.perturb = np.array([c.get('perturb', '') for c in clips])
        self.augment = np.array([bool(c.get('augment')) for c in clips])
        self.base = np.array([c.get('base', c['id']) for c in clips])
        self.is_layer = self.perturb == 'layer'
        self.bed_group = np.array([c.get('bed_group', c['group']) for c in clips])
        self.bed_src = np.array([c.get('bed_src', c['src']) for c in clips])
        self.Xc = np.stack([clap[i] for i in self.ids]) if len(clips) else np.zeros((0, 512))
        self.Xs = np.array([[np.nan if sig[i] is None else sig[i].get(k, np.nan) for k in SIG_INPUTS] for i in self.ids], float)
        self.Xa = np.stack([ast[i] for i in self.ids]) if ast is not None and len(clips) else None

    def sub(self, mask):
        d = object.__new__(Data)
        for k, v in self.__dict__.items():
            d.__dict__[k] = (v[mask] if isinstance(v, np.ndarray) else [c for c, m in zip(v, mask) if m]) if v is not None else None
        return d

    def __len__(self):
        return len(self.ids)


def load(require_ast=False, robust=True):
    man = json.load(open(os.path.join(OUT, 'manifest.json')))
    clips = [c for k, part in man.items() for c in part if robust or k != 'robust']
    # a few recordings sit in two asset folders (e.g. foley and toy): keep them in the first group only, so no
    # recording is ever on both sides of a held-out split
    owner, seen, drop = {}, set(), set()
    for c in clips:
        if c['label'] == 1 and 'perturb' not in c:
            key = os.path.basename(c['src']).split('_')[0]
            if owner.setdefault(key, c['group']) != c['group'] or c['id'] in seen:
                drop.add(c['id'])
            seen.add(c['id'])
    clips = [c for c in clips if c['id'] not in drop and c.get('base') not in drop]
    sig = json.load(open(os.path.join(FEATS, 'signal.json')))
    z = np.load(os.path.join(FEATS, 'clap.npz'), allow_pickle=False)
    clap = dict(zip(z['ids'], z['emb']))
    ast = None
    if require_ast:
        za = np.load(os.path.join(FEATS, 'ast.npz'), allow_pickle=False)
        ast = dict(zip(za['ids'], za['emb']))
    keep = [c for c in clips if c['id'] in clap and c['id'] in sig and sig[c['id']] is not None and (ast is None or c['id'] in ast)]
    return Data(keep, clap, sig, ast)


def weights(d, groups, aug_weight=AUG_WEIGHT):
    """each class weighs 1, split equally over its groups; inside a group an augmented or layered clip weighs
    aug_weight of a clean clip"""
    w = np.zeros(len(d))
    u = np.where(d.augment | d.is_layer, aug_weight, 1.0)
    for lab in (0, 1):
        gs = sorted(set(groups[d.y == lab]))
        for g in gs:
            m = (d.y == lab) & (groups == g)
            w[m] = u[m] / (len(gs) * u[m].sum())
    return w * len(d) / w.sum()


def logreg(X, y, w, C, signs=None):
    """weighted L2 logistic regression (sklearn's objective: C * sum w*logloss + |b|^2 / 2, free intercept);
    signs (+1 / -1 / 0 per column) bound coefficients to that sign (0 = free)"""
    n, k = X.shape
    s = 2 * y - 1.0

    def f(b):
        m = s * (X @ b[:k] + b[k])
        lo = np.logaddexp(0, -m)
        g = -s * w / (1 + np.exp(np.clip(m, -50, 50)))
        return C * np.sum(w * lo) + 0.5 * b[:k] @ b[:k], np.r_[C * (X.T @ g) + b[:k], C * g.sum()]
    bounds = None
    if signs is not None:
        bounds = [(0, None) if v > 0 else (None, 0) if v < 0 else (None, None) for v in signs] + [(None, None)]
    r = minimize(f, np.zeros(k + 1), jac=True, method='L-BFGS-B', bounds=bounds, options={'maxiter': 3000})
    return r.x[:k], float(r.x[k])


def fit_head(X, y, w, C, signs=None):
    mean = np.nanmean(X, axis=0)
    X = np.where(np.isfinite(X), X, mean)
    lo, hi = np.percentile(X, 0.5, axis=0), np.percentile(X, 99.5, axis=0)
    X = np.clip(X, lo, hi)
    scale = X.std(axis=0) + 1e-6
    coef, b0 = logreg((X - mean) / scale, y, w, C, signs)
    return {'mean': mean.tolist(), 'scale': scale.tolist(), 'clip': [lo.tolist(), hi.tolist()],
            'coef': coef.tolist(), 'intercept': b0}


def head_logits(h, X):
    mean = np.asarray(h['mean'])
    X = np.where(np.isfinite(X), X, mean)
    X = np.clip(X, np.asarray(h['clip'][0]), np.asarray(h['clip'][1]))
    return ((X - mean) / np.asarray(h['scale'])) @ np.asarray(h['coef']) + h['intercept']


def head_signs(name):
    return [SIGNS[k] for k in SIG_INPUTS] if name == 'signal' else None


def head_inputs(d, name):
    return {'clap': d.Xc, 'signal': d.Xs, 'ast': None if d.Xa is None else np.log(np.clip(d.Xa, 1e-4, 1))}[name]


def inner_folds(groups, k=5):
    """split the distinct groups into k folds (deterministic, round-robin over sorted names)"""
    gs = sorted(set(groups))
    return [np.isin(groups, gs[i::k]) for i in range(min(k, len(gs)))]


def fit_model(d, groups, heads=('clap', 'signal'), c_clap=None, inner=5, aug_weight=AUG_WEIGHT):
    """heads + fusion on data d. c_clap: fixed C for the CLAP head, else chosen by inner grouped CV (log loss on
    clean clips: augmented copies help training but are not what the model is scored on)."""
    w = weights(d, groups, aug_weight)
    clean = ~d.augment & ~d.is_layer
    folds = inner_folds(groups, inner)
    cs = {'signal': [C_SIGNAL], 'ast': [C_AST], 'clap': [c_clap] if c_clap else list(C_CLAP)}
    oof = {}
    chosen = {}
    for name in heads:
        X = head_inputs(d, name)
        best = None
        for C in cs[name]:
            z = np.zeros(len(d))
            for te in folds:
                tr = ~te
                if len(set(d.y[tr])) < 2:
                    z[te] = 0.0
                    continue
                z[te] = head_logits(fit_head(X[tr], d.y[tr], w[tr], C, head_signs(name)), X[te])
            p = 1 / (1 + np.exp(-np.clip(z, -30, 30)))
            ll = -np.sum((w * (d.y * np.log(p + 1e-9) + (1 - d.y) * np.log(1 - p + 1e-9)))[clean]) / w[clean].sum()
            if best is None or ll < best[0]:
                best = (ll, C, z)
        chosen[name], oof[name] = best[1], best[2]
    F = np.stack([oof[h] for h in heads], axis=1)
    coef, b0 = logreg(F[clean], d.y[clean], w[clean], 1.0, [1] * len(heads))
    model = {'heads': {}, 'fusion': {'heads': list(heads), 'coef': coef.tolist(), 'intercept': b0}, 'C': chosen}
    for name in heads:
        model['heads'][name] = fit_head(head_inputs(d, name), d.y, w, chosen[name], head_signs(name))
    return model


def predict(model, d, return_heads=False):
    fu = model['fusion']
    L = {h: head_logits(model['heads'][h], head_inputs(d, h)) for h in fu['heads']}
    s = fu['intercept'] + sum(c * L[h] for h, c in zip(fu['heads'], fu['coef']))
    p = 1 / (1 + np.exp(-np.clip(s, -30, 30)))
    return (p, L) if return_heads else p


def logo(d, groups, train_mask=None, augment=True, layers=False, bed_groups=None, **kw):
    """leave-one-group-out predictions: every clip is scored by a model that never saw its group (nor augmented
    copies of its clips). train_mask limits which clips may ever be used for training (e.g. never the retired score);
    augment=False trains on clean clips only; layers=True also trains on layered clips whose layer group and bed group
    (bed_groups) are both outside the held-out group. Returns p (n,) and head logits (augmented and layered clips: NaN)."""
    p = np.full(len(d), np.nan)
    L = {}
    tm = np.ones(len(d), bool) if train_mask is None else train_mask
    bg = groups if bed_groups is None else bed_groups
    usable = (d.perturb == '') | (d.augment & augment) | (d.is_layer & layers)
    for g in sorted(set(groups[~d.is_layer])):
        te = (groups == g) & ~d.augment & ~d.is_layer
        tr = (groups != g) & (bg != g) & tm & usable
        sub = d.sub(tr)
        m = fit_model(sub, groups[tr], **kw)
        pt, Lt = predict(m, d.sub(te), True)
        p[te] = pt
        for h, v in Lt.items():
            L.setdefault(h, np.full(len(d), np.nan))[te] = v
    return p, L