Ears for coding agents: let AI hear, review and make sound. CPU-only, open source.
"""Evaluate the launch profile and the critique scorecard.
.venv/Scripts/python.exe eval/launch/run_eval.py --only ref_1,ref_2 --fresh # measure films (cold), in chunks
.venv/Scripts/python.exe eval/launch/run_eval.py --neg quiet,hot,mono # known-bad mixes, in chunks
.venv/Scripts/python.exe eval/launch/run_eval.py --build # + write earshot/data/launch_profile.json
.venv/Scripts/python.exe eval/launch/run_eval.py # LOO + Clawd + RESULTS.md (cached films)
1. per film: earshot.profile.film_metrics for the 7 references and the two Clawd films; --fresh first deletes this
module's audio-pass cache for the film (--fresh-ast also the AudioSet pass) so that pass is timed cold. Times and
metrics go to eval/launch/films.json (numbers only).
2. --build: the shipped profile (earshot/data/launch_profile.json) from the 7 references, ids and styles from refs.json.
3. leave-one-out: each reference scored against a profile of the other six (do the ranges generalise?), plus the
in-sample check against all seven.
4. the Clawd film with its real soundtrack vs the same film with the retired synthesized soundtrack: full critique
against the 7-film profile (with earshot.realness when its model is installed).
5. sensitivity: known-bad mixes (the Clawd real soundtrack degraded one way at a time, its SFX / ambience stems alone,
the retired synthesized bed, a sine, pink noise; scratch audio in out/launch/neg/, never committed): is the metric
the defect should move flagged, and what is the verdict? Results go to eval/launch/negatives.json.
6. the visual-event detector (earshot.video frame difference) vs ffmpeg scene cuts (video.cuts(method='scene')), and
a check that the core detector reproduces this module's first-build picture pass exactly.
Writes eval/launch/results.json and eval/launch/RESULTS.md.
"""
import hashlib
import argparse
import json
import os
import sys
import time
import subprocess # noqa: E402
import numpy as np # noqa: E402
HERE = os.path.dirname(os.path.abspath(__file__))
ROOT = os.path.dirname(os.path.dirname(HERE))
sys.path.insert(0, ROOT)
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) # eval/: local data locations
import evalpaths as EP # noqa: E402
from earshot import audio, config, video # noqa: E402
from earshot import profile as P # noqa: E402
from earshot.critique import SPEC, critique, real_thresholds, scorecard # noqa: E402
REFS = json.load(open(os.path.join(HERE, 'refs.json'), encoding='utf-8'))
for _r in REFS['refs']:
_r['path'] = EP.ref(_r['id']) # local files, see eval/README.md
for _r in REFS['tests']:
_r['path'] = f"{EP.FILM}/{_r['file']}"
FILMS = os.path.join(HERE, 'films.json')
NEGS = os.path.join(HERE, 'negatives.json')
RESULTS = os.path.join(HERE, 'results.json')
NEG_DIR = os.path.join(ROOT, 'out', 'launch', 'neg')
ST = f'{EP.FILM}/out/audio/soundtrack.wav'
STEMS = f'{EP.FILM}/out/audio/stems_pro/'
FILM = f'{EP.FILM}/out/Clawd_1080p60.mp4'
# id: (source or lavfi graph, ffmpeg -af filter, models (AudioSet + realness), picture, metrics the defect should flag,
# what it is). Expectations were written down before the first run.
NEGATIVES = {
'base': (ST, None, False, None, (), 'control: the real Clawd soundtrack untouched'),
'quiet': (ST, 'volume=-12dB', False, None, ('lufs_i',), 'unmastered: 12 dB quieter'),
'hot': (ST, 'volume=12dB', False, None, ('lufs_i', 'true_peak'), '12 dB hotter, clipped by the 16-bit file'),
'mono': (ST, 'pan=stereo|c0=0.5*c0+0.5*c1|c1=0.5*c0+0.5*c1', False, None, ('side_to_mid_db',), 'folded to mono'),
'muffled': (ST, 'lowpass=f=3000,lowpass=f=3000', False, None, ('band_presence', 'band_air'),
'low-passed at 3 kHz (12 dB/oct)'),
'late': (ST, 'adelay=2500:all=1,atrim=0:60', False, None, ('first_sound_s',), '2.5 s of silence before the start'),
'deadair': (ST, 'apad=pad_dur=4', False, None, ('end_silence_s',), '4 s of silence after the end'),
'fadeout': (ST, 'afade=t=out:st=48:d=12', False, None, ('fade_s', 'final_lu', 'end_lu'), '12 s fade-out'),
'hole': (ST, "volume=enable='between(t,25,27.5)':volume=0", False, None, ('max_gap_s',), '2.5 s muted mid-film'),
'squashed': (ST, 'acompressor=threshold=-36dB:ratio=20:attack=1:release=60:makeup=12,alimiter=limit=0.7', False,
None, ('lra', 'crest_lu'), 'over-compressed: 20:1 from -36 dB + limiter'),
'sfx_only': (STEMS + 'sfx.wav', None, True, FILM, ('music_frac',), 'the SFX stem alone, with the picture'),
'amb_only': (STEMS + 'amb.wav', None, True, None, ('music_frac', 'onset_rate', 'hit_rate'), 'the ambience stem alone'),
'synth_bed': (f'{EP.FILM}/audio/_retired_synth/bed.wav', None, True, None, ('p_real',),
'the retired code-synthesized music bed'),
'sine': ('lavfi:sine=f=440:d=30:sample_rate=48000', 'volume=0.3', True, None, ('p_real', 'music_frac', 'onset_rate'),
'30 s 440 Hz sine'),
'pink': ('lavfi:anoisesrc=color=pink:d=30:a=0.3:r=48000', None, True, None, ('p_real', 'music_frac', 'onset_rate'),
'30 s pink noise'),
}
def _all():
return REFS['refs'] + REFS['tests']
def _clear(path, ast=False):
for k, f in P.cache_paths(path).items():
if os.path.exists(f) and (ast or k != 'ast'):
os.remove(f)
def measure(only, fresh, fresh_ast=False):
films = json.load(open(FILMS)) if os.path.exists(FILMS) else {}
for r in _all():
if only and r['id'] not in only:
continue
if fresh:
_clear(r['path'], fresh_ast)
t0 = time.time()
f = P.film_metrics(r['path'])
cold = round(time.time() - t0, 1)
t0 = time.time()
P.film_metrics(r['path'])
warm = round(time.time() - t0, 1)
old = films.get(r['id'], {})
films[r['id']] = {'id': r['id'], 'style': r['style'], 'duration': f['duration'], 'fps': f['fps'],
'lossy': f['lossy'],
'metrics': f['metrics'], 'chance': f['chance'], 'arc': f['arc'],
'video': bool(f['video']),
'n_visual': len(f['events']['visual']), 'n_hits': len(f['events']['hits']),
'seconds_cold': cold if fresh else old.get('seconds_cold'), 'seconds_warm': warm,
'timing_cold': f['timing'] if fresh else old.get('timing_cold'),
'audioset_cold_s': (f['timing'].get('audioset') if fresh_ast else
old.get('audioset_cold_s', (old.get('timing_cold') or {}).get('audioset')))}
print(f"{r['id']}: {f['duration']:.1f} s film, {'cold ' + str(cold) + ' s, ' if fresh else ''}warm {warm} s | "
f"{f['summary']}", flush=True)
json.dump(films, open(FILMS, 'w'), indent=1)
return films
def loo(films):
refs = [films[r['id']] for r in REFS['refs']]
full = P.aggregate(refs)
out = []
for i, f in enumerate(refs):
others = refs[:i] + refs[i + 1:]
sc = scorecard(f['metrics'], P.aggregate(others), f['chance'], f['lossy'])
ins = scorecard(f['metrics'], full, f['chance'], f['lossy'])
rows = [r for r in sc['metrics'] if r['weight'] != 'info' and r['status'] != 'n.a.']
out.append({'id': f['id'], 'verdict': sc['verdict'], 'score': sc['score'], 'in_range': sc['in_range'],
'scored': sc['scored'], 'critical': sc['critical_issues'], 'major': sc['major_issues'],
'out': [f"{r['metric']} {r['status']}" for r in rows if r['status'] != 'ok'],
'refs_basis_in_range': sum(1 for r in rows if r['basis'] == 'references' and r['status'] == 'ok'),
'refs_basis_scored': sum(1 for r in rows if r['basis'] == 'references'),
'in_sample_out': [f"{r['metric']} {r['status']}" for r in ins['metrics']
if r['weight'] != 'info' and r['status'] in ('low', 'high')]})
per_metric = {}
for k in SPEC:
if SPEC[k][0] == 0 or k == 'p_real':
continue
st = []
for i, f in enumerate(refs):
sc = scorecard(f['metrics'], P.aggregate(refs[:i] + refs[i + 1:]), lossy=f['lossy'])
st.append(next(r['status'] for r in sc['metrics'] if r['metric'] == k))
per_metric[k] = {'ok': st.count('ok'), 'low': st.count('low'), 'high': st.count('high'), 'na': st.count('n.a.')}
return out, per_metric
def clawd():
res = {}
for r in REFS['tests']:
t0 = time.time()
c = critique(r['path'])
res[r['id']] = {'style': r['style'], 'verdict': c['verdict'], 'score': c['score'], 'in_range': c['in_range'],
'scored': c['scored'], 'critical': c['critical_issues'], 'major': c['major_issues'],
'actions': c['actions'], 'tips': c['tips'],
'status': {m['metric']: [m['value'], m['status']] for m in c['metrics']},
'realness': (c.get('realness') or {}).get('summary') or (c.get('realness') or {}).get('error'),
'seconds_critique': round(time.time() - t0, 1)}
print(f"{r['id']}: {c['summary']}", flush=True)
return res
def detector_check():
out = []
for r in _all():
fps, d = video.frame_diff(r['path'])
ev = video.diff_events(d, fps) / fps
hc = video.cuts(r['path'], method='scene')
row = {'id': r['id'], 'scene_cuts': len(hc), 'visual_events': int(len(ev))}
if len(hc) >= 5:
row['recall_of_scene_cuts'] = round(P.near_frac(hc, ev, 1.0 / fps + 1e-3), 2)
st = os.stat(r['path']) # the first build's cache key used the file id of that time (path, size, mtime only)
fid = hashlib.sha1(f"{os.path.abspath(r['path'])}|{st.st_size}|{st.st_mtime_ns}".encode()).hexdigest()[:20]
k = hashlib.sha1('|'.join(map(str, (fid, 96, 54, 1))).encode()).hexdigest()[:20]
old = os.path.join(config.EMB_CACHE, 'launch', f'frames_{k}.npz')
if os.path.exists(old):
with np.load(old) as z:
row['same_as_first_build'] = bool(np.array_equal(z['d'], d))
out.append(row)
return out
def _make_neg(nid):
src, af = NEGATIVES[nid][:2]
if not af and not src.startswith('lavfi:'):
return src
os.makedirs(NEG_DIR, exist_ok=True)
out = os.path.join(NEG_DIR, f'{nid}.wav')
if not os.path.exists(out):
inp = ['-f', 'lavfi', '-i', src[6:]] if src.startswith('lavfi:') else ['-i', src]
subprocess.run(['ffmpeg', '-v', 'error', '-y', *inp, *(['-af', af] if af else []), '-ac', '2', '-ar', '48000',
'-c:a', 'pcm_s16le', out], check=True)
return out
def negatives(ids):
res = json.load(open(NEGS)) if os.path.exists(NEGS) else {}
for nid in ids:
src, af, models, vid, expect, what = NEGATIVES[nid]
path = _make_neg(nid)
t0 = time.time()
c = critique(path, video=vid, use_ast=models, use_realness=models)
st = {m['metric']: m['status'] for m in c['metrics']}
flagged = [f"{m['metric']} {m['status']} ({m['weight']})" for m in c['metrics']
if m['status'] in ('low', 'high')]
res[nid] = {'what': what, 'models': models, 'picture': bool(vid), 'expect': list(expect),
'caught': [k for k in expect if st.get(k) in ('low', 'high')], 'verdict': c['verdict'],
'partial': c.get('partial'), 'score': c['score'], 'critical': c['critical_issues'],
'major': c['major_issues'], 'flagged': flagged,
'first_action': c['actions'][0]['action'] if c['actions'] else None,
'p_real': next((m['value'] for m in c['metrics'] if m['metric'] == 'p_real'), None),
'seconds': round(time.time() - t0, 1)}
print(f"{nid}: {c['summary'][:300]}", flush=True)
json.dump(res, open(NEGS, 'w'), indent=1)
return res
def hit_timing_check():
"""tuning data: detected hits / onsets of the Clawd real soundtrack vs the sample placements in its mix report"""
mr = f'{EP.FILM}/out/qa/mix_report.json'
film = next(r['path'] for r in REFS['tests'] if r['id'] == 'clawd_real')
if not os.path.exists(mr):
return None
pl = sorted(p['t'] for p in json.load(open(mr))['placements'])
f = P.film_metrics(film)
on = P._audio_pass(film)['onsets']
hits = f['events']['hits']
off = [1000 * min(abs(p - t) for p in pl) for t in hits]
row = {'placements': len(pl), 'hits': len(hits), 'onsets': len(on),
'hit_offset_median_ms': round(float(sorted(off)[len(off) // 2]), 1) if off else None,
'hits_within_2ms': sum(1 for o in off if o <= 2.0)}
for tol in (0.02, 0.05, 0.1):
row[f'hit_precision_{int(tol * 1000)}ms'] = round(P.near_frac(hits, pl, tol), 2)
row[f'hit_chance_{int(tol * 1000)}ms'] = round(P.chance_frac(hits, pl, tol, f['duration']), 2)
row[f'onset_precision_{int(tol * 1000)}ms'] = round(P.near_frac(on, pl, tol), 2)
row[f'placement_recall_{int(tol * 1000)}ms'] = round(P.near_frac(pl, on, tol), 2)
return row
def _fmt(k, v):
return 'n.a.' if v is None else P.fmt_value(k, v) if k in P.METRICS else str(v)
def report(films, loo_rows, per_metric, cl, det, ht, neg=None):
L = ['# Launch profile + critique: evaluation', '',
'Generated by `eval/launch/run_eval.py` (numbers only; the films are local reference files of the author, '
'never copied). Profile = `earshot/data/launch_profile.json`, built from the 7 references below.', '',
'## Commands', '',
'```',
'EARSHOT_THREADS=2 .venv/Scripts/python.exe eval/launch/run_eval.py --only ref_1,ref_2,ref_3 --fresh --measure-only',
'EARSHOT_THREADS=2 .venv/Scripts/python.exe eval/launch/run_eval.py --only ref_4,ref_5,ref_6,ref_7,clawd_real,'
'clawd_synth --fresh --measure-only',
'EARSHOT_THREADS=2 .venv/Scripts/python.exe eval/launch/run_eval.py --build # profile, LOO, Clawd, this file',
'EARSHOT_THREADS=2 .venv/Scripts/python.exe eval/launch/run_eval.py --neg base,quiet,hot,mono,muffled,late,deadair,'
'fadeout,hole,squashed',
'EARSHOT_THREADS=2 .venv/Scripts/python.exe eval/launch/run_eval.py --neg sfx_only,amb_only,synth_bed',
'EARSHOT_THREADS=2 .venv/Scripts/python.exe eval/launch/run_eval.py --neg sine,pink',
'EARSHOT_THREADS=2 .venv/Scripts/python.exe eval/launch/run_eval.py # rewrite this file from the caches',
'```', '',
'## What was tuned on what', '',
'- **Detector thresholds** (salient hit: >= 6 dB jump of the 100 ms level within 80 ms, landing within 8 LU '
'of integrated, one hit per run, timed at the attack; visual event: frame-difference peak >= 4x the median '
'of the surrounding second or >= 0.15, now earshot.video.diff_events; stop-down: >= 10 dB dip for 0.15-1.5 s) '
'were chosen by looking at plots of all 9 films (7 refs + 2 Clawd), at the Clawd mix-report placements (hits) '
'and at ffmpeg scene cuts (visual events). The Clawd real soundtrack is therefore partly tuning data for the '
'hit detector.',
'- **Music threshold** (AudioSet Music >= 0.3 per 5 s window) was set after looking at the per-window '
'probabilities of all 9 films (music under voice or SFX scored 0.4-0.7, one sound-design stretch 0.1-0.15).',
'- **Changed after the first leave-one-out run** (first build; not a blind test of these choices): true peak '
'went from critical to major and is judged against 0 dBTP for decoded lossy files (the -1 dBTP master rule '
'failed all 7 AAC-decoded references); a critical rule missed by <= 2 tolerances counts as major; the metric '
'max_gap_s was added; the music_frac tolerance went from 0.1 to 0.05 after an SFX-only stem (music in 42 % of '
'windows) came out as only a near miss.',
'- **Changed after the independent review, before the known-bad set below was first run:** p_real uses the '
"realness model's own thresholds (ok at its 'real or pro-produced' threshold, critical below 0.5, minor in "
'between; no near-miss downgrade); a density range that would reach 0 keeps a floor at half the sparsest '
'reference (salient hits >= 0.05 /s); a mono mix (identical channels) counts as -40 dB stereo width instead '
'of n.a.; files under 10 s get no loudness-shape rows; brightness_hz replaces the power-weighted centroid '
'(info only). The known-bad expectations were written into run_eval.py before its first run; nothing was '
'changed after seeing its results.',
'- **Not adjusted after seeing results:** the other tolerances, the weights of the other metrics and the '
'verdict thresholds (90 / 75 / 50 %).',
"- **Ranges** are data: min-max of the reference films widened by each metric's tolerance. Loudness and true "
'peak use web-delivery practice (-16..-9 LUFS; <= -1 dBTP for a master, <= 0 dBTP for decoded AAC / MP3) '
'instead, so references can fail them.', '',
'## Per-film measurements', '',
'| film | dur s | LUFS | dBTP | LRA | music | speech | onsets/s | hits/s | vis events/s | events w/ sound '
'(chance) | audio pass s (cold) | picture pass s | AudioSet pass s (cold, first build) | warm s |',
'|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|']
for r in _all():
f = films.get(r['id'])
if not f:
continue
m, c, tc = f['metrics'], f['chance'], f.get('timing_cold') or {}
L.append(f"| {r['id']} | {f['duration']:.1f} | {m['lufs_i']} | {m['true_peak']} | {m['lra']} | "
f"{_fmt('music_frac', m['music_frac'])} | {_fmt('speech_frac', m['speech_frac'])} | {m['onset_rate']} | "
f"{m['hit_rate']} | {m['cut_rate']} | {_fmt('cuts_with_sound', m['cuts_with_sound'])} "
f"({_fmt('cuts_with_sound', c.get('cuts_with_sound'))}) | {tc.get('audio')} | {tc.get('picture')} | "
f"{f.get('audioset_cold_s')} | {f.get('seconds_warm')} |")
L += ['', 'Audio pass = decode + features.measure(light) + librosa onsets / beats + loudness curves, timed cold in '
"this round (this module's cache deleted; the core decode cache was cold too after its v2 bump). Picture "
'pass = earshot.video.frame_diff (core cache, shared with timeline / sheet): 0.0 means another module had '
'already decoded that film; ref_6 (4K) and clawd_synth were cold. AudioSet = 5 s windows through AST, '
'measured cold in the first build (same code and cache key, so not re-run): ~4-9 s per 5 s window on this '
'laptop under load from 3-4 other model jobs; `earshot critique --fast` skips it and realness. Warm = '
'everything cached. EARSHOT_THREADS=2.', '',
'## Leave-one-out: each reference vs a profile of the other six', '',
'| film | verdict | weighted score | scored metrics in range | reference-range metrics in range | out of range |',
'|---|---|---|---|---|---|']
n_ok = n_all = r_ok = r_all = 0
for x in loo_rows:
n_ok += x['in_range']; n_all += x['scored']; r_ok += x['refs_basis_in_range']; r_all += x['refs_basis_scored']
L.append(f"| {x['id']} | {x['verdict']} | {100 * x['score']:.0f}% | {x['in_range']}/{x['scored']} | "
f"{x['refs_basis_in_range']}/{x['refs_basis_scored']} | {', '.join(x['out']) or '-'} |")
L += ['', f'Total: {n_ok}/{n_all} scored metric checks in range ({100 * n_ok / max(1, n_all):.0f} %); '
f'reference-range metrics only: {r_ok}/{r_all} ({100 * r_ok / max(1, r_all):.0f} %). For exchangeable data '
'a held-out film falls outside the min-max of six others with probability 2/7 (29 %) per continuous metric '
'before tolerance, so ~71 % is the no-overfit baseline; the tolerance adds a little. p_real is not part of '
'this check (the realness model was trained on these films).', '',
'In-sample (each reference vs the full 7-film profile): every reference-range metric is in range by '
'construction; the only failures are web-practice rules: ' +
'; '.join(f"{x['id']}: {', '.join(x['in_sample_out'])}" for x in loo_rows if x['in_sample_out']) + '.', '',
'### Per metric (7 leave-one-out checks each)', '', '| metric | weight | ok | low | high | n.a. |',
'|---|---|---|---|---|---|']
for k, st in per_metric.items():
L.append(f"| {k} | {['info', 'minor', 'major', 'critical'][SPEC[k][0]]} | {st['ok']} | {st['low']} | "
f"{st['high']} | {st['na']} |")
if neg:
bad = {k: v for k, v in neg.items() if v['expect']}
caught = sum(1 for v in bad.values() if v['caught'])
L += ['', '## Sensitivity: known-bad mixes', '',
'Each row is one defect made on purpose (ffmpeg, scratch files in out/launch/neg/), critiqued against the '
"7-film profile. 'Should flag' was written down before the first run; 'caught' = at least one of those "
'metrics came out low / high. Rows without models skip AudioSet and realness (partial verdicts); the '
'control is the untouched real soundtrack.', '',
'| mix | what | should flag | caught | verdict | weighted score | all flagged rows | p_real | s (critique, cold) |',
'|---|---|---|---|---|---|---|---|---|']
for k, v in neg.items():
verdict = v['verdict'] + (' (partial)' if v.get('partial') else '')
L.append(f"| {k} | {v['what']} | {', '.join(v['expect']) or '-'} | {', '.join(v['caught']) or ('no' if v['expect'] else '-')} | "
f"{verdict} | {100 * (v['score'] or 0):.0f}% | {'; '.join(v['flagged']) or '-'} | "
f"{'-' if v['p_real'] is None else v['p_real']} | {v['seconds']} |")
miss = [k for k, v in bad.items() if not v['caught']]
L += ['', f'Caught: {caught} of {len(bad)} known-bad mixes have the expected metric flagged' +
(f" (missed: {', '.join(miss)})" if miss else '') + '. Verdicts: ' +
', '.join(f"{n} {sum(1 for v in bad.values() if v['verdict'] == n)}" for n in
('launch-grade', 'close', 'needs work', 'not launch-grade')) +
f" (control: {neg.get('base', {}).get('verdict', 'n.a.')}). A single structural defect on a mix that "
'is already close usually stays close: the verdict is coarse, the flagged rows and FIX FIRST actions '
'are the signal. Over-compression is not a defect by these references: two of them (LRA 1.7 / 3.3 LU, '
'loudest 3 s only 1.1-1.2 LU above integrated) are as dense as the squashed mix.']
L += ['', '## The Clawd film: real soundtrack vs the retired synthesized soundtrack', '']
r, s_ = cl.get('clawd_real'), cl.get('clawd_synth')
if r and s_:
pr, ps = r['status']['p_real'][0], s_['status']['p_real'][0]
L += [f"Reading: the scorecard rates the real soundtrack {r['verdict']} ({100 * r['score']:.0f}%) and the "
f"synthesized one {s_['verdict']} ({100 * s_['score']:.0f}%). The structural metrics separate them only "
'a little: both are loud enough, continuous and on the picture; the synth version fades away at the end, '
'is sparser and has a long fade, while the real one starts late (a 1 s near-silent cold open) and has '
'one loudness peak that sticks out. The difference a listener hears most - synthesized timbre - is not '
'structural; it is what p_real measures' +
(f": real {pr:.2f} vs synth {ps:.2f} (earshot.realness, verdict thresholds lo {real_thresholds()['lo']} "
f"/ hi {real_thresholds()['hi']} at this run); below 0.5 the p_real row is critical, which is what "
"moves the synth version's verdict." if pr is not None and ps is not None else
'; the realness model was not installed when this ran.') +
' Caveat: the realness model (another module) was trained partly on these 7 launch films, on the Clawd '
"project's real asset library and on the retired synthesized stems (eval/realness/), so p_real on the "
'Clawd films is not an independent test.', '']
for k, c in cl.items():
L += [f"### {k}: {c['style']}", '',
f"Verdict **{c['verdict']}**, weighted score {100 * c['score']:.0f}%, {c['in_range']}/{c['scored']} in range, "
f"{c['critical']} critical / {c['major']} major issues; critique call {c['seconds_critique']} s with the "
f"measurement passes cached (realness runs inside it).", '']
for a in c['actions']:
L.append(f"- [{a['weight']}] {a['action']}")
for t in c['tips']:
L.append(f'- tip: {t}')
L.append(f"- realness: {c['realness']}")
L.append('')
if ht:
L += ['## Hit / onset timing vs ground truth (tuning data: the Clawd real soundtrack)', '',
f"The mix report lists {ht['placements']} sample placements (every key tick, pop, whoosh, hit, music "
f"motif note ...). earshot found {ht['hits']} salient hits and {ht['onsets']} onsets in the mixed film "
f"audio; {ht['hits_within_2ms']} of the {ht['hits']} hits sit within 2 ms of a placement (median distance "
f"{ht['hit_offset_median_ms']} ms).", '',
'| tolerance | hits on a placement (chance) | onsets on a placement | placements with an onset |',
'|---|---|---|---|']
for ms in (20, 50, 100):
L.append(f"| +-{ms} ms | {100 * ht[f'hit_precision_{ms}ms']:.0f}% ({100 * ht[f'hit_chance_{ms}ms']:.0f}%) | "
f"{100 * ht[f'onset_precision_{ms}ms']:.0f}% | {100 * ht[f'placement_recall_{ms}ms']:.0f}% |")
L += ['', 'Chance = the same hits shifted off the placements by fixed offsets. Hit and onset times are moved to '
'the steepest rise of the 2 ms level (attack), which is how a mixer aligns a sample.', '']
L += ['## Visual-event detector (earshot.video frame difference) vs ffmpeg scene cuts', '',
"Scene cuts = `video.cuts(path, method='scene')` (ffmpeg scene score 0.3). 'Same as first build' = the core "
"video.frame_diff signal is bit-identical to this module's first-build picture pass, so moving the detector "
'into core changed no number.', '',
'| film | scene cuts | visual events | share of scene cuts also found (+-1 frame) | same as first build |',
'|---|---|---|---|---|']
for d in det:
rec = d.get('recall_of_scene_cuts', 'n.a. (< 5 scene cuts)')
L.append(f"| {d['id']} | {d['scene_cuts']} | {d['visual_events']} | {rec} | "
f"{d.get('same_as_first_build', 'n.a.')} |")
return '\n'.join(L) + '\n'
def main():
ap = argparse.ArgumentParser()
ap.add_argument('--only', help='comma-separated ids to (re)measure')
ap.add_argument('--fresh', action='store_true', help='delete the audio-pass cache first (cold timings)')
ap.add_argument('--fresh-ast', action='store_true', help='with --fresh: also delete the AudioSet cache')
ap.add_argument('--neg', help='comma-separated known-bad mixes to critique (or "all"): ' + ','.join(NEGATIVES))
ap.add_argument('--build', action='store_true', help='write earshot/data/launch_profile.json')
ap.add_argument('--measure-only', action='store_true')
a = ap.parse_args()
only = set(a.only.split(',')) if a.only else None
if a.neg:
negatives(list(NEGATIVES) if a.neg == 'all' else a.neg.split(','))
return
films = (measure(only, a.fresh, a.fresh_ast) if (only or a.fresh or not os.path.exists(FILMS)) else
json.load(open(FILMS)))
if a.measure_only:
return
if a.build:
P.profile([r['path'] for r in REFS['refs']], styles=[r['style'] for r in REFS['refs']], out=P.PROFILE_PATH)
loo_rows, per_metric = loo(films)
cl = clawd()
det = detector_check()
ht = hit_timing_check()
neg = json.load(open(NEGS)) if os.path.exists(NEGS) else {}
json.dump({'loo': loo_rows, 'per_metric': per_metric, 'clawd': cl, 'detector': det, 'hit_timing': ht},
open(RESULTS, 'w'), indent=1)
open(os.path.join(HERE, 'RESULTS.md'), 'w', encoding='utf-8').write(
report(films, loo_rows, per_metric, cl, det, ht, neg))
print('wrote', RESULTS, 'and RESULTS.md')
if __name__ == '__main__':
main()