Skip to content

Ears for coding agents: let AI hear, review and make sound. CPU-only, open source.

20.9 KB
Raw Blame
"""skills/real-sound: the scripts run end to end on a synthetic project (test signals only, never committed media).

The project is the design template (templates/design_fragment.json) + the cue template, with synthetic stand-ins at
every file path, so these tests also prove the templates render. A second tiny project carries deliberate defects
(a cut into a playing sine, a masked tick, a partly sounded class of action, a trim on digital silence, a riser that
ends long before its hit) that qa.py must catch. The end-to-end tests start the scripts as subprocesses (mix, qa,
arrange: tens of seconds each on this laptop) and are marked slow; the fast ones run in-process.
"""
import json
import os
import shutil
import subprocess
import sys

import numpy as np
import pytest
import soundfile as sf
from scipy import signal

SR = 48000
SKILL = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'skills', 'real-sound')
SCRIPTS = os.path.join(SKILL, 'scripts')
sys.path.insert(0, SCRIPTS)
import licences  # noqa: E402
import variant  # noqa: E402

RNG = np.random.RandomState(7)


def run(script, *args, cwd=None):
    p = subprocess.run([sys.executable, '-I', os.path.join(SCRIPTS, script), *args], capture_output=True, text=True,
                       cwd=cwd, timeout=600)
    return p.returncode, p.stdout + p.stderr


def hits(dur, times, sr=SR, tau=0.012, amp=0.5, floor=1e-4):
    n = int(dur * sr)
    x = RNG.randn(n) * floor
    for t in times:
        i = int(t * sr)
        k = np.arange(min(n - i, int(0.25 * sr)))
        x[i:i + len(k)] += RNG.randn(len(k)) * np.exp(-k / sr / tau) * amp
    return x.astype(np.float32)


def write(root, rel, x, sr=SR):
    p = os.path.join(root, rel)
    os.makedirs(os.path.dirname(p), exist_ok=True)
    sf.write(p, x, sr, format='WAV', subtype='PCM_16')  # WAV bytes; ffmpeg probes content, not the extension
    return p


def music(dur, sr=22050, start=0.5, bpm=120):
    """a steady test track: a low thump on every downbeat, a tick on every beat, a different chord per bar"""
    n = int(dur * sr)
    t = np.arange(n) / sr
    x = np.zeros(n)
    beat = 60 / bpm
    for k in range(int((dur - start) / beat)):
        i = int((start + k * beat) * sr)
        m = np.arange(min(n - i, int(0.3 * sr)))
        if k % 4 == 0:
            x[i:i + len(m)] += 0.6 * np.sin(2 * np.pi * 55 * m / sr) * np.exp(-m / sr / 0.12)
        x[i:i + len(m)] += 0.15 * RNG.randn(len(m)) * np.exp(-m / sr / 0.01)
    for b in range(int((dur - start) / (4 * beat))):
        a0, a1 = int((start + 4 * beat * b) * sr), int((start + 4 * beat * (b + 1)) * sr)
        for f in ([220, 277, 330], [196, 247, 294], [175, 220, 262], [165, 208, 247])[b % 4]:
            x[a0:a1] += 0.05 * np.sin(2 * np.pi * f * t[a0:a1])
    return x.astype(np.float32), sr


@pytest.fixture(scope='module')
def project(tmp_path_factory):
    """the design template + cue template with synthetic recordings at every path, mixed once"""
    root = str(tmp_path_factory.mktemp('rs_project'))
    write(root, 'audio/assets/ui/mk-0000_mouse-click.wav', hits(0.3, [0.1, 0.185]))
    write(root, 'audio/assets/keyboard/fs-000001_mech-spacebar.ogg', hits(1.0, [0.5], tau=0.03))
    write(root, 'audio/assets/motion/fs-000002_stick-swish.ogg',
          (RNG.randn(int(0.6 * SR)) * np.hanning(int(0.6 * SR)) * 0.3).astype(np.float32))
    write(root, 'audio/assets/ui/fs-000003_cork-pop.ogg', hits(0.3, [0.03], tau=0.02))
    write(root, 'audio/assets/motion/fs-000004_cymbal-roll.ogg',
          (RNG.randn(int(2.5 * SR)) * np.linspace(0.02, 0.3, int(2.5 * SR))).astype(np.float32))
    write(root, 'audio/assets/impacts/mk-0005_deep-drum-impact.wav', hits(2.0, [0.0], tau=0.25, amp=0.8))
    write(root, 'audio/assets/impacts/fs-000006_wood-slam.ogg', hits(0.5, [0.02], tau=0.04))
    tt = np.arange(3 * SR) / SR
    write(root, 'audio/assets/magic/fs-000007_celesta-E5.ogg',
          (0.4 * np.sin(2 * np.pi * 659.3 * tt) * np.exp(-tt / 0.5)).astype(np.float32))
    write(root, 'audio/assets/keyboard/fs-000008_mech-typing.ogg', hits(10.0, np.arange(0.6, 9.5, 0.17), tau=0.01))
    write(root, 'audio/assets/ambience/fs-000009_quiet-office-roomtone.ogg',
          (RNG.randn(46 * 8000) * 0.01).astype(np.float32), sr=8000)
    m, msr = music(40.0)
    write(root, 'audio/assets/music/sb-0000_bright-launch-track.mp3', m, sr=msr)
    os.makedirs(os.path.join(root, 'out'), exist_ok=True)
    shutil.copy(os.path.join(SKILL, 'templates', 'cues.json'), os.path.join(root, 'out', 'cues.json'))
    os.makedirs(os.path.join(root, 'audio'), exist_ok=True)
    shutil.copy(os.path.join(SKILL, 'templates', 'design_fragment.json'), os.path.join(root, 'audio', 'design.json'))
    code, out = run('mix.py', '--root', root, '--design', 'audio/design.json', '--trace', 'out/qa/adds.json')
    assert code == 0, out
    return root


@pytest.mark.slow
def test_template_mix_renders(project):
    rep = json.load(open(os.path.join(project, 'out/qa/mix_report.json')))
    cues = json.load(open(os.path.join(project, 'out/cues.json')))
    assert rep['unsounded_events'] == []
    assert len(rep['placements']) == 24  # 23 rule placements on 19 events (layered) + 1 free placement
    x, sr = sf.read(os.path.join(project, 'out/audio/soundtrack.wav'))
    assert sr == SR and abs(len(x) / SR - cues['total'] / cues['fps']) < 1e-3
    master = json.load(open(os.path.join(project, 'out/qa/master.json')))
    assert abs(master['lufs'] - (-14)) <= 0.5
    trace = json.load(open(os.path.join(project, 'out/qa/adds.json')))
    assert len([a for a in trace if 'sample' in a]) == len(rep['placements'])
    assert 'audio/assets/music/sb-0000_bright-launch-track.mp3' in rep['files_used']


@pytest.mark.slow
def test_solo_and_dump(project, tmp_path):
    out = str(tmp_path / 'solo' / 'design.wav')
    code, txt = run('mix.py', '--root', project, '--design', 'audio/design.json', '--only', 'design', '--out', out)
    assert code == 0, txt
    assert os.path.exists(out)  # the unmastered solo mix is written to --out itself
    code, txt = run('mix.py', '--root', project, '--design', 'audio/design.json', '--dump', str(tmp_path / 'dump'))
    assert code == 0, txt
    rows = json.load(open(tmp_path / 'dump' / '_samples.json'))
    names = {r.get('name') for r in rows}
    assert 'ui.press' in names and any(n and n.startswith('kb.type#') for n in names)


@pytest.mark.slow
def test_qa_on_template(project):
    code, txt = run('qa.py', '--root', project, '--design', 'audio/design.json', '--out', 'out/qa/qa.json')
    res = json.load(open(os.path.join(project, 'out/qa/qa.json')))
    for k in ('clicks', 'loudness', 'coverage', 'edges', 'audibility', 'harshness', 'sync', 'density'):
        assert k in res['sections'] and res['sections'][k]['summary']
    assert res['sections']['coverage']['status'] == 'ok'
    assert res['sections']['loudness']['lufs'] == pytest.approx(-14, abs=0.5)
    assert txt.startswith('QA ')


@pytest.mark.slow
def test_qa_catches_defects(tmp_path):
    root = str(tmp_path)
    t = np.arange(3 * SR) / SR
    write(root, 'a/sine.wav', (0.3 * np.sin(2 * np.pi * 880 * t)).astype(np.float32))
    write(root, 'a/tick.wav', hits(0.3, [0.05], amp=0.5))
    write(root, 'a/silence.wav', np.zeros(SR, np.float32))
    ramp = RNG.randn(int(2.6 * SR)) * np.r_[np.linspace(0.01, 0.4, 2 * SR), np.full(int(0.6 * SR), 0.004)]
    write(root, 'a/riser.wav', ramp.astype(np.float32))  # its body ends 0.6 s before the file does
    bed = signal.sosfilt(signal.butter(2, 2000, 'low', fs=SR, output='sos'), RNG.randn(4 * SR)) * 0.3
    write(root, 'a/bed.wav', bed.astype(np.float32))  # a dark bed: masks the quiet tick, leaves the click audible
    cues = {'fps': 60, 'total': 240, 'events': [{'f': 60, 'kind': 'pop', 'id': 't:cut'},
                                               {'f': 120, 'kind': 'tick', 'id': 't:quiet'},
                                               {'f': 150, 'kind': 'land', 'id': 't:step:0'},
                                               {'f': 180, 'kind': 'land', 'id': 't:step:1'},
                                               {'f': 200, 'kind': 'boop', 'id': 't:silent'},
                                               {'f': 228, 'kind': 'riser', 'id': 't:riser'}]}
    design = {'duration': 4.0, 'samples': {
        'cut': {'file': 'a/sine.wav', 'start': 1.0003, 'end': 1.3, 'fade_in': 0, 'fade_out': 0.05},
        'quiet': {'file': 'a/tick.wav', 'norm_peak_db': -60},
        'step': {'file': 'a/tick.wav', 'norm_peak_db': -6},
        'sil': {'file': 'a/silence.wav', 'start': 0.2, 'end': 0.5},
        'rise': {'file': 'a/riser.wav', 'fade_out': 0.01}},
        'pools': {'cut': ['cut'], 'quiet': ['quiet'], 'step': ['step'], 'sil': ['sil'], 'rise': ['rise']},
        'rules': [{'name': 'cut', 'match': {'kind': 'pop'}, 'pool': 'cut', 'mode': 'fixed'},
                  {'name': 'quiet', 'match': {'kind': 'tick'}, 'pool': 'quiet', 'mode': 'fixed'},
                  {'name': 'step', 'match': {'kind': 'land', 'to': 2.9}, 'pool': 'step', 'mode': 'fixed'},
                  {'name': 'sil', 'match': {'kind': 'boop'}, 'pool': 'sil', 'mode': 'fixed'},
                  {'name': 'rise', 'match': {'kind': 'riser'}, 'pool': 'rise', 'mode': 'fixed', 'align': 'end',
                   'gain_db': -12}],
        'ambience': [{'file': 'a/bed.wav', 'at': 0, 'until': 4.0, 'fade_in': 0.2, 'fade_out': 0.2}],
        'master': {'lufs': -14}}
    os.makedirs(os.path.join(root, 'out'), exist_ok=True)
    json.dump(cues, open(os.path.join(root, 'out/cues.json'), 'w'))
    json.dump(design, open(os.path.join(root, 'd.json'), 'w'))
    code, txt = run('mix.py', '--root', root, '--design', 'd.json', '--trace', 'out/qa/adds.json')
    assert code == 0, txt
    code, txt = run('qa.py', '--root', root, '--design', 'd.json', '--out', 'out/qa/qa.json')
    s = json.load(open(os.path.join(root, 'out/qa/qa.json')))['sections']
    cut = [c for c in s['clicks']['items'] if c['owners'] and c['owners'][0].get('what') == 'cut']
    assert cut and cut[0]['owners'][0]['where'] == 'start'  # the cut into a playing sine clicks, at its start
    assert s['clicks']['status'] == 'fail' and code == 1
    edge = [it for it in s['edges']['items'] if it['name'] == 'cut']
    assert edge and any('already playing' in m for m in edge[0]['issues'])
    quiet = [r for r in s['audibility']['events'] if r['label'] == 't:quiet']
    assert quiet and quiet[0]['verdict'] == 'masked'
    assert s['coverage']['status'] == 'warn' and s['coverage']['partial'][0]['id'] == 't:step'
    silent = [r for r in s['audibility']['events'] if r['label'] == 't:silent']
    assert silent and silent[0]['verdict'] == 'silent' and s['audibility']['status'] == 'fail'
    assert [q['id'] for q in s['coverage']['placed_but_silent']] == ['t:silent']
    riser = [r for r in s['sync']['items'] if r['label'] == 't:riser']
    assert riser and riser[0]['test'] == 'end' and riser[0]['end_ms'] < -500  # the riser placed by its end is checked
    code, txt = run('qa.py', '--root', root, '--design', 'd.json', '--report', 'out/nope.json')
    assert code == 2 and 'mix report not found' in txt and 'Traceback' not in txt


def test_licence_classes():
    c = licences.classify
    assert c('CC0 1.0')['status'] == 'allowed' and c('Creative Commons 0')['key'] == 'cc0'
    assert c('CC BY 4.0')['key'] == 'by4' and c('Attribution 4.0')['attribution']
    assert c('Mixkit Sound Effects Free License')['key'] == 'mixkit-sfx'
    assert c('Mixkit Stock Music Free License')['key'] == 'mixkit-music'
    assert c('Pixabay Content License')['status'] == 'allowed'
    for bad in ('CC BY-NC 4.0', 'CC BY-ND 4.0', 'CC BY-SA 4.0', 'Sampling+ 1.0', 'personal use only',
                'DISPUTED: chosic says CC BY 3.0; artist says custom'):
        assert c(bad)['status'] == 'forbidden', bad
    assert c('Royalty free')['status'] == 'unknown' and c('')['status'] == 'unknown'
    assert c('Creative Commons Attribution 4.0 International (copyrighted by the author)')['key'] == 'by4'
    assert c('All rights reserved')['status'] == 'forbidden' and c('CC BY 4.0, all rights reserved')['status'] == 'unknown'


def manifests(root):
    m = os.path.join(root, 'audio/assets/manifests')
    os.makedirs(m, exist_ok=True)
    ents = [
        {'id': 'fs-111', 'category': 'ui', 'file': 'a/tick.wav', 'title': 'Tick', 'author': 'alice', 'license': 'CC BY 4.0',
         'page_url': 'https://freesound.org/people/alice/sounds/111/',
         'download_url': 'https://cdn.freesound.org/previews/0/111_42-hq.ogg',
         'attribution': '"Tick" by alice (https://freesound.org/s/111/), licensed under CC BY 4.0'},
        {'id': 'fs-222', 'category': 'ui', 'file': 'a/sine.wav', 'title': 'Tone', 'author': 'bob', 'license': 'CC BY-NC 4.0',
         'page_url': 'https://freesound.org/people/bob/sounds/223/'},
        {'id': 'mk-5', 'category': 'amb', 'file': 'a/bed.wav', 'title': 'Bed', 'author': 'Mixkit',
         'license': 'Mixkit Sound Effects Free License', 'page_url': 'https://mixkit.co/free-sound-effects/'},
    ]
    json.dump(ents, open(os.path.join(m, 'ui.json'), 'w'))
    json.dump([dict(ents[2], category='dup')], open(os.path.join(m, 'zz.json'), 'w'))
    return m


def test_audit_and_credits(tmp_path):
    root = str(tmp_path)
    write(root, 'a/tick.wav', hits(0.3, [0.05]))
    write(root, 'a/sine.wav', hits(0.3, [0.05]))
    write(root, 'a/bed.wav', hits(1.0, [0.1]))
    m = manifests(root)
    os.makedirs(os.path.join(root, 'out/qa'), exist_ok=True)
    json.dump({'files_used': ['a/tick.wav', 'a/bed.wav', 'a/sine.wav']}, open(os.path.join(root, 'out/qa/mix_report.json'), 'w'))
    code, txt = run('audit.py', '--root', root, '--report', 'out/qa/mix_report.json')
    rows = {r['id']: r for r in json.load(open(os.path.join(m, 'audit.json'))) if not r.get('duplicate_of')}
    assert code == 1
    assert rows['fs-222']['severity'] == 'blocker' and any('forbidden' in p for p in rows['fs-222']['problems'])
    assert any('page is sound 223' in w for w in rows['fs-222']['warnings'])  # the id does not match the page
    assert any('no licence record' in p for p in rows['fs-111']['problems'])
    assert rows['mk-5']['in_scope'] and rows['mk-5']['file_check']['decodes_clean']
    assert sum(1 for r in json.load(open(os.path.join(m, 'audit.json'))) if r.get('duplicate_of')) == 1
    agent = json.load(open(os.path.join(m, 'audit.json')))
    for r in agent:  # the auditor records what it read on each page; audit.py keeps it on the next run
        if r['id'] in ('fs-111', 'mk-5'):
            r['licence_seen'] = 'page shows Attribution 4.0' if r['id'] == 'fs-111' else 'Mixkit Sound Effects Free License (read)'
            r['licence_confirmed'] = True
    json.dump(agent, open(os.path.join(m, 'audit.json'), 'w'))
    code, txt = run('audit.py', '--root', root, '--report', 'out/qa/mix_report.json')
    rows = {r['id']: r for r in json.load(open(os.path.join(m, 'audit.json'))) if not r.get('duplicate_of')}
    assert rows['fs-111']['severity'] == 'ok' and rows['mk-5']['severity'] == 'ok' and rows['fs-222']['severity'] == 'blocker'
    code, txt = run('credits.py', '--root', root, '--audit', 'audio/assets/manifests/audit.json')
    assert code == 1 and 'PROBLEM' in txt  # the NC recording is used
    md = open(os.path.join(root, 'CREDITS.md'), encoding='utf-8').read()
    assert 'by alice' in md and 'creativecommons.org/licenses/by/4.0/' in md  # the licence link is completed
    desc = open(os.path.join(root, 'out/CREDITS_description.txt'), encoding='utf-8').read()
    assert 'Sound effects (CC BY):' in desc and 'Mixkit' in desc
    assert 'were modified' in desc  # CC BY 4.0 s.3(a)(1)(B): the description says the works were changed
    assert os.path.exists(os.path.join(m, 'audit.prev.json'))  # the earlier audit is kept before it is rewritten


def test_audit_inputs(tmp_path):
    code, txt = run('audit.py', '--root', str(tmp_path), '--manifests', 'nope', '--out', 'a.json')
    assert code == 2 and 'manifest folder not found' in txt
    m = tmp_path / 'm'
    m.mkdir()
    write(str(tmp_path), 'a/x.wav', hits(0.3, [0.05]))
    json.dump([{'id': 'fs-1', 'file': 'a/x.wav', 'title': 'x', 'author': 'a', 'license': 'CC0 1.0',
                'page_url': 'https://freesound.org/people/a/sounds/1/'},
               {'id': 'fs-2', 'file': 'a/x.wav', 'title': 'x', 'author': 'a', 'license': 'CC BY-SA 4.0'}], open(m / 's.json', 'w'))
    code, txt = run('audit.py', '--root', str(tmp_path), '--manifests', 'm', '--out', 'a.json', '--decode', 'none')
    rows = json.load(open(tmp_path / 'a.json'))
    assert code == 1 and any('forbidden licence' in p for p in rows[0]['problems'])  # the same file, a second licence


@pytest.mark.slow
def test_arrange_grid(tmp_path):
    x, sr = music(24.0)
    p = str(tmp_path / 'track.wav')
    sf.write(p, x, sr)
    code, txt = run('arrange.py', p, '--bars', '6', '--drop-film-bar', '4', '--out', str(tmp_path / 'a.json'))
    assert code == 0, txt
    r = json.load(open(tmp_path / 'a.json'))
    assert r['drop_source_bar'] == r['drop_candidates'][0] and r['map'][3] == r['drop_source_bar']
    assert abs(r['bpm'] - 120) < 0.5
    ph = (r['bars'][0]['t'] - 0.5) % 2.0
    assert min(ph, 2.0 - ph) < 0.02, r['bars'][0]  # downbeats on the thumps (0.5 + 2k s), within 20 ms
    assert len(r['map']) == 6 and r['music']['segments'][0]['at'] == 0.0


@pytest.mark.skipif(shutil.which('node') is None or shutil.which('ffmpeg') is None, reason='needs node and ffmpeg')
def test_mux(tmp_path):
    v, a, o = str(tmp_path / 'v.mp4'), str(tmp_path / 'a.wav'), str(tmp_path / 'o.mp4')
    subprocess.run(['ffmpeg', '-v', 'error', '-y', '-f', 'lavfi', '-i', 'testsrc=size=160x90:rate=30:duration=2', '-c:v',
                    'mpeg4', v], check=True)
    sf.write(a, hits(2.5, [0.5, 1.0]), SR)
    p = subprocess.run(['node', os.path.join(SCRIPTS, 'mux.mjs'), '--video', v, '--audio', a, '--out', o],
                       capture_output=True, text=True)
    assert p.returncode == 0, p.stderr
    assert 'longer than the picture' in p.stdout
    j = json.loads(subprocess.run(['ffprobe', '-v', 'error', '-show_entries', 'stream=codec_type,codec_name:format=duration',
                                   '-of', 'json', o], capture_output=True, text=True).stdout)
    kinds = {s['codec_type']: s['codec_name'] for s in j['streams']}
    assert kinds == {'video': 'mpeg4', 'audio': 'aac'} and abs(float(j['format']['duration']) - 2.0) < 0.1


def test_templates_parse():
    tdir = os.path.join(SKILL, 'templates')
    for f in os.listdir(tdir):
        if f.endswith('.json'):
            json.load(open(os.path.join(tdir, f), encoding='utf-8'))
    arc = json.load(open(os.path.join(tdir, 'arc.json')))
    assert len(arc['arc']) >= arc['drop_film_bar']
    top = json.load(open(os.path.join(tdir, 'sounddesign.json')))
    assert top['master']['lufs'] == -14 and all(i.startswith('audio/design/') for i in top['include'])


def test_variant_patch_keeps_format():
    text = ('{\n  "rules": [\n    {"name": "a",   "gain_db": -3.0, "pitch": [1, 1]},\n'
            '    {"name": "b",   "gain_db": -6},\n    {"name": "c",   "gain_db": -9}\n  ],\n'
            '  "samples": {"s1": {"file": "x.ogg", "end": 0.250}}\n}\n')
    new = json.loads(text)
    del new['rules'][1]
    new['rules'].append({'name': 'd'})  # a delete plus an add keeps the length: must not patch pairwise
    new['rules'][0]['gain_db'] = -4
    new['samples']['s1']['eq'] = {'lp': 9000}
    out = variant.patch_text(text, new)
    assert json.loads(out) == new
    assert '{"name": "a",   "gain_db": -4, "pitch": [1, 1]}' in out and '"end": 0.250' in out  # spacing and 0.250 kept
    assert '{"name": "c",   "gain_db": -9}' in out and '"b"' not in out
    assert variant.change('x', 0.3, 0.3).endswith('unchanged (0.3)')


@pytest.mark.slow
def test_variant_ops(project, tmp_path):
    top = {'duration': 12.0, 'cues': 'out/cues.json', 'include': ['audio/design.json'], 'master': {'lufs': -14}}
    json.dump(top, open(os.path.join(project, 'audio/top.json'), 'w'))
    ops = [{'f': 'design', 'rule': 'fx.hit', 'set': {'gain_db': -5}}, {'f': 'design', 'sample': 'fx.pop', 'eq': {'lp': 9000}},
           {'sd': 'master.lufs', 'value': -15}]
    json.dump(ops, open(tmp_path / 'ops.json', 'w'))
    out = str(tmp_path / 'v1')
    code, txt = run('variant.py', '--root', project, '--design', 'audio/top.json', '--ops', str(tmp_path / 'ops.json'), '--out', out)
    assert code == 0 and '3 change(s)' in txt, txt
    v = json.load(open(os.path.join(out, 'design', 'design.json')))
    assert [r for r in v['rules'] if r['name'] == 'fx.hit'][0]['gain_db'] == -5
    assert v['samples']['fx.pop']['eq'] == {'hp': 250, 'peq': [[3200, 1.2, -2]], 'lp': 9000}
    assert json.load(open(os.path.join(out, 'sounddesign.json')))['master']['lufs'] == -15
    orig = json.load(open(os.path.join(project, 'audio/design.json')))
    assert [r for r in orig['rules'] if r['name'] == 'fx.hit'][0]['gain_db'] == -2  # the design itself is untouched
    json.dump([{'f': 'design', 'rule': 'no.such.rule', 'set': {'gain_db': 0}}], open(tmp_path / 'bad.json', 'w'))
    code, txt = run('variant.py', '--root', project, '--design', 'audio/top.json', '--ops', str(tmp_path / 'bad.json'),
                    '--out', str(tmp_path / 'v2'))
    assert code != 0 and 'nothing written' in txt and not os.path.exists(tmp_path / 'v2')