Skip to content

Make videos on the computer you already own. FreeVideo runs MiniMax H3 in as little as 8 GB of VRAM and 16 GB of RAM, and adapts its acceleration path to your hardware.

33.5 KB
Raw Blame
"""Sequential full text-to-video stability tests; retain every artifact locally."""
import argparse
import csv
import html
import json
import math
import os
from pathlib import Path
import signal
import shutil
import subprocess
import sys
import time
import traceback
from urllib.parse import quote

from . import execution_paths
from .bootstrap import inventory, digest, GiB, PACKAGE, DEFAULT_ROOT, existing_parent
from .geometry import geometry
from .locking import runtime_lock, LOCK_ENV
from .monitoring import Monitor, save
from .ram import ProcessMemory
from .terminal_ui import TerminalUI, LogProgress
from .validation import read_case_reports, validate_metrics, validate_artifacts
from . import processes
from .system import (windows, system_memory, cpu_info, nvidia_smi, memory_peak, memory_sample,
                     inference_headroom, inference_memory_sample)


def case_plan(suite='standard'):
    prompts = json.loads((PACKAGE / 'test_prompts.json').read_text(encoding='utf-8'))
    w, h, frames = (768, 448, 90) if suite == 'quick' else (1344, 768, 243)
    specs = [('01-first', 'first request; cache state observed, not forced cold', w, h, frames, prompts[0]),
             ('02-repeat', 'second request, identical prompt and seed', w, h, frames, prompts[0]),
             ('03-new-prompt', 'third request, changed prompt', w, h, frames, prompts[1]),
             ('04-resolution', 'changed resolution only, relative to first request',
              512 if suite == 'quick' else 1024, 288 if suite == 'quick' else 576, frames, prompts[0]),
             ('05-duration', 'changed duration only, relative to first request', w, h,
              56 if suite == 'quick' else 124, prompts[0])]
    if suite == 'stress':
        specs += [('06-higher-resolution', 'larger canvas stress case', 1536, 864, 243, prompts[0]),
                  ('07-longer', 'longer duration stress case', 1344, 768, 362, prompts[0])]
    return [dict(id=name, purpose=purpose, width=w, height=h, frames=f, prompt=p, seed=2026090901)
            for name, purpose, w, h, f, p in specs]


def validate_cases(cases):
    if not isinstance(cases, list) or not cases:
        raise ValueError('Case file must contain a nonempty JSON list.')
    ids = set()
    for case in cases:
        name = case.get('id') if isinstance(case, dict) else None
        if not isinstance(name, str) or not name or any(c not in 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_' for c in name) or name in ids:
            raise ValueError('Each case needs a unique path-safe id.')
        ids.add(name)
        if not isinstance(case.get('prompt'), str) or not case['prompt'].strip():
            raise ValueError('Every test case must include a nonempty raw text prompt.')
        geometry(case.get('width', 1344), case.get('height', 768),
                 frames=case.get('frames'), seconds=case.get('seconds'))
    return cases


def cgroup_snapshot():
    if windows():
        return {'unavailable': 'Linux cgroups do not apply to native Windows; no hard capacity limit was created.'}
    try:
        group = next(line.split(':', 2)[2] for line in Path('/proc/self/cgroup').read_text(encoding='utf-8').splitlines() if line.startswith('0::'))
        root = Path('/sys/fs/cgroup') / group.lstrip('/')
        result = {'path': str(root), 'scope': 'Existing shared cgroup; peaks are lifetime values, not case attribution.'}
        for name in ('memory.current', 'memory.peak', 'memory.max', 'memory.swap.current', 'memory.swap.max'):
            path = root / name
            raw = path.read_text(encoding='utf-8').strip() if path.is_file() else 'max'
            result[name] = None if raw == 'max' else int(raw)
        result['events'] = dict((k, int(v)) for k, v in (line.split() for line in (root / 'memory.events').read_text(encoding='utf-8').splitlines()))
        return result
    except (OSError, ValueError, StopIteration) as error:
        return {'unavailable': repr(error)}


def swap_snapshot():
    if windows():
        return {}  # Windows commit is captured in each RAM sample, not Linux swap counters.
    return {k: int(v) for k, v in (line.split() for line in Path('/proc/vmstat').read_text(encoding='utf-8').splitlines())
            if k in ('pswpin', 'pswpout', 'pgmajfault', 'oom_kill')}


def stop_tree(process):
    if windows():
        return processes.stop(process, grace=20)
    import psutil
    try:
        children = psutil.Process(process.pid).children(recursive=True)
    except psutil.NoSuchProcess:
        children = []
    # generate catches SIGTERM and waits for its separate encoder/video group.
    if process.poll() is None:
        process.terminate()
    try:
        process.wait(timeout=20)
    except subprocess.TimeoutExpired:
        process.kill()
        process.wait()
    for child in children:
        try:
            if child.is_running():
                child.kill()
        except psutil.NoSuchProcess:
            pass
    psutil.wait_procs(children, timeout=5)


class CaseProgress:
    def __init__(self, directory, ui):
        self.directory, self.ui = directory, ui
        self.log = LogProgress(directory / 'video.engine.log')
        self.phase = None
        self.ram_peak = None

    def update(self, sample, monitor):
        phase = 'Starting request'
        for name, stages in [('encoding', {'encoder_import': 'Loading encoder libraries', 'encoder_load': 'Loading text encoder',
                                          'encode': 'Encoding prompt'}),
                             ('engine', {'load': 'Loading transformer', 'sample': 'Denoising', 'decode': 'Decoding video and audio',
                                         'complete': 'Writing request report'})]:
            try:
                value = json.loads((self.directory / ('video.' + name + '.json')).read_text(encoding='utf-8'))
                phase = stages.get(value.get('phase'), 'Releasing text encoder' if name == 'encoding' else phase)
            except (OSError, ValueError):
                pass
        if phase != self.phase:
            self.ui.update(self.directory.name, done=0, total=8 if phase == 'Denoising' else 0, detail=phase)
            self.phase = phase
        current_ram = memory_sample(sample)
        if current_ram is not None:
            self.ram_peak = max(self.ram_peak or 0, current_ram)
        now = '—' if current_ram is None else '%.2f' % (current_ram/GiB)
        peak = '—' if self.ram_peak is None else '%.2f' % (self.ram_peak/GiB)
        self.ui.update(self.directory.name, **self.log.read(), resource='GPU %.2f GiB · peak %.2f GiB · RAM now %s GiB · peak %s GiB · available %.2f GiB' %
            ((monitor.last if monitor else 0)/GiB, (monitor.peak if monitor else 0)/GiB,
             now, peak, inference_headroom(sample)/GiB))


def run_process(command, directory, env, gpu, timeout, minimum_ram, *, sample_gpu=True, pass_fds=(), ui=None,
                maximum_gpu=None, maximum_working_ram=None):
    memory = ProcessMemory()
    monitor = None
    gpu_error = None
    if sample_gpu:
        try:
            monitor = Monitor(directory / 'gpu.csv', device=gpu).start()
        except Exception as error:
            gpu_error = repr(error)
    initial_cgroup, initial_swap = cgroup_snapshot(), swap_snapshot()
    tick = time.perf_counter()
    status = 'failed'
    reason = None
    process = None
    progress = CaseProgress(directory, ui) if ui else None
    try:
        with (directory / 'run.log').open('w', encoding='utf-8') as log, (directory / 'ram.jsonl').open('w', buffering=1, encoding='utf-8') as samples:
            process = processes.popen(command, stdout=log, stderr=subprocess.STDOUT, env=env,
                                       start_new_session=True, pass_fds=pass_fds)
            heartbeat = time.monotonic()
            while process.poll() is None:
                sample = memory.sample(process.pid)
                samples.write(json.dumps(dict(sample, epoch_seconds=time.time(), elapsed_seconds=time.perf_counter()-tick)) + '\n')
                if progress:
                    progress.update(sample, monitor)
                if ((maximum_gpu is not None and monitor and monitor.peak > maximum_gpu)
                        or (maximum_working_ram is not None and inference_memory_sample(sample) is not None
                            and inference_memory_sample(sample) > maximum_working_ram)):
                    reason, status = 'Optimization trial exceeded its observed memory budget.', 'trial_memory_guard'
                    stop_tree(process)
                    break
                if memory_sample(sample) is None:
                    reason, status = 'Process-tree memory monitoring became incomplete; details retained in ram.jsonl.', 'ram_monitor_failed'
                    stop_tree(process)
                    break
                if inference_headroom(sample) <= 0 or inference_headroom(sample) < minimum_ram:
                    reason, status = 'Available system RAM crossed the configured emergency floor.', 'ram_guard'
                    stop_tree(process)
                    break
                if time.perf_counter() - tick > timeout:
                    reason, status = 'Case exceeded timeout.', 'timeout'
                    stop_tree(process)
                    break
                try:
                    # Short model-load peaks can disappear between 500 ms
                    # samples. Keep this bounded; it is observed, not exact, RAM.
                    process.wait(timeout=.2)
                except subprocess.TimeoutExpired:
                    pass
                if not ui and time.monotonic() - heartbeat >= 30:
                    print('[%s] %.0fs; RAM now %.2f GiB; peak %.2f GiB; available %.2f GiB' %
                          (directory.name, time.perf_counter()-tick, (memory_sample(sample) or 0)/GiB,
                           (memory_peak(memory.result()) or 0)/GiB,
                           inference_headroom(sample)/GiB), flush=True)
                    heartbeat = time.monotonic()
            if reason is None:
                status = 'complete' if process.returncode == 0 else 'failed'
    except BaseException as error:
        reason, status = repr(error), 'interrupted' if isinstance(error, KeyboardInterrupt) else 'failed'
        if process is not None:
            stop_tree(process)
    finally:
        measured = monitor.stop() if monitor else {'sampling_errors': [gpu_error or 'GPU sampling disabled'], 'gpu_peak_bytes': None}
    elapsed = time.perf_counter() - tick
    final_cgroup, final_swap = cgroup_snapshot(), swap_snapshot()
    return {'status': status, 'error': reason, 'returncode': process.returncode if process else None,
            'wall_seconds': elapsed, 'gpu': measured,
            'ram': dict(memory.result(), ram_poll_interval_seconds=.2),
            'cgroup_before': initial_cgroup, 'cgroup_after': final_cgroup,
            'cgroup_event_delta': {k: v - initial_cgroup.get('events', {}).get(k, 0)
                                   for k, v in final_cgroup.get('events', {}).items()},
            'system_vmstat_delta': {k: final_swap[k] - initial_swap[k] for k in final_swap},
            'hard_memory_limits_created': False,
            'timing_scope': 'Fresh request process: text encoding, model loads, sampling, decode, all artifact writes and process startup/exit. Media validation and artifact hashing excluded.'}


def environment(machine):
    """The installation's defaults, not an override of the caller's choices.

    The recorded GPU UUID keeps the host and its children on the device that
    passed initialization and survives a Windows display-adapter reorder, so it
    is the right default. Forcing it also pins the installation to one physical
    GPU: a caller that selected another device silently lost it, and on any
    other machine that UUID does not exist, so CUDA reported no device at all.
    The compiler caches behave the same way; several measurements on one host
    need their own.
    """
    inherited = lambda name, fallback: os.environ.get(name) or fallback
    env = dict(os.environ, FREEVIDEO_HOME=machine['root'], FREEVIDEO_VDN_ROOT=machine['vdn_root'],
                PATH=(str(Path(machine['git']).parent) + os.pathsep if machine.get('git') else '') + os.environ.get('PATH', ''),
                FREEVIDEO_MODEL_ROOT=machine['model_root'], FREEVIDEO_COMFY_ROOT=machine['comfy_root'],
                FREEVIDEO_COMFY_PYTHON=machine['comfy_python'],
                PYTHONUNBUFFERED='1', PYTHONUTF8='1', PYTHONIOENCODING='utf-8',
                FREEVIDEO_LOCK_PATH=inherited('FREEVIDEO_LOCK_PATH', str(Path(machine['root']) / 'engine.lock')),
                OMP_NUM_THREADS='8', MKL_NUM_THREADS='8')
    if machine.get('device_backend') == 'mps':
        from .macos_bootstrap import environment as native_environment
        return native_environment(machine['root'], dict(env, OMP_NUM_THREADS='2', MKL_NUM_THREADS='2'))
    env['CUDA_VISIBLE_DEVICES'] = inherited('CUDA_VISIBLE_DEVICES', machine['gpu_uuid'])
    from .triton_compat import environment as compiler_environment
    return compiler_environment(machine['root'], env)


def command_for(machine, case, destination, args):
    command = [machine['python'], '-m', 'freevideo_engine', 'generate', '--prompt-file', str(destination / 'prompt.txt'),
        '--cache', machine['cache'], '--base', machine['base'], '--checkpoint', machine['checkpoint'],
        '--encoder-python', machine['comfy_python'], '--encoder-root', machine['comfy_root'],
        '--model-paths', machine['model_paths'], '--encoder', machine['encoder'],
        '--out', str(destination / 'video.mp4'), '--seed', str(case.get('seed', 2026090901)),
        '--width', str(case.get('width', 1344)), '--height', str(case.get('height', 768)),
        '--attention', args.attention]
    if getattr(args, 'no_tuning', False):
        command.append('--no-tuning')
    if case.get('seconds') is not None:
        command += ['--seconds', str(case['seconds'])]
    else:
        command += ['--frames', str(case.get('frames', 243))]
    for key in ('vram_gib', 'ram_gib'):
        value = getattr(args, key) if getattr(args, key) is not None else machine.get(key)
        if value is not None:
            command += ['--' + key.replace('_', '-'), str(value)]
    return command


def artifact_manifest(directory):
    files = []
    for path in sorted(directory.rglob('*')):
        if path.is_file() and path.name != 'artifacts.json':
            files.append({'path': path.relative_to(directory).as_posix(), 'bytes': path.stat().st_size, 'sha256': digest(path)})
    save(directory / 'artifacts.json', {'files': files, 'total_bytes': sum(f['bytes'] for f in files)})


def artifact_estimate(cases):
    canvases = [geometry(c.get('width', 1344), c.get('height', 768), frames=c.get('frames'), seconds=c.get('seconds')) for c in cases]
    raw_bytes = sum(c['width'] * c['height'] * c['frames'] * 3 for c in canvases)
    # RGB is the largest retained output; include latents, audio, MP4 and logs.
    return int(raw_bytes * 1.5 + len(cases) * 128 * 2**20 + GiB)


def flatten(row):
    g, request = row['geometry'], row.get('request', {})
    video, encoding = row.get('engine', {}), row.get('encoding', {})
    gpu, ram = row.get('gpu', {}), row.get('ram', {})
    profile = request.get('profile', {})
    return dict(case=row['id'], status=row['status'], width=g['width'], height=g['height'],
        frames=g['frames'], duration_seconds=g['seconds'], wall_seconds=row.get('wall_seconds'),
        text_encode_seconds=encoding.get('work_seconds'), conditioning_cache_hit=bool(encoding.get('cache_hit')),
        tuning_profile=request.get('tuning', {}).get('profile_id'),
        baseline_wall_seconds=(row.get('comparison') or {}).get('baseline_wall_seconds'),
        wall_gain_percent=(row.get('comparison') or {}).get('wall_gain_percent'),
        sampling_gain_percent=(row.get('comparison') or {}).get('sampling_gain_percent'),
        model_load_seconds=video.get('load_seconds'),
        sample_seconds=video.get('sample_seconds'), vae_load_seconds=video.get('vae_load_seconds'),
        video_decode_seconds=video.get('video_decode_seconds'), audio_decode_seconds=video.get('audio_load_decode_seconds'),
        mp4_save_seconds=video.get('encode_seconds'), latent_save_seconds=video.get('latent_save_seconds'),
        rgb_audio_save_seconds=video.get('decoded_artifact_save_seconds'),
        gpu_baseline_gib=gpu.get('gpu_baseline_bytes', 0)/GiB,
        gpu_peak_gib=gpu['gpu_peak_bytes']/GiB if gpu.get('gpu_peak_bytes') is not None else None,
        process_peak_rss_gib=ram.get('process_tree_peak_rss_bytes', 0)/GiB,
        process_peak_pss_gib=ram['process_tree_peak_pss_bytes']/GiB if ram.get('process_tree_peak_pss_bytes') is not None else None,
        process_peak_private_commit_gib=ram['process_tree_peak_private_commit_bytes']/GiB if ram.get('process_tree_peak_private_commit_bytes') is not None else None,
        process_peak_private_working_set_gib=ram['process_tree_peak_private_working_set_bytes']/GiB if ram.get('process_tree_peak_private_working_set_bytes') is not None else None,
        process_peak_guard_gib=(memory_peak(ram) or 0)/GiB, ram_guard_metric=ram.get('ram_guard_metric', 'PSS'),
        system_min_commit_available_gib=ram['system_min_commit_available_bytes']/GiB if ram.get('system_min_commit_available_bytes') is not None else None,
        system_min_available_gib=(ram.get('system_min_available_bytes') or 0)/GiB,
        attention=profile.get('engine', {}).get('attention'), error=row.get('error'))


def report(directory, value):
    save(directory / 'report.json', value)
    flat = [flatten(row) for row in value['cases']]
    if flat:
        with (directory / 'report.csv').open('w', encoding='utf-8-sig', newline='') as stream:
            writer = csv.DictWriter(stream, fieldnames=list(flat[0]))
            writer.writeheader()
            writer.writerows(flat)
    columns = ('case', 'status', 'width', 'height', 'frames', 'wall_seconds', 'text_encode_seconds',
               'conditioning_cache_hit', 'sample_seconds', 'wall_gain_percent', 'sampling_gain_percent',
               'gpu_peak_gib', 'process_peak_guard_gib', 'ram_guard_metric', 'system_min_available_gib')
    def cell(v):
        return '—' if v is None else ('%.2f' % v if isinstance(v, float) else html.escape(str(v)))
    labels = {'gpu_peak_gib': 'GPU peak (GiB)', 'process_peak_guard_gib': 'RAM peak (GiB)',
              'ram_guard_metric': 'RAM measurement', 'system_min_available_gib': 'System available minimum (GiB)'}
    table = '<tr>' + ''.join('<th>' + html.escape(labels.get(k, k)) + '</th>' for k in columns) + '</tr>'
    table += ''.join('<tr>' + ''.join('<td>' + cell(row.get(k)) + '</td>' for k in columns) + '</tr>' for row in flat)
    media = ''
    for row in value['cases']:
        path = directory / row['id'] / 'video.mp4'
        media += '<h2>%s — %s</h2><p>%s</p>' % (html.escape(row['id']), html.escape(row['status']), html.escape(row.get('purpose', '')))
        if path.is_file():
            relative = quote(path.relative_to(directory).as_posix())
            media += '<video controls preload="metadata" src="%s"></video>' % relative
        media += '<p><a href="%s/case.json">Metrics</a> · <a href="%s/run.log">Log</a> · <a href="%s/artifacts.json">All artifacts</a></p>' % ((quote(row['id']),) * 3)
        if row.get('error'):
            media += '<pre>' + html.escape(row['error']) + '</pre>'
    (directory / 'report.html').write_text('<!doctype html><html lang="en"><meta charset="utf-8"><title>FreeVideo test report</title>'
        '<style>body{font:16px system-ui;margin:32px;background:#111827;color:#e5e7eb}a{color:#93c5fd}'
        'table{border-collapse:collapse;font-size:13px}td,th{padding:8px;border:1px solid #4b5563}video{max-width:800px;width:100%}'
        'pre{white-space:pre-wrap}section{overflow:auto}</style><h1>FreeVideo test report</h1>'
        '<p>Status: ' + html.escape(value['status']) + '. All videos, audio, raw RGB, latents, conditions and logs are retained locally.</p>'
        '<p>Whole GPU usage includes other applications. Linux uses PSS; Windows separately records working set and private commit, and guards their maximum. '
        'Neither includes all system file cache. Windows available memory also checks commit headroom. RAM/VRAM budgets are placement guidance; '
        'these runs create no hard capacity limit. Each request reloads models; validated conditioning reuse is shown separately. '
        'Gains compare matched prompts, seeds, geometry, base placement and dependencies from the optimization baseline; OS/JIT cache and storage warmth can affect wall time.</p>'
        '<p><a href="report.json">Full JSON</a> · <a href="report.csv">CSV</a> · <a href="inventory.json">Machine and versions</a></p>'
        '<section><table>' + table + '</table></section>' + media + '</html>', encoding='utf-8')


def provenance(machine, env):
    result = inventory(machine['gpu_uuid'])
    result['machine_configuration'] = machine
    result['source_files'] = {p.relative_to(PACKAGE.parent).as_posix(): digest(p)
                              for p in sorted([*PACKAGE.glob('*.py'), *PACKAGE.glob('*.json')])}
    result['cpu_info'] = cpu_info()
    result['dependency_revisions'] = json.loads((PACKAGE / 'dependencies.json').read_text(encoding='utf-8'))
    result['fp8_cache_manifest_sha256'] = digest(Path(machine['cache']) / 'manifest.json')
    for name, command in [('git_head', ['git', '-C', str(PACKAGE.parent), 'rev-parse', 'HEAD']),
                          ('git_status', ['git', '-C', str(PACKAGE.parent), 'status', '--porcelain']),
                          ('nvidia_smi', [nvidia_smi(), '-q']),
                          ('engine_packages', [machine['python'], '-m', 'pip', 'freeze']),
                          ('encoder_packages', [machine['comfy_python'], '-m', 'pip', 'freeze'])]:
        p = execution_paths.run(command, capture_output=True, text=True, env=env, timeout=30, encoding='utf-8', errors='replace')
        result[name] = {'returncode': p.returncode, 'stdout': p.stdout, 'stderr': p.stderr}
    return result


def _main(argv=None):
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('--root', type=Path, default=Path(os.environ.get('FREEVIDEO_HOME', DEFAULT_ROOT)), help='Setup installation directory')
    parser.add_argument('--config', type=Path, help='Override root/machine.json')
    parser.add_argument('--suite', choices=('standard', 'quick', 'stress'), default='standard')
    parser.add_argument('--cases', type=Path, help='Custom JSON list of raw-text cases')
    parser.add_argument('--out', type=Path, help='New report directory; existing paths are refused')
    parser.add_argument('--plan', action='store_true')
    parser.add_argument('--json', action='store_true', help='Print --plan as JSON')
    parser.add_argument('--plain', action='store_true', help='Disable live terminal rendering')
    parser.add_argument('--no-color', action='store_true', help='Disable colors (also respects NO_COLOR)')
    parser.add_argument('--timeout-seconds', type=float, default=3600)
    parser.add_argument('--min-available-ram-gib', type=float, default=.25)
    parser.add_argument('--vram-gib', type=float)
    parser.add_argument('--ram-gib', type=float)
    parser.add_argument('--attention', default='auto')
    parser.add_argument('--no-tuning', action='store_true', help='Bypass saved optimization and text reuse for a baseline run')
    args = parser.parse_args(argv)
    if args.json and not args.plan:
        parser.error('--json requires --plan')
    if not all(math.isfinite(v) for v in (args.timeout_seconds, args.min_available_ram_gib)) or args.timeout_seconds <= 0 or args.min_available_ram_gib < 0:
        parser.error('Use a positive timeout and a nonnegative emergency RAM floor.')
    config = (args.config or args.root / 'machine.json').expanduser().resolve()
    machine = json.loads(config.read_text(encoding='utf-8'))
    if not machine.get('ready'):
        raise ValueError('Setup is incomplete.')
    cases = validate_cases(json.loads(args.cases.read_text(encoding='utf-8')) if args.cases else case_plan(args.suite))
    destination = (args.out or Path(machine['root']) / 'test-runs' / (time.strftime('%Y%m%dT%H%M%SZ', time.gmtime()) + '-' + str(os.getpid()))).resolve()
    disk_free, disk_needed = shutil.disk_usage(existing_parent(destination)).free, artifact_estimate(cases)
    ui = TerminalUI('Engine test / ' + (args.suite if not args.cases else 'custom'), plain=args.plain, no_color=args.no_color)
    test_plan = {'cases': cases, 'geometry': [geometry(c.get('width', 1344), c.get('height', 768), frames=c.get('frames'), seconds=c.get('seconds')) for c in cases],
                 'estimated_artifact_bytes': disk_needed, 'disk_free_bytes': disk_free, 'output': str(destination),
                 'artifacts': 'All retained', 'scope': 'Native text conditioning (validated reuse when enabled), complete default 8 + 3 video/audio generation.'}
    if args.json:
        print(json.dumps(test_plan, indent=2))
    else:
        ui.panel('FreeVideo / Engine test plan', [('GPU', machine['gpu_uuid']),
            ('Cases', '%d · text encode → 8 + 3 denoising steps → video + audio → validation' % len(cases)),
            ('Cache', 'Fresh request processes; persistent compatible inputs ' + ('bypassed' if args.no_tuning else 'reused automatically; no eviction')),
            ('Disk', '~%.2f GiB for retained artifacts · %.2f GiB free' % (disk_needed/GiB, disk_free/GiB)),
            ('Outputs', destination)] + [(c['id'], '%d×%d · %d frames · %.2fs' % (g['width'], g['height'], g['frames'], g['seconds']))
                                       for c, g in zip(cases, test_plan['geometry'])])
    if disk_free < disk_needed:
        print('Insufficient free disk space for all retained test artifacts.', file=sys.stderr)
        return 1
    if args.plan:
        return 0
    env = environment(machine)
    with runtime_lock(path=Path(env['FREEVIDEO_LOCK_PATH'])) as fd:
        if json.loads(config.read_text(encoding='utf-8')) != machine:
            raise ValueError('Installation changed during preflight. Rerun the test command.')
        return run_suite(args, machine, cases, destination, ui, env, fd)


def run_suite(args, machine, cases, destination, ui, env, fd):
    destination.mkdir(parents=True, exist_ok=False)
    value = {'schema_version': 1, 'suite': args.suite if not args.cases else 'custom', 'status': 'running',
             'created_epoch': time.time(), 'cases': [], 'plan': cases,
             'artifacts_retained': True, 'full_model_residency_between_requests': False,
             'cache_policy': 'Natural OS file and persistent JIT caches; no global cache flushing; first is not claimed storage-cold.',
             'physical_gpu_scope': 'Measured on the GPU in inventory.json. Nominal capacity overrides do not emulate another GPU.'}
    save(destination / 'plan.json', value)
    report(destination, value)
    def interrupted(signum, frame):
        raise KeyboardInterrupt('Test interrupted by signal %s' % signum)
    previous = processes.termination_handler(interrupted)
    try:
        ui.start(destination)
        ui.phase('Capture hardware, source and dependency versions', 0, len(cases))
        save(destination / 'inventory.json', provenance(machine, env))
        for index, case in enumerate(cases):
            directory = destination / case['id']
            directory.mkdir()
            (directory / 'prompt.txt').write_text(case['prompt'], encoding='utf-8')
            canvas = geometry(case.get('width', 1344), case.get('height', 768), frames=case.get('frames'), seconds=case.get('seconds'))
            command = command_for(machine, case, directory, args)
            row = dict(case, geometry=canvas, command=command, status='running')
            value['cases'].append(row)
            save(directory / 'case.json', row)
            report(destination, value)
            ui.phase('Case %d of %d' % (index + 1, len(cases)), index, len(cases))
            ui.begin(case['id'], '%s · %d×%d · %d frames' % (case['id'], canvas['width'], canvas['height'], canvas['frames']), detail='Starting exclusive Engine request')
            row['before'] = inventory(machine['gpu_uuid'])
            child_env = dict(env, **{LOCK_ENV: str(fd)})
            row.update(run_process(command, directory, child_env, machine['gpu_uuid'], args.timeout_seconds,
                                   args.min_available_ram_gib * GiB, pass_fds=(fd,), ui=ui))
            read_case_reports(directory, row)
            if row['status'] == 'complete':
                ui.update(case['id'], done=0, total=0, detail='Validate every video frame, audio and retained artifacts')
                try:
                    from .media import inspect
                    validate_metrics(row, canvas)
                    row['artifact_validation'] = validate_artifacts(directory, canvas)
                    row['media'] = inspect(directory / 'video.mp4', **{k: canvas[k] for k in ('width', 'height', 'frames', 'fps')})
                except Exception as error:
                    row.update(status='validation_failed', error=repr(error))
            elif not row.get('error'):
                row['error'] = row.get('request', {}).get('error') or (directory / 'run.log').read_text(errors='replace', encoding='utf-8')[-2500:]
            if row['status'] != 'complete' and row.get('request', {}).get('tuning', {}).get('profile_id'):
                from .tuning import disable_profile
                # The child may have been killed before its own finally block.
                old_root = os.environ.get('FREEVIDEO_HOME')
                os.environ['FREEVIDEO_HOME'] = machine['root']
                try:
                    disable_profile(row['request']['tuning']['profile_id'], row.get('error', row['status']))
                finally:
                    if old_root is None:
                        os.environ.pop('FREEVIDEO_HOME', None)
                    else:
                        os.environ['FREEVIDEO_HOME'] = old_root
            if row['status'] == 'complete':
                row['comparison'] = baseline_comparison(row)
            save(directory / 'case.json', row)
            ui.update(case['id'], done=0, total=0, detail='Hash retained artifacts and write reports')
            artifact_manifest(directory)
            report(destination, value)
            ui.end(case['id'], success=row['status'] == 'complete', detail='%.2fs request · GPU peak %.2f GiB · RAM peak %.2f GiB' %
                   (row['wall_seconds'], (row.get('gpu', {}).get('gpu_peak_bytes') or 0)/GiB, (memory_peak(row['ram']) or 0)/GiB))
            if row['status'] == 'interrupted':
                break
        value['status'] = 'complete' if len(value['cases']) == len(cases) and all(r['status'] == 'complete' for r in value['cases']) else 'failed'
    except BaseException as error:
        value.update(status='failed', error=repr(error), traceback=traceback.format_exc())
        if value['cases'] and value['cases'][-1]['status'] == 'running':
            last = value['cases'][-1]
            last.update(status='interrupted' if isinstance(error, KeyboardInterrupt) else 'failed', error=repr(error))
            ui.end(last['id'], success=False, detail=str(error))
            save(destination / last['id'] / 'case.json', last)
            artifact_manifest(destination / last['id'])
    finally:
        try:
            value['finished_epoch'] = time.time()
            report(destination, value)
        except OSError as error:
            value.update(status='failed', report_error=repr(error))
            print('Could not save final test report: ' + str(error), file=sys.stderr)
        finally:
            processes.restore_handlers(previous)
            try:
                ui.phase('Tests ' + value['status'], len([r for r in value['cases'] if r['status'] != 'running']), len(cases))
            finally:
                ui.close()
    print('\nReport: %s\nStatus: %s' % (destination / 'report.html', value['status']), flush=True)
    return 0 if value['status'] == 'complete' else 1


def baseline_comparison(row):
    reference = row.get('request', {}).get('tuning', {}).get('baseline_report')
    if not reference:
        return None
    try:
        baseline = json.loads(Path(reference).read_text(encoding='utf-8'))
        for previous in baseline.get('cases', []):
            if previous.get('status') != 'complete' or any(row.get(k) != previous.get(k) for k in ('prompt', 'seed', 'geometry')):
                continue
            before, after = previous['request']['profile'], row['request']['profile']
            from .tuning import PATCH_KEYS
            if (before['decoder'] != after['decoder'] or any(before.get(k) != after.get(k) for k in ('gpu_budget_gb', 'inference_ram_budget_gb'))
                    or {k: v for k, v in before['engine'].items() if k not in PATCH_KEYS} !=
                       {k: v for k, v in after['engine'].items() if k not in PATCH_KEYS}):
                continue
            return {'baseline_report': reference, 'case': previous['id'], 'baseline_wall_seconds': previous['wall_seconds'],
                    'wall_gain_percent': 100 * (1 - row['wall_seconds'] / previous['wall_seconds']),
                    'sampling_gain_percent': 100 * (1 - row['engine']['sample_seconds'] / previous['engine']['sample_seconds']),
                    'conditioning_reused': bool(row['encoding'].get('cache_hit')),
                    'scope': 'Same input, seed, geometry and base resource placement. Storage/JIT warmth can affect timings.'}
    except (OSError, ValueError, KeyError, ZeroDivisionError, TypeError):
        return None
    return None


def main(argv=None):
    try:
        return _main(argv)
    except (OSError, ValueError, RuntimeError, KeyError) as error:
        print('Test could not start: ' + str(error), file=sys.stderr)
        return 1
    except KeyboardInterrupt:
        print('Test cancelled.', file=sys.stderr)
        return 130


if __name__ == '__main__':
    raise SystemExit(main())