Skip to content

Make videos on the computer you already own. FreeVideo runs MiniMax H3 in as little as 8 GB of VRAM and 16 GB of RAM, and adapts its acceleration path to your hardware.

26.2 KB
Raw Blame
"""Whole-device NVML sampling; values include driver reservations."""
import bisect
import csv
import ctypes
import json
import os
from pathlib import Path
import threading
import time
import tempfile
from .system import windows
from .long_paths import io_path, plain_data


def save(path, value, *, staging=None, plain_paths=True):
    path = io_path(path)
    path.parent.mkdir(parents=True, exist_ok=True)
    # Unique, closed staging file; Windows readers/antivirus may briefly hold
    # the destination. Keep the previous good report and the failed staging file.
    # ``staging`` names it for a caller whose folder cannot take a longer name
    # than the file itself.
    with tempfile.NamedTemporaryFile(mode='w', encoding='utf-8', newline='\n',
            prefix=path.name + '.' if staging is None else staging, suffix='.tmp',
            dir=path.parent, delete=False) as stream:
        temporary = Path(stream.name)
        stream.write(json.dumps(plain_data(value) if plain_paths else value, ensure_ascii=False, indent=2) + '\n')
    for attempt in range(8):
        try:
            temporary.replace(path)
            return
        except PermissionError:
            if not windows() or attempt == 7:
                raise
            time.sleep(.1 * (attempt + 1))


class Memory(ctypes.Structure):
    _fields_ = [('total', ctypes.c_ulonglong), ('free', ctypes.c_ulonglong),
               ('used', ctypes.c_ulonglong)]


class Utilization(ctypes.Structure):
    _fields_ = [('gpu', ctypes.c_uint), ('memory', ctypes.c_uint)]


class Activity:
    """Constant-space driver observations; unsupported sensors remain unknown."""
    fields = ('gpu_util_percent', 'temperature_c', 'power_mw', 'sm_clock_mhz', 'memory_clock_mhz')
    clock_reasons = {'sw_power_cap': 0x04, 'hw_slowdown': 0x08,
                     'sw_thermal': 0x20, 'hw_thermal': 0x40, 'hw_power_brake': 0x80}

    def __init__(self):
        self.samples = 0
        self.values = {}
        self.reason_samples = 0
        self.reason_counts = {key: 0 for key in self.clock_reasons}
        self.reason_mask = 0

    def add(self, util, auxiliary, reasons):
        self.samples += 1
        for name, value in zip(self.fields, (util, *auxiliary)):
            if type(value) not in (int, float) or not 0 <= value < float('inf'):
                continue
            row = self.values.setdefault(name, dict(samples=0, total=0, minimum=value, maximum=value))
            row['samples'] += 1
            row['total'] += value
            row['minimum'] = min(row['minimum'], value)
            row['maximum'] = max(row['maximum'], value)
        if type(reasons) is int and reasons >= 0:
            self.reason_samples += 1
            self.reason_mask |= reasons
            for key, mask in self.clock_reasons.items():
                self.reason_counts[key] += bool(reasons & mask)

    def result(self):
        result = dict(samples=self.samples, clock_reason_samples=self.reason_samples)
        for name, row in self.values.items():
            result[name] = dict(samples=row['samples'], minimum=row['minimum'], maximum=row['maximum'],
                                mean=row['total']/row['samples'])
        if self.reason_samples:
            result['clock_reason_mask'] = self.reason_mask
            result['clock_reason_counts'] = dict(self.reason_counts)
        return result


def thermal_slowdown(measured, *, share=.5, clock_ratio=.7, minimum_samples=200):
    """The worker's busy time held back by driver thermal slowdown, or None.

    Counted only when thermal reasons cover at least `share` of the busy
    samples and the mean SM clock stays below `clock_ratio` of the highest
    one observed, so a brief hot spell or a normal power cap is not reported.
    """
    busy = measured.get('busy_activity') if isinstance(measured, dict) else None
    if not isinstance(busy, dict):
        return None
    samples = _count(busy.get('clock_reason_samples'))
    counts = busy.get('clock_reason_counts')
    counts = counts if isinstance(counts, dict) else {}
    clock = busy.get('sm_clock_mhz')
    clock = clock if isinstance(clock, dict) else {}
    mean, peak = _count(clock.get('mean')), _count(clock.get('maximum'))
    if samples < minimum_samples or not mean or not peak:
        return None
    thermal = min(samples, max(_count(counts.get('sw_thermal')), _count(counts.get('hw_thermal'))))
    if thermal < share * samples or mean > clock_ratio * peak:
        return None
    temperature = busy.get('temperature_c')
    temperature = temperature.get('mean') if isinstance(temperature, dict) else None
    return dict(share=round(thermal / samples, 3), sm_clock_mean_mhz=round(mean), sm_clock_max_mhz=round(peak),
                temperature_mean_c=round(temperature, 1) if type(temperature) in (int, float) and 0 <= temperature <= 1000 else None)


class OtherPrograms:
    """Other programs' share of the GPU's cores while a worker computes.

    A game, a browser tab or another generation on the same GPU slows every
    step, yet the device still reads fully busy at its full clock, exactly as
    when the worker has it alone. NVML's per-process samples tell the two
    apart where the driver reports them. Our processes are the one that
    monitors, the worker and their descendants. Constant space: counts and
    sums over the checks made while the worker was computing.
    """
    seconds = 1.
    refresh_seconds = 5.
    share_percent = 20
    # The worker's own share below which a check is not counted: while it
    # loads files, another program's work delays nothing of ours.
    own_percent = 15
    failures_allowed = 5

    def __init__(self, lib, handle, pid=None):
        self.lib, self.handle, self.pid = lib, handle, pid
        self.since = int(time.time() * 1e6)
        self.ours, self.seen = set(), set()
        self.checked = self.refreshed = 0.
        self.supported = None
        self.own_reported = False
        self.failures = 0
        self.checks = self.shared = 0
        self.other_total = self.own_total = self.other_peak = 0.

    def read(self):
        """{pid: mean % of cores} over the sample periods newer than the last ones read; None when unavailable.

        The driver leaves a process out of a period in which it did nothing,
        so the mean is over the periods the busiest reporter has, not over a
        process's own rows.
        """
        from .background_room import UtilizationSample, NOT_FOUND, INSUFFICIENT_SIZE
        function = getattr(self.lib, 'nvmlDeviceGetProcessUtilization', None)
        if function is None:
            return None
        count = ctypes.c_uint(0)
        rc = function(self.handle, None, ctypes.byref(count), ctypes.c_ulonglong(self.since))
        if rc == NOT_FOUND:
            return {}
        if rc != INSUFFICIENT_SIZE:
            return None
        count = ctypes.c_uint(count.value + 16)
        rows = (UtilizationSample * count.value)()
        rc = function(self.handle, rows, ctypes.byref(count), ctypes.c_ulonglong(self.since))
        if rc == NOT_FOUND:
            return {}
        if rc:
            return None
        totals, periods = {}, {}
        for row in rows[:count.value]:
            if row.sm <= 100:
                totals[row.pid] = totals.get(row.pid, 0) + row.sm
                periods[row.pid] = periods.get(row.pid, 0) + 1
            self.since = max(self.since, row.timestamp)
        number = max(periods.values(), default=1)
        return {pid: total / number for pid, total in totals.items()}

    def refresh(self):
        """A worker must never turn into another program for want of a process list: the last known ones stay ours."""
        from .background_room import family
        try:
            self.ours = family(os.getpid()) | (family(self.pid) if self.pid else set())
        except OSError:
            self.ours = self.ours or {pid for pid in (os.getpid(), self.pid) if pid}

    def check(self, util):
        """Read once a second; count the reading when the worker is computing on a busy device.

        Returns the other programs' share of the cores in percent for a reading that was counted, else None.
        """
        now = time.monotonic()
        if self.supported is False or now - self.checked < self.seconds:
            return None
        self.checked = now
        found = self.read()
        if found is None:
            self.failures += 1
            if self.supported is None and self.failures >= self.failures_allowed:
                self.supported = False
            return None
        self.supported = True
        if not found or util is None or util < 50:
            return None
        # A process we have not seen may be a child the worker just started.
        if now - self.refreshed >= self.refresh_seconds or any(pid not in self.seen for pid in found):
            self.refresh()
            self.refreshed, self.seen = now, set(found)
        other = min(100., sum(value for pid, value in found.items() if pid not in self.ours))
        own = min(100., sum(value for pid, value in found.items() if pid in self.ours))
        self.own_reported = self.own_reported or own > 0
        if not self.own_reported:
            # Some drivers report no samples for a compute process: what the
            # device did beyond the other programs was the worker's.
            own = max(0., util - other)
        if own < self.own_percent:
            return None
        self.checks += 1
        self.shared += other >= self.share_percent
        self.other_total += other
        self.own_total += own
        self.other_peak = max(self.other_peak, other)
        return other

    def result(self):
        result = dict(supported=bool(self.supported), own_reported=self.own_reported, checks=self.checks,
                      shared_checks=self.shared, check_seconds=self.seconds, share_percent=self.share_percent)
        if self.checks:
            result.update(other_mean_percent=self.other_total / self.checks, other_peak_percent=self.other_peak,
                          own_mean_percent=self.own_total / self.checks)
        return result


def shared_gpu(measured, *, share=.5, minimum_checks=30):
    """Other programs holding a real part of the GPU for most of the time the worker computed, or None.

    Counted only when they took at least `share_percent` of the cores in at
    least `share` of the checks, so a desktop that draws now and then or a
    short video is not reported.
    """
    programs = measured.get('other_programs') if isinstance(measured, dict) else None
    if not isinstance(programs, dict):
        return None
    checks, shared, mean = (programs.get(key) for key in ('checks', 'shared_checks', 'other_mean_percent'))
    if (any(type(value) not in (int, float) for value in (checks, shared, mean))
            or not minimum_checks <= checks <= 1e9 or not 0 <= shared <= checks or not 0 <= mean <= 100
            or shared < share * checks):
        return None
    return dict(share=round(shared / checks, 3), other_mean_percent=round(mean))


# Other programs' mean share of the GPU, in percent, from which the result says so:
# the run took at least about a quarter longer than with the GPU to itself.
SHARED_NOTE_PERCENT = 20


def shared_gpu_note(measured):
    """{other_mean_percent} for the note under a result, or None.

    Shown when other programs averaged at least SHARED_NOTE_PERCENT of the
    GPU, and shared_gpu holds: at least 30 checks, with other programs using
    at least 20% in at least half of them.
    """
    if shared_gpu(measured) is None:
        return None
    mean = measured['other_programs']['other_mean_percent']
    return dict(other_mean_percent=mean) if mean >= SHARED_NOTE_PERCENT else None


def _count(value):
    return value if type(value) in (int, float) and 0 <= value <= 1e12 else 0


def combined(rows):
    """Several attempts' busy-time clock reasons and other programs' checks as one observation.

    Only what thermal_slowdown and shared_gpu read: counts add up, means are
    weighted by their samples and peaks are the highest.
    """
    result = {}
    busy = [row['busy_activity'] for row in rows if isinstance(row.get('busy_activity'), dict)]
    if busy:
        merged = dict(clock_reason_samples=sum(_count(item.get('clock_reason_samples')) for item in busy),
                      clock_reason_counts={key: sum(_count(item['clock_reason_counts'].get(key)) for item in busy
                                                    if isinstance(item.get('clock_reason_counts'), dict))
                                           for key in Activity.clock_reasons})
        for field in ('sm_clock_mhz', 'temperature_c'):
            parts = [(item[field], _count(item[field].get('samples')) or _count(item.get('clock_reason_samples')))
                     for item in busy if isinstance(item.get(field), dict) and _count(item[field].get('mean'))]
            weight = sum(samples for _, samples in parts)
            if weight:
                merged[field] = dict(mean=sum(_count(part.get('mean')) * samples for part, samples in parts) / weight,
                                     maximum=max(_count(part.get('maximum')) for part, _ in parts))
        result['busy_activity'] = merged
    programs = [row['other_programs'] for row in rows if isinstance(row.get('other_programs'), dict)]
    if programs:
        counted = [(item, _count(item.get('checks'))) for item in programs]
        checks = sum(number for _, number in counted)
        merged = dict(supported=any(item.get('supported') is True for item in programs),
                      own_reported=any(item.get('own_reported') is True for item in programs),
                      checks=checks, shared_checks=sum(_count(item.get('shared_checks')) for item in programs))
        if checks:
            merged.update(other_mean_percent=sum(_count(item.get('other_mean_percent')) * number for item, number in counted) / checks,
                          other_peak_percent=max(_count(item.get('other_peak_percent')) for item in programs),
                          own_mean_percent=sum(_count(item.get('own_mean_percent')) * number for item, number in counted) / checks)
        result['other_programs'] = merged
    return result


def worker_gpu(report):
    """The video worker's GPU telemetry for a whole request, or None.

    A request's report keeps it in `resources.gpu` and a written diagnostic
    report in `memory.video.gpu`, but that is the last attempt's alone: after
    a retry that only decodes, the attempt that sampled is in
    `resource_attempts`. Several attempts are combined, so a share of the
    request's busy time is a share of all of it.
    """
    report = report if isinstance(report, dict) else {}
    attempts = report.get('resource_attempts')
    rows = [row['resources']['gpu'] for row in attempts if isinstance(row, dict) and isinstance(row.get('resources'), dict)
            and isinstance(row['resources'].get('gpu'), dict) and row['resources']['gpu']] if isinstance(attempts, list) else []
    if len(rows) > 1:
        return combined(rows)
    video = report.get('memory').get('video') if isinstance(report.get('memory'), dict) else None
    for section in (report.get('resources'), video):
        if isinstance(section, dict) and isinstance(section.get('gpu'), dict) and section['gpu']:
            return section['gpu']
    return None


def nvml_device(device=None):
    """Initialize NVML and select one GPU by UUID or index; returns (library, handle, device)."""
    if windows():
        candidates = [Path(os.environ.get('SystemRoot', r'C:\Windows')) / 'System32/nvml.dll',
                      Path(os.environ.get('ProgramFiles', r'C:\Program Files')) / 'NVIDIA Corporation/NVSMI/nvml.dll']
        library = next((p for p in candidates if p.is_file()), None)
        if library is None:
            raise RuntimeError('NVML DLL is missing from the NVIDIA driver installation; repair the driver and rerun test.')
        lib = ctypes.CDLL(str(library))
    else:
        lib = ctypes.CDLL('libnvidia-ml.so.1')
    rc = lib.nvmlInit_v2()
    if rc:
        raise RuntimeError(f'NVML initialization returned {rc}')
    handle = ctypes.c_void_p()
    device = str(device if device is not None else os.environ.get('CUDA_VISIBLE_DEVICES', '0').split(',')[0])
    if device.startswith('GPU-'):
        rc = lib.nvmlDeviceGetHandleByUUID(device.encode(), ctypes.byref(handle))
    else:
        rc = lib.nvmlDeviceGetHandleByIndex_v2(int(device), ctypes.byref(handle))
    if rc:
        lib.nvmlShutdown()
        raise RuntimeError(f'NVML could not select {device}: {rc}')
    return lib, handle, device


class DeviceMemory:
    """Whole-device memory in use by every process, read on demand.

    Windows' per-process budget and CUDA's free memory leave out what other
    applications hold until the device is already full; NVML counts them.
    """
    def __init__(self, device=None):
        self.lib, self.handle, self.device = nvml_device(device)

    def sample(self):
        m = Memory()
        rc = self.lib.nvmlDeviceGetMemoryInfo(self.handle, ctypes.byref(m))
        if rc:
            raise RuntimeError(f'NVML memory read returned {rc}')
        if m.used > m.total or not m.total:
            raise RuntimeError('Driver returned unavailable NVML device memory')
        return dict(used_bytes=m.used, total_bytes=m.total)

    def close(self):
        lib, self.lib = self.lib, None
        if lib is not None:
            lib.nvmlShutdown()


class Monitor:
    def __init__(self, path, interval=0.05, device=None, pid=None):
        self.lib, self.handle, device = nvml_device(device)
        self.device = device
        self.programs = OtherPrograms(self.lib, self.handle, pid)
        self.path, self.interval = Path(path), interval
        self.stop_event = threading.Event()
        self.peak = 0
        self.baseline = 0
        self.last = 0
        self.count = 0
        self.errors = []
        self.max_temperature = None
        self.max_power_mw = None
        self.activity, self.busy_activity = Activity(), Activity()

    def sample(self):
        m, u = Memory(), Utilization()
        rc = self.lib.nvmlDeviceGetMemoryInfo(self.handle, ctypes.byref(m))
        if rc:
            raise RuntimeError(f'NVML memory read returned {rc}')
        if m.used > m.total or not m.total:
            raise RuntimeError('Driver returned unavailable NVML device memory; no zero/estimated peak will be reported.')
        try:
            rc = self.lib.nvmlDeviceGetUtilizationRates(self.handle, ctypes.byref(u))
        except AttributeError:
            rc = -1
        return m.used, m.total, u.gpu if rc == 0 and u.gpu <= 100 else None

    def start(self):
        self.baseline = self.sample()[0]
        self.peak = self.baseline
        self.thread = threading.Thread(target=self.loop, daemon=True)
        self.thread.start()
        return self

    def auxiliary(self):
        values = []
        for name, extra in [('nvmlDeviceGetTemperature', (0,)), ('nvmlDeviceGetPowerUsage', ()),
                            ('nvmlDeviceGetClockInfo', (1,)), ('nvmlDeviceGetClockInfo', (2,))]:
            value = ctypes.c_uint()
            try:
                rc = getattr(self.lib, name)(self.handle, *extra, ctypes.byref(value))
                values.append(value.value if rc == 0 else None)
            except AttributeError:
                values.append(None)
        return values

    def clock_reasons(self):
        for name in ('nvmlDeviceGetCurrentClocksEventReasons', 'nvmlDeviceGetCurrentClocksThrottleReasons'):
            try:
                value = ctypes.c_ulonglong()
                if getattr(self.lib, name)(self.handle, ctypes.byref(value)) == 0:
                    return value.value
            except AttributeError:
                pass
        return None

    def loop(self):
        with self.path.open('w', buffering=1, encoding='utf-8') as f:
            writer = csv.writer(f)
            writer.writerow(['epoch_seconds', 'monotonic_seconds', 'used_bytes', 'total_bytes', 'gpu_util_percent',
                             'temperature_c', 'power_mw', 'sm_clock_mhz', 'memory_clock_mhz', 'clock_reason_mask',
                             'other_programs_percent'])
            while not self.stop_event.is_set():
                try:
                    used, total, util = self.sample()
                    self.peak = max(self.peak, used)
                    self.last = used
                    self.count += 1
                    auxiliary = self.auxiliary()
                    reasons = self.clock_reasons()
                    self.activity.add(util, auxiliary, reasons)
                    if util is not None and util >= 50:
                        self.busy_activity.add(util, auxiliary, reasons)
                    if auxiliary[0] is not None:
                        self.max_temperature = max(self.max_temperature or 0, auxiliary[0])
                    if auxiliary[1] is not None:
                        self.max_power_mw = max(self.max_power_mw or 0, auxiliary[1])
                    try:
                        other = self.programs.check(util)
                    except Exception:
                        other = None
                    writer.writerow([time.time(), time.monotonic(), used, total, util, *auxiliary, reasons,
                                     '' if other is None else '%.1f' % other])
                except Exception as e:
                    if str(e) not in self.errors and len(self.errors) < 20:
                        self.errors.append(str(e))
                self.stop_event.wait(self.interval)

    def stop(self):
        self.stop_event.set()
        self.thread.join(timeout=5)
        self.lib.nvmlShutdown()
        return {'device': self.device, 'gpu_peak_bytes': self.peak, 'gpu_peak_mib': self.peak / 2**20,
                'gpu_baseline_bytes': self.baseline, 'gpu_last_bytes': self.last,
                'max_temperature_c': self.max_temperature, 'max_power_mw': self.max_power_mw,
                'activity': self.activity.result(), 'busy_activity': self.busy_activity.result(),
                'busy_util_threshold_percent': 50,
                'activity_scope': 'Whole device during this worker request, including load and decode. '
                                  'Sample means, not duration-weighted measurements. Busy subset has GPU utilization >= 50%; '
                                  'clock reasons are driver observations, not attribution of elapsed time.',
                'other_programs': self.programs.result(),
                'other_programs_scope': 'Per-process driver samples read once a second while the worker was computing on a '
                                        'busy device: the share of its cores used by processes outside this worker and its '
                                        'monitor. shared_checks counts readings at or above share_percent. Without own_reported '
                                        "the worker's share is the device's utilization beyond the other programs. A driver "
                                        'without per-process samples reports no checks.',
                'sampling_interval_seconds': self.interval, 'samples': self.count,
                'sampling_errors': self.errors, 'sample_csv': str(self.path)}



def step_activity(csv_path, steps, *, limit_bytes=64 * 2**20):
    """Add each sampling step's whole-device NVML samples to it as `device_activity`.

    A request-wide mean cannot show one slow step: whether the device was full,
    another process kept it busy or clocks fell during that step. Steps carry
    the monotonic time they ended; the monitor's CSV, written by the parent
    process, carries the same clock. Returns how many steps were annotated.
    Missing, oversized or partial files add nothing.
    """
    windows = sorted((row['monotonic_seconds'] - row['seconds'], row['monotonic_seconds'], index)
                     for index, row in enumerate(steps if isinstance(steps, list) else [])
                     if isinstance(row, dict) and all(type(row.get(key)) in (int, float) and row[key] >= 0
                                                      for key in ('monotonic_seconds', 'seconds')))
    if not windows:
        return 0
    try:
        if Path(csv_path).stat().st_size > limit_bytes:
            return 0
        with open(csv_path, newline='', encoding='utf-8') as handle:
            samples = list(csv.DictReader(handle))
    except (OSError, ValueError, csv.Error):
        return 0
    def number(sample, key):
        try:
            value = float(sample.get(key) or 'nan')
        except ValueError:
            return None
        return value if 0 <= value < float('inf') else None
    starts = [start for start, _, _ in windows]
    found = {}
    for sample in samples:
        moment = number(sample, 'monotonic_seconds')
        if moment is None:
            continue
        position = bisect.bisect_right(starts, moment) - 1
        if position < 0 or moment > windows[position][1]:
            continue
        item = found.setdefault(windows[position][2], dict(samples=0, busy_samples=0, sums={}))
        item['samples'] += 1
        util = number(sample, 'gpu_util_percent')
        item['busy_samples'] += util is not None and util >= 50
        for key, value in (('gpu_util_mean_percent', util), ('sm_clock_mean_mhz', number(sample, 'sm_clock_mhz')),
                           ('power_mean_mw', number(sample, 'power_mw'))):
            if value is not None:
                total = item['sums'].setdefault(key, [0., 0])
                total[0] += value
                total[1] += 1
        other = number(sample, 'other_programs_percent')
        if other is not None and other <= 100:
            item['other_programs_checks'] = item.get('other_programs_checks', 0) + 1
            item['other_programs_percent_peak'] = max(item.get('other_programs_percent_peak', 0.), other)
            total = item['sums'].setdefault('other_programs_percent_mean', [0., 0])
            total[0] += other
            total[1] += 1
        used, size = number(sample, 'used_bytes'), number(sample, 'total_bytes')
        if used is not None:
            item['device_used_peak_bytes'] = max(item.get('device_used_peak_bytes', 0), int(used))
            item['device_used_minimum_bytes'] = min(item.get('device_used_minimum_bytes', int(used)), int(used))
        if size is not None:
            item['device_total_bytes'] = int(size)
    for index, item in found.items():
        for key, (total, count) in item.pop('sums').items():
            item[key] = total / count
        steps[index]['device_activity'] = item
    return len(found)