Make videos on the computer you already own. FreeVideo runs MiniMax H3 in as little as 8 GB of VRAM and 16 GB of RAM, and adapts its acceleration path to your hardware.
"""Exclude setup, tests and GPU work; children retain their parent's lease.
GPU work is device-scoped, so it takes the machine lease shared and its own
device lease exclusively. Setup still takes the machine lease exclusively and
therefore still excludes every device. Without this split one measurement
occupied an entire multi-GPU host.
"""
from contextlib import contextmanager
import math
import os
from pathlib import Path
import re
import sys
import threading
import time
from .paths import data_root
from .system import windows
from .processes import inherited_fd
LOCK_PATH = Path(os.environ.get('FREEVIDEO_LOCK_PATH', data_root() / 'engine.lock'))
LOCK_ENV = 'FREEVIDEO_RUNTIME_LOCK_FD'
ENGINE_BUSY = 'Setup, testing or generation is already using this installation/GPU lock. Retry after it finishes.'
def lock_holders(paths, *, deadline=None):
"""Read-only Linux diagnostics, including orphans retaining inherited FDs."""
if windows() or not Path('/proc/self/fd').is_dir():
return []
identities = set()
for path in paths:
if deadline is not None and time.monotonic() >= deadline:
break
try:
item = Path(path).stat()
identities.add((item.st_dev, item.st_ino))
except OSError:
pass
holders = []
if not identities:
return []
for directory in Path('/proc').iterdir():
if deadline is not None and time.monotonic() >= deadline:
break
if not directory.name.isdigit():
continue
try:
for fd in (directory / 'fd').iterdir():
if deadline is not None and time.monotonic() >= deadline:
break
try:
item = fd.stat()
if (item.st_dev, item.st_ino) in identities:
holders.append(int(directory.name))
break
except OSError:
continue
except OSError:
continue
return sorted(holders)
# Only these small, public command identifiers may leave process diagnostics.
_COMMAND_WORD = re.compile(r'[a-z][a-z0-9_.-]{0,40}\Z')
_EXECUTABLE = re.compile(r'[A-Za-z0-9_][A-Za-z0-9_.-]{0,80}\Z')
_HOLDER_TIMEOUT = 2.0
def _executable_basename(value):
name = value.replace('\\', '/').rsplit('/', 1)[-1]
return name if _EXECUTABLE.fullmatch(name) else 'unknown'
def _lock_command(cmdline):
"""Pure allowlist parser: no scripts, option values, prompts or environment."""
arguments = cmdline.decode('utf-8', errors='replace').rstrip('\0').split('\0')
executable = _executable_basename(arguments[0])
index = 1
# Do not mistake a script/-c argument or an option's value for Python's -m.
while index < len(arguments) and arguments[index] in (
'-B', '-u', '-E', '-s', '-S', '-I', '-O', '-OO', '-q', '-P'):
index += 1
words = []
if index < len(arguments) and arguments[index] == '-m':
# Words after the module are that program's own arguments. Only our own
# modules have subcommands known not to carry private values.
own = arguments[index + 1].startswith('freevideo_engine') if index + 1 < len(arguments) else False
for word in arguments[index + 1:index + (4 if own else 2)]:
if not _COMMAND_WORD.fullmatch(word):
break # Never skip an option/path to expose its value as a word.
words.append(word)
return executable, ' '.join(words)
def _holder_line(pid, executable, command='', seconds=None):
line = f'Lock file open: pid {int(pid)}, {_executable_basename(executable)}'
if command:
line += f' ({command})'
if seconds is not None and math.isfinite(seconds) and seconds >= 0:
line += f', running {int(seconds)} s'
return line
def _linux_holder(pid):
executable, command, seconds = 'unknown', '', None
try:
with (Path('/proc') / str(pid) / 'cmdline').open('rb') as stream:
executable, command = _lock_command(stream.read(65536))
except Exception:
pass
try:
# comm may itself contain spaces and parentheses; starttime is field 22.
stat = (Path('/proc') / str(pid) / 'stat').read_text()
ticks = int(stat.rsplit(')', 1)[1].split()[19])
uptime = float(Path('/proc/uptime').read_text().split()[0])
seconds = uptime - ticks / os.sysconf('SC_CLK_TCK')
except Exception:
pass
return _holder_line(pid, executable, command, seconds)
def _windows_lock_holders(paths, deadline, lines):
"""Restart Manager is read-only here: never request a shutdown or restart."""
import ctypes
from ctypes import POINTER, c_int32, c_uint32, c_void_p, c_wchar, c_wchar_p
class FileTime(ctypes.Structure):
_fields_ = [('low', c_uint32), ('high', c_uint32)]
class UniqueProcess(ctypes.Structure):
_fields_ = [('pid', c_uint32), ('started', FileTime)]
class ProcessInfo(ctypes.Structure):
_fields_ = [('process', UniqueProcess), ('app', c_wchar * 256),
('service', c_wchar * 64), ('kind', c_int32),
('status', c_uint32), ('session', c_uint32), ('restartable', c_int32)]
manager = ctypes.WinDLL('rstrtmgr.dll', use_last_error=True)
kernel = ctypes.WinDLL('kernel32.dll', use_last_error=True)
signatures = (
(manager.RmStartSession, [POINTER(c_uint32), c_uint32, c_wchar_p], c_uint32),
(manager.RmRegisterResources, [c_uint32, c_uint32, POINTER(c_wchar_p), c_uint32,
POINTER(UniqueProcess), c_uint32, POINTER(c_wchar_p)], c_uint32),
(manager.RmGetList, [c_uint32, POINTER(c_uint32), POINTER(c_uint32),
POINTER(ProcessInfo), POINTER(c_uint32)], c_uint32),
(manager.RmEndSession, [c_uint32], c_uint32),
(kernel.OpenProcess, [c_uint32, c_int32, c_uint32], c_void_p),
(kernel.QueryFullProcessImageNameW, [c_void_p, c_uint32, c_wchar_p, POINTER(c_uint32)], c_int32),
(kernel.CloseHandle, [c_void_p], c_int32),
)
for function, arguments, result in signatures:
function.argtypes, function.restype = arguments, result
session = c_uint32()
if time.monotonic() >= deadline or manager.RmStartSession(ctypes.byref(session), 0, ctypes.create_unicode_buffer(33)):
return
try:
resources = (c_wchar_p * len(paths))(*(str(Path(path).absolute()) for path in paths))
if not resources or time.monotonic() >= deadline:
return
if manager.RmRegisterResources(session, len(resources), resources, 0, None, 0, None):
return
needed, count, reason = c_uint32(), c_uint32(), c_uint32()
items = None
# The list can grow between calls. Bound both allocation and retries.
for _ in range(3):
if time.monotonic() >= deadline:
return
status = manager.RmGetList(session, ctypes.byref(needed), ctypes.byref(count), items, ctypes.byref(reason))
if status == 0:
break
if status != 234 or not 0 < needed.value <= 4096: # ERROR_MORE_DATA
return
count.value = needed.value
items = (ProcessInfo * count.value)()
else:
return
holders = {item.process.pid: item for item in (items or ())[:count.value]
if item.process.pid and item.process.pid != os.getpid()}
for pid in sorted(holders)[:5]:
if time.monotonic() >= deadline:
return
item = holders[pid]
executable = _executable_basename(item.app)
seconds = None
try:
ticks = (item.process.started.high << 32) | item.process.started.low
if ticks:
seconds = time.time() - (ticks / 10_000_000 - 11_644_473_600)
except Exception:
pass
# RM's app name is the fallback when the process has exited or access
# is denied. QueryFullProcessImageNameW returns no command arguments.
position = len(lines)
lines.append(_holder_line(pid, executable, seconds=seconds))
handle = None
try:
handle = kernel.OpenProcess(0x1000, False, pid) # QUERY_LIMITED_INFORMATION
if handle and time.monotonic() < deadline:
image = ctypes.create_unicode_buffer(32768)
size = c_uint32(len(image))
if kernel.QueryFullProcessImageNameW(handle, 0, image, ctypes.byref(size)):
lines[position] = _holder_line(pid, image.value, seconds=seconds)
except Exception:
pass
finally:
if handle:
kernel.CloseHandle(handle)
if len(holders) > 5:
lines.append(f'Lock file open: and {len(holders) - 5} more')
finally:
manager.RmEndSession(session)
def describe_lock_holders(paths) -> list[str]:
"""Best-effort, redacted diagnostics; even a stalled OS call gets two seconds.
A daemon thread is deliberate: native calls have no timeout, and an attempt
to cancel Restart Manager must not delay the busy exit or interpreter exit.
Completed rows remain available if a later process/API query fails or hangs.
"""
lines = []
def query():
try:
if windows():
_windows_lock_holders(paths, deadline, lines)
else:
holders = sorted(set(lock_holders(paths, deadline=deadline)) - {os.getpid()})
for pid in holders[:5]:
if time.monotonic() >= deadline:
return
lines.append(_linux_holder(pid))
if len(holders) > 5:
lines.append(f'Lock file open: and {len(holders) - 5} more')
except Exception:
pass
finally:
done.set()
try:
deadline = time.monotonic() + _HOLDER_TIMEOUT
done = threading.Event()
threading.Thread(target=query, daemon=True, name='lock-holder-diagnostics').start()
done.wait(max(0, deadline - time.monotonic()))
except Exception:
pass
return lines[:]
def device_lock_path(device, path=None):
"""The lease for one device, beside the machine lease it shares a root with."""
base = Path(path or os.environ.get('FREEVIDEO_LOCK_PATH', data_root() / 'engine.lock'))
token = re.sub(r'[^A-Za-z0-9-]', '_', str(device))[:64] or 'unknown'
return base.with_name(base.stem + '-gpu-' + token + base.suffix)
def name_busy_lock(error, path, shared, wait_seconds):
"""Say which lease a refused acquisition wanted; the bare errno names nothing."""
path = Path(path)
token = path.stem.partition('-gpu-')[2]
error.busy_lock = dict(lock=('device lease ' + path.name.replace(token, token[:8]) if token
else 'installation lease ' + path.name),
shared=bool(shared), waited_seconds=round(wait_seconds, 3))
error.busy_lock_path = path
def busy_lock_details(error):
"""Name, wait and current holders of the lease an acquisition was refused, else {}."""
details = dict(getattr(error, 'busy_lock', None) or {})
if details:
try:
details['holders'] = describe_lock_holders([error.busy_lock_path])
except Exception:
pass
return details
def failure_frames(error, limit=12):
"""The traceback as 'file:line in function' rows: no source text, no folders, no values."""
import traceback
return ['%s:%d in %s' % (Path(frame.filename).name, frame.lineno, frame.name)
for frame in traceback.extract_tb(error.__traceback__)[-limit:]]
@contextmanager
def runtime_lock(path=None, *, inherit=True, shared=False, wait_seconds=0):
"""Take a lease, optionally waiting a bounded time for acquisition only."""
if not math.isfinite(wait_seconds) or wait_seconds < 0:
raise ValueError('Runtime lock wait must be finite and nonnegative')
path = Path(path or os.environ.get('FREEVIDEO_LOCK_PATH', data_root() / 'engine.lock'))
path.parent.mkdir(parents=True, exist_ok=True)
inherited = inherited_fd() if inherit else None
if inherited is None:
stream = path.open('a+b')
else:
descriptor = int(inherited)
actual, expected = os.fstat(descriptor), path.stat()
if (actual.st_dev, actual.st_ino) != (expected.st_dev, expected.st_ino):
raise ValueError('Inherited runtime lock descriptor does not match the GPU-work lock')
stream = os.fdopen(os.dup(descriptor), 'a+b')
try:
# An inherited descriptor shares the parent's open-file description.
# Close our descriptor on exit; do not unlock the parent's live work.
deadline = time.monotonic() + wait_seconds
while True:
try:
if windows():
if inherited is None:
import msvcrt
if path.stat().st_size == 0:
stream.write(b'\0')
stream.flush()
stream.seek(0)
try:
msvcrt.locking(stream.fileno(),
msvcrt.LK_NBRLCK if shared else msvcrt.LK_NBLCK, 1)
except OSError as error:
raise BlockingIOError('Another process holds the installation lease') from error
# The parent holds the byte-range lock; its Job Object owns this
# worker tree. Closing a duplicated child handle must not unlock it.
else:
import fcntl
fcntl.flock(stream, (fcntl.LOCK_SH if shared else fcntl.LOCK_EX) | fcntl.LOCK_NB)
break
except BlockingIOError as error:
remaining = deadline - time.monotonic()
if remaining <= 0:
name_busy_lock(error, path, shared, wait_seconds)
raise
time.sleep(min(0.075, remaining))
# A delayed wakeup must not start another attempt after budget.
if time.monotonic() >= deadline:
name_busy_lock(error, path, shared, wait_seconds)
raise
# The body is outside the retry loop: a failed request must never rerun.
yield stream.fileno()
finally:
# Releasing the lease must not discard completed work. Taking it still
# fails loudly, but once the request has run, a close that fails has
# nothing left to protect: the kernel drops the descriptor when this
# process exits, and the lease is advisory, so the next holder waits on
# the file rather than on this handle. Observed once in 120 concurrent
# eight-GPU measurements on a WekaFS share, as OSError EBADFD raised by
# the close itself after a full video had been sampled, decoded and
# written; the exit code threw that video away.
try:
stream.close()
except OSError as error:
print('Could not release the runtime lease at %s: %s. The request is '
'unaffected; the descriptor is dropped when this process exits.'
% (path, error), file=sys.stderr, flush=True)