Make videos on the computer you already own. FreeVideo runs MiniMax H3 in as little as 8 GB of VRAM and 16 GB of RAM, and adapts its acceleration path to your hardware.
"""Persistent ImageRunner owner; stdin/stdout contain one JSON object per line."""
import argparse
from contextlib import ExitStack, redirect_stdout
import json
import os
from pathlib import Path
import queue
import sys
import threading
import time
from . import keep
from .failure import Context, classify_exception, exception_details, host_memory_shortage, safe_plan
class Inbox:
"""Read cancellation while the main thread is inside a denoising step."""
def __init__(self, stream, send):
self.stream = stream
self.send = send
self.rows = queue.Queue()
self.cancel = threading.Event()
self.closed = threading.Event()
self.mutex = threading.Lock()
self.busy = False
self.reader = threading.Thread(target=self.read, daemon=True)
self.reader.start()
def read(self):
try:
while not self.closed.is_set():
line = self.stream.readline(64 * 1024 + 1)
if not line:
break
try:
if len(line.encode('utf-8')) > 64 * 1024:
raise ValueError('invalid_request')
value = json.loads(line)
if not isinstance(value, dict):
raise ValueError('invalid_request')
action = value.get('action')
with self.mutex:
if action == 'cancel':
if self.busy:
self.cancel.set()
elif action == 'close':
break
elif action == 'generate':
if self.busy:
self.send(phase='error', code='busy')
else:
self.busy = True
self.cancel.clear()
self.rows.put(value)
else:
raise ValueError('invalid_request')
except (ValueError, UnicodeError):
self.send(phase='error', code='invalid_request')
finally:
self.closed.set()
self.cancel.set()
self.rows.put(None)
def _flatten(image):
"""RGB copy; transparency is shown on white, as the encoder's vision tower sees it."""
from PIL import Image
if image.mode in ('RGBA', 'LA') or 'transparency' in image.info:
rgba = image.convert('RGBA')
base = Image.new('RGBA', rgba.size, (255, 255, 255, 255))
return Image.alpha_composite(base, rgba).convert('RGB')
return image.convert('RGB')
def next_request(inbox, wait, crowding=None):
"""The next row, or None when the pipe closed, `wait` seconds passed, or the card is crowded.
Without `crowding` this is one blocking read, as for an ordinary idle wait."""
if crowding is None:
try:
return inbox.rows.get(timeout=wait)
except queue.Empty:
return None
deadline = time.monotonic() + wait
while True:
remaining = deadline - time.monotonic()
if remaining <= 0:
return None
try:
return inbox.rows.get(timeout=min(remaining, keep.POLL_SECONDS))
except queue.Empty:
if crowding:
return None
def hold(runner):
"""Start the long idle wait: hand back the allocator's cached memory, then watch the card."""
torch = sys.modules.get('torch')
if torch is not None:
torch.cuda.empty_cache()
return keep.Crowding(lambda: keep.card_free_bytes(runner.device))
def load_runner(pack_dir, device, *, progress=None):
# Importing this module in ComfyUI or in CPU protocol tests never loads torch.
from .image_runner import ImageRunner
return ImageRunner(pack_dir, device=device, progress=progress)
def serve(pack_dir, *, device='cuda', idle_seconds=45., stream=None, output=None, factory=None):
stream = sys.stdin if stream is None else stream
output = sys.stdout if output is None else output
factory = factory or load_runner
output_mutex = threading.Lock()
def send(**row):
with output_mutex:
output.write(json.dumps(row, ensure_ascii=True, allow_nan=False) + '\n')
output.flush()
inbox = Inbox(stream, send)
runner = None
back_detector = None
back_detector_loaded = False
wait, crowding = idle_seconds, None
try:
while not inbox.closed.is_set():
value = next_request(inbox, wait, crowding)
if value is None or inbox.closed.is_set():
break
started = time.monotonic()
context = Context('generate', value)
result = None
try:
from PIL import Image
with ExitStack() as stack, redirect_stdout(sys.stderr):
if inbox.cancel.is_set():
raise InterruptedError('cancelled')
def progress(stage, fraction):
context.timing.enter(stage)
send(phase='progress', stage=stage, fraction=fraction)
if runner is None:
context.timing.enter('load')
runner = factory(pack_dir, device, progress=progress)
if hasattr(runner, 'last_plan'):
runner.last_plan = None
context.timing.enter('start')
references = []
for path in value.get('images', []):
image = stack.enter_context(Image.open(path))
references.append(_flatten(image))
region = {}
if value.get('mask'):
region['edit_mask'] = stack.enter_context(Image.open(value['mask']))
region['edit_mask'].load()
loras = value.get('loras') or None
back_check = None
def generate(seed):
return runner.generate(
value['prompt'], width=value['width'], height=value['height'],
seed=seed, reference_images=references, **region,
**(dict(loras=loras) if loras else {}),
progress=progress, cancel=inbox.cancel.is_set)
if value.get('cast_view') == 'back' and len(references) == 1 and not region:
from .back_view import generate_back, load_detector
if not back_detector_loaded:
back_detector = load_detector(pack_dir)
back_detector_loaded = True
image, back_check = generate_back(generate, value.get('seed'), back_detector,
cancel=inbox.cancel.is_set)
else:
image = generate(value.get('seed'))
if inbox.cancel.is_set():
raise InterruptedError('cancelled')
# The parent chooses a fresh path under ComfyUI input.
# Exclusive creation never replaces an existing user file.
with Path(value['path']).open('xb') as target:
image.save(target, format='PNG')
plan = runner.stats().get('plan')
if back_check is not None:
plan = dict(plan or {}, back_check=back_check)
result = dict(phase='result', path=value['path'], seconds=time.monotonic() - started, plan=plan)
try:
result.update(attention_report(runner, context.redactor))
except Exception:
# Diagnostics must not turn a saved picture into an error.
pass
except Exception as error:
code = classify_exception(error, stage=context.timing.stage, cancelled=inbox.cancel.is_set())
plan = None
if runner is not None:
try:
plan = safe_plan(runner.stats().get('plan'), context.redactor)
except Exception:
# Diagnostics must not replace the inference exception.
pass
reload = not getattr(runner, 'at_boundary', lambda: True)()
result = dict(phase='error', code=code, stage=context.timing.stage, reloaded=reload,
seconds=context.timing.seconds(), stages=context.timing.snapshot(), plan=plan,
error=exception_details(error, context.redactor, code))
if code == 'no_memory' and host_memory_shortage(error):
result['memory'] = 'host'
try:
result.update(attention_report(runner, context.redactor))
except Exception:
pass
finally:
if result is not None and result.get('reloaded'):
# A failure inside a layer pass leaves the weights streamer part-way through it and the
# next request would be refused; the model is loaded again for that one.
with redirect_stdout(sys.stderr):
try:
runner.close()
except Exception as error:
print('image runner close failed: %s' % type(error).__name__, file=sys.stderr, flush=True)
runner = None
import gc
gc.collect()
torch = sys.modules.get('torch')
if torch is not None and torch.cuda.is_available():
torch.cuda.empty_cache()
with inbox.mutex:
inbox.busy = False
if result is not None:
send(**result)
wait, crowding = idle_seconds, None
if result is not None and result.get('phase') == 'result' and runner is not None:
wait = keep.wait_seconds(value.get('keep_seconds'), runner, idle_seconds)
if wait > idle_seconds:
try:
crowding = hold(runner)
except Exception as error:
# The picture is already sent; a hold that cannot start is the normal short wait.
print('image worker hold failed: %s' % type(error).__name__, file=sys.stderr, flush=True)
wait = idle_seconds
finally:
inbox.closed.set()
inbox.cancel.set()
if runner is not None:
with redirect_stdout(sys.stderr):
runner.close()
def attention_report(runner, redactor=None):
"""{'attention': ...} on Macs where MLX attention was installed (why it fell back, if it did); {} elsewhere."""
if runner is None:
return {}
from . import mlx_attention, qwen_dit
value = mlx_attention.summary(qwen_dit.attention)
if value is None:
return {}
if redactor is not None and 'fallback_reason' in value:
value['fallback_reason'] = redactor.text(value['fallback_reason'])[:300]
return dict(attention=value)
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('pack_dir', type=Path)
parser.add_argument('--device', default='cuda')
parser.add_argument('--idle-seconds', type=float, default=45.)
args = parser.parse_args()
if not 0 < args.idle_seconds < float('inf'):
parser.error('--idle-seconds must be positive and finite')
# A native crash (access violation, driver fault) leaves no Python error: the crash
# handler writes the Python stack to a file the service reads once the worker is gone.
crash_file = os.environ.get('FREEVIDEO_CRASH_FILE')
if crash_file:
try:
import faulthandler
faulthandler.enable(file=open(crash_file, 'w'), all_threads=False)
except (OSError, RuntimeError, ValueError):
pass
stream = None
if os.name == 'nt':
from ..win32 import private_stdin
try:
stream = private_stdin()
except OSError as error:
print('private stdin unavailable, reading stdin directly: %s' % error, file=sys.stderr, flush=True)
serve(args.pack_dir, device=args.device, idle_seconds=args.idle_seconds, stream=stream)
return 0
if __name__ == '__main__':
raise SystemExit(main())