Make videos on the computer you already own. FreeVideo runs MiniMax H3 in as little as 8 GB of VRAM and 16 GB of RAM, and adapts its acceleration path to your hardware.
"""The running ComfyUI server as seen by agent jobs: queue, history, folders and local services."""
import json
import os
from pathlib import Path, PurePosixPath
import shutil
import stat
from urllib.parse import urlencode
import uuid
from . import spec as specs
from .jobs import JobError
class Request:
"""Just enough of an aiohttp request for ComfyUI's own POST /prompt handler."""
method, path, content_type, query, match_info = 'POST', '/prompt', 'application/json', {}, {}
def __init__(self, body):
from multidict import CIMultiDict, CIMultiDictProxy
self.body = body
self.headers = CIMultiDictProxy(CIMultiDict({'Content-Type': 'application/json'}))
async def json(self):
return self.body
class ComfyPort:
def __init__(self, server):
import folder_paths
self.server = server
self.folders = folder_paths
self.handler = None
# Folders -------------------------------------------------------------------------------------------
@property
def input_root(self):
return Path(self.folders.get_input_directory()).resolve()
@property
def output_root(self):
return Path(self.folders.get_output_directory()).resolve()
def _inside(self, root, relative, create):
parts = PurePosixPath(relative).parts
if not relative or PurePosixPath(relative).is_absolute() or '..' in parts or '\\' in relative or ':' in relative:
raise JobError('invalid_media', 'Not a relative path: %s' % relative)
path = (root / relative).resolve()
if not path.is_relative_to(root):
raise JobError('invalid_media', 'Path leaves the folder: %s' % relative)
if create:
path.parent.mkdir(parents=True, exist_ok=True)
return path
def input_path(self, relative, create=False):
return self._inside(self.input_root, relative, create)
def output_path(self, relative, create=False):
return self._inside(self.output_root, relative, create)
def output_relative(self, path):
return Path(path).resolve().relative_to(self.output_root).as_posix()
def voice_window(self, path, seconds):
"""Speech in the first seconds of an uploaded sound, or None without the speech model."""
from ..comfy_bridge import installation
from ..image_gen import image_pack
from ..speech import MODEL_FILE
from ..voice_window import inspect
model = image_pack.base(installation()[1]['model_root']) / MODEL_FILE
return inspect(path, seconds, model, self.output_root / 'FreeVideo' / '.editor' / 'speech')
def library_video(self, identity):
from ..comfy_library import _video, _report
try:
path = _video(self.output_root, identity)
info = path.stat()
if not stat.S_ISREG(info.st_mode) or not info.st_size:
raise ValueError('Video is not a nonempty regular file')
_report(path)
except (OSError, ValueError):
raise JobError('invalid_media', 'No finished saved video %s' % identity) from None
return path
def file_view(self, relative, kind='output'):
from .assets import view
return view(relative, kind)
def chain_runner(self):
from ..comfy_chain import RUNNER
return RUNNER
async def capability_view(self, query):
from ..comfy_capabilities import _query
from ..capabilities import UNAVAILABLE
key = _query(query)
cache = getattr(self.server, '_freevideo_capabilities', None)
if not hasattr(cache, 'get'):
return dict(UNAVAILABLE)
try:
await cache.check()
return cache.get(key)
except Exception:
cache._log_once()
return dict(UNAVAILABLE)
def capability_snapshot(self, query):
from ..comfy_capabilities import _query
from ..capabilities import UNAVAILABLE
cache = getattr(self.server, '_freevideo_capabilities', None)
return cache.get(_query(query)) if hasattr(cache, 'get') else dict(UNAVAILABLE)
def output_summary(self, relative):
from .jobs import library_id
from ..comfy_library import _report
from ..comfy_assets import output_summary
path = self.library_video(library_id(relative))
if path != self.output_path(relative):
raise ValueError('Invalid library output')
report = _report(path)
summary = output_summary(report, Path(relative))
if summary.get('geometry'):
from .media import saved_geometry
summary['geometry'] = saved_geometry(path, summary['geometry'])
if report.get('result_cache_hit') is not None:
summary['result_cache_hit'] = report['result_cache_hit']
return summary
# Checks at submission (fast; files are checked again when the job starts) --------------------------------
def check(self, spec):
from .schema import diagnostic, editor_issues
errors = []
names = set(self.folders.get_filename_list('loras')) if spec.get('loras') else set()
for i, row in enumerate(spec.get('loras', [])):
if row['name'] not in names:
errors.append(diagnostic('/loras/%d/name' % i, 'an installed LoRA listed by GET /freevideo/agent/v1', row['name']))
def walk(value, pointer=''):
if isinstance(value, list):
for i, row in enumerate(value):
walk(row, pointer + '/' + str(i))
elif isinstance(value, dict):
source = None
for key in ('file', 'path', 'library'):
if key in value:
try:
source = (self.input_path(value[key]) if key == 'file' else
self.library_video(value[key]) if key == 'library' else Path(value[key]).expanduser())
if not source.is_file():
raise ValueError
except (OSError, ValueError, JobError):
errors.append(diagnostic(pointer + '/' + key, 'an existing uploaded input or saved library asset', value[key]))
if value.get('edit') is not None:
size = (spec.get('width', 1344), spec.get('height', 768))
if source is not None and source.is_file():
from .media import inspect
try:
info = inspect(source)
size = (info['width'], info['height'])
except (OSError, ValueError, KeyError):
errors.append(diagnostic(pointer, 'a readable image or video', None))
errors.extend(editor_issues(value['edit'], pointer + '/edit', source_size=size,
target=(spec.get('width', 1344), spec.get('height', 768)), resolve=self.input_path))
for key, child in value.items():
if key not in ('metadata', 'edit'):
walk(child, pointer + '/' + key)
for key in ('media', 'clips', 'questions', 'refine_from'):
if key in spec:
walk(spec[key], '/' + key)
if errors:
raise specs.SpecError(errors)
# Queue ---------------------------------------------------------------------------------------------
def scheduler(self):
return getattr(self.server, '_freevideo_front', None)
def snapshot(self):
if self.scheduler() is not None:
running, pending = self.scheduler().current_queue()
return [row[1] for row in running], [row[1] for row in pending]
running, pending = self.server.prompt_queue.get_current_queue()
return [row[1] for row in running], [row[1] for row in sorted(pending, key=lambda r: r[0])]
def editor_exports(self):
"""The editor page's export queue (editor_export.register), shared so exports take turns."""
return getattr(self.server, '_freevideo_editor_export', None)
def editor_ears(self):
"""The auto-subtitle ears queue (subtitle_ears.register): time lines against the sound or write down speech."""
return getattr(self.server, '_freevideo_subtitle_ears', None)
def queue_busy(self):
running, pending = self.snapshot()
return bool(running or pending)
def prompt_state(self, prompt_id):
running, pending = self.snapshot()
return 'running' if prompt_id in running else 'pending' if prompt_id in pending else 'missing'
def position(self, prompt_id):
running, pending = self.snapshot()
return pending.index(prompt_id) + 1 if prompt_id in pending else None
def history(self, prompt_id):
if self.scheduler() is not None:
return self.scheduler().history(prompt_id).get(prompt_id)
return self.server.prompt_queue.get_history(prompt_id=prompt_id).get(prompt_id)
def progress(self, node, prompt_id=None):
from ..comfy_progress import STATE
with STATE.lock:
key = (str(prompt_id), str(node)) if self.scheduler() is not None else str(node)
row = STATE.rows.get(key)
if not row:
return None
# How old the message is, as a reopened page gets it, so a bar can go on from where the engine is now.
return dict(json.loads(json.dumps(row['message'])), age_seconds=max(0., STATE.clock() - row['measured']))
async def submit(self, graph, extra):
if self.scheduler() is not None:
reply = await self.scheduler().submit(dict(prompt=graph, extra_data=extra,
client_id=extra.get('client_id', '')))
return reply['prompt_id']
if self.handler is None:
for route in self.server.routes:
if getattr(route, 'method', None) == 'POST' and getattr(route, 'path', None) == '/prompt':
self.handler = route.handler
if self.handler is None:
raise JobError('unsupported_comfyui', 'This ComfyUI has no POST /prompt route')
prompt_id = str(uuid.uuid4())
value = dict(prompt=graph, prompt_id=prompt_id, extra_data=extra)
try:
response = await self.handler(Request(value))
status, body = response.status, json.loads(response.text or '{}')
except (AttributeError, TypeError):
# A newer handler reads more of the request: post to this server instead.
status, body = await self.post_prompt(value)
if status != 200:
error = body.get('error') or {}
messages = [row.get('message') or row.get('details') for node in (body.get('node_errors') or {}).values()
for row in node.get('errors', [])]
message = '; '.join(m for m in messages if m) or error.get('message') or 'ComfyUI rejected the request'
details = [row.get('details') for node in (body.get('node_errors') or {}).values()
for row in node.get('errors', []) if row.get('details')]
not_ready = any('setup' in (m or '').lower() or 'install' in (m or '').lower() for m in messages + details)
raise JobError('not_ready' if not_ready else 'invalid_request',
'; '.join(d for d in details if d) or message)
return body.get('prompt_id', prompt_id)
async def post_prompt(self, value):
import aiohttp
from comfy.cli_args import args
host = (args.listen or '127.0.0.1').split(',')[0]
host = '127.0.0.1' if host in ('0.0.0.0', '') else '::1' if host == '::' else host
scheme = 'https' if getattr(args, 'tls_keyfile', None) else 'http'
url = '%s://%s:%d/prompt' % (scheme, '[%s]' % host if ':' in host else host, args.port)
async with aiohttp.ClientSession() as session:
async with session.post(url, json=value, ssl=False) as response:
return response.status, await response.json(content_type=None)
def cancel(self, prompt_id, node):
if self.scheduler() is not None:
return self.scheduler().cancel(prompt_id, node)
import nodes
from ..comfy_progress import cancel_request
return cancel_request(self.server.prompt_queue, prompt_id, node, nodes.interrupt_processing)
def generation_error(self, history, node):
messages = (history.get('status') or {}).get('messages') or []
text = None
for kind, data in messages:
if kind == 'execution_interrupted':
return dict(code='cancelled', message='Cancelled')
if kind == 'execution_error':
text = str(data.get('exception_message') or data.get('exception_type') or '').strip()
error = dict(code='generation_failed', message=(text or 'The generation did not finish')[:4000])
run = self.run_output(node)
if run is not None:
error['run'] = self.output_relative(run.parent)
report = _read(run.with_suffix('.request.json'))
engine = _read(run.with_suffix('.engine.json'))
if report.get('resource_error'):
details = report['resource_error'] if isinstance(report['resource_error'], dict) else {}
error.update(code='insufficient_memory', retryable=False,
alternatives=(details.get('details') or details).get('alternatives') or [])
failure = engine.get('failure') or {}
if failure.get('kind') in ('gpu_oom', 'ram_pressure'):
error.update(code='out_of_memory', retryable=True)
elif failure.get('kind'):
error.setdefault('failure_kind', failure['kind'])
debug = run.with_suffix('.debug.json')
if debug.is_file():
error['report'] = self.file_view(self.output_relative(debug))
return error
def run_output(self, node):
from ..comfy_progress import REPORTS
message = self.progress(node) or {}
with REPORTS.lock:
path = REPORTS.runs.get(message.get('report_id'))
return path
# Local VLM -----------------------------------------------------------------------------------------
@property
def vlm(self):
service = getattr(self.server, '_freevideo_prompt_vlm', None)
if service is None:
raise JobError('vlm_unavailable', 'The prompt VLM service is not loaded')
return service
def vlm_busy(self):
service = getattr(self.server, '_freevideo_prompt_vlm', None)
return service is not None and service.busy()
def vlm_start(self, action, value):
try:
return self.vlm.start(action, value)['job']
except ValueError as error:
from .jobs import VLM_ERRORS
code = str(error) if str(error) in VLM_ERRORS or str(error).isidentifier() else 'invalid_request'
raise JobError(code, VLM_ERRORS.get(code, str(error)), retryable=code == 'busy') from None
def vlm_status(self, key):
return self.vlm.status(key)
def vlm_cancel(self, key):
try:
self.vlm.cancel(key)
except ValueError:
pass
# Local image generation ----------------------------------------------------------------------------
def _image(self):
from ..image_gen.service import get_service
service = get_service(self.server)
if (service is not None and getattr(service, 'folders', None) is None
and getattr(self, 'folders', None) is not None):
service.folders = self.folders
return service
@property
def image(self):
service = self._image()
if service is None:
raise JobError('image_unavailable', 'The image service is not loaded')
return service
def image_busy(self):
service = self._image()
return service is not None and service.busy()
def image_ready(self):
try:
service = self._image()
return service is not None and bool(service.info()['ready'])
except Exception: # noqa: BLE001 - an optional service must not prevent video generation
return False
def image_start(self, action, value):
try:
return self.image.start(action, value)['job']
except ValueError as error:
from .jobs import IMAGE_ERRORS, IMAGE_WAIT_ERRORS
code = getattr(error, 'code', str(error))
code = code if code in IMAGE_ERRORS else 'invalid_request'
raise JobError(code, IMAGE_ERRORS[code], retryable=code in IMAGE_WAIT_ERRORS,
kind='image', report_id=getattr(error, 'report_id', None),
other_usage=getattr(error, 'status', {}).get('other_usage')) from None
def image_status(self, key):
return self.image.status(key)
def image_cancel(self, key):
try:
self.image.cancel(key)
except ValueError:
pass
# Media tools ---------------------------------------------------------------------------------------
def import_file(self, source, target):
from ..comfy_assets import media_kind
media_kind(source.name)
if source.stat().st_size > 2 * 1024 ** 3:
raise JobError('invalid_media', 'Media files are limited to 2 GiB')
try:
os.link(source, target)
except OSError:
shutil.copyfile(source, target)
def extract_frame(self, video, out, at):
from . import media
try:
return media.extract_frame(video, out, at=at)
except ValueError as error:
raise JobError('invalid_media', str(error)) from None
def sample_frames(self, video, out_dir, count):
from . import media
return media.sample_frames(video, out_dir, count=count)
def edit(self, source, recipe, out, target=None):
"""Render a recipe (or an editor document) of the input file `source`; keep the document beside it."""
from . import edit
from PIL import Image, ImageOps
resolve = lambda name: self.input_path(name)
try:
if recipe.get('version') == 2 and 'layers' in recipe:
doc = recipe
else:
with Image.open(resolve(source)) as image:
size = ImageOps.exif_transpose(image).size
doc = edit.compile_recipe(recipe, source=source, source_size=size, target=target, resolve=resolve)
made = edit.save(doc, resolve, out)
except edit.EditError as error:
raise JobError('invalid_edit', str(error), path=getattr(error, 'path', None)) from None
out.with_suffix('.edit.json').write_text(json.dumps(doc), encoding='utf-8')
return dict(width=made['width'], height=made['height'], doc=doc)
def assemble(self, clips, out, **options):
from . import media
return media.assemble(clips, out, **options)
def _read(path):
try:
value = json.loads(Path(path).read_text(encoding='utf-8'))
return value if isinstance(value, dict) else {}
except (OSError, ValueError):
return {}