Skip to content

Make videos on the computer you already own. FreeVideo runs MiniMax H3 in as little as 8 GB of VRAM and 16 GB of RAM, and adapts its acceleration path to your hardware.

Keep the image worker on the card longer for people who mostly make pictures

Bowen Xue15d6187b48fa
diff --git a/freevideo_engine/comfy_nodes.py b/freevideo_engine/comfy_nodes.py
index 5e259dc..93a1231 100644
--- a/freevideo_engine/comfy_nodes.py
+++ b/freevideo_engine/comfy_nodes.py
@@ -258,6 +258,11 @@ class FreeVideoGenerate(io.ComfyNode):
stop_music_for_generation(PromptServer.instance)
except Exception as error:
logging.warning('[FreeVideo] Could not stop music before the video: %s', error)
+ try: # the picture worker keeps the card longer only for people who mostly make pictures
+ from .image_gen.keep import note, path
+ note('video', path(comfy_bridge.installation_root()))
+ except Exception as error:
+ logging.warning('[FreeVideo] Could not record the video for the picture worker: %s', error)
prewarm = {}
result_reused = [False]
result_report = [None]
diff --git a/freevideo_engine/comfy_residency.py b/freevideo_engine/comfy_residency.py
index aa32995..d9dff57 100644
--- a/freevideo_engine/comfy_residency.py
+++ b/freevideo_engine/comfy_residency.py
@@ -76,6 +76,16 @@ def release_native(server, *, timeout=10.):
reserved_after_bytes=int(torch.cuda.memory_reserved(device)))
+def release_image_worker():
+ """Let an idle image worker give back the card; one that is making a picture is left alone."""
+ try:
+ from server import PromptServer
+ from .image_gen.service import release_idle_worker
+ release_idle_worker(PromptServer.instance)
+ except Exception:
+ pass # Native models must load whether or not the image option is installed.
+
+
def install(memory):
if getattr(memory.free_memory, '_freevideo_resident_cooperation', False) is True:
return
@@ -90,6 +100,8 @@ def install(memory):
OWNER.process is not None and OWNER.process.poll() is None):
if device is None or memory_required > memory.get_free_memory(device):
OWNER.release('comfy_model')
+ if device is None or memory_required > memory.get_free_memory(device):
+ release_image_worker()
return original(memory_required, device, *args, **kwargs)
free_memory._freevideo_resident_cooperation = True
diff --git a/freevideo_engine/image_gen/keep.py b/freevideo_engine/image_gen/keep.py
new file mode 100644
index 0000000..a00325b
--- /dev/null
+++ b/freevideo_engine/image_gen/keep.py
@@ -0,0 +1,78 @@
+"""How long an idle image worker keeps the model on the card.
+
+A worker that made a picture waits IDLE_SECONDS for the next one. For someone who mostly makes
+pictures it waits KEEP_SECONDS, but only while the whole model sits on the card, and it lets go at
+once when another program starts taking card memory. Video always takes the card first.
+"""
+
+from .. import usage_habit
+
+IDLE_SECONDS = 45.
+KEEP_SECONDS = 600.
+# Operations looked at (two thirds of them must be pictures), and how few are enough to judge at all.
+WINDOW = 6
+MINIMUM = 3
+STORED = usage_habit.OPS_STORED
+# Card memory other programs may take while the worker waits before it lets go.
+YIELD_BYTES = 1 << 30
+POLL_SECONDS = .5
+
+
+def path(root=None):
+ """Where the habit is kept: the usage habit file under the installation directory."""
+ return usage_habit.path(root)
+
+
+def load(file=None):
+ """The last operations ('image' or 'video'), oldest first; empty when the file cannot be trusted."""
+ return usage_habit.ops(file)
+
+
+def note(kind, file=None):
+ """Remember one operation. Only the kind is stored, nothing from the request."""
+ usage_habit.note_op(kind, file)
+
+
+def mostly_images(file=None):
+ """True when most of the recent operations were pictures and the last one was too."""
+ recent = load(file)[-WINDOW:]
+ return len(recent) >= MINIMUM and recent[-1] == 'image' and recent.count('image') * 3 >= len(recent) * 2
+
+
+def keep_seconds(file=None):
+ """The idle limit the worker is asked to use for the request being sent."""
+ return KEEP_SECONDS if mostly_images(file) else IDLE_SECONDS
+
+
+def card_free_bytes(device):
+ """Free memory of the card the worker's model is on, or None when it cannot be read."""
+ torch = __import__('sys').modules.get('torch')
+ if torch is None or getattr(device, 'type', str(device).split(':')[0]) != 'cuda':
+ return None
+ try:
+ return int(torch.cuda.mem_get_info(device)[0])
+ except Exception:
+ return None
+
+
+class Crowding:
+ """True once other programs have taken YIELD_BYTES of card memory since the wait began.
+
+ The worker's own memory does not change while it waits, so a drop in free memory is theirs."""
+
+ def __init__(self, read, limit=YIELD_BYTES):
+ self.read = read
+ self.limit = limit
+ self.start = read()
+
+ def __bool__(self):
+ now = self.read()
+ return self.start is not None and now is not None and self.start - now >= self.limit
+
+
+def wait_seconds(requested, runner, idle=IDLE_SECONDS):
+ """Idle limit for the worker after a picture: the longer one only for a model whole on a readable card."""
+ if (type(requested) in (int, float) and requested > idle and getattr(runner, 'full_card', False) is True
+ and card_free_bytes(getattr(runner, 'device', None)) is not None):
+ return min(float(requested), KEEP_SECONDS)
+ return idle
diff --git a/freevideo_engine/image_gen/service.py b/freevideo_engine/image_gen/service.py
index 3255b9f..6a71abb 100644
--- a/freevideo_engine/image_gen/service.py
+++ b/freevideo_engine/image_gen/service.py
@@ -16,7 +16,7 @@ import threading
import time
import uuid
-from . import image_pack
+from . import image_pack, keep
from ..download_settings import observed_speed
from .failure import (Context, HOST_MESSAGE_ZH, host_memory_shortage, ERRORS, FailureError, MESSAGE_ZH, NO_REPORT, STAGES, UNIFIED_MESSAGE_ZH,
classify_exception)
@@ -135,6 +135,7 @@ class Service:
self.worker_restarts = 0
self.worker_started = False
self.queued = None
+ self.habit_root = None
self.queue_thread = None
self.queue_wake = threading.Event()
self.finished = OrderedDict()
@@ -882,6 +883,7 @@ class Service:
def work(self, root, machine):
from ..locking import runtime_lock
from .. import processes, resident_process
+ self.habit_root = root
session = None
failure = 'worker_exit'
failure_error = None
@@ -1082,6 +1084,11 @@ class Service:
def infer(self, session, value):
context = self.context or Context('generate', value)
heard = False
+ habit = keep.path(self.habit_root)
+ if 'keep_seconds' not in value:
+ keep.note('image', habit)
+ # How long the worker may wait for the next picture; it applies that only with the whole model on the card.
+ value['keep_seconds'] = keep.keep_seconds(habit)
try:
session.send(action='generate', **value)
except OSError:
diff --git a/freevideo_engine/image_gen/worker.py b/freevideo_engine/image_gen/worker.py
index f9c565a..0b549e3 100644
--- a/freevideo_engine/image_gen/worker.py
+++ b/freevideo_engine/image_gen/worker.py
@@ -9,6 +9,7 @@ import sys
import threading
import time
+from . import keep
from .failure import Context, classify_exception, exception_details, host_memory_shortage, safe_plan
@@ -71,6 +72,35 @@ def _flatten(image):
return image.convert('RGB')
+def next_request(inbox, wait, crowding=None):
+ """The next row, or None when the pipe closed, `wait` seconds passed, or the card is crowded.
+
+ Without `crowding` this is one blocking read, as for an ordinary idle wait."""
+ if crowding is None:
+ try:
+ return inbox.rows.get(timeout=wait)
+ except queue.Empty:
+ return None
+ deadline = time.monotonic() + wait
+ while True:
+ remaining = deadline - time.monotonic()
+ if remaining <= 0:
+ return None
+ try:
+ return inbox.rows.get(timeout=min(remaining, keep.POLL_SECONDS))
+ except queue.Empty:
+ if crowding:
+ return None
+
+
+def hold(runner):
+ """Start the long idle wait: hand back the allocator's cached memory, then watch the card."""
+ torch = sys.modules.get('torch')
+ if torch is not None:
+ torch.cuda.empty_cache()
+ return keep.Crowding(lambda: keep.card_free_bytes(runner.device))
+
+
def load_runner(pack_dir, device, *, progress=None):
# Importing this module in ComfyUI or in CPU protocol tests never loads torch.
from .image_runner import ImageRunner
@@ -92,12 +122,10 @@ def serve(pack_dir, *, device='cuda', idle_seconds=45., stream=None, output=None
runner = None
back_detector = None
back_detector_loaded = False
+ wait, crowding = idle_seconds, None
try:
while not inbox.closed.is_set():
- try:
- value = inbox.rows.get(timeout=idle_seconds)
- except queue.Empty:
- break
+ value = next_request(inbox, wait, crowding)
if value is None or inbox.closed.is_set():
break
started = time.monotonic()
@@ -195,6 +223,16 @@ def serve(pack_dir, *, device='cuda', idle_seconds=45., stream=None, output=None
inbox.busy = False
if result is not None:
send(**result)
+ wait, crowding = idle_seconds, None
+ if result is not None and result.get('phase') == 'result' and runner is not None:
+ wait = keep.wait_seconds(value.get('keep_seconds'), runner, idle_seconds)
+ if wait > idle_seconds:
+ try:
+ crowding = hold(runner)
+ except Exception as error:
+ # The picture is already sent; a hold that cannot start is the normal short wait.
+ print('image worker hold failed: %s' % type(error).__name__, file=sys.stderr, flush=True)
+ wait = idle_seconds
finally:
inbox.closed.set()
inbox.cancel.set()
diff --git a/freevideo_engine/usage_habit.py b/freevideo_engine/usage_habit.py
index 334a7be..b7f1154 100644
--- a/freevideo_engine/usage_habit.py
+++ b/freevideo_engine/usage_habit.py
@@ -4,6 +4,9 @@ Nothing derived from a prompt, image or seed is stored. A transition between
two successive successful videos is one of three kinds of change, and the file
keeps the last three transitions (the four-request window) plus running counts.
A file that is missing, unreadable or malformed counts as no habit at all.
+
+The same file keeps the last OPS_STORED operations ('image' or 'video') under "ops", which the image
+worker reads to decide how long to stay on the card. Reading and saving the habit leave them as they are.
"""
import json
import os
@@ -11,14 +14,18 @@ from pathlib import Path
WINDOW = 4
KINDS = ('images', 'prompt', 'rerun')
+OPS = ('image', 'video')
+OPS_STORED = 8
-def path():
+def path(root=None):
override = os.environ.get('FREEVIDEO_USAGE_HABIT')
if override:
return Path(override)
- from .system import install_root
- return install_root() / '.freevideo' / 'usage-habit.json'
+ if root is None:
+ from .system import install_root
+ root = install_root()
+ return Path(root) / '.freevideo' / 'usage-habit.json'
def transition(previous, current):
@@ -64,20 +71,46 @@ def load(file=None):
or not isinstance(counts, dict)
or any(type(counts.get(name)) is not int or counts[name] < 0 for name in (*KINDS, 'requests'))):
return None
+ ops = value.get('ops', [])
+ if not isinstance(ops, list) or len(ops) > OPS_STORED or any(kind not in OPS for kind in ops):
+ ops = []
return dict(version=1, window=WINDOW, recent=[list(kinds) for kinds in recent],
- counts={name: counts[name] for name in (*KINDS, 'requests')})
+ counts={name: counts[name] for name in (*KINDS, 'requests')}, ops=list(ops))
except (OSError, ValueError, KeyError, TypeError, AttributeError):
return None
+def empty():
+ return dict(version=1, window=WINDOW, recent=[], counts=dict.fromkeys((*KINDS, 'requests'), 0), ops=[])
+
+
def record(kinds, file=None):
"""Append one transition (None for the first request after a restart) and save."""
from .monitoring import save
- value = load(file) or dict(version=1, window=WINDOW, recent=[], counts=dict.fromkeys((*KINDS, 'requests'), 0))
+ value = load(file) or empty()
value['counts']['requests'] += 1
if kinds is not None:
value['recent'] = (value['recent'] + [list(kinds)])[-(WINDOW - 1):]
for kind in kinds:
value['counts'][kind] += 1
- save(Path(file or path()), value)
+ save(Path(file or path()), value if value['ops'] else {name: item for name, item in value.items() if name != 'ops'})
return value
+
+
+def ops(file=None):
+ """The last operations ('image' or 'video'), oldest first; empty when nothing can be trusted."""
+ value = load(file)
+ return value['ops'] if value else []
+
+
+def note_op(kind, file=None):
+ """Remember one operation next to the habit. Only the kind is stored, nothing from the request."""
+ if kind not in OPS:
+ raise ValueError(kind)
+ from .monitoring import save
+ value = load(file) or empty()
+ value['ops'] = (value['ops'] + [kind])[-OPS_STORED:]
+ try:
+ save(Path(file or path()), value)
+ except OSError:
+ pass # A habit that cannot be saved is only forgotten at restart.