Make videos on the computer you already own. FreeVideo runs MiniMax H3 in as little as 8 GB of VRAM and 16 GB of RAM, and adapts its acceleration path to your hardware.
Launcher asks before freeing video model files the engine no longer reads
Bowen Xuee65f005f0c47
diff --git a/freevideo_engine/adaln_reclaim.py b/freevideo_engine/adaln_reclaim.pynew file mode 100644index 0000000..b95112a--- /dev/null+++ b/freevideo_engine/adaln_reclaim.py@@ -0,0 +1,437 @@+"""Reclaim the 26 GB of optional AdaLN projection sources that tables made unused.++Before 0.4.0 a slim installation that ran a sampling schedule without a+published table fetched the 50 original projection files (adaln/00..49) and+computed its own tables from them, on the step grid of that version. A newer+engine uses a different grid on some platforms, so those tables are never read+again and the sources only occupy disk. They can be fetched again on demand+(restore_projections), so removing them is recoverable. Everything below must+hold, otherwise nothing is deleted:++1. the installation is a slim export. In a full installation the files are part+ of the model itself;+2. the 50 files are exactly the ones the manifest lists: same names, sizes and+ SHA-256, regular, unshared, reached without links or junctions;+3. at least one complete table of this model, computed on the old step grid,+ is in the cache (its timesteps are the old grid and not the engine grid);+4. no LoRA variant that changes modulation sits beside the cache. Those hardlink+ or copy the projection groups. An unreadable variant counts as one.++Nothing is removed on the engine's own initiative. The background precompile+only checks the conditions and leaves a small offer file; the launcher asks the+user, and the removal runs only when the user agrees. A "later" answer is+recorded against the release line (0.4, 0.5, ...) and the offer comes back with+the next one.++The old-grid tables go after the sources, under the same conditions, one file+at a time and only when the folder holds nothing but that table's files. A+removal that was interrupted resumes where it stopped. Hashing 26 GB is read+only and may be interrupted at any file; it resumes from a receipt of the files+already verified. The removal itself runs under the GPU-work lock, never during+a generation. Nothing here fails a request.+"""+import json+import os+from pathlib import Path+import re+import stat+import time++from . import adaln_assets as assets+from .long_paths import io_path+from .variant_cleanup import inside, redirected++RECEIPT = 'adaln-reclaim.json'+OFFER = 'adaln-reclaim-offer.json'+SOURCES = 'adaln'+COUNT = 50+SOURCE_NAME = re.compile(r'adaln/(\d{2})\.safetensors')+VARIANT = re.compile(r'vdn-fp8-lora-[0-9a-f]{24}')+TABLE_FOLDER = re.compile(r'adaln-tables-v2-[0-9a-f]{24}')+TABLE_FILE = re.compile(r'identity\.json|producer\.json|(\d{2})\.(safetensors|json)')+IDENTITY_LIMIT = 65536+BLOCK = 8 * 1024 * 1024+++class Stopped(Exception):+ """The caller asked to give way; nothing was deleted."""+++def _json(path, limit=IDENTITY_LIMIT):+ path = io_path(path)+ info = path.lstat()+ if redirected(info) or not stat.S_ISREG(info.st_mode) or info.st_size > limit:+ raise ValueError('Not a small regular file')+ return json.loads(path.read_text(encoding='utf-8'))+++def _regular(path):+ try:+ info = io_path(path).lstat()+ except OSError:+ return None+ return None if redirected(info) or not stat.S_ISREG(info.st_mode) else info+++def _contained(cache, path):+ """True when path, fully resolved, lies strictly inside this installation's own resolved cache."""+ try:+ root, real = (os.path.realpath(io_path(item)) for item in (cache, path))+ return real != root and os.path.commonpath([root, real]) == root+ except (OSError, ValueError):+ return False+++def engine_schedules(steps):+ """(task, old grid rows, engine grid rows) for every task of one step count."""+ from .adaln import schedule_timesteps+ from .media_request import TASKS+ rows = []+ for task in TASKS:+ old, new = (schedule_timesteps(steps, device='cpu', task=task, upstream_grid=grid) for grid in (True, False))+ rows.append((task, [t.float().tolist() for t in old], [t.float().tolist() for t in new]))+ return rows+++def _recording(schedules, consulted):+ """`schedules` that also keeps what it answered, for the offer: the launcher has no torch to ask again."""+ def answer(steps):+ rows = schedules(steps)+ consulted[str(steps)] = [[task, old, new] for task, old, new in rows]+ return rows+ return answer+++def _offered_schedules(cache):+ """The grids recorded with the standing offer when they were computed by this very engine release, else None."""+ value = offer(cache)+ release, stored = value.get('engine_release'), value.get('schedules')+ if not isinstance(release, str) or release != installed_release() or not isinstance(stored, dict):+ return None+ def answer(steps):+ rows = stored.get(str(steps))+ if not isinstance(rows, list) or not rows:+ raise KeyError(steps)+ return [(task, old, new) for task, old, new in rows]+ return answer+++def old_grid_tables(cache, manifest, schedules=None):+ """Folders of this model's tables that sit on the old step grid, as (folder, complete)."""+ schedules = schedules or engine_schedules+ weights = assets.weight_identity(manifest)+ count = len(assets.projection_groups(manifest))+ found, memo = [], {}+ for folder in sorted(io_path(cache).glob('adaln-tables-v2-*')):+ if not TABLE_FOLDER.fullmatch(folder.name) or not inside(cache, folder) or not folder.is_dir():+ continue+ try:+ value = _json(folder / 'identity.json')+ steps = len(value['timesteps'])+ if value.get('weights') != weights or assets.directory(value) != folder.name:+ continue+ if steps not in memo:+ memo[steps] = schedules(steps)+ if any(value['timesteps'] == new for _, _, new in memo[steps]):+ continue # A table the engine reads: not ours to remove.+ if not any(value['timesteps'] == old != new for _, old, new in memo[steps]):+ continue+ complete = all(_regular(folder / ('%02d.safetensors' % i)) is not None+ and _regular(folder / ('%02d.json' % i)) is not None for i in range(count))+ except (OSError, ValueError, KeyError, TypeError):+ continue+ found.append((folder, complete))+ return found+++def modulation_variants(cache):+ """True when a LoRA variant beside the cache may hold or share the projection groups."""+ parent = io_path(cache).parent+ try:+ entries = list(parent.iterdir())+ except OSError:+ return True+ for entry in entries:+ if not VARIANT.fullmatch(entry.name):+ continue+ try:+ if (entry / SOURCES).exists() or (entry / SOURCES).is_symlink():+ return True+ manifest = _json(entry / 'manifest.json', 64 * 1024 * 1024)+ if any(str(row.get('group', '')).startswith('adaln/') for row in manifest.get('groups', [])):+ return True+ except (OSError, ValueError, TypeError, AttributeError):+ return True # Cannot tell: keep.+ return False+++def release_line(version):+ """'0.4' for '0.4.0' (None when the version is not one): a decline holds for this line."""+ match = re.match(r'(\d{1,4})\.(\d{1,4})(?:\.|$)', version) if isinstance(version, str) else None+ return '%s.%s' % match.groups() if match else None+++def installed_release():+ """The product version this engine was released as, or None."""+ try:+ value = _json(Path(__file__).with_name('release_notes.json'), 64 * 1024 * 1024)+ except (OSError, ValueError):+ return None+ return value.get('product_version') if isinstance(value, dict) else None+++def offer(cache):+ """The standing offer {'bytes', 'manifest', 'declined'} or {}; small regular file only."""+ try:+ value = _json(Path(cache) / OFFER)+ except (OSError, ValueError):+ return {}+ return value if isinstance(value, dict) else {}+++def _write_offer(cache, **fields):+ from .monitoring import save+ save(Path(cache) / OFFER, dict(format=1, **fields))+++def decline(cache, version):+ """Record "later" for this release line; the offer stays visible in the file, not in the dialog."""+ current = {key: value for key, value in offer(cache).items() if key != 'format'}+ current.update(bytes=current.get('bytes', 0), manifest=current.get('manifest'), declined=release_line(version))+ _write_offer(cache, **current)+++def _tables_size(folders):+ total = 0+ for folder, _ in folders:+ try:+ total += sum(entry.lstat().st_size for entry in io_path(folder).iterdir() if _regular(entry) is not None)+ except OSError:+ continue+ return total+++def _source_paths(cache, manifest):+ rows = assets.projection_groups(manifest)+ names = sorted(row.get('file') for row in rows)+ if names != ['adaln/%02d.safetensors' % i for i in range(COUNT)]:+ return None+ return [(row, io_path(cache) / row['file']) for row in sorted(rows, key=lambda r: r['file'])]+++def _hash(path, should_stop):+ import hashlib+ digest = hashlib.sha256()+ with io_path(path).open('rb') as stream:+ for block in iter(lambda: stream.read(BLOCK), b''):+ if should_stop():+ raise Stopped()+ digest.update(block)+ return digest.hexdigest()+++def _receipt(cache):+ try:+ value = _json(Path(cache) / RECEIPT)+ return value if isinstance(value, dict) else {}+ except (OSError, ValueError):+ return {}+++def _save(cache, value):+ from .monitoring import save+ save(Path(cache) / RECEIPT, value)+++def verify_sources(cache, manifest, should_stop=lambda: False):+ """None when the 50 files are exactly the manifest's, otherwise the reason."""+ from .storage import fingerprint+ listed = _source_paths(cache, manifest)+ if listed is None:+ return 'unexpected_manifest'+ receipt, stamps = _receipt(cache), {}+ previous = receipt.get('verified', {}) if receipt.get('manifest') == assets.weight_identity(manifest) else {}+ present = [path for _, path in listed if _regular(path) is not None]+ if not present:+ return 'absent'+ if len(present) != len(listed) and not receipt.get('deleting'):+ return 'partial'+ if not inside(cache, io_path(cache) / SOURCES) or not _contained(cache, io_path(cache) / SOURCES):+ return 'redirected'+ if any(not _contained(cache, path) for _, path in listed if _regular(path) is not None):+ return 'redirected'+ for row, path in listed:+ info = _regular(path)+ if info is None:+ continue # Removed by an interrupted run of this cleanup.+ if info.st_size != row['bytes'] or info.st_nlink != 1:+ return 'changed' if info.st_size != row['bytes'] else 'shared'+ stamps[row['file']] = dict(fingerprint(path), sha256=row['sha256'])+ verified = {}+ for row, path in listed:+ if _regular(path) is None:+ continue+ stamp = stamps[row['file']]+ saved = previous.get(row['file'])+ if saved is not None and {k: v for k, v in saved.items()} == stamp:+ verified[row['file']] = saved+ continue+ if should_stop():+ raise Stopped()+ if _hash(path, should_stop) != row['sha256']:+ return 'changed'+ verified[row['file']] = stamp+ _save(cache, dict(manifest=assets.weight_identity(manifest), verified=verified,+ deleting=bool(receipt.get('deleting'))))+ return None+++def _remove_tables(cache, folders):+ removed = 0+ for folder, _ in folders:+ if not _contained(cache, folder):+ continue+ files = [entry for entry in io_path(folder).iterdir()]+ if any(_regular(entry) is None or not TABLE_FILE.fullmatch(entry.name) or not _contained(cache, entry)+ for entry in files):+ continue # Anything else in it keeps the whole folder.+ size = 0+ for entry in sorted(files, key=lambda p: p.suffix == '.json'): # Receipts last.+ size += entry.lstat().st_size+ entry.unlink()+ folder.rmdir()+ removed += size+ return removed+++def reclaim(cache, *, should_stop=lambda: False, schedules=None, lock=None, ask=False, release=None):+ """Remove the sources and the old-grid tables when every condition holds.++ Returns {action: 'deleted'|'offered'|'kept', reason, bytes, tables_bytes}. `lock`+ is a context-manager factory held while deleting (the GPU-work lock). With+ `ask` nothing is deleted: the conditions are checked, the offer file is+ written and the result is 'offered'. `release` is the installed version,+ which a recorded "later" is compared against.+ """+ cache = Path(cache)+ kept = lambda reason: dict(action='kept', reason=reason, bytes=0, tables_bytes=0)+ try:+ manifest = _json(cache / 'manifest.json', 64 * 1024 * 1024)+ except (OSError, ValueError):+ return kept('no_manifest')+ if manifest.get('format') not in assets.SLIM_FORMATS:+ return kept('full_install') # The files are part of the model.+ if ask and release_line(release) is not None and offer(cache).get('declined') == release_line(release):+ return kept('declined')+ try:+ listed = _source_paths(cache, manifest)+ if listed is None:+ return kept('unexpected_manifest')+ if modulation_variants(cache):+ return kept('modulation_lora')+ consulted = {}+ tables = old_grid_tables(cache, manifest, _recording(schedules or engine_schedules, consulted))+ if not any(complete for _, complete in tables):+ return kept('no_old_grid_table')+ reason = verify_sources(cache, manifest, should_stop)+ except Stopped:+ return kept('stopped')+ if reason not in (None, 'absent'):+ if ask:+ _forget_offer(cache)+ return kept(reason)+ if ask:+ present = sum(row['bytes'] for row, path in listed if _regular(path) is not None)+ tables_bytes = _tables_size(tables)+ _write_offer(cache, bytes=present, tables_bytes=tables_bytes, manifest=assets.weight_identity(manifest),+ declined=offer(cache).get('declined'), engine_release=release, schedules=consulted)+ return dict(action='offered', reason='ask', bytes=present, tables_bytes=tables_bytes)+ freed = 0+ try:+ with (lock() if lock is not None else _nothing()):+ # Nothing may have changed while hashing: decide again under the lock.+ try:+ if should_stop():+ return kept('stopped')+ if modulation_variants(cache):+ return kept('modulation_lora')+ if reason is None:+ again = verify_sources(cache, manifest, should_stop)+ if again not in (None, 'absent'):+ return kept(again)+ except Stopped:+ return kept('stopped')+ if reason is None:+ _save(cache, dict(_receipt(cache), manifest=assets.weight_identity(manifest), deleting=True))+ for row, path in listed:+ info = _regular(path)+ if info is None:+ continue+ if info.st_size != row['bytes'] or info.st_nlink != 1:+ return kept('changed')+ if not _contained(cache, path):+ return kept('redirected')+ freed += info.st_size+ io_path(path).unlink()+ try:+ (io_path(cache) / SOURCES).rmdir()+ except OSError:+ pass # Other content stays.+ tables_bytes = _remove_tables(cache, old_grid_tables(cache, manifest, schedules))+ io_path(Path(cache) / RECEIPT).unlink(missing_ok=True)+ _forget_offer(cache)+ except BlockingIOError:+ return kept('busy')+ return dict(action='deleted', reason='reclaimed', bytes=freed, tables_bytes=tables_bytes)+++def _forget_offer(cache):+ try:+ io_path(Path(cache) / OFFER).unlink(missing_ok=True)+ except OSError:+ pass+++class _nothing:+ def __enter__(self):+ return self++ def __exit__(self, *args):+ return False+++def apply(machine, *, should_stop=lambda: False, schedules=None):+ """The user agreed: check everything again under the installation's GPU-work lock and delete. Never raises.++ The launcher has no torch, so which grids are old comes from the offer the engine wrote; an offer+ from another engine release, or without them, is dropped and made again by the next background check.+ """+ from .locking import runtime_lock+ cache = machine['cache']+ path = Path(machine['root']) / 'engine.lock' if machine.get('root') else None+ schedules = schedules or _offered_schedules(cache)+ if schedules is None:+ _forget_offer(cache)+ return dict(action='kept', reason='stale_offer', bytes=0, tables_bytes=0)+ try:+ result = reclaim(cache, should_stop=should_stop, schedules=schedules,+ lock=lambda: runtime_lock(path, inherit=False))+ except Exception as error:+ return dict(action='kept', reason='error:' + type(error).__name__, bytes=0, tables_bytes=0)+ if result['action'] != 'deleted' and result['reason'] not in ('busy', 'stopped'):+ _forget_offer(cache) # The conditions no longer hold: there is nothing to offer.+ return result+++def run(machine, *, should_stop=lambda: False):+ """Entry point of the background precompile: check and leave the offer, report one event, never raise."""+ started = time.time()+ try:+ result = reclaim(machine['cache'], should_stop=should_stop, ask=True, release=installed_release())+ except Exception as error: # Best effort: the installation works either way.+ result = dict(action='kept', reason='error:' + type(error).__name__, bytes=0, tables_bytes=0)+ if result['action'] == 'kept' and result['reason'] not in ('declined', 'stopped', 'busy') \+ and not result['reason'].startswith('error:'):+ _forget_offer(machine['cache']) # An offer is only standing while every condition holds.+ result['seconds'] = round(time.time() - started, 1)+ return resultdiff --git a/freevideo_engine/launcher/Main.qml b/freevideo_engine/launcher/Main.qmlindex 3a17c59..1d05b04 100644--- a/freevideo_engine/launcher/Main.qml+++ b/freevideo_engine/launcher/Main.qml@@ -941,6 +941,23 @@ ApplicationWindow {}}+ FPopup {+ id: adalnDialog; objectName: "adalnDialog"+ visible: s.adaln_dialog.remind && !settings.visible && !modelDialog.visible && !updateDialog.visible && !(s.archives_dialog && s.archives_dialog.remind) && !s.notice+ width: Math.min(480, win.width - 40); closePolicy: Popup.NoAutoClose+ contentItem: ColumnLayout {+ spacing: 14+ FText { objectName: "adalnDialogTitle"; text: s.adaln_dialog.title; font.pixelSize: theme.section + 2; font.weight: Font.DemiBold; Layout.fillWidth: true }+ FText { objectName: "adalnDialogText"; text: s.adaln_dialog.body; color: theme.muted; Layout.fillWidth: true }+ RowLayout {+ Layout.fillWidth: true; spacing: 8+ FButton { objectName: "adalnLaterButton"; text: s.adaln_dialog.later_text; flat: true; enabled: !s.adaln_dialog.busy; onClicked: backend.action("keep_adaln", false) }+ Item { Layout.fillWidth: true }+ FButton { objectName: "adalnDeleteButton"; text: s.adaln_dialog.delete_text; primary: true; enabled: !s.adaln_dialog.busy; onClicked: backend.action("delete_adaln", false) }+ }+ }+ }+FPopup {id: settings; objectName: "settingsDialog"visible: settingsOpen; onClosed: settingsOpen = falsediff --git a/freevideo_engine/launcher_session.py b/freevideo_engine/launcher_session.pyindex a28cf97..3ca99ac 100644--- a/freevideo_engine/launcher_session.py+++ b/freevideo_engine/launcher_session.py@@ -58,6 +58,9 @@ class Session:page = 'comfy'selected = installed_versions = engine_identity = bundled_identity = Nonerelease_details = None+ adaln_thread = adaln_result = adaln_found = None+ adaln_polled = 0.+ adaln_hidden = adaln_retry = False# The source whose earlier copies were removed once ComfyUI ran it.old_versions_retired = Nonearchives_busy = False@@ -407,6 +410,10 @@ class Session:self.queue_state = Noneself._update_engine_when_idle()return+ if name == 'keep_adaln':+ self.keep_adaln(); return+ if name == 'delete_adaln':+ self.delete_adaln(); returnif name == 'back':self.page = 'comfy' if self.page == 'models' else 'models'return@@ -1153,6 +1160,79 @@ class Session:remove(self.bridge)self._closed = True+ # -- AdaLN sources the engine no longer reads: the user decides ---------------------+ def adaln_offer(self):+ """The background check's standing offer for this installation, or None; read at most every 5 s."""+ now = time.monotonic()+ if now - self.adaln_polled < 5:+ return self.adaln_found+ self.adaln_polled, self.adaln_found = now, None+ try:+ from . import adaln_reclaim+ machine = json.loads((io_path(self.engine_root()) / 'machine.json').read_text(encoding='utf-8'))+ offer = adaln_reclaim.offer(machine['cache'])+ line = adaln_reclaim.release_line(self.installed_product_version() or adaln_reclaim.installed_release())+ size = sum(offer.get(key, 0) for key in ('bytes', 'tables_bytes'))+ if (all(type(offer.get(key, 0)) is int for key in ('bytes', 'tables_bytes')) and size >= 2**30+ and offer.get('manifest') and (line is None or offer.get('declined') != line)):+ self.adaln_found = dict(bytes=size, machine=machine, line=line)+ except (OSError, ValueError, KeyError, TypeError, AttributeError):+ pass+ return self.adaln_found++ def adaln_idle(self):+ return (self.controller.state.get('status') == 'open'+ and not (self.controller.busy or self.importer.busy or self.cleaner.busy or self.closing+ or self.update_intent or self.engine_updating or self.update_restarting+ or self.updater and self.updater.busy or self.upgrade.busy or self.model_switch)+ and self._queue_busy() is False)++ def keep_adaln(self):+ """"Later": the question does not come back before the next release line."""+ offer = self.adaln_offer()+ if self.adaln_thread is not None or not offer:+ return+ if offer['line']:+ from . import adaln_reclaim+ try:+ adaln_reclaim.decline(offer['machine']['cache'], offer['line'])+ except OSError:+ pass+ self.adaln_hidden = True++ def delete_adaln(self):+ offer = self.adaln_offer()+ if self.adaln_thread is not None or not offer or not self.adaln_idle():+ return+ machine = offer['machine']+ def work():+ from . import adaln_reclaim+ self.adaln_result = adaln_reclaim.apply(machine)+ self.adaln_retry = False+ self.adaln_thread = threading.Thread(target=work, name='freevideo-adaln-cleanup', daemon=True)+ self.adaln_thread.start()++ def adaln_dialog_view(self, error, update):+ thread = self.adaln_thread+ if thread is not None and not thread.is_alive():+ result, self.adaln_thread, self.adaln_result = self.adaln_result or {}, None, None+ thread = None+ # Busy GPU: ask again; anything else (deleted, or the conditions stopped holding) ends the question.+ self.adaln_retry = result.get('reason') in ('busy', 'stopped')+ self.adaln_hidden = not self.adaln_retry+ self.adaln_polled, self.adaln_found = 0., None+ offer = None if self.adaln_hidden else self.adaln_offer()+ remind = bool(offer and self.page == 'launcher' and not error and not update['remind']+ and (thread is not None or self.adaln_idle()))+ size = size_text((offer or {}).get('bytes', 0), decimal_sizes=sys.platform == 'darwin')+ return dict(remind=remind, busy=thread is not None,+ title=self.t('Delete model files no longer in use?', '删除不再使用的模型文件?'),+ body=self.t('This frees %s of disk space. If the files are needed later, they will be downloaded again automatically.',+ '可释放 %s 磁盘空间。以后如需要,会自动重新下载。') % size,+ delete_text=(self.t('Deleting…', '正在删除…') if thread is not None else+ self.t('Try again', '再试一次') if self.adaln_retry else self.t('Delete', '删除')),+ later_text=self.t('Not now', '暂不删除'))+def tick(self):if isinstance(self.controller, Controller) and not self.controller.busy:self.controller.poll_access_proxy()@@ -1641,6 +1721,7 @@ class Session:models=len(self.form['offline_models']), guide=package_instructions(self.form['new_comfy'] and self.form['environment_method'] == 'manual'and not self.form['offline_runtime'], zh)),video_model_guide=video_instructions(zh),+ adaln_dialog=self.adaln_dialog_view(error, update),selected=bool(self.selected), can_repair=self.can_repair(), repair_offered=repair_offered, error=error, retry_kind=self.retry_kind(), notice=self.notice,# Qt reads a list as an array; LEVELS is a tuple.compatibility=dict(self.compatibility, levels=list(self.compatibility.get('levels') or ())),diff --git a/freevideo_engine/precompile.py b/freevideo_engine/precompile.pyindex b9c4d4c..7495902 100644--- a/freevideo_engine/precompile.py+++ b/freevideo_engine/precompile.py@@ -508,6 +508,16 @@ def lower_priority():os.nice(19)+def reclaim(machine):+ """Free the AdaLN sources a table made unused; disk only, so the GPU may be busy (adaln_reclaim)."""+ from .adaln_reclaim import run as reclaim_sources+ result = reclaim_sources(machine, should_stop=CANCELLED.is_set)+ if result['action'] == 'deleted' or result['reason'] not in ('full_install', 'no_manifest',+ 'no_old_grid_table', 'unexpected_manifest'):+ print(json.dumps(dict(event='adaln_reclaim', **result)), flush=True)+ return result++def main(argv=None):parser = argparse.ArgumentParser(description=__doc__)parser.add_argument('--config', type=Path, required=True, help='The installation machine.json')@@ -531,6 +541,7 @@ def main(argv=None):own_temporary_environment(machine.get('root'))os.environ.update(execution_environment(environment(machine)))owner = os.environ.get(OWNER_ENV)+ reclaim(machine)receipt = run(machine, background=args.background, owner=int(owner) if owner else None)except Exception as error:if args.background and isinstance(error, BlockingIOError):