Skip to content

Make videos on the computer you already own. FreeVideo runs MiniMax H3 in as little as 8 GB of VRAM and 16 GB of RAM, and adapts its acceleration path to your hardware.

Check offline packages once and show progress throughout offline installation (0.3.11) (#112)

Bowen Xue50bac4620bc5
diff --git a/freevideo_engine/launcher_session.py b/freevideo_engine/launcher_session.py
index 8fde211..415665d 100644
--- a/freevideo_engine/launcher_session.py
+++ b/freevideo_engine/launcher_session.py
@@ -1006,6 +1006,18 @@ class Session:
task = row.get('task', {})
progress = progress_view(task.get('progress') or {}, zh)
overall = progress_view(row.get('overall') or task.get('phase_progress') or {}, zh)
+ if self.importer.busy:
+ # The sidebar's Working card follows the offline import, not the idle installer.
+ state = self.importer.state
+ if state.get('status') == 'preparing':
+ label = self.t('Checking packages', '检查配套包')
+ else:
+ label = self.t('Importing offline packages', '导入离线包')
+ if (state.get('count') or 0) > 1:
+ label += ' %d/%d' % (state.get('index') or 1, state['count'])
+ if state.get('detail'):
+ label += ' · ' + display(state['detail'], zh)
+ overall = dict(label=label, done=state.get('done'), total=state.get('total'))
errors = [self.error, self.browser_error, self.page_error, row.get('error', ''), *row.get('errors', [])]
shortcut = row.get('shortcut') or {}
if shortcut.get('status') == 'failed':
@@ -1072,7 +1084,8 @@ class Session:
model_upgrade=dict(self.upgrade.state, busy=self.upgrade.busy, switching=bool(self.model_switch)),
offline=dict(progress_view(self.importer.state, zh), runtime=bool(self.form['offline_runtime']),
runtime_supported=runtime_packages_supported(),
- models=len(self.form['offline_models']), guide=package_instructions(self.form['new_comfy'], zh)),
+ models=len(self.form['offline_models']), guide=package_instructions(self.form['new_comfy'] and self.form['environment_method'] == 'manual'
+ and not self.form['offline_runtime'], zh)),
video_model_guide=video_instructions(zh),
selected=bool(self.selected), error=error, retry_kind=self.retry_kind(), notice=self.notice, compatibility=self.compatibility,
report=dict(self.report),
diff --git a/freevideo_engine/model_guidance.py b/freevideo_engine/model_guidance.py
index 290514d..0fe173d 100644
--- a/freevideo_engine/model_guidance.py
+++ b/freevideo_engine/model_guidance.py
@@ -9,8 +9,13 @@ def runtime_packages_supported():
return sys.platform != 'darwin'
-def package_instructions(new_comfy, zh=False):
- """Describe the payloads this host can actually import."""
+def package_instructions(runtime_needed, zh=False):
+ """Describe the payloads this host can actually import.
+
+ runtime_needed: a new installation with the offline environment chosen and
+ no Environment ZIP imported yet; once it is in, asking again only invites
+ importing it a second time.
+ """
if sys.platform == 'darwin':
return (
'Get the four model packs (video model, text encoder, video & audio decoder, sampling caches); they also work on Mac. '
@@ -21,7 +26,7 @@ def package_instructions(new_comfy, zh=False):
'The reference audio pack is optional, for audio references.',
'下载「视频模型」「文本编码器」「视频与音频解码器」「采样缓存」四个模型包;'
'「音频参考缓存」可选,用音频做参考时才需要。')[bool(zh)]
- if new_comfy:
+ if runtime_needed:
text += '\n' + ('For a new installation, also get the Environment package.',
'全新安装还需「运行环境」包。')[bool(zh)]
return text
diff --git a/freevideo_engine/offline_packages.py b/freevideo_engine/offline_packages.py
index 88e52ea..a30da1e 100644
--- a/freevideo_engine/offline_packages.py
+++ b/freevideo_engine/offline_packages.py
@@ -6,10 +6,11 @@ from pathlib import Path
import shutil
import stat
import threading
+import time
import zipfile
from .monitoring import save
-from .portable import inside
+from .portable import bundle_name, inside, listing
PREFIX = 'FreeVideo-Windows/'
# Optional packs (reference-audio tables) carry sampling tables outside the
@@ -84,7 +85,7 @@ def inventory(rows):
seen = set()
for row in rows:
name = row['path']
- inside(Path('/unused'), name)
+ bundle_name(name)
# Windows strips trailing dots/spaces and recognizes device names.
for part in name.split('/'):
if (part.rstrip(' .') != part or part.split('.')[0].upper() in
@@ -158,51 +159,112 @@ def inspect_archive(path):
return dict(value, marker=marker, raw=raw, id=hashlib.sha256(raw).hexdigest())
-def import_archive(path, destination, progress=lambda **kw: None, cancelled=lambda: False):
+# Size and modification time of each file an import has hashed. A later check
+# of the same unchanged file trusts them instead of hashing tens of GiB again.
+VERIFIED = 'import-verified.json'
+
+
+def stamps(root):
+ try:
+ value = json.loads((Path(root) / VERIFIED).read_text(encoding='utf-8'))
+ return value if isinstance(value, dict) else {}
+ except (OSError, ValueError):
+ return {}
+
+
+def stamp(path):
+ info = Path(path).stat()
+ return [info.st_size, info.st_mtime_ns]
+
+
+def unchanged(path, row, known, current=None):
+ """Still the file its import checked: the size matches, and the recorded time too.
+
+ Only a file whose record says it was rewritten since is hashed. `current`
+ is its [size, mtime] when a directory walk already read it.
+ """
+ try:
+ current = current or stamp(path)
+ except OSError:
+ return False
+ if current[0] != row['bytes']:
+ return False
+ if known.get(row['path']) in (None, current):
+ return True
from .network import hash_file
+ return hash_file(path) == row['sha256']
+
+
+def import_archive(path, destination, progress=lambda **kw: None, cancelled=lambda: False):
+ """Extract once, checked by the ZIP's own CRC-32 and each file's size.
+
+ A file is written under a temporary name and renamed only when complete,
+ so a file found under its own name was checked when it was written.
+ """
path = Path(path).resolve()
+ progress(done=0, total=None, detail=path.name)
value = inspect_archive(path)
root = Path(destination).resolve() / '.freevideo-packages' / value['id'] / 'FreeVideo-Windows'
total = sum(r['bytes'] for r in value['files'])
- missing = sum(r['bytes'] for r in value['files'] if not inside(root, r['path']).is_file())
+ found = listing(root) if root.is_dir() else {}
+ present = {row['path']: found[row['path']] for row in value['files'] if row['path'] in found}
+ missing = sum(r['bytes'] for r in value['files'] if r['path'] not in present)
parent = root
while not parent.exists():
parent = parent.parent
if shutil.disk_usage(parent).free < missing + 64 * 2**20:
raise ValueError('磁盘空间不足,导入此包还需约 %.1f GiB。' % (missing / 2**30))
- done = 0
+ known, verified = stamps(root), {}
+ for row in value['files']:
+ current = present.get(row['path'])
+ if current is None:
+ continue
+ if current[0] != row['bytes'] or not unchanged(root / row['path'], row, known, current):
+ raise ValueError('已有导入文件已改变,未覆盖:' + str(root / row['path']))
+ verified[row['path']] = current
+ marker = root / value['marker']
+ if not missing and marker.is_file() and marker.read_bytes() == value['raw']:
+ # Imported before and complete: nothing to read again.
+ save(root / VERIFIED, verified)
+ progress(done=total, total=total, detail=path.name)
+ return dict(root=str(root), kind=value['kind'], variant=value['variant'], name=path.name)
+ done = sum(r['bytes'] for r in value['files'] if r['path'] in verified)
root.mkdir(parents=True, exist_ok=True)
+ progress(done=done, total=total, detail=path.name)
with zipfile.ZipFile(path) as archive:
for row in value['files']:
if cancelled():
raise InterruptedError('已暂停导入,ZIP 与已校验文件保留。')
- target = inside(root, row['path'])
- if target.is_file():
- if target.stat().st_size != row['bytes'] or hash_file(target) != row['sha256']:
- raise ValueError('已有导入文件已改变,未覆盖:' + str(target))
- done += row['bytes']
- progress(done=done, total=total, detail=path.name)
+ if row['path'] in verified:
continue
+ # The walk above refused links anywhere in this folder; rows passed
+ # the lexical path rules in inspect_archive.
+ target = root / row['path']
target.parent.mkdir(parents=True, exist_ok=True)
partial = target.with_name(target.name + '.importing')
- digest = hashlib.sha256()
- with archive.open(PREFIX + row['path']) as source, partial.open('wb') as output:
- while True:
- if cancelled():
- raise InterruptedError('已暂停导入,ZIP 与已校验文件保留。')
- data = source.read(4 * 2**20)
- if not data:
- break
- output.write(data); digest.update(data); done += len(data)
- progress(done=done, total=total, detail=path.name)
- if partial.stat().st_size != row['bytes'] or digest.hexdigest() != row['sha256']:
+ try:
+ # Reading a member to its end checks its CRC-32 (BadZipFile otherwise).
+ with archive.open(PREFIX + row['path']) as source, partial.open('wb') as output:
+ while True:
+ if cancelled():
+ raise InterruptedError('已暂停导入,ZIP 与已校验文件保留。')
+ data = source.read(4 * 2**20)
+ if not data:
+ break
+ output.write(data); done += len(data)
+ progress(done=done, total=total, detail=path.name)
+ except zipfile.BadZipFile as error:
+ raise ValueError('离线包已损坏,请重新下载:' + path.name) from error
+ if partial.stat().st_size != row['bytes']:
raise ValueError('离线包校验失败,已保留文件:' + path.name)
partial.replace(target)
- inside(root, value['marker']).write_bytes(value['raw'])
+ verified[row['path']] = stamp(target)
+ save(root / VERIFIED, verified)
+ marker.write_bytes(value['raw'])
return dict(root=str(root), kind=value['kind'], variant=value['variant'], name=path.name)
-def assemble(runtime, models, source):
+def assemble(runtime, models, source, progress=lambda **kw: None):
"""Bind imported components to the current EXE's engine, without running it."""
from .desktop_runtime import materialize_source
check_runtime_platform()
@@ -225,39 +287,49 @@ def assemble(runtime, models, source):
if any(not r['path'].startswith('models/') for r in model['files']):
raise ValueError('Invalid model inventory')
from .network import hash_file
+ known = {root: stamps(root) for root in models}
+ # One directory walk per folder: no file is opened to be checked.
+ walks, here = {root: listing(root) for root in models}, listing(runtime)
+ total, done = sum(r['bytes'] for r in model['files']), 0
+ progress(done=0, total=total, detail='检查配套包')
+
+ def source_for(row):
+ for root in models:
+ current = walks[root].get(row['path'])
+ if current and unchanged(Path(root) / row['path'], row, known[root], current):
+ return Path(root) / row['path']
+ return None
+
+ def place(src, target):
+ target.parent.mkdir(parents=True, exist_ok=True)
+ try:
+ os.link(src, target)
+ except OSError:
+ shutil.copyfile(src, target)
+
for row in model['files']:
- target = inside(runtime, row['path'])
- sources = [inside(Path(p), row['path']) for p in models]
- src = next((p for p in sources if p.is_file() and p.stat().st_size == row['bytes']
- and hash_file(p) == row['sha256']), None)
+ target, src = runtime / row['path'], source_for(row)
if src is None:
raise ValueError('还需导入「%s」包,缺少文件:%s' % (pack_name(row['path']), row['path']))
- if target.exists():
- if hash_file(target) != row['sha256']:
+ if row['path'] in here:
+ if not os.path.samefile(target, src) and hash_file(target) != row['sha256']:
raise ValueError('离线安装已有不同模型,未覆盖:' + str(target))
else:
- target.parent.mkdir(parents=True, exist_ok=True)
- try:
- os.link(src, target)
- except OSError:
- shutil.copyfile(src, target)
+ place(src, target)
+ done += row['bytes']
+ progress(done=done, total=total, detail='检查配套包')
listed = {row['path'] for row in model['files']}
for root in models:
package = Path(root) / 'offline-package.json'
if not package.is_file():
continue
for row in json.loads(package.read_text(encoding='utf-8')).get('files', []):
- if row['path'] in listed or not row['path'].startswith(OPTIONAL_PREFIX):
+ if row['path'] in listed or not row['path'].startswith(OPTIONAL_PREFIX) or row['path'] in here:
continue
- src, target = inside(Path(root), row['path']), inside(runtime, row['path'])
- if (target.exists() or not src.is_file() or src.stat().st_size != row['bytes']
- or hash_file(src) != row['sha256']):
+ current = walks[root].get(row['path'])
+ if not current or not unchanged(Path(root) / row['path'], row, known[root], current):
continue # The engine fetches a table it cannot find when a request needs it.
- target.parent.mkdir(parents=True, exist_ok=True)
- try:
- os.link(src, target)
- except OSError:
- shutil.copyfile(src, target)
+ place(Path(root) / row['path'], runtime / row['path'])
listed.add(row['path'])
code = materialize_source(source, runtime / 'engine/launcher/source')
variant = model['variant']
@@ -268,6 +340,14 @@ def assemble(runtime, models, source):
files=config['files'] + model['files'])
inventory(value['files'])
save(runtime / 'portable.json', value)
+ # Every model file was verified above and the environment's files at import;
+ # the first start takes that instead of reading tens of GiB once more.
+ imported, final = stamps(runtime), listing(runtime)
+ trusted = {row['path'] for row in model['files']}
+ trusted |= {row['path'] for row in config['files'] if imported.get(row['path']) == final.get(row['path'])}
+ save(runtime / 'engine' / 'portable-verified.json',
+ {row['path']: dict(bytes=final[row['path']][0], mtime_ns=final[row['path']][1], sha256=row['sha256'])
+ for row in value['files'] if row['path'] in trusted and row['path'] in final})
return runtime
@@ -286,8 +366,10 @@ class Importer:
return
self.state = dict(status='preparing', done=0, total=None, detail='检查配套包', packages=[])
def work():
+ def update(**row):
+ self.state = dict(self.state, **row)
try:
- root = assemble(runtime, models, source)
+ root = assemble(runtime, models, source, update)
self.state = dict(self.state, status='prepared', ready_root=str(root))
except RuntimeMismatch as error:
# An earlier import no longer fits after an update: the models stay.
@@ -307,7 +389,8 @@ class Importer:
def work():
# Take every ZIP that fits; one that does not never holds back the others.
packages, skipped, failed = [], False, []
- for path in paths:
+ for index, path in enumerate(paths):
+ update(index=index + 1, count=len(paths))
try:
packages.append(import_archive(path, destination, update, self.cancelled.is_set))
except RuntimeMismatch:
diff --git a/freevideo_engine/portable.py b/freevideo_engine/portable.py
index 24ebd40..7a1cea3 100644
--- a/freevideo_engine/portable.py
+++ b/freevideo_engine/portable.py
@@ -26,10 +26,19 @@ def child_lease(descriptor):
os.environ[LOCK_ENV] = previous
-def inside(root, name):
+def bundle_name(name):
+ """The lexical rules for a file inside a bundle, without touching the disk.
+
+ Inventories list ~58,000 files; resolving each (inside) opens it on Windows.
+ """
if (not isinstance(name, str) or not name or '\\' in name or ':' in name
or PurePosixPath(name).is_absolute() or any(p in ('', '.', '..') for p in name.split('/'))):
raise ValueError('Invalid bundle path: ' + str(name))
+ return name
+
+
+def inside(root, name):
+ bundle_name(name)
root = Path(root).resolve()
path = root / name
if any(p.is_symlink() or (hasattr(p, 'is_junction') and p.is_junction())
@@ -40,13 +49,41 @@ def inside(root, name):
return path
+def listing(root):
+ """Size and modification time of every file under root, from one directory walk.
+
+ Directory entries carry both, so on Windows no file is opened; opening
+ each of ~58,000 files (for stat or its NTFS change time) took minutes under
+ real-time scanning. Links are refused.
+ """
+ root = os.path.join(str(root), '')
+ found, pending = {}, [root]
+ while pending:
+ current = pending.pop()
+ try:
+ entries = os.scandir(current)
+ except FileNotFoundError:
+ continue
+ with entries:
+ for entry in entries:
+ if entry.is_symlink() or entry.is_junction():
+ raise ValueError('Bundle files must not redirect outside their directory: ' + entry.path)
+ if entry.is_dir(follow_symlinks=False):
+ pending.append(entry.path)
+ elif entry.is_file(follow_symlinks=False):
+ info = entry.stat(follow_symlinks=False)
+ found[entry.path[len(root):].replace(os.sep, '/')] = [info.st_size, info.st_mtime_ns]
+ return found
+
+
def manifest(root):
value = json.loads((Path(root) / 'portable.json').read_text(encoding='utf-8'))
if value.get('schema_version') != 1 or value.get('variant') not in ('rowwise', 'per_tensor', 'int8_convrot'):
raise ValueError('Invalid FreeVideo portable bundle')
seen = set()
for row in value['files']:
- inside(root, row['path'])
+ # Links anywhere in the bundle are refused by the directory walk in verify.
+ bundle_name(row['path'])
if row['path'].casefold() in seen or type(row['bytes']) is not int or row['bytes'] < 0:
raise ValueError('Invalid or duplicate bundle file')
seen.add(row['path'].casefold())
@@ -58,8 +95,7 @@ def manifest(root):
def verify(root, value, progress=lambda **kw: None):
- """Stream bytes once; only settled file identities can reuse verification."""
- from .storage import fingerprint
+ """Hash a file only when it is not as last verified (size, modification time)."""
from .network import hash_file
root = Path(root)
ledger = root / 'engine' / 'portable-verified.json'
@@ -67,30 +103,34 @@ def verify(root, value, progress=lambda **kw: None):
prior = json.loads(ledger.read_text(encoding='utf-8'))
except (OSError, ValueError):
prior = {}
+ present = listing(root)
stamps = {}
total = sum(r['bytes'] for r in value['files'])
done, started, last_event = 0, time.monotonic(), [0.]
+ def report(detail, current=0):
+ elapsed = time.monotonic()-started
+ if elapsed-last_event[0] >= .2:
+ last_event[0] = elapsed
+ progress(stage='verify', done=done+current, total=total, detail=detail,
+ elapsed_seconds=elapsed, bytes_per_second=(done+current)/max(.001, elapsed))
for row in value['files']:
- path = inside(root, row['path'])
- if not path.is_file() or path.stat().st_size != row['bytes']:
+ found = present.get(row['path'])
+ if found is None or found[0] != row['bytes']:
raise ValueError('Missing or incomplete bundled file: ' + row['path'])
- stamp = dict(fingerprint(path), sha256=row['sha256'])
- settled = ('change_time_ns' not in stamp or
- (stamp['change_time_ns'] is not None and max(stamp['mtime_ns'], stamp['change_time_ns']) < time.time_ns()-10**9))
- if not settled or prior.get(row['path']) != stamp:
- def update(current, length):
- elapsed = time.monotonic()-started
- if elapsed-last_event[0] < .2:
- return
- last_event[0] = elapsed
- progress(stage='verify', done=done+current, total=total, detail=row['path'],
- elapsed_seconds=elapsed, bytes_per_second=(done+current)/max(.001, elapsed))
- if hash_file(path, discard_cache=True, progress=update) != row['sha256']:
+ stamp = dict(bytes=found[0], mtime_ns=found[1], sha256=row['sha256'])
+ entry = prior.get(row['path'])
+ # Earlier ledgers kept more fields; size, time and digest decide.
+ known = isinstance(entry, dict) and all(entry.get(k) == v for k, v in stamp.items())
+ if not known or found[1] >= time.time_ns()-10**9:
+ path = inside(root, row['path'])
+ if hash_file(path, discard_cache=True, progress=lambda current, length: report(row['path'], current)) != row['sha256']:
raise ValueError('Bundled file failed verification: ' + row['path'])
- if stamp != dict(fingerprint(path), sha256=row['sha256']):
+ info = path.stat()
+ if [info.st_size, info.st_mtime_ns] != found:
raise ValueError('Bundled file changed during verification: ' + row['path'])
stamps[row['path']] = stamp
done += row['bytes']
+ report(row['path'])
save(ledger, stamps)
progress(stage='verify', done=total, total=total, detail='All bundled files verified')
diff --git a/freevideo_engine/release_notes.json b/freevideo_engine/release_notes.json
index 8029315..2240169 100644
--- a/freevideo_engine/release_notes.json
+++ b/freevideo_engine/release_notes.json
@@ -1,26 +1,20 @@
{
"schema": 1,
- "product_version": "0.3.10",
+ "product_version": "0.3.11",
"en": {
- "summary": "Long videos on Windows cards with 6–16 GB run faster, Environment ZIPs from earlier versions import again, and creations made with prompt enhancement can be deleted.",
+ "summary": "Importing offline packages is faster and shows its progress throughout.",
"changes": [
- "On Windows, int8 attention needs about a quarter less GPU memory. More long or large videos now keep attention results on the graphics card instead of in system memory, which makes their steps faster. The output is unchanged.",
- "When little system memory is free, weights are now read ahead while the previous ones compute, from 1 GB of spare memory instead of 3 GB. Steps that read weights from disk get faster.",
- "An offline Environment ZIP from an earlier version is accepted again. It now only has to contain the environment this version needs, whichever version it came with.",
- "When several offline ZIPs are imported together, every one that fits is imported. One that cannot be used no longer stops the others; an environment that does not fit is installed automatically, and the imported models are still used.",
- "Creations made with prompt enhancement can be deleted from Your creations again.",
- "Each downloaded diagnostic report is named by when it was saved (FreeVideo-report-date-time.json), so downloading several no longer produces \"video.debug (1).json\", \"(2)\" and so on."
+ "Offline packages are checked once, while they are unpacked. Importing a package again, Checking packages before installing and the first start no longer read every file again.",
+ "The Working card shows which package is being imported and how far it is; Checking packages and Verify local files show their progress as well.",
+ "After the Environment package is imported, the model step no longer asks for it again."
]
},
"zh": {
- "summary": "Windows 上 6–16 GB 显卡生成长视频更快;早先版本的运行环境包重新可以导入;用过提示词增强的作品可以正常删除。",
+ "summary": "离线包导入更快,全程都能看到进度。",
"changes": [
- "Windows 上 int8 注意力占用的显存减少约四分之一,更多长视频或大尺寸视频的注意力结果可以留在显卡上,不再放到内存里,每步更快。生成结果不变。",
- "内存余量较少时,现在只要有 1 GB 空闲(原来需要 3 GB)就会在计算的同时预读下一批权重,需要从硬盘读权重的步骤会更快。",
- "早先版本附带的离线运行环境包重新可以导入。现在只检查包里的运行环境是否符合当前版本,不再看它来自哪个版本。",
- "同时导入多个离线包时,能用的都会导入,某个包用不了也不会再挡住其他包;运行环境不符合时会改为自动安装,已导入的模型照常使用。",
- "用过提示词增强的作品,现在可以在「你的作品」里正常删除。",
- "下载的诊断报告按保存时间命名(FreeVideo-report-日期-时间.json),多次下载不会再出现「video.debug (1).json」「(2)」这样的文件名。"
+ "离线包只在解压时校验一次;再次导入同一个包、安装前「检查配套包」和首次启动都不再把所有文件重读一遍。",
+ "左侧「正在处理」卡片会显示正在导入第几个包和进度;「检查配套包」和「校验本地文件」也有进度条。",
+ "导入运行环境包后,模型这一步不再提示还需要运行环境包。"
]
}
}