Skip to content

Make videos on the computer you already own. FreeVideo runs MiniMax H3 in as little as 8 GB of VRAM and 16 GB of RAM, and adapts its acceleration path to your hardware.

Recover small-GPU out-of-memory failures and return the encoder's GPU memory (#26)

Bowen Xue7cd07b5485e5
diff --git a/docs/execution-planning.md b/docs/execution-planning.md
index d496730..1df235e 100644
--- a/docs/execution-planning.md
+++ b/docs/execution-planning.md
@@ -34,7 +34,7 @@ VRAM budget: free VRAM minus a reserve of 2.5% of free VRAM, clamped to 0.5–1
- Budgets are re-measured before every request.
- Completed requests store their timings and memory peaks in `resource-history.sqlite3`. The history drives the forecast shown before each request and `./freevideo predict`, and replaces the placement when an alternative is predicted to be at least 2% faster.
- If the minimum working set does not fit, the request is rejected before the transformer loads, and the shortfall is reported.
-- An out-of-memory failure is retried in a fresh process with a lighter placement, up to two times (`--resource-retries`).
+- An out-of-memory failure is retried in a fresh process, up to two times (`--resource-retries`). The retry first lightens the placement. When nothing is left to move, it shrinks the compute partitions: head group, feed-forward and projection chunks, then attention outputs staged in host memory. This can change the floating-point reduction order. A failure in the second pass of two-pass sampling reuses the completed first pass, and the next identical request on the same machine starts with the partitions that completed.
- `./freevideo optimize` searches compute settings for the machine and applies them only after a complete validation run.
## Inspection
diff --git a/docs/zh-CN/execution-planning.md b/docs/zh-CN/execution-planning.md
index bb9e65b..35cae26 100644
--- a/docs/zh-CN/execution-planning.md
+++ b/docs/zh-CN/execution-planning.md
@@ -34,7 +34,7 @@ FreeVideo Adaptive Execution Planner(自适应执行规划器)根据设备
- 每个请求前重新测量预算。
- 完成的请求把耗时和内存峰值写入 `resource-history.sqlite3`,用于请求前的预测和 `./freevideo predict`;预计另一种放置方案快 2% 以上时,替换当前方案。
- 最小工作集放不下时,请求在 Transformer 加载前被拒绝,并报告缺口。
-- 内存不足的失败会在新进程中以更轻的放置方案重试,最多两次(`--resource-retries`)。
+- 内存不足的失败会在新进程中重试,最多两次(`--resource-retries`)。重试先减轻放置方案;无可再减时缩小计算分块:头分组、前馈与投影分块,再把注意力输出暂存到主机内存,浮点累加顺序可能因此改变。二次采样的第二轮失败时复用已完成的第一轮,同一台机器上的相同请求下次直接使用已成功的分块。
- `./freevideo optimize` 为本机搜索计算设置,只有完整验证运行通过后才启用。
## 查看执行计划
diff --git a/freevideo_engine/adaptive.py b/freevideo_engine/adaptive.py
index f73e2d0..d4f8b88 100644
--- a/freevideo_engine/adaptive.py
+++ b/freevideo_engine/adaptive.py
@@ -285,13 +285,109 @@ def nonlocal_exhausted(failure):
and not gpu.get('allocator_ooms'))
+# Compute partitions a sampling OOM may shrink once placement is exhausted.
+# Smaller slices change floating-point reduction order, never the requested
+# video: geometry, frames, steps, seed, precision and attention backend stay.
+# A 6 GiB RTX 3060 Laptop (2026.10.4.15209) completed the 288x480 first pass of
+# a 544x960 two-pass request, then the 544x960 second pass ran out of memory
+# in the VDN linear branch with zero resident blocks; with no smaller
+# placement left the whole 13-minute request failed. Those branch states scale
+# with frames x heads x head_dim^2, not with tokens, so a smaller canvas does
+# not shrink them, while a smaller head group does.
+COMPUTE_RECOVERY = (
+ # Keep attention outputs on the GPU first: host outputs cost 19-36% of
+ # sampling time in the small band. FF and projection slices take the
+ # small-card values.
+ dict(head_chunk=2, ff_chunk=512, projection_chunk=512, window_batch=1, head_parallelism=1),
+ # The smallest bounded path: one head at a time, attention outputs in
+ # grouped host buffers and the residual stream staged in host memory.
+ dict(head_chunk=1, ff_chunk=512, projection_chunk=512, window_batch=1, head_parallelism=1,
+ attention_cpu_outputs=True, grouped_attention_outputs=True, residual_offload=True),
+)
+COMPUTE_RECOVERY_KEYS = ('head_chunk', 'ff_chunk', 'projection_chunk', 'window_batch', 'head_parallelism',
+ 'attention_cpu_outputs', 'grouped_attention_outputs', 'residual_offload')
+# Absent options run with these values; a chunk of 0 is unchunked, the widest.
+COMPUTE_DEFAULTS = dict(window_batch=1, head_parallelism=1)
+
+
+def compute_recovery(engine):
+ """The next smaller compute partition for the bounded inference path, or None.
+
+ Integers only shrink and flags only switch on, so recovery never widens a
+ partition that already failed.
+ """
+ if engine.get('inference_kernels') is not True or type(engine.get('head_chunk')) is not int or engine['head_chunk'] <= 0:
+ return None
+ for rung in COMPUTE_RECOVERY:
+ updates = {}
+ for name, target in rung.items():
+ current = engine.get(name, COMPUTE_DEFAULTS.get(name))
+ if type(target) is bool:
+ if current is not True:
+ updates[name] = True
+ elif type(current) is not int or current <= 0 or current > target:
+ updates[name] = target
+ if updates:
+ return updates
+ return None
+
+
+def recovered_compute(profile, history, identity, canvas):
+ """Start with the compute partitions this machine needed for the same request.
+
+ Only a verified complete request counts whose attempt was a
+ compute-partition recovery, with the identical software identity and
+ geometry, so a request that failed once does not fail first every time.
+ The partitions only shrink, and only while the budget is not more than
+ 0.5 GB above the recovered one; every other plan stays exactly as chosen.
+ """
+ decision = dict(applied=False)
+ engine = profile.get('engine') or {}
+ if history is None or identity is None or canvas is None or compute_recovery(engine) is None:
+ return profile, decision
+ try:
+ rows = history.observations(identity=identity, geometry=attempt_geometry(canvas, engine),
+ limit=16, include_details=True)
+ except Exception:
+ return profile, decision # Unreadable history is the same as none.
+ for row in reversed(rows):
+ evidence = ((row.get('details') or {}).get('decision_evidence') or {})
+ config = row.get('config') or {}
+ observed = config.get('engine') or {}
+ budget = config.get('gpu_budget_gb')
+ if (evidence.get('numerical_class') != 'compute-partition' or type(budget) not in (int, float)
+ or profile.get('gpu_budget_gb', 0) > budget + .5):
+ continue
+ updates = {}
+ for name in COMPUTE_RECOVERY_KEYS:
+ value, current = observed.get(name), engine.get(name, COMPUTE_DEFAULTS.get(name))
+ if type(value) is bool:
+ if value and current is not True:
+ updates[name] = True
+ elif type(value) is int and value > 0 and (type(current) is not int or current <= 0 or value < current):
+ updates[name] = value
+ if not updates:
+ return profile, decision
+ value = copy.deepcopy(profile)
+ value['engine'].update(updates)
+ if isinstance(value.get('policy'), dict):
+ value['policy']['engine'] = dict(value['engine'])
+ return value, dict(applied=True, source_attempt=row.get('id'), changes=updates,
+ numerical_class='compute-partition',
+ reason='A complete request with this geometry needed these partitions on this machine.')
+ return profile, decision
+
+
def next_placement(profile, failure, phase, canvas=None, *, previous_ram_retries=0, previous_gpu_retries=0,
sampling_complete=False):
- """A bounded neutral fallback; never change request, backend or chunking.
+ """Bounded recovery after a resource failure; never change the request.
Free activation space in one useful step instead of spending many full
- requests lowering residency two blocks at a time. This is recovery only;
- success does not declare the fallback faster than the initial candidate.
+ requests lowering residency two blocks at a time. Placement comes first.
+ When a sampling OOM leaves no smaller placement, shrink compute partitions
+ (``COMPUTE_RECOVERY``): a slightly different rounding is better than no
+ video. This is recovery only; success does not declare the fallback faster
+ than the initial candidate.
"""
value = copy.deepcopy(profile)
engine, decoder = value['engine'], value['decoder']
@@ -310,6 +406,7 @@ def next_placement(profile, failure, phase, canvas=None, *, previous_ram_retries
changes[section+'.'+name] = {'from': options.get(name), 'to': new}
options[name] = new
host_pins = failure['kind'] == 'gpu_oom' and nonlocal_exhausted(failure)
+ compute = None
if failure['kind'] == 'gpu_oom':
if host_pins and phase in ('load', 'sample') and (engine.get('pin_host_gb', 0.) > 0 or engine.get('pin_host_weights')):
# Page-locked memory ran out, not VRAM: return pinned weights and
@@ -367,6 +464,10 @@ def next_placement(profile, failure, phase, canvas=None, *, previous_ram_retries
and engine.get('attention_cpu_outputs') and engine.get('grouped_attention_outputs')
and engine.get('window_batch', 1) == 1):
change('engine', 'residual_offload', True)
+ if not changes and phase == 'sample':
+ compute = compute_recovery(engine)
+ for name, new in (compute or {}).items():
+ change('engine', name, new)
# Offloading more GPU weights must not turn the next attempt
# into a whole-model RAM allocation on a 16 GiB machine.
from .system import weight_cache_headroom, residual_host_headroom
@@ -418,11 +519,15 @@ def next_placement(profile, failure, phase, canvas=None, *, previous_ram_retries
changes={}, numerical_class='placement-only', measurement=failure,
ram_recovery=ram_recovery)
if not changes:
- return None, {'reason': 'No further numerical-neutral placement recovery is available; '
- 'chunk changes require local complete-request equivalence.', 'changes': {}}
+ return None, {'reason': 'No further placement or compute-partition recovery is available.', 'changes': {}}
if 'policy' in value:
value['policy'].update(engine=dict(engine), decoder=dict(decoder))
cause = 'page-locked memory exhausted the WDDM non-local budget' if host_pins else failure['kind']
+ if compute:
+ return value, dict(reason='Fresh worker after %s with no smaller placement left; use smaller compute '
+ 'partitions. Geometry, steps, seed, precision and attention are preserved; '
+ 'floating-point reduction order may differ.' % cause, changes=changes,
+ numerical_class='compute-partition', measurement=failure, ram_recovery=ram_recovery)
return value, dict(reason='Fresh worker after %s; preserve geometry, steps, seed, precision, '
'attention and chunk arithmetic.' % cause, changes=changes,
numerical_class='same-shape-ff-recompute' if recompute_recovery else 'placement-only',
diff --git a/freevideo_engine/decode_resume.py b/freevideo_engine/decode_resume.py
index 392faae..35ed891 100644
--- a/freevideo_engine/decode_resume.py
+++ b/freevideo_engine/decode_resume.py
@@ -14,6 +14,11 @@ ENGINE_PLACEMENT = frozenset(('resident_blocks', 'pin_host_gb', 'pin_host_weight
'preload_host', 'stream_weights', 'prefetch', 'offload_refiner', 'adaln_disk_cache'))
DECODER_PLACEMENT = frozenset(('offload', 'prefetch', 'preload', 'pin_weights',
'stream_output', 'stream_weights', 'resident_blocks'))
+# Compute partitions change floating-point reduction order, not what a pass
+# computes. An OOM retry shrinks them for the second pass, so a retained first
+# pass stays valid across them.
+COMPUTE_PARTITION = frozenset(('head_chunk', 'window_batch', 'ff_chunk', 'projection_chunk', 'head_parallelism',
+ 'attention_cpu_outputs', 'grouped_attention_outputs', 'fp8_ff_recompute', 'residual_offload', 'query_chunk'))
SAMPLE_FIELDS = ('config', 'sample_seconds', 'step_seconds', 'finite_latents', 'geometry',
'conditioning_info', 'conditioning_shape', 'offload', 'residual_offload_steps',
'attention_backend_calls', 'head_execution', 'text_refinement', 'sampling_memory',
@@ -157,3 +162,89 @@ def retain_latents(source, artifacts):
except OSError:
with source.open('rb') as incoming, destination.open('xb') as outgoing:
shutil.copyfileobj(incoming, outgoing, length=1024 * 1024)
+
+
+def refine_provenance(request):
+ """Identity of a completed first pass: the sampling identity without compute partitions or decoding."""
+ result = sampling_provenance(request)
+ if result.get('version') != 2:
+ raise ValueError('A retained first pass requires a two-pass sampling plan')
+ result['engine_math'] = {key: value for key, value in result['engine_math'].items() if key not in COMPUTE_PARTITION}
+ result.pop('decoder_math', None)
+ return dict(result, scope='first-pass')
+
+
+def save_refine_input(value, artifacts, torch):
+ """Retain the upscaled, cropped first pass atomically before the second pass."""
+ artifacts = Path(artifacts)
+ artifacts.mkdir(parents=True, exist_ok=True)
+ with tempfile.NamedTemporaryFile(prefix='refine-input.', suffix='.tmp', dir=artifacts, delete=False) as stream:
+ temporary = Path(stream.name)
+ try:
+ torch.save(value, temporary)
+ temporary.replace(artifacts / 'refine-input.pt')
+ finally:
+ temporary.unlink(missing_ok=True)
+
+
+def refine_checkpoint_complete(metrics):
+ """A failed attempt retained its full first pass, latent upscale and crop."""
+ if (not isinstance(metrics, dict) or metrics.get('success') is not False
+ or metrics.get('refine_checkpoint_complete') is not True
+ or not isinstance(metrics.get('refine_provenance'), dict)):
+ return False
+ plan, passes = metrics.get('sampling_plan'), metrics.get('sampling_passes')
+ if not isinstance(plan, dict) or not plan.get('enabled') or not isinstance(passes, list) or not passes:
+ return False
+ first, lifted = passes[0], metrics.get('latent_upscale')
+ if not isinstance(first, dict) or not isinstance(lifted, dict):
+ return False
+ steps = first.get('step_seconds')
+ seconds = [first.get('sample_seconds'), lifted.get('stage_seconds')]
+ return (isinstance(steps, list) and len(steps) == plan.get('base_steps')
+ and all(type(value) in (int, float) and math.isfinite(value) and value > 0 for value in steps)
+ and all(type(value) in (int, float) and math.isfinite(value) and value >= 0 for value in seconds))
+
+
+def load_refine_input(request, torch):
+ """Load a verified retained first pass on the CPU; the caller moves it to the GPU."""
+ resume = request['resume_refine']
+ if not isinstance(resume, dict) or any(not isinstance(resume.get(key), str) or not resume[key]
+ for key in ('input', 'metrics', 'request')):
+ raise ValueError('Invalid first-pass retry source')
+ paths = {key: Path(resume[key]).resolve() for key in ('input', 'metrics', 'request')}
+ if len({path.parent for path in paths.values()}) != 1 or len(set(paths.values())) != 3:
+ raise ValueError('First-pass retry artifacts must belong to one retained attempt')
+ prior = json.loads(paths['request'].read_text(encoding='utf-8'))
+ previous = json.loads(paths['metrics'].read_text(encoding='utf-8'))
+ if not isinstance(prior, dict) or not refine_checkpoint_complete(previous):
+ raise ValueError('First-pass retry requires a complete retained first pass')
+ provenance = refine_provenance(request)
+ if provenance != refine_provenance(prior) or provenance != previous['refine_provenance']:
+ raise ValueError('Retained first pass does not match this request and its current inputs')
+ canvas = request['geometry']
+ value = torch.load(str(paths['input']), map_location='cpu', weights_only=True)
+ if (not isinstance(value, dict) or type(value.get('seed')) is not int or value['seed'] != request['seed']
+ or value.get('geometry') != canvas or value.get('refine_provenance') != provenance):
+ raise ValueError('Retained first-pass provenance differs from its receipt')
+ width, height, frames, fps = (canvas[key] for key in ('width', 'height', 'frames', 'fps'))
+ if (any(type(n) is not int or n <= 0 for n in (width, height, frames, fps))
+ or width % 32 or height % 32 or frames % 17 != 5 or fps != 24):
+ raise ValueError('Invalid retained H3 first-pass geometry')
+ # The second pass refines at the requested canvas: same layouts as the
+ # final latents (video C=24, stride 16; stereo audio C=32 at 40/s).
+ shapes = {'video': (1, 24, (frames - 5) // 17 * 5 + 2, height // 16, width // 16),
+ 'audio': (2, 32, round(frames / fps * 40))}
+ for key, shape in shapes.items():
+ tensor = value.get(key)
+ if (not isinstance(tensor, torch.Tensor) or tensor.device.type != 'cpu'
+ or tuple(tensor.shape) != shape or not tensor.is_floating_point()
+ or not bool(torch.isfinite(tensor).all())):
+ raise ValueError('Retained first-pass %s latents are malformed or non-finite' % key)
+ first = copy.deepcopy(previous['sampling_passes'][0])
+ lifted = copy.deepcopy(previous['latent_upscale'])
+ receipt = dict(first_pass_reused=True, refine_provenance=provenance,
+ first_pass_source_attempt=previous.get('refine_source_attempt', prior.get('resource_attempt')),
+ first_pass_scope='Reused the retained first pass, latent upscale and crop; their times and '
+ 'peaks belong to the source attempt.')
+ return value['video'], value['audio'], first, lifted, receipt
diff --git a/freevideo_engine/encode_worker.py b/freevideo_engine/encode_worker.py
index b149576..b10aa5c 100644
--- a/freevideo_engine/encode_worker.py
+++ b/freevideo_engine/encode_worker.py
@@ -345,7 +345,8 @@ def _encode(args, request, resident, trace):
# Finish async forward work at the existing conditioning boundary, so
# it is not attributed to packing/saving on the host.
torch.cuda.synchronize()
- from .encoder_memory import snapshot
+ from .encoder_memory import release_cast_buffers, snapshot
+ loading['cast_buffers'] = release_cast_buffers(memory, torch)
phase('encoder_conditioning_pack', gpu=snapshot(torch, clip, memory))
finally:
del clip.load_model
diff --git a/freevideo_engine/encoder_diagnostics.py b/freevideo_engine/encoder_diagnostics.py
index 7fe4aaf..b781411 100644
--- a/freevideo_engine/encoder_diagnostics.py
+++ b/freevideo_engine/encoder_diagnostics.py
@@ -27,7 +27,7 @@ def cached_metrics(value):
value['cache_source_timings'] = timings
value.update({key: 0. for key in timings})
value['cache_source_metrics'] = {key: value.pop(key) for key in ('timing', 'runtime',
- 'load_stages', 'gpu', 'checkpoint', 'encoder_attempts', 'token_summary',
+ 'load_stages', 'gpu', 'checkpoint', 'encoder_attempts', 'token_summary', 'cast_buffers',
'resident_encoder_cache_hit', 'resident_encoder_gpu_ready_at_start') if key in value}
value.pop('phase', None)
return value
diff --git a/freevideo_engine/encoder_memory.py b/freevideo_engine/encoder_memory.py
index 3735536..afcda3f 100644
--- a/freevideo_engine/encoder_memory.py
+++ b/freevideo_engine/encoder_memory.py
@@ -127,6 +127,27 @@ def token_counts(tokens):
return dict(sequences=len(lengths), max_sequence_tokens=max(lengths, default=0), total_tokens=sum(lengths))
+def release_cast_buffers(manager, torch):
+ """Free the streaming buffers the native library keeps after an encode.
+
+ Without dynamic VRAM, which this worker disables, ComfyUI copies every
+ weight it did not load through one cached buffer per offload stream, sized
+ to the largest weight so far. Unloading models never frees them, and in
+ this mode ComfyUI itself never does: its executor resets them only for
+ dynamic VRAM. The resident worker runs the transformer next in the same
+ process. On a 6 GiB RTX 3060 Laptop, with no encoder weights loaded, they
+ held 0.80 GiB allocated and 1.68 GiB reserved, and evicting the encoder
+ released nothing. The next encode allocates them again.
+ """
+ reset = getattr(manager, 'reset_cast_buffers', None)
+ if not callable(reset):
+ return None
+ allocated, reserved = torch.cuda.memory_allocated(), torch.cuda.memory_reserved()
+ reset() # The native reset also empties the CUDA cache.
+ return dict(released_allocated_bytes=max(0, allocated - torch.cuda.memory_allocated()),
+ released_reserved_bytes=max(0, reserved - torch.cuda.memory_reserved()))
+
+
def encode_with_recovery(clip, tokens, manager, torch, phase, *, max_attempts=3):
from .adaptive import classify_failure
original_reserve = manager.EXTRA_RESERVED_VRAM
diff --git a/freevideo_engine/generate.py b/freevideo_engine/generate.py
index 2534ea4..5ec5d13 100644
--- a/freevideo_engine/generate.py
+++ b/freevideo_engine/generate.py
@@ -206,16 +206,38 @@ def automatic_profile(args, canvas, *, stage, evidence, descriptor=None, environ
type(commit) is int and commit >= 0 and
commit < mapped + 2 * 2**30)
encoder_idle = state and any(m.get('role') == 'encoder' for m in state.get('models', []))
+ # The plan counted the idle encoder's memory as reclaimable. If it
+ # still streams every transformer block from disk with nothing
+ # retained, actually reclaim it: a 16 GiB RTX 3060 Laptop kept the
+ # encoder's 4.5 GiB while each step reread 21.6 GB of weights with
+ # no room left for host read-ahead. Machines that retain weights
+ # keep the encoder for the next prompt.
+ from .policy import BLOCK_BYTES
+ engine = profile['engine']
+ before, after = getattr(raw, 'ram_available', None), getattr(hardware, 'ram_available', None)
+ credited = after - before if type(before) is int and type(after) is int else 0
+ retention_starved = (engine.get('stream_weights') is True and credited >= 2 * 2**30
+ and engine.get('pin_host_gb', 0.) * 1e9 < BLOCK_BYTES)
+ # The plan also counted the idle worker's GPU memory as free, but
+ # only a process exit is sure to return memory the model bank does
+ # not own. On a 6 GiB RTX 3060 Laptop the lowvram encoder's
+ # streaming buffers held 0.80 GiB allocated (1.68 GiB reserved),
+ # evicting the encoder released nothing, and the transformer,
+ # planned with zero resident blocks, ran out of memory in its second
+ # pass. The worker now frees those buffers after encoding; this
+ # covers anything else it still holds.
+ idle_gpu = state.get('reclaimable_gpu_bytes') if state else None
+ gpu_starved = (type(idle_gpu) is int and idle_gpu >= 256 * 2**20
+ and engine.get('resident_blocks', 0) == 0)
+ working_set_short = type(used) is int and used >= 0 and used + 2*2**30 > row['budget_bytes']['ram']
reclaim = (stage == 'after-encoding' and not attempt and encoder_idle and environment.get(ENV)
- and ((type(used) is int and used >= 0
- and used + 2*2**30 > row['budget_bytes']['ram'])
- or mapped_commit_pressure))
+ and (working_set_short or mapped_commit_pressure or retention_starved or gpu_starved))
if not reclaim:
return hardware, state, profile
- reason = ('Idle encoder working set leaves insufficient model-loading room'
- if type(used) is int and used >= 0
- and used + 2*2**30 > row['budget_bytes']['ram']
- else 'Idle encoder mapping leaves insufficient Windows Commit headroom')
+ reason = ('Idle encoder working set leaves insufficient model-loading room' if working_set_short
+ else 'Idle encoder mapping leaves insufficient Windows Commit headroom' if mapped_commit_pressure
+ else 'Idle encoder GPU memory leaves the transformer no spare VRAM' if gpu_starved
+ else 'Idle encoder memory leaves the transformer rereading every weight from disk')
row.update(status='reclaim-before-load', reason=reason,
mapped_bytes=mapped if type(mapped) is int else None,
commit_available_bytes=commit if type(commit) is int else None)
@@ -547,6 +569,11 @@ def _run(args):
text_tokens=tokens)
report['history_placement'] = decision
report['profile'] = profile
+ from .adaptive import recovered_compute
+ profile, report['recovered_compute'] = recovered_compute(profile, history, resource_identity, canvas)
+ report['profile'] = profile
+ if report['recovered_compute']['applied']:
+ print(json.dumps(dict(event='recovered_compute', **report['recovered_compute'])), flush=True)
from .compatibility import Store as CompatibilityStore, apply as apply_compatibility
compatibility_state = CompatibilityStore(history.path.parent).status(resource_identity, persist=True)
profile, report['compatibility'] = apply_compatibility(profile, compatibility_state)
@@ -593,7 +620,7 @@ def _run(args):
report['benchmark_limit'] = {'pytorch_allocator_bytes': request['allocator_limit_bytes'],
'scope': 'PyTorch CUDA allocator only; other CUDA allocations and whole-device usage are measured separately.'}
request_path = temporary / 'video.json'
- resume_decode = None
+ resume_decode = resume_refine = None
# Keep explicit dependency overlays while keeping the snapshotted engine first.
engine_env = dict(env, PYTHONPATH=os.pathsep.join((str(repo), str(vdn_root()),
env.get('PYTHONPATH', ''))))
@@ -627,6 +654,10 @@ def _run(args):
request['resume_decode'] = dict(resume_decode)
else:
request.pop('resume_decode', None)
+ if resume_decode is None and resume_refine is not None:
+ request['resume_refine'] = dict(resume_refine)
+ else:
+ request.pop('resume_refine', None)
save(request_path, request)
print(json.dumps({'event': 'decode_resume' if resume_decode else 'video_start', 'resource_attempt': attempt,
'engine': selected['engine'], 'decoder': selected['decoder']}), flush=True)
@@ -656,13 +687,13 @@ def _run(args):
raise
return json.loads(destination.with_suffix('.engine.json').read_text(encoding='utf-8')), telemetry
def archive(index, row):
- nonlocal resume_decode
+ nonlocal resume_decode, resume_refine
retained = artifacts / 'attempts' / ('%02d-%s' % (index + 1, row['id']))
retained.mkdir(parents=True, exist_ok=False)
paths = [destination] + [destination.with_suffix('.engine.' + ext)
for ext in ('json','log','memory.json','gpu.json','gpu.csv')]
paths.append(destination.with_suffix('.sampling-memory.json'))
- paths += [artifacts/name for name in ('latents.pt','rgb.npy','audio.npy','audio.wav','video.json')]
+ paths += [artifacts/name for name in ('latents.pt','refine-input.pt','rgb.npy','audio.npy','audio.wav','video.json')]
for path in paths:
if path.exists():
path.rename(retained / path.name)
@@ -688,11 +719,27 @@ def _run(args):
metrics=str(sample_metrics), request=str(retained / 'video.json'))
except (OSError, ValueError, TypeError):
pass # Older/incomplete artifacts retain full-request recovery.
+ # A second-pass failure keeps its completed first pass. Later
+ # retries reuse the first retained one; the worker validates
+ # its provenance and tensors again before loading it.
+ if resume_decode is None and resume_refine is None:
+ try:
+ sample_metrics = retained / destination.with_suffix('.engine.json').name
+ sampled = json.loads(sample_metrics.read_text(encoding='utf-8'))
+ from .decode_resume import refine_checkpoint_complete
+ source = retained / 'refine-input.pt'
+ if (refine_checkpoint_complete(sampled) and source.is_file() and source.stat().st_size > 0
+ and (retained / 'video.json').is_file()):
+ resume_refine = dict(input=str(source), metrics=str(sample_metrics),
+ request=str(retained / 'video.json'))
+ except (OSError, ValueError, TypeError):
+ pass
return str(retained)
def retry(index, row, selected, decision):
retry_state = dict(attempt=index + 2, max_attempts=getattr(args, 'resource_retries', 2) + 1,
failed_phase=row.get('phase'), kind=row['failure']['kind'],
- reuse_sampling=resume_decode is not None)
+ reuse_sampling=resume_decode is not None,
+ reuse_first_pass=resume_decode is None and resume_refine is not None)
row['recovery'] = dict(decision, next_profile=selected, **retry_state)
persist()
print(json.dumps(dict(event='resource_retry', retained=row['retained'], retry=retry_state)), flush=True)
diff --git a/freevideo_engine/offload.py b/freevideo_engine/offload.py
index e077bd1..d9e373b 100644
--- a/freevideo_engine/offload.py
+++ b/freevideo_engine/offload.py
@@ -167,9 +167,17 @@ class Slot:
compute_done: object = None
+# Live RAM that must stay free beside the one prepared block host read-ahead
+# holds. It used to borrow the 8 GiB weight-cache allowance, which no 16 GiB
+# machine reaches while sampling: a 16 GiB RTX 3060 Laptop reread all 21.6 GB
+# of weights every step (about 80 s) with reads and compute never overlapping.
+# The runtime check in _queue_host stops read-ahead again under live pressure.
+HOST_PREFETCH_HEADROOM = 3 * 2**30
+
+
def _host_prefetch_fits(extra_bytes):
"""Speculative staging must fit both live physical RAM and Windows commit."""
- from .system import system_memory, HOST_WEIGHT_HEADROOM
+ from .system import system_memory
try:
memory = system_memory()
available = memory.get('physical_available_bytes')
@@ -178,7 +186,7 @@ def _host_prefetch_fits(extra_bytes):
commit = memory.get('commit_available_bytes')
if commit is not None:
available = min(available, commit)
- return available >= HOST_WEIGHT_HEADROOM + extra_bytes
+ return available >= HOST_PREFETCH_HEADROOM + extra_bytes
except (OSError, ValueError, TypeError, KeyError):
return False
diff --git a/freevideo_engine/resident_models.py b/freevideo_engine/resident_models.py
index ac248f2..c384c36 100644
--- a/freevideo_engine/resident_models.py
+++ b/freevideo_engine/resident_models.py
@@ -103,6 +103,11 @@ def release_encoder(clip, manager):
return _native(device_to=None, unpatch_weights=unpatch_weights)
target.unpatch_model = discard
manager.unload_all_models()
+ # Unloading leaves the native streaming buffers allocated; see
+ # encoder_memory.release_cast_buffers. The caller empties the cache.
+ reset = getattr(manager, 'reset_cast_buffers', None)
+ if callable(reset):
+ reset()
finally:
for target, existed, original in reversed(saved):
if existed:
diff --git a/freevideo_engine/two_pass_metrics.py b/freevideo_engine/two_pass_metrics.py
index 4ae14f1..a9ff1eb 100644
--- a/freevideo_engine/two_pass_metrics.py
+++ b/freevideo_engine/two_pass_metrics.py
@@ -23,7 +23,7 @@ def combine(first, second, upscale, sampling_plan):
step_seconds=list(first['step_seconds']) + list(second['step_seconds']),
sample_seconds=first['sample_seconds'] + upscale['stage_seconds'] + second['sample_seconds'])
for key in ('torch_peak_allocated_bytes', 'torch_peak_reserved_bytes'):
- result[key] = max(first[key], second[key], upscale.get(key, 0))
+ result[key] = max(first.get(key, 0), second[key], upscale.get(key, 0))
for key in ('attention_backend_calls', 'fp8_kernel_calls'):
rows = [value[key] for value in (first, second) if value.get(key) is not None]
result[key] = ({name: sum(row.get(name, 0) for row in rows)
diff --git a/freevideo_engine/worker.py b/freevideo_engine/worker.py
index e9a9f68..cf47f2f 100644
--- a/freevideo_engine/worker.py
+++ b/freevideo_engine/worker.py
@@ -194,6 +194,8 @@ def generate(request, resident=None):
'geometry': canvas, 'sampling_provenance': provenance}, artifacts, torch)
metrics.update(sampling_provenance=provenance, sampling_source_attempt=request.get('resource_attempt'),
latent_save_seconds=time.perf_counter() - tick, sampling_checkpoint_complete=True)
+ # The final latents supersede this attempt's retained first pass.
+ (artifacts / 'refine-input.pt').unlink(missing_ok=True)
sample_finalize_phase('offload_release')
try:
sampling_plan = request.get('sampling_plan', {})
@@ -203,96 +205,124 @@ def generate(request, resident=None):
from .latent_upscale import upscale
if engine.steps != sampling_plan['base_steps']:
raise ValueError('Engine steps disagree with the requested sampling plan')
- pass_budget = (min(request['gpu_budget_bytes'],
- metrics['device_memory'].get('effective_allocator_limit_bytes') or request['gpu_budget_bytes'])
- if request.get('automatic_pass_cache', False) else None)
- first_options = request.get('first_pass_policy', {}).get('profile', {}).get('engine')
- metrics['first_pass_policy'] = request.get('first_pass_policy')
- latents, audio, first = engine.sample(request['conditioning'], request['seed'],
- **sampling_plan['first'], allow_smaller_canvas=True,
- pass_cache_budget_bytes=pass_budget, gpu_reserve_bytes=request.get('gpu_reserve_bytes', 0),
- compute_options=first_options,
- pass_resident_blocks=first_options.get('resident_blocks') if first_options else None,
- step_callback=memory_diagnostics.complete, progress_total=sampling_plan['total_steps'],
- budget_refresh=refresh_budget if live_budget is not None else None)
- metrics.update(sampling_passes=[first], sample_stage='latent_upscale')
- save(request['metrics'], metrics)
- print(json.dumps(dict(event='latent_upscale', completed_steps=sampling_plan['base_steps'],
- total=sampling_plan['total_steps'])), flush=True)
- tick = time.perf_counter()
- if live_budget is not None:
- refresh_budget('upscale')
- need = upscale_workspace(sampling_plan['upscale_target']) if sampling_plan['upscaler_sha256'] else 0
- if need and resident is not None and not resident.ram_fits(2 * 2**30):
- resident.make_room('latent_upscaler', need, ram_need=2 * 2**30)
- torch.cuda.empty_cache()
- def upscale_available():
- free, _ = torch.cuda.mem_get_info()
- budget = metrics['device_memory'].get('effective_allocator_limit_bytes') or request['gpu_budget_bytes']
- return min(free - request.get('gpu_reserve_bytes', 0), budget - torch.cuda.memory_allocated())
- available = upscale_available()
- # Prefer buffer reuse before discarding useful host weights.
- # The estimate selects this path, not permission to run.
- # Actual allocation remains bounded by the worker's CUDA
- # allocator limit. Keep the completed first-pass latents.
- memory_saving = available < need
- torch.cuda.reset_peak_memory_stats()
- lift_canvas = sampling_plan['upscale_target']
- retry_upscale = False
- upscale_failures = []
- metrics['latent_upscale'] = dict(estimated_workspace_bytes=need,
- available_workspace_bytes=max(0, available), attempted_below_estimate=memory_saving,
- buffer_reuse=memory_saving, allocation_retries=0, allocation_failures=upscale_failures)
- save(request['metrics'], metrics)
- try:
- kwargs = dict(memory_saving=True) if memory_saving else {}
- lifted_video, lifted = upscale(latents, request.get('upscaler_checkpoint'),
- lift_canvas['width'], lift_canvas['height'], **kwargs)
- except Exception as error:
- from .adaptive import classify_failure
- if classify_failure(error)['kind'] != 'gpu_oom' or (engine.closed and memory_saving):
- raise
- from .diagnostic_resources import error_details
- upscale_failures.append(error_details(error))
- retry_upscale = True
- if retry_upscale:
- # Leave the failed call's traceback before retrying so
- # its tensors no longer occupy the allocator. Do not
- # sample again or change the temporal normalization.
- if not engine.closed:
- if resident is not None:
- resident.drop('engine', 'Latent upscale allocation needs additional workspace')
- else:
- engine.close()
- if resident is not None:
+ if request.get('resume_refine'):
+ # A retry after a second-pass failure: this request's
+ # retained first pass, latent upscale and crop are
+ # reused instead of sampling them again.
+ from .decode_resume import load_refine_input
+ metrics.update(sample_stage='first_pass_load')
+ save(request['metrics'], metrics)
+ tick = time.perf_counter()
+ latents, audio, first, lifted, reused = load_refine_input(request, torch)
+ latents, audio = latents.to('cuda'), audio.to('cuda')
+ metrics.update(reused, sampling_passes=[first], latent_upscale=lifted,
+ first_pass_load_seconds=time.perf_counter() - tick)
+ print(json.dumps(dict(event='first_pass_reused', source_attempt=reused['first_pass_source_attempt'],
+ completed_steps=sampling_plan['base_steps'],
+ total=sampling_plan['total_steps'])), flush=True)
+ else:
+ pass_budget = (min(request['gpu_budget_bytes'],
+ metrics['device_memory'].get('effective_allocator_limit_bytes') or request['gpu_budget_bytes'])
+ if request.get('automatic_pass_cache', False) else None)
+ first_options = request.get('first_pass_policy', {}).get('profile', {}).get('engine')
+ metrics['first_pass_policy'] = request.get('first_pass_policy')
+ latents, audio, first = engine.sample(request['conditioning'], request['seed'],
+ **sampling_plan['first'], allow_smaller_canvas=True,
+ pass_cache_budget_bytes=pass_budget, gpu_reserve_bytes=request.get('gpu_reserve_bytes', 0),
+ compute_options=first_options,
+ pass_resident_blocks=first_options.get('resident_blocks') if first_options else None,
+ step_callback=memory_diagnostics.complete, progress_total=sampling_plan['total_steps'],
+ budget_refresh=refresh_budget if live_budget is not None else None)
+ metrics.update(sampling_passes=[first], sample_stage='latent_upscale')
+ save(request['metrics'], metrics)
+ print(json.dumps(dict(event='latent_upscale', completed_steps=sampling_plan['base_steps'],
+ total=sampling_plan['total_steps'])), flush=True)
+ tick = time.perf_counter()
+ if live_budget is not None:
+ refresh_budget('upscale')
+ need = upscale_workspace(sampling_plan['upscale_target']) if sampling_plan['upscaler_sha256'] else 0
+ if need and resident is not None and not resident.ram_fits(2 * 2**30):
resident.make_room('latent_upscaler', need, ram_need=2 * 2**30)
- import gc
- gc.collect()
torch.cuda.empty_cache()
- metrics['latent_upscale'].update(buffer_reuse=True, allocation_retries=1)
+ def upscale_available():
+ free, _ = torch.cuda.mem_get_info()
+ budget = metrics['device_memory'].get('effective_allocator_limit_bytes') or request['gpu_budget_bytes']
+ return min(free - request.get('gpu_reserve_bytes', 0), budget - torch.cuda.memory_allocated())
+ available = upscale_available()
+ # Prefer buffer reuse before discarding useful host weights.
+ # The estimate selects this path, not permission to run.
+ # Actual allocation remains bounded by the worker's CUDA
+ # allocator limit. Keep the completed first-pass latents.
+ memory_saving = available < need
+ torch.cuda.reset_peak_memory_stats()
+ lift_canvas = sampling_plan['upscale_target']
+ retry_upscale = False
+ upscale_failures = []
+ metrics['latent_upscale'] = dict(estimated_workspace_bytes=need,
+ available_workspace_bytes=max(0, available), attempted_below_estimate=memory_saving,
+ buffer_reuse=memory_saving, allocation_retries=0, allocation_failures=upscale_failures)
save(request['metrics'], metrics)
- lifted_video, lifted = upscale(latents, request.get('upscaler_checkpoint'),
- lift_canvas['width'], lift_canvas['height'], memory_saving=True)
- latents = lifted_video
- del lifted_video
- latents = crop_latents(latents, sampling_plan)
- lifted.update(stage_seconds=time.perf_counter() - tick,
- crop=dict(sampling_plan['crop']),
- estimated_workspace_bytes=need,
- available_workspace_bytes=max(0, available),
- attempted_below_estimate=memory_saving,
- allocation_retries=len(upscale_failures),
- allocation_failures=upscale_failures,
- torch_peak_allocated_bytes=torch.cuda.max_memory_allocated(),
- torch_peak_reserved_bytes=torch.cuda.max_memory_reserved(),
- transformer_reloaded=engine.closed)
- metrics['latent_upscale'] = lifted
- if engine.closed:
- tick = time.perf_counter()
- engine, _ = resident.engine(request, factory) if resident is not None else (factory(), False)
- reload_seconds = time.perf_counter() - tick
- metrics['load_seconds'] += reload_seconds
- metrics['load_breakdown']['two_pass_reload_seconds'] = reload_seconds
+ try:
+ kwargs = dict(memory_saving=True) if memory_saving else {}
+ lifted_video, lifted = upscale(latents, request.get('upscaler_checkpoint'),
+ lift_canvas['width'], lift_canvas['height'], **kwargs)
+ except Exception as error:
+ from .adaptive import classify_failure
+ if classify_failure(error)['kind'] != 'gpu_oom' or (engine.closed and memory_saving):
+ raise
+ from .diagnostic_resources import error_details
+ upscale_failures.append(error_details(error))
+ retry_upscale = True
+ if retry_upscale:
+ # Leave the failed call's traceback before retrying so
+ # its tensors no longer occupy the allocator. Do not
+ # sample again or change the temporal normalization.
+ if not engine.closed:
+ if resident is not None:
+ resident.drop('engine', 'Latent upscale allocation needs additional workspace')
+ else:
+ engine.close()
+ if resident is not None:
+ resident.make_room('latent_upscaler', need, ram_need=2 * 2**30)
+ import gc
+ gc.collect()
+ torch.cuda.empty_cache()
+ metrics['latent_upscale'].update(buffer_reuse=True, allocation_retries=1)
+ save(request['metrics'], metrics)
+ lifted_video, lifted = upscale(latents, request.get('upscaler_checkpoint'),
+ lift_canvas['width'], lift_canvas['height'], memory_saving=True)
+ latents = lifted_video
+ del lifted_video
+ latents = crop_latents(latents, sampling_plan)
+ lifted.update(stage_seconds=time.perf_counter() - tick,
+ crop=dict(sampling_plan['crop']),
+ estimated_workspace_bytes=need,
+ available_workspace_bytes=max(0, available),
+ attempted_below_estimate=memory_saving,
+ allocation_retries=len(upscale_failures),
+ allocation_failures=upscale_failures,
+ torch_peak_allocated_bytes=torch.cuda.max_memory_allocated(),
+ torch_peak_reserved_bytes=torch.cuda.max_memory_reserved(),
+ transformer_reloaded=engine.closed)
+ metrics['latent_upscale'] = lifted
+ if engine.closed:
+ tick = time.perf_counter()
+ engine, _ = resident.engine(request, factory) if resident is not None else (factory(), False)
+ reload_seconds = time.perf_counter() - tick
+ metrics['load_seconds'] += reload_seconds
+ metrics['load_breakdown']['two_pass_reload_seconds'] = reload_seconds
+ if artifacts:
+ # Retain the second pass's input so an out-of-memory
+ # retry starts there. A failed save only loses that.
+ try:
+ from .decode_resume import refine_provenance, save_refine_input
+ provenance = refine_provenance(request)
+ save_refine_input({'video': latents.cpu(), 'audio': audio.cpu(), 'seed': request['seed'],
+ 'geometry': canvas, 'refine_provenance': provenance}, artifacts, torch)
+ metrics.update(refine_checkpoint_complete=True, refine_provenance=provenance,
+ refine_source_attempt=request.get('resource_attempt'))
+ except (OSError, RuntimeError, ValueError) as error:
+ metrics['refine_checkpoint_error'] = str(error)
metrics['sample_stage'] = 'refine'
save(request['metrics'], metrics)
def refined_complete(video, sound, receipt):