Skip to content

Make videos on the computer you already own. FreeVideo runs MiniMax H3 in as little as 8 GB of VRAM and 16 GB of RAM, and adapts its acceleration path to your hardware.

Mac encoder: slice limits follow the live encoder pool, and projection slices keep a row floor

Bowen Xue8533de0acabd
diff --git a/freevideo_engine/macos_encoder.py b/freevideo_engine/macos_encoder.py
index 3bdc43f..20bc20c 100644
--- a/freevideo_engine/macos_encoder.py
+++ b/freevideo_engine/macos_encoder.py
@@ -45,7 +45,7 @@ def nvfp4_weight(weight, dtype, device, *, rows=128, row_range=None):
return result[:count if row_range is not None else params.orig_shape[0], :params.orig_shape[1]]
-def install_weight_reader(clip, backend, checkpoint, *, decoder=nvfp4_weight):
+def install_weight_reader(clip, backend, checkpoint, *, decoder=nvfp4_weight, pool_bytes=None):
"""Adapt only this encoder instance; never patch the shared Comfy library."""
import torch
from comfy.quant_ops import QuantizedTensor, get_layout_class
@@ -119,7 +119,8 @@ def install_weight_reader(clip, backend, checkpoint, *, decoder=nvfp4_weight):
stats['projection_slices'], slice_handles = install(
[module for module, function in selected if function is linear], model.layers,
decoder=decoder, quantized_type=QuantizedTensor, backend=backend,
- guard=guard, release=lambda: release_mapped_pages([checkpoint]), reader_stats=stats)
+ guard=guard, release=lambda: release_mapped_pages([checkpoint]), reader_stats=stats,
+ pool_bytes=pool_bytes)
handles.extend(slice_handles)
return stats, restores, handles
@@ -198,9 +199,11 @@ def encode(prompt, output, *, root, budget_bytes=3 * 2**30, reserve_bytes=2 * 2*
clip = load_clip([checkpoint], clip_type=comfy.sd.CLIPType.MINIMAX,
model_options=dict(initial_device=torch.device('cpu'), load_device=torch.device('mps'),
offload_device=torch.device('cpu'), dtype=torch.bfloat16), disable_dynamic=True)
- stats, restores, handles = install_weight_reader(clip, backend, checkpoint, decoder=decoder)
+ pool_bytes = policy['effective_allocator_limit_bytes']
+ stats, restores, handles = install_weight_reader(clip, backend, checkpoint, decoder=decoder,
+ pool_bytes=pool_bytes)
from .macos_encoder_attention import install
- stats['attention'], attention_handles = install(clip)
+ stats['attention'], attention_handles = install(clip, pool_bytes)
handles.extend(attention_handles)
# Placement is handled by the instance reader; native tokenization,
# forward and output packing still execute through the CLIP API.
diff --git a/freevideo_engine/macos_encoder_attention.py b/freevideo_engine/macos_encoder_attention.py
index bb77e34..351e25d 100644
--- a/freevideo_engine/macos_encoder_attention.py
+++ b/freevideo_engine/macos_encoder_attention.py
@@ -13,7 +13,7 @@ FULL_ATTENTION_MAX_BYTES = 640 * 2**20
SLICE_WORKING_BYTES = 256 * 2**20
-def slice_plan(tokens, heads, dim, kv_heads=None):
+def slice_plan(tokens, heads, dim, kv_heads=None, pool=1):
"""Pinned batch-one FP32 contract; include full output and both tile outputs.
GQA visits one KV head's query heads at a time, with zero-stride KV views.
@@ -30,17 +30,17 @@ def slice_plan(tokens, heads, dim, kv_heads=None):
output_bytes = tokens * heads * dim * 4
per_row = 2 * group_heads * (tokens + dim) * 4
# Even if a whole GQA group fits, large calls still split the query axis.
- rows = min((tokens + 1) // 2, max(1, (SLICE_WORKING_BYTES - output_bytes) // per_row))
+ rows = min((tokens + 1) // 2, max(1, (SLICE_WORKING_BYTES * pool - output_bytes) // per_row))
return dict(query_rows=rows, head_groups=groups,
slice_working_bytes=output_bytes + rows * per_row,
minimum_slice_bytes=output_bytes + per_row)
-def plan(tokens, heads, dim, kv_heads=None):
+def plan(tokens, heads, dim, kv_heads=None, pool=1):
kv_heads = heads if kv_heads is None else kv_heads
- selected = slice_plan(tokens, heads, dim, kv_heads)
+ selected = slice_plan(tokens, heads, dim, kv_heads, pool)
full_bytes = heads * tokens * tokens * 4
- full = full_bytes <= FULL_ATTENTION_MAX_BYTES
+ full = full_bytes <= FULL_ATTENTION_MAX_BYTES * pool
if full:
selected.update(query_rows=tokens, head_groups=1, slice_working_bytes=None)
return dict(tokens=tokens, heads=heads, kv_heads=kv_heads, head_dim=dim,
@@ -99,7 +99,7 @@ def sliced_basic(q, k, v, heads, rows, basic, *, mask=None, skip_reshape=True, e
return output
-def install(clip):
+def install(clip, pool_bytes=None):
"""Return shared dispatch statistics and removable instance hooks.
Counts are attempts/planned basic invocations, not successful completions.
@@ -113,16 +113,18 @@ def install(clip):
or any((m.num_heads, m.head_dim) != (16, 72) for m in vision)
or any((m.num_heads, m.num_kv_heads, m.head_dim) != (64, 8, 128) for m in language)):
raise ValueError('Expected the pinned H3 vision and language attention geometry')
- stats = dict(full_calls=0, sliced_calls=0, slices=0, plans=[],
- full_attention_max_bytes=FULL_ATTENTION_MAX_BYTES,
- slice_working_bytes=SLICE_WORKING_BYTES,
+ from .macos_encoder_linear import pool_scale
+ pool = pool_scale(pool_bytes)
+ stats = dict(pool_scale=pool, full_calls=0, sliced_calls=0, slices=0, plans=[],
+ full_attention_max_bytes=FULL_ATTENTION_MAX_BYTES * pool,
+ slice_working_bytes=SLICE_WORKING_BYTES * pool,
kernel='Comfy attention_basic: einsum, scale, additive mask, softmax, einsum; FP32')
seen = set()
def attention(q, k, v, heads, *, mask=None, skip_reshape=False, enable_gqa=False):
if q.device.type != 'mps' or not skip_reshape:
raise ValueError('Bounded native encoder attention requires MPS and pre-shaped heads')
- selected = plan(q.shape[2], heads, q.shape[3], k.shape[1])
+ selected = plan(q.shape[2], heads, q.shape[3], k.shape[1], pool)
stats['sliced_calls'] += 1
stats['slices'] += ((q.shape[2] + selected['query_rows'] - 1) // selected['query_rows']
* selected['head_groups'])
@@ -136,7 +138,7 @@ def install(clip):
x = args[0] if args else kwargs['x' if kind == 'vision' else 'hidden_states']
tokens = x.shape[0] if kind == 'vision' else x.shape[1]
selected = plan(tokens, module.num_heads, module.head_dim,
- module.num_heads if kind == 'vision' else module.num_kv_heads)
+ module.num_heads if kind == 'vision' else module.num_kv_heads, pool)
key = (kind, tokens)
if key not in seen:
seen.add(key)
diff --git a/freevideo_engine/macos_encoder_linear.py b/freevideo_engine/macos_encoder_linear.py
index 1995170..7f90053 100644
--- a/freevideo_engine/macos_encoder_linear.py
+++ b/freevideo_engine/macos_encoder_linear.py
@@ -14,14 +14,29 @@ import types
FULL_LINEAR_MAX_BYTES = 128 * 2**20
LINEAR_WORKING_BYTES = 128 * 2**20
MLP_WORKING_BYTES = 256 * 2**20
+# The limits above were sized for the 3 GiB trial pool. A larger live pool
+# scales them by the same ratio; a smaller one keeps them as they are.
+REFERENCE_POOL_BYTES = 3 * 2**30
+# Output rows of one slice never drop below this, however large the assembled
+# output is: one row per slice means one decode, fence and cache release each.
+MIN_SLICE_ROWS = 256
-def linear_plan(tokens, inputs, outputs, itemsize=4, *, packed_columns=None, packed_rows=None):
+def pool_scale(pool_bytes):
+ if pool_bytes is None:
+ return 1
+ if type(pool_bytes) is not int or pool_bytes < 1:
+ raise ValueError('Expected a positive encoder pool size')
+ return max(1, pool_bytes // REFERENCE_POOL_BYTES)
+
+
+def linear_plan(tokens, inputs, outputs, itemsize=4, *, packed_columns=None, packed_rows=None, pool=1):
"""Bound full output + decoded slab + packed/scales + slab result + bias.
Packed geometry can include padding on the input axis. A result whose full
- output alone exceeds the budget still uses one row and reports that minimum
- explicitly; the parent MLP's token plan avoids this for the H3 workload.
+ output alone exceeds the budget still slices at least MIN_SLICE_ROWS output
+ rows (or all outputs, if fewer) and reports the over-budget estimate in
+ slice_working_bytes; minimum_slice_bytes is the theoretical one-row bound.
"""
if min(tokens, inputs, outputs, itemsize) < 1:
raise ValueError('Expected positive linear dimensions')
@@ -34,8 +49,9 @@ def linear_plan(tokens, inputs, outputs, itemsize=4, *, packed_columns=None, pac
upload = 0 if packed_columns is None else columns // 2 + columns // 16 * 4
per_row = columns * itemsize + upload + (tokens + 1) * itemsize
scale = 4 if packed_columns is not None else 0
- rows = min(outputs, max(1, (LINEAR_WORKING_BYTES - output - scale) // per_row))
- full = weight + output <= FULL_LINEAR_MAX_BYTES
+ floor = min(outputs, MIN_SLICE_ROWS)
+ rows = min(outputs, max(floor, (LINEAR_WORKING_BYTES * pool - output - scale) // per_row))
+ full = weight + output <= FULL_LINEAR_MAX_BYTES * pool
return dict(tokens=tokens, inputs=inputs, outputs=outputs, itemsize=itemsize,
decoded_weight_bytes=weight, output_bytes=output,
full_weight_output_bytes=weight + output,
@@ -44,7 +60,7 @@ def linear_plan(tokens, inputs, outputs, itemsize=4, *, packed_columns=None, pac
minimum_slice_bytes=output + per_row + scale)
-def mlp_plan(tokens, hidden, intermediate, itemsize=4):
+def mlp_plan(tokens, hidden, intermediate, itemsize=4, pool=1):
"""Three simultaneous intermediate arrays plus two hidden-sized row arrays.
The full assembled MLP output stays live across chunks. Linear workspace is
@@ -54,8 +70,9 @@ def mlp_plan(tokens, hidden, intermediate, itemsize=4):
raise ValueError('Expected positive MLP dimensions')
output = tokens * hidden * itemsize
per_row = (3 * intermediate + 2 * hidden) * itemsize
- full = tokens * per_row <= MLP_WORKING_BYTES
- rows = min(tokens, max(1, (MLP_WORKING_BYTES - output) // per_row))
+ budget = MLP_WORKING_BYTES * pool
+ full = tokens * per_row <= budget
+ rows = min(tokens, max(1, (budget - output) // per_row))
return dict(tokens=tokens, hidden=hidden, intermediate=intermediate, itemsize=itemsize,
mode='full' if full else 'sliced', token_rows=tokens if full else rows,
full_activation_bytes=tokens * per_row,
@@ -121,16 +138,19 @@ class _Restore:
setattr(self.module, self.attribute, self.original)
-def install(linears, layers, *, decoder, quantized_type, backend, guard, release, reader_stats):
+def install(linears, layers, *, decoder, quantized_type, backend, guard, release, reader_stats,
+ pool_bytes=None):
"""Select before the native forward looks up its projection/MLP callable.
Below the thresholds do not assign anything: the fourth-version weight
reader and vendor MLP function objects remain intact. MLP selection occurs
at the parent layer because Module reads its own forward before pre-hooks.
"""
- stats = dict(linear_sliced_calls=0, linear_slices=0, mlp_sliced_calls=0, mlp_slices=0,
- plans=[], full_linear_max_bytes=FULL_LINEAR_MAX_BYTES,
- linear_working_bytes=LINEAR_WORKING_BYTES, mlp_working_bytes=MLP_WORKING_BYTES)
+ pool = pool_scale(pool_bytes)
+ stats = dict(pool_scale=pool, linear_sliced_calls=0, linear_slices=0, mlp_sliced_calls=0, mlp_slices=0,
+ plans=[], full_linear_max_bytes=FULL_LINEAR_MAX_BYTES * pool,
+ linear_working_bytes=LINEAR_WORKING_BYTES * pool,
+ mlp_working_bytes=MLP_WORKING_BYTES * pool)
handles, seen = [], set()
def record(kind, selected):
@@ -151,7 +171,8 @@ def install(linears, layers, *, decoder, quantized_type, backend, guard, release
columns = module.weight._qdata.shape[1] * 2 if quantized else None
selected = linear_plan(math.prod(x.shape[:-1]), module.in_features,
module.out_features, x.element_size(), packed_columns=columns,
- packed_rows=module.weight._qdata.shape[0] if quantized else None)
+ packed_rows=module.weight._qdata.shape[0] if quantized else None,
+ pool=pool)
record('linear', selected)
if selected['mode'] == 'full':
return None
@@ -210,7 +231,7 @@ def install(linears, layers, *, decoder, quantized_type, backend, guard, release
nonlocal active
x = args[0] if args else kwargs['x']
selected = mlp_plan(math.prod(x.shape[:-1]), mlp.down_proj.out_features,
- mlp.down_proj.in_features, x.element_size())
+ mlp.down_proj.in_features, x.element_size(), pool)
record('mlp', selected)
if selected['mode'] == 'full':
return None