Skip to content

Make videos on the computer you already own. FreeVideo runs MiniMax H3 in as little as 8 GB of VRAM and 16 GB of RAM, and adapts its acceleration path to your hardware.

Open image generation and editing on Macs with Apple GPUs

Bowen Xue95108b079adb
diff --git a/freevideo_engine/image_gen/failure.py b/freevideo_engine/image_gen/failure.py
index a29582f..2c90f4b 100644
--- a/freevideo_engine/image_gen/failure.py
+++ b/freevideo_engine/image_gen/failure.py
@@ -13,14 +13,15 @@ from ..exit_diagnostics import clean_reading, clean_sample
from ..failure_details import Redactor
-ERRORS = frozenset(('no_memory', 'busy_video', 'busy', 'pack_missing', 'model_unreadable',
+ERRORS = frozenset(('no_memory', 'small_memory', 'busy_video', 'busy', 'pack_missing', 'model_unreadable',
'device_unavailable', 'worker_exit', 'timeout', 'cancelled',
'invalid_request', 'invalid_media', 'invalid_points', 'segment_not_installed', 'generation_failed'))
-NO_REPORT = frozenset(('busy', 'invalid_request', 'invalid_media', 'invalid_points', 'segment_not_installed', 'cancelled'))
+NO_REPORT = frozenset(('small_memory', 'busy', 'invalid_request', 'invalid_media', 'invalid_points', 'segment_not_installed', 'cancelled'))
STAGES = ('start', 'load', 'text', 'encode', 'denoise', 'decode')
KINDS = ('generate', 'edit', 'cast', 'compose', 'download', 'install')
MESSAGE_ZH = {
'no_memory': '显存不够,最小尺寸也没能生成。关掉其他占用显卡的程序后再试。',
+ 'small_memory': '这台 Mac 的内存不够生成图片,需要 16 GB 或以上的内存。',
'busy_video': '视频正在生成,完成后再试。',
'busy': '已有一张图在生成,请等它完成。',
'pack_missing': '请先在启动器里下载“图像生成模型”。',
@@ -47,7 +48,10 @@ _MODEL = re.compile(
r'safetensors?|(?:deserializ|invalid|incomplete|too large).*header|'
r'(?:checkpoint|weights?).*(?:cannot|could not|unable|failed|corrupt|unreadable|mismatch)|'
r'pytorchstreamreader|unexpected (?:end of file|eof)', re.I)
+_MPS_MEMORY = re.compile(r'MPS backend out of memory', re.I)
_LIMIT = 16 * 1024
+# On a Mac the GPU's memory is the machine's: the same failure, in its words.
+UNIFIED_MESSAGE_ZH = {'no_memory': '内存不够,最小尺寸也没能生成。关掉其他程序后再试。'}
class FailureError(ValueError):
@@ -70,7 +74,7 @@ def classify_exception(error, *, stage='start', cancelled=False):
error = error.__cause__ or (None if error.__suppress_context__ else error.__context__)
for item in chain:
names = {cls.__name__ for cls in type(item).__mro__}
- if isinstance(item, MemoryError) or 'OutOfMemoryError' in names:
+ if isinstance(item, MemoryError) or 'OutOfMemoryError' in names or _MPS_MEMORY.search(str(item)):
return 'no_memory'
for item in chain:
cuda_init = any(frame.f_globals.get('__name__') == 'torch.cuda'
diff --git a/freevideo_engine/image_gen/image_policy.py b/freevideo_engine/image_gen/image_policy.py
index 32914af..75e4ddb 100644
--- a/freevideo_engine/image_gen/image_policy.py
+++ b/freevideo_engine/image_gen/image_policy.py
@@ -82,6 +82,19 @@ MIN_MARGIN = 1.5 * GiB # allocator fragmentation, stage hand-over, dr
MARGIN_FRACTION = 0.06
MIN_HOST_WORKING = 4 * GiB # beyond the weights: images, tokenizer, the process itself
+# Apple GPUs draw on the host's memory, where the weights are mapped file pages the
+# system can drop. Planning against half of it kept a 24 GB M5 with a browser open
+# out of heavy swapping; planning against what was free at the time let the second
+# request keep 16 encoder layers resident and swap 4 GB. A Mac below 16 GB cannot
+# plan even with every block streamed (4 GiB), and closing programs does not change it.
+UNIFIED_SHARE = 0.5
+UNIFIED_MIN_TOTAL = 15 * GiB # 16 GB Macs report exactly 16 GiB
+
+
+def unified_budget(total_bytes):
+ """What a request on an Apple GPU plans against, from the machine's total memory."""
+ return int(total_bytes * UNIFIED_SHARE)
+
@dataclass(frozen=True)
class ImagePlan:
diff --git a/freevideo_engine/image_gen/image_runner.py b/freevideo_engine/image_gen/image_runner.py
index b73d021..e4e11ca 100644
--- a/freevideo_engine/image_gen/image_runner.py
+++ b/freevideo_engine/image_gen/image_runner.py
@@ -97,11 +97,12 @@ class _Placement:
runner.dit_stack.set_resident(_kept(runner.last_plan))
self._trim()
- @staticmethod
- def _trim():
+ def _trim(self):
# Stage peaks are planned on allocated bytes; give the cached blocks back so the
# next stage's workspace is not fragmented by the previous one's.
- if torch.cuda.is_available():
+ if self.runner.device.type == 'mps':
+ torch.mps.empty_cache()
+ elif torch.cuda.is_available():
torch.cuda.empty_cache()
@@ -117,6 +118,7 @@ class ImageRunner:
"""
full_card = False
+ mapped = False
def __init__(self, pack_dir, *, device='cuda', gpu_budget_bytes=None, log=None, progress=None):
from .qwen_pipeline import QwenImageGenerator
@@ -136,9 +138,12 @@ class ImageRunner:
loading = _LoadProgress(progress or (lambda stage, fraction: None))
started = time.monotonic()
# A card that holds every weight plus a request's working memory keeps no host copy.
- self.full_card = self._fits_whole_model()
+ # An Apple GPU shares the host's memory: its layers stay views of the mapped files (mapped.py).
+ self.mapped = self.device.type == 'mps'
+ self.full_card = not self.mapped and self._fits_whole_model()
self.transformer = load_dit(self.pack_dir / _DIT_FILE, device=self.device if self.full_card else 'cpu',
- progress=lambda done, total: loading.report('dit', done, total))
+ progress=lambda done, total: loading.report('dit', done, total),
+ **({'mapped': True} if self.mapped else {}))
self.load_seconds['dit'] = time.monotonic() - started
loading.complete('dit')
started = time.monotonic()
@@ -150,7 +155,8 @@ class ImageRunner:
config_dir = self.pack_dir / 'text_encoder'
processor_dir = self.pack_dir / 'processor'
text = load_text_encoder(self.pack_dir / _TE_FILE, config_path=config_dir / 'config.json', device='cpu',
- progress=lambda done, total: loading.report('text', done, total))
+ progress=lambda done, total: loading.report('text', done, total),
+ **({'mapped': True} if self.mapped else {}))
from transformers import Qwen3VLProcessor
processor = Qwen3VLProcessor.from_pretrained(str(processor_dir), local_files_only=True)
self.load_seconds['text'] = time.monotonic() - started
@@ -183,7 +189,15 @@ class ImageRunner:
return lambda done, total: loading.report(
'pin', offset + stack_bytes[index] * (done / total if total else 0.), pin_total)
- if self.full_card:
+ if self.mapped:
+ from .mapped import MappedStack
+ self.dit_stack = MappedStack(stacks[0], name='dit', device=self.device)
+ self.te_stack = MappedStack(stacks[1], name='text_encoder', device=self.device)
+ from . import mlx_attention, qwen_dit
+ if mlx_attention.available() and not getattr(qwen_dit.attention, 'mlx', False):
+ qwen_dit.attention = mlx_attention.routed(qwen_dit.attention)
+ qwen_dit.attention.mlx = True
+ elif self.full_card:
# Move the encoder layers over one at a time so the host copy is freed as it goes.
for layer in stacks[1]:
layer.to(self.device)
@@ -246,6 +260,9 @@ class ImageRunner:
def budget(self):
if self.gpu_budget_bytes is not None:
return int(self.gpu_budget_bytes)
+ if self.device.type == 'mps':
+ from ..system import system_memory
+ return image_policy.unified_budget(system_memory()['total_bytes'])
free, _ = torch.cuda.mem_get_info(self.device)
return int(free + torch.cuda.memory_reserved(self.device))
@@ -267,7 +284,7 @@ class ImageRunner:
memory = system_memory()
free = memory.get('physical_available_bytes') or memory['available_bytes']
host_available = free + self.dit_stack.host_bytes + self.te_stack.host_bytes
- if self.full_card:
+ if self.full_card or self.mapped:
host_available = None
except Exception:
host_available = None
diff --git a/freevideo_engine/image_gen/int8_linear.py b/freevideo_engine/image_gen/int8_linear.py
new file mode 100644
index 0000000..92d92ba
--- /dev/null
+++ b/freevideo_engine/image_gen/int8_linear.py
@@ -0,0 +1,114 @@
+"""The int8 projections of the image model, per platform.
+
+``ops()`` is the engine's CUDA ``int8_ops`` everywhere but macOS, where Triton
+does not exist. There the same prepared ConvRot rows run on the Apple GPU: on
+an M5 or newer through the TensorOps int8 kernels of ``backends.mps_int8``
+(one int8 scale per activation row, like CUDA); on older Apple GPUs the
+activation is rotated and multiplied by the int8 rows widened to BF16, with
+the row scales applied to the output.
+"""
+import sys
+
+import torch
+from torch.nn import functional as F
+
+GROUP = 256
+_H16 = {}
+
+
+def ops():
+ """The module providing ``Int8Linear`` and ``install_cached_linears`` on this platform."""
+ if sys.platform == 'darwin':
+ return sys.modules[__name__]
+ from .. import int8_ops
+ return int8_ops
+
+
+def hadamard16(device):
+ """H4 (x) H4 / 4 with H4 = [[1,1,1,-1],[1,1,-1,1],[1,-1,1,1],[-1,1,1,1]] (FP32), as in int8_ops."""
+ key = str(device)
+ if key not in _H16:
+ h4 = torch.tensor([[1, 1, 1, -1], [1, 1, -1, 1], [1, -1, 1, 1], [-1, 1, 1, 1]], dtype=torch.float32)
+ _H16[key] = (torch.kron(h4, h4) / 4).to(device).contiguous()
+ return _H16[key]
+
+
+def rotate(rows):
+ """[M, K] rows times the group-256 ConvRot rotation H256 = H16 (x) H16, in FP32."""
+ h16 = hadamard16(rows.device)
+ groups = rows.float().reshape(rows.shape[0], -1, 16, 16)
+ return (h16 @ groups @ h16).reshape(rows.shape)
+
+
+def _tensor_ops(linear):
+ from ..backends import mps_int8
+ return (mps_int8.available() and linear.in_features <= mps_int8.CONVROT_DEPTH
+ and linear.out_features % mps_int8.TILE == 0)
+
+
+class Int8Linear(torch.nn.Module):
+ """A prepared ConvRot int8 Linear on an Apple GPU; the buffers are those of int8_ops.Int8Linear.
+
+ The most recent prepared input is shared by every Int8Linear, so Q/K/V
+ quantize (or rotate) the same activation once; the entry keeps its source alive.
+ """
+ _last = {}
+
+ @property
+ def bias(self):
+ return self.original.bias
+
+ def prepared(self, rows, quantize):
+ key = (rows.data_ptr(), rows._version, tuple(rows.shape), tuple(rows.stride()), str(rows.device), quantize)
+ last = Int8Linear._last
+ if last.get('key') != key:
+ last.clear()
+ if quantize:
+ from ..backends import mps_int8
+ value = mps_int8.quantize_rows(rows, convrot=True)
+ else:
+ value = rotate(rows).to(rows.dtype)
+ last.update(key=key, source=rows, value=value)
+ return last['value']
+
+ def forward(self, value):
+ rows = value.reshape(-1, value.shape[-1])
+ if rows.stride(-1) != 1:
+ rows = rows.contiguous()
+ scales = self.weight_scale.reshape(-1)
+ quantize = value.dtype in (torch.bfloat16, torch.float16) and _tensor_ops(self)
+ if quantize:
+ from ..backends import mps_int8
+ output = mps_int8.matmul(self.prepared(rows, True), self.weight_int8, scales, dtype=value.dtype)
+ else:
+ # int8 values are exact in BF16; the row scales go on the output, in FP32.
+ output = F.linear(self.prepared(rows, False), self.weight_int8.to(value.dtype))
+ output = output.float().mul_(scales).to(value.dtype)
+ if self.bias is not None:
+ output = output + self.bias
+ return output.reshape(*value.shape[:-1], self.out_features)
+
+
+def install_cached_linears(model, linears, rotation):
+ """Replace prepared int8 Linears with meta int8 placeholders for layer loading (as int8_ops)."""
+ if rotation is None or rotation.get('kind') != 'convrot' or rotation.get('group') != GROUP:
+ raise ValueError('Int8 projections need ConvRot group-256 weights')
+ for name, spec in linears.items():
+ if spec.get('storage') != 'int8' or spec.get('convrot_group') != GROUP:
+ raise ValueError('Int8 cache entry is not a ConvRot int8 matrix: ' + name)
+ original = model.get_submodule(name)
+ if not isinstance(original, torch.nn.Linear):
+ raise ValueError('Int8 cache target is not a Linear: ' + name)
+ if list(original.weight.shape) != spec['weight_shape'] or original.in_features % GROUP:
+ raise ValueError('Int8 cache shape differs from the official model: ' + name)
+ replacement = Int8Linear.__new__(Int8Linear)
+ torch.nn.Module.__init__(replacement)
+ replacement.register_buffer('weight_int8', torch.empty(spec['weight_shape'], dtype=torch.int8, device='meta'))
+ replacement.register_buffer('weight_scale', torch.empty(spec['scale_shape'], dtype=torch.float32, device='meta'))
+ replacement.original = torch.nn.Module()
+ replacement.original.register_parameter('bias', original.bias)
+ replacement.in_features = original.in_features
+ replacement.out_features = original.out_features
+ replacement.input_dtype = getattr(torch, spec.get('input_dtype', 'bfloat16'))
+ prefix, _, child = name.rpartition('.')
+ setattr(model.get_submodule(prefix) if prefix else model, child, replacement)
diff --git a/freevideo_engine/image_gen/mapped.py b/freevideo_engine/image_gen/mapped.py
new file mode 100644
index 0000000..1fd8454
--- /dev/null
+++ b/freevideo_engine/image_gen/mapped.py
@@ -0,0 +1,130 @@
+"""File-backed image weights for Apple GPUs.
+
+On a Mac the card's memory is the host's. Weights copied into ordinary memory
+and onto the GPU heap stay wired while the GPU uses them, and a 24 GB machine
+with a browser open swapped the other programs out at gigabytes per second to
+hold the 15 GB model. Here the stacked layers' weights are read-only views of
+the checkpoint's own mapped pages, which the system can drop and read again
+instead of swapping, and ``MappedStack`` lends each layer to the GPU for its
+pass. Only ``set_resident`` layers keep their GPU copy between passes.
+"""
+import json
+import mmap
+import struct
+import warnings
+
+import torch
+
+_DTYPES = {'BF16': torch.bfloat16, 'F16': torch.float16, 'F32': torch.float32, 'I8': torch.int8,
+ 'U8': torch.uint8, 'I32': torch.int32, 'I64': torch.int64}
+
+
+class MappedCheckpoint:
+ """A safetensors reader whose ``get_tensor`` returns views of the mapped file.
+
+ Everything else (keys, slices, metadata) is the wrapped reader's, so the
+ loaders validate exactly what they validate without it.
+ """
+
+ def __init__(self, reader, path):
+ self.reader = reader
+ with open(path, 'rb') as stream:
+ size, = struct.unpack('<Q', stream.read(8))
+ header = json.loads(stream.read(size))
+ self.mapping = mmap.mmap(stream.fileno(), 0, access=mmap.ACCESS_READ)
+ self.start = 8 + size
+ self.header = {key: value for key, value in header.items() if key != '__metadata__'}
+ with warnings.catch_warnings():
+ # Read-only checkpoint bytes: the views are only ever copy sources.
+ warnings.simplefilter('ignore', UserWarning)
+ self.data = torch.frombuffer(self.mapping, dtype=torch.uint8)
+
+ def __getattr__(self, name):
+ return getattr(self.reader, name)
+
+ def get_tensor(self, key):
+ entry = self.header.get(key)
+ dtype = _DTYPES.get(entry['dtype']) if entry else None
+ if dtype is None:
+ return self.reader.get_tensor(key)
+ begin, end = entry['data_offsets']
+ begin += self.start
+ size = torch.empty((), dtype=dtype).element_size()
+ if begin % size:
+ return self.reader.get_tensor(key) # unaligned: an ordinary copy
+ return self.data[begin:self.start + end].view(dtype).reshape(entry['shape'])
+
+
+def _stores(layer):
+ for module in layer.modules():
+ for store, parameter in ((module._parameters, True), (module._buffers, False)):
+ for key, tensor in store.items():
+ if tensor is not None:
+ yield store, key, tensor, parameter
+
+
+class MappedStack:
+ """Layers whose weights live on the host (mapped or not), moved to the GPU for each pass."""
+
+ def __init__(self, layers, *, name, device):
+ self.name = name
+ self.layers = list(layers)
+ if not self.layers:
+ raise ValueError('MappedStack needs at least one layer')
+ self.device = torch.device(device)
+ self.host = [list(_stores(layer)) for layer in self.layers]
+ self.layer_bytes = max(sum(t.numel() * t.element_size() for _, _, t, _ in entries) for entries in self.host)
+ # Mapped pages are the file's, not this process's: the system takes them back when it needs them.
+ self.host_bytes = 0
+ self.resident = 0
+ self.bound = set()
+ self.copied_bytes = 0
+ self.hooks = []
+ for index, layer in enumerate(self.layers):
+ self.hooks.append(layer.register_forward_pre_hook(lambda module, args, index=index: self._bind(index)))
+ self.hooks.append(layer.register_forward_hook(
+ lambda module, args, output, index=index: self._release(index)))
+
+ at_boundary = True
+
+ @property
+ def offloader(self):
+ return self
+
+ def _bind(self, index):
+ if index in self.bound:
+ return
+ for store, key, tensor, parameter in self.host[index]:
+ value = tensor.to(self.device)
+ store[key] = torch.nn.Parameter(value, requires_grad=False) if parameter else value
+ self.copied_bytes += tensor.numel() * tensor.element_size()
+ self.bound.add(index)
+
+ def _unbind(self, index):
+ for store, key, tensor, _ in self.host[index]:
+ store[key] = tensor
+ self.bound.discard(index)
+
+ def _release(self, index):
+ if index >= self.resident:
+ self._unbind(index)
+
+ def set_resident(self, count):
+ """Keep the first ``count`` layers' GPU copies between passes (filled on their next use)."""
+ count = max(0, min(int(count), len(self.layers)))
+ for index in sorted(self.bound):
+ if index >= count:
+ self._unbind(index)
+ self.resident = count
+ return count
+
+ def stats(self):
+ return dict(name=self.name, layers=len(self.layers), layer_bytes=self.layer_bytes, pinned_layers=0,
+ pinned_bytes=0, resident_layers=self.resident, cached_bytes=len(self.bound) * self.layer_bytes,
+ copied_bytes=self.copied_bytes, mapped=True)
+
+ def close(self):
+ for hook in self.hooks:
+ hook.remove()
+ for index in list(self.bound):
+ self._unbind(index)
diff --git a/freevideo_engine/image_gen/mlx_attention.py b/freevideo_engine/image_gen/mlx_attention.py
new file mode 100644
index 0000000..3163e11
--- /dev/null
+++ b/freevideo_engine/image_gen/mlx_attention.py
@@ -0,0 +1,44 @@
+"""Image attention through MLX's fused Metal kernel on Apple GPUs.
+
+At the 1024 x 1024 shape (4125 keys, 32 heads of 128) Torch's MPS attention
+ran at about 3.5 TFLOPS on an M5; this kernel is the one the Mac video engine
+already uses. Q/K/V and the output share their Metal buffers through DLPack.
+"""
+import math
+
+import torch
+
+from ..backends.mps_mlx_attention import to_mlx_many, to_torch
+
+
+def available():
+ try:
+ import mlx.core # noqa: F401
+ except Exception:
+ return False
+ return True
+
+
+def routed(fallback, *, min_queries=256):
+ """``attention`` for image-sized calls; short text segments, where the hand-over costs more, keep ``fallback``."""
+ def run(query, key, value, attn_mask=None):
+ if query.shape[1] < min_queries:
+ return fallback(query, key, value, attn_mask=attn_mask)
+ return attention(query, key, value, attn_mask=attn_mask)
+ return run
+
+
+def attention(query, key, value, attn_mask=None):
+ """Same contract as ``qwen_dit.attention``: (batch, sequence, heads, dim), boolean mask or None."""
+ import mlx.core as mx
+ tensors = (query, key, value) if attn_mask is None else (query, key, value, attn_mask)
+ arrays = to_mlx_many(tensors)
+ q, k, v = (mx.swapaxes(array, 1, 2) for array in arrays[:3])
+ mask = arrays[3] if attn_mask is not None else None
+ out = mx.fast.scaled_dot_product_attention(q, k, v, scale=1 / math.sqrt(query.shape[-1]), mask=mask)
+ out = mx.contiguous(mx.swapaxes(out, 1, 2))
+ result = to_torch(out, query.device)
+ del arrays, q, k, v, mask, out
+ # One unified pool: do not let MLX keep the freed buffers from Torch.
+ mx.clear_cache()
+ return result
diff --git a/freevideo_engine/image_gen/qwen_pipeline.py b/freevideo_engine/image_gen/qwen_pipeline.py
index 15ebbc9..b0eeb89 100644
--- a/freevideo_engine/image_gen/qwen_pipeline.py
+++ b/freevideo_engine/image_gen/qwen_pipeline.py
@@ -400,6 +400,7 @@ class QwenImageGenerator:
self.text_encoder = self.transformer = self.vae = None
self._closed = True
# Int8Linear keeps the last activation alive for Q/K/V sharing.
- module = sys.modules.get("freevideo_engine.int8_ops")
- if module is not None:
- module.Int8Linear._last.clear()
+ for name in ("freevideo_engine.int8_ops", "freevideo_engine.image_gen.int8_linear"):
+ module = sys.modules.get(name)
+ if module is not None:
+ module.Int8Linear._last.clear()
diff --git a/freevideo_engine/image_gen/qwen_te_weights.py b/freevideo_engine/image_gen/qwen_te_weights.py
index 86d6299..7765a9f 100644
--- a/freevideo_engine/image_gen/qwen_te_weights.py
+++ b/freevideo_engine/image_gen/qwen_te_weights.py
@@ -117,7 +117,8 @@ def _install_text_weights(model, st, *, dtype, chunk_rows, reads=None):
_dequantize_vocabulary(st, name, shape, dtype, chunk_rows, reads), requires_grad=False
)
else:
- from ..int8_ops import Int8Linear
+ from .int8_linear import ops
+ Int8Linear = ops().Int8Linear
weight = st.get_tensor(name + ".weight")
if weight.dtype != torch.int8 or weight.shape != shape or shape[1] % 256:
@@ -158,7 +159,7 @@ def _install_text_weights(model, st, *, dtype, chunk_rows, reads=None):
def load_text_encoder(path, *, config=None, config_path=None, device="cpu", dtype=torch.bfloat16,
- chunk_rows=8192, progress=None):
+ chunk_rows=8192, progress=None, mapped=False):
"""Return a Qwen3VLForConditionalGeneration with int8 language projections.
``config`` accepts a transformers Qwen3VLConfig or its dictionary. Otherwise
@@ -196,6 +197,9 @@ def load_text_encoder(path, *, config=None, config_path=None, device="cpu", dtyp
with safe_open(str(path), framework="pt", device="cpu") as st:
reads = (TensorProgress(st, progress, (key for key in st.keys()
if not key.startswith(("model.visual.", "lm_head.")))) if progress is not None else None)
+ if mapped: # CPU only: tensors stored as they are on disk stay views of the mapped file
+ from .mapped import MappedCheckpoint
+ st = MappedCheckpoint(st, path)
_install_text_weights(model, st, dtype=dtype, chunk_rows=chunk_rows, reads=reads)
model.eval().requires_grad_(False).to(device=device)
model_ref = weakref.ref(model)
diff --git a/freevideo_engine/image_gen/qwen_weights.py b/freevideo_engine/image_gen/qwen_weights.py
index 662d12c..fe64360 100644
--- a/freevideo_engine/image_gen/qwen_weights.py
+++ b/freevideo_engine/image_gen/qwen_weights.py
@@ -92,15 +92,19 @@ def _quant_targets(model, name):
return targets, modules
-def load_dit(path, device="cpu", *, config=None, progress=None):
+def load_dit(path, device="cpu", *, config=None, progress=None, mapped=False):
"""Load a Comfy Turbo int8_convrot DiT; CPU supports storage, CUDA execution.
The optional config override is for small models/tests. Production files
without JSON config metadata use the official 32-block Qwen-Image 2.1 config.
An unknown marker, tensor layout, missing key, or extra key is an error.
``progress(done_bytes, total_bytes)`` reports completed source tensor batches.
+ ``mapped`` (CPU only) leaves tensors stored as they are on disk as views of the mapped file.
"""
with _open_safetensors(path, framework="pt", device="cpu") as checkpoint:
+ if mapped:
+ from .mapped import MappedCheckpoint
+ checkpoint = MappedCheckpoint(checkpoint, path)
reads = TensorProgress(checkpoint, progress) if progress is not None else None
with torch.device("meta"):
model = QwenImageDiT(**_config(checkpoint.metadata() or {}, QwenImageDiT, config))
@@ -159,7 +163,8 @@ def load_dit(path, device="cpu", *, config=None, progress=None):
for key in ordinary:
_check_tensor(checkpoint, keys[key], expected[key].shape, {"BF16", "F32"})
# Reuse the engine's wrapper construction and exact runtime member names.
- from ..int8_ops import install_cached_linears
+ from .int8_linear import ops
+ install_cached_linears = ops().install_cached_linears
specs = {}
for targets in quantized.values():
for target in targets:
diff --git a/freevideo_engine/image_gen/service.py b/freevideo_engine/image_gen/service.py
index c351c58..6a0dcb6 100644
--- a/freevideo_engine/image_gen/service.py
+++ b/freevideo_engine/image_gen/service.py
@@ -17,7 +17,8 @@ import uuid
from . import image_pack
from ..download_settings import observed_speed
-from .failure import Context, ERRORS, FailureError, MESSAGE_ZH, NO_REPORT, STAGES, classify_exception
+from .failure import (Context, ERRORS, FailureError, MESSAGE_ZH, NO_REPORT, STAGES, UNIFIED_MESSAGE_ZH,
+ classify_exception)
BODY_LIMIT = 64 * 1024
CANCEL_GRACE = 5.
@@ -156,12 +157,28 @@ class Service:
segment=dict(ready=image_pack.ready(machine['model_root'], group='segment'),
max_points=16, device='cpu',
install=self.segment_installer.state() if self.segment_installer else None),
- bytes_per_second=None if ready else observed_speed(engine_root))
+ bytes_per_second=None if ready else observed_speed(engine_root),
+ **({'unified_memory': True} if machine.get('device_backend') == 'mps' else {}))
+
+ @staticmethod
+ def unified_total(machine):
+ """Total memory of a Mac whose Apple GPU shares it; None for a separate card."""
+ if machine.get('device_backend') != 'mps':
+ return None
+ from ..system import system_memory
+ try:
+ return int(system_memory()['total_bytes'])
+ except (OSError, KeyError, TypeError, ValueError):
+ return None
def size_tiers(self, machine):
"""The sizes this card offers: the standard one always, the high one only with enough card memory."""
total = None
- if machine.get('device_backend', 'cuda') == 'cuda':
+ unified = self.unified_total(machine)
+ if unified is not None:
+ from .image_policy import unified_budget
+ total = unified_budget(unified)
+ elif machine.get('device_backend', 'cuda') == 'cuda':
from ..background_room import Gpu
from ..gpu_budget import open_bounded
@@ -495,6 +512,10 @@ class Service:
try:
if not image_pack.ready(machine['model_root']):
raise ValueError('pack_missing')
+ from .image_policy import UNIFIED_MIN_TOTAL
+ unified = self.unified_total(machine)
+ if unified is not None and unified < UNIFIED_MIN_TOTAL:
+ raise ValueError('small_memory')
requested = {name: value.get(name) for name in ('width', 'height')} if isinstance(value, dict) else {}
if action == 'compose':
from . import compose
@@ -598,6 +619,10 @@ class Service:
def memory_budget(self, machine):
"""Bound the driver query, without importing Torch or starting a worker."""
+ unified = self.unified_total(machine)
+ if unified is not None:
+ from .image_policy import unified_budget
+ return unified_budget(unified)
if machine.get('device_backend', 'cuda') != 'cuda':
return None
from ..background_room import Gpu, family
@@ -679,6 +704,8 @@ class Service:
result = dict(phase='failed', error=code, report_id=None, stage=stage)
if code in MESSAGE_ZH:
result['message_zh'] = MESSAGE_ZH[code]
+ if context.machine.get('device_backend') == 'mps' and code in UNIFIED_MESSAGE_ZH:
+ result['message_zh'] = UNIFIED_MESSAGE_ZH[code]
if code not in NO_REPORT:
if video_busy is None:
try:
diff --git a/web/image_generation.js b/web/image_generation.js
index 2481c6c..fb18dcc 100644
--- a/web/image_generation.js
+++ b/web/image_generation.js
@@ -18,8 +18,15 @@ export const IMAGE_GEN_POLL_MS = 500;
export const IMAGE_GEN_INFO_POLL_MS = 10000;
export const IMAGE_PACK_POLL_MS = 2000;
export const BLANK_CANVAS_RATIOS = Object.freeze(['16:9', '9:16', '1:1', '4:3', '3:4']);
-export function imageGenerationSupported(platform = globalThis.navigator?.userAgentData?.platform || globalThis.navigator?.platform || '') {
- return !/Mac|iPhone|iPad/i.test(platform);
+// The one switch the editor, media panel and cast library consult. Every engine platform
+// generates pictures now; a Mac too small for it gets the service's small_memory reason.
+export function imageGenerationSupported() {
+ return true;
+}
+// On a Mac the GPU's memory is the machine's: the same notes without the graphics card.
+function memoryWords([en, zh], unified) {
+ return unified ? [en.replace('GPU memory', 'memory').replace(' using the GPU', ''),
+ zh.replace('显存', '内存').replace('占用显卡的程序', '程序')] : [en, zh];
}
export function blankCanvasRatio(target) {
return BLANK_CANVAS_RATIOS.find(ratio => {
@@ -162,23 +169,26 @@ export function imageGenError(code, info, t) {
invalid_media: ['The reference image cannot be read. Choose another image.', '参考图读不出来,请换一张图。'],
reload: ['Image generation was interrupted. Try again.', '图片生成中途被打断,请重试。'],
generation_failed: ['Image generation ran into an error. Try again.', '图像生成出错了,请再试一次。'],
+ small_memory: ['This Mac does not have enough memory to generate pictures. It needs 16 GB or more.',
+ '这台 Mac 的内存不够生成图片,需要 16 GB 或以上的内存。'],
};
+ const say = (en, zh) => t(...memoryWords([en, zh], info?.unified));
if (code === 'pack_missing') return imagePackUnavailableMessage(info, t);
if (code === 'no_memory') {
// Where it ran out decides what helps: the reference pictures are read at a fixed size,
// so a smaller canvas does not change that stage.
const stage = info?.stage, size = info?.fitted;
if (stage === 'encode')
- return t('Not enough GPU memory to load the reference images. Use fewer reference images, or close other apps using the GPU, then try again.',
+ return say('Not enough GPU memory to load the reference images. Use fewer reference images, or close other apps using the GPU, then try again.',
'显存不够,参考图没能载入。少用几张参考图,或关掉其他占用显卡的程序后再试。');
if (stage === 'load' || stage === 'text')
- return t('Not enough GPU memory to load the model. Close other apps using the GPU, then try again.',
+ return say('Not enough GPU memory to load the model. Close other apps using the GPU, then try again.',
'显存不够,模型没能载入。关掉其他占用显卡的程序后再试。');
if (size?.width && size?.height)
- return t(`Not enough GPU memory, even at ${size.width}\u00a0×\u00a0${size.height}. Close other apps using the GPU, then try again.`,
+ return say(`Not enough GPU memory, even at ${size.width}\u00a0×\u00a0${size.height}. Close other apps using the GPU, then try again.`,
`显存不够,${size.width}\u00a0×\u00a0${size.height} 也没能生成。关掉其他占用显卡的程序后再试。`);
}
- return t(...(Object.hasOwn(reasons, code) ? reasons[code] : reasons.generation_failed));
+ return say(...(Object.hasOwn(reasons, code) ? reasons[code] : reasons.generation_failed));
}
// Keep image failure copy here for editorial review. The submit card and its
@@ -188,8 +198,8 @@ export const QUEUE_LABELS = Object.freeze({
image: ['Added to the queue. It starts when the previous one finishes.', '已加入队列,前面的完成后自动开始。'],
video: ['Queued. It starts when the video finishes.', '已排队,视频生成完后自动开始。'],
});
-export const fittedNote = (size, t) => t(`Not enough GPU memory. Made at ${size.width}\u00a0×\u00a0${size.height} instead.`,
- `显存不足,已改为 ${size.width}\u00a0×\u00a0${size.height} 生成。`);
+export const fittedNote = (size, t, unified = false) => t(...memoryWords([`Not enough GPU memory. Made at ${size.width}\u00a0×\u00a0${size.height} instead.`,
+ `显存不足,已改为 ${size.width}\u00a0×\u00a0${size.height} 生成。`], unified));
export const REPORT_LABELS = Object.freeze({
download: ['Diagnostic report', '诊断报告'],
description: ['Diagnostic report: hardware, settings and timings', '诊断报告:硬件、设置和耗时'],
@@ -199,7 +209,7 @@ export const REPORT_LABELS = Object.freeze({
// Shared by the editor and the other image entry points. Only the opaque report
// ID goes in the request; raw service messages are never shown in the page.
-export function renderImageFailure(host, {error, report_id, stage, fitted} = {}, t,
+export function renderImageFailure(host, {error, report_id, stage, fitted, info} = {}, t,
{productVersion = () => null, fetchApi = globalThis.fetch.bind(globalThis)} = {}) {
host.replaceChildren();
if (!error || error === 'cancelled') return null;
@@ -210,7 +220,7 @@ export function renderImageFailure(host, {error, report_id, stage, fitted} = {},
}
const card = doc.createElement('div'); card.className = 'fv-result-links';
const reason = doc.createElement('div'); reason.className = 'fv-image-failure-reason';
- reason.setAttribute('role', 'status'); reason.textContent = imageGenError(error, {stage, fitted}, t);
+ reason.setAttribute('role', 'status'); reason.textContent = imageGenError(error, {stage, fitted, unified: info?.unified_memory}, t);
card.append(reason); host.append(card);
if (!report_id) return card;
const fetchReport = async () => {
@@ -723,10 +733,10 @@ export function createImageGeneration({client, renderer, t, getDoc, getSelection
: missing ? imagePackUnavailableMessage(state.info, t)
: promptRequired ? t('Enter a prompt first.', '请先填写提示词。')
: state.running && state.waiting ? t(...(QUEUE_LABELS[state.waiting] || QUEUE_LABELS.image))
- : state.running && state.fitted ? t(`Not enough GPU memory. Retrying at ${state.fitted.width}\u00a0×\u00a0${state.fitted.height}`,
- `显存不足,改用 ${state.fitted.width}\u00a0×\u00a0${state.fitted.height} 重试`) + ' · ' + percent
+ : state.running && state.fitted ? t(...memoryWords([`Not enough GPU memory. Retrying at ${state.fitted.width}\u00a0×\u00a0${state.fitted.height}`,
+ `显存不足,改用 ${state.fitted.width}\u00a0×\u00a0${state.fitted.height} 重试`], state.info?.unified_memory)) + ' · ' + percent
: state.running ? (state.stage === 'load' ? t('Loading the model', '加载模型') + ' ' : '') + percent
- : state.fitted ? fittedNote(state.fitted, t) : '';
+ : state.fitted ? fittedNote(state.fitted, t, state.info?.unified_memory) : '';
}
note.hidden = !note.textContent;
bar.dataset.running = String(state.running);