Make videos on the computer you already own. FreeVideo runs MiniMax H3 in as little as 8 GB of VRAM and 16 GB of RAM, and adapts its acceleration path to your hardware.
"""Prepared storage for the official VDN FP8 arithmetic and scale granularity.
Quantization calls the upstream Fp8Linear constructor on one CUDA matrix at a
time. Loading binds those exact bytes to the original Fp8Linear class. The BF16
backup is omitted, as in the separately verified compact-storage wrapper.
"""
import argparse
from contextlib import closing
import gc
import hashlib
import json
import os
from pathlib import Path
import time
from safetensors.torch import save_file
import torch
from .weights import sha256, skeleton, Sources, group_name
from .preparation import cache_location, expected_groups, reusable_group, source_paths
from .tensor_io import open_tensors
def input_dtype(module):
"""Input dtype of an ordinary Linear or a cached official FP8 Linear."""
return module.input_dtype if hasattr(module, 'input_dtype') else module.weight.dtype
def quantize_group(values, eligible, official):
"""Same official quantization for both retained-BF16 and streamed setup."""
converted, linears = {}, {}
items = values.items() if hasattr(values, 'items') else values
for name, value in items:
prefix, _, suffix = name.rpartition('.')
if prefix not in eligible:
converted[name] = value
elif suffix == 'bias':
converted[prefix + '.original.bias'] = value
elif suffix == 'weight':
module = torch.nn.Linear(value.shape[1], value.shape[0], bias=False, device='meta', dtype=value.dtype)
module.weight = torch.nn.Parameter(value.to('cuda'), requires_grad=False)
quantized = official.Fp8Linear(module)
converted[prefix + '.weight_fp8'] = quantized.weight_fp8.cpu()
converted[prefix + '.weight_scale'] = quantized.weight_scale.cpu()
linears[prefix] = {'weight_shape': list(value.shape),
'scale_shape': list(quantized.weight_scale.shape),
'input_dtype': str(value.dtype).removeprefix('torch.')}
del quantized, module
else:
raise ValueError('Unknown eligible Linear state: ' + name)
# Do not keep the preceding matrix while the iterator merges/loads the
# next one. Only this group's converted output survives until saving.
del value
return converted, linears
def _merged_tensors(sources, entries):
for name, parameter in entries:
final_dtype = torch.bfloat16 if '.attn.' in name and name.startswith('transformer_blocks.') else parameter.dtype
value = sources.tensor(name, parameter.dtype, final_dtype)
try:
if value.shape != parameter.shape:
raise ValueError('Weight shape mismatch: ' + name)
yield name, value
finally:
del value
def _file_tensors(path):
with open_tensors(path) as stream:
for name in stream.keys():
value = stream.get_tensor(name)
try:
yield name, value
finally:
del value
@torch.no_grad()
def prepare_streamed(base, checkpoint, cache_root):
"""Merge and quantize one group at a time, without a second BF16 disk cache.
Sources.tensor retains the original CPU LoRA merge order and dtypes. The
final group is hashed and checkpointed before advancing; retries reuse only
verified complete groups. Original source files remain read-only here.
"""
from . import weights
from .paths import add_vdn
add_vdn()
from src.models.ops import fp8_linear as official
base, checkpoint, cache_root = map(Path, (base, checkpoint, cache_root))
started = time.monotonic()
identity = {'preparation': 'streamed_cpu_merge_official_cuda_fp8',
'implementation_sha256': sha256(__file__), 'merge_implementation_sha256': sha256(weights.__file__),
'official_fp8_sha256': sha256(official.__file__),
'base_config_sha256': sha256(base / 'transformer/config.json'),
'model_spec_sha256': sha256(checkpoint / 'model_spec.json'),
'torch': torch.__version__, 'scale_granularity': 'per_tensor' if official.per_tensor_gemm() else 'rowwise',
'min_width': official.MIN_WIDTH, 'skip_end_blocks': 0,
'sources': [{'path': str(p), 'bytes': p.stat().st_size, 'mtime_ns': p.stat().st_mtime_ns}
for p in source_paths(base, checkpoint)]}
output, identity, key = cache_location(cache_root, 'vdn-fp8-streamed-', identity)
output.mkdir(parents=True, exist_ok=True)
marker = output / 'source.json'
if marker.exists() and json.loads(marker.read_text(encoding='utf-8')) != identity:
raise ValueError('Streamed FP8 cache identity conflict')
if not marker.exists():
marker.write_text(json.dumps(identity, indent=2) + '\n', encoding='utf-8')
group_names = expected_groups(base)
# Even the meta model and source-key index are unnecessary on a full hit.
groups = sources = eligible = None
records, linears = [], {}
reused_groups = source_tensors_loaded = 0
for index, group in enumerate(group_names, 1):
destination = output / (group + '.safetensors')
destination.parent.mkdir(parents=True, exist_ok=True)
progress = destination.with_suffix('.json')
print(json.dumps({'event': 'prepare_fp8_phase', 'phase': 'verify_cache', 'group': group,
'index': index, 'total': len(group_names)}), flush=True)
saved = reusable_group(output, group)
if saved is not None:
reused_groups += 1
records.append(saved)
linears.update(saved['linears'])
print(json.dumps({'event': 'prepared_fp8_group', 'preparation': 'streamed', 'group': group,
'index': index, 'total': len(group_names), 'reused': True}), flush=True)
continue
if destination.is_file() and progress.is_file():
rejected = '.hash-rejected-' + str(time.time_ns())
destination.rename(destination.with_name(destination.name + rejected))
progress.rename(progress.with_name(progress.name + rejected))
print(json.dumps({'event': 'rebuild_fp8_group', 'group': group,
'reason': 'Corrupt group retained before rebuilding'}), flush=True)
print(json.dumps({'event': 'prepare_fp8_phase', 'phase': 'read_source_weights', 'group': group,
'index': index, 'total': len(group_names)}), flush=True)
if groups is None:
model, sources = skeleton(base, checkpoint), Sources(base, checkpoint)
params = dict(model.named_parameters())
if params.keys() != sources.weights.keys():
raise ValueError({'missing': sorted(params.keys() - sources.weights.keys()),
'unused': sorted(sources.weights.keys() - params.keys())})
eligible = {name for name, module in model.named_modules()
if isinstance(module, torch.nn.Linear)
and min(module.in_features, module.out_features) >= official.MIN_WIDTH}
groups = {}
for name, parameter in params.items():
if not parameter.is_meta:
raise ValueError('Preparation skeleton materialized a parameter: ' + name)
groups.setdefault(group_name(name), []).append((name, parameter))
if sorted(groups) != group_names:
raise ValueError('Unexpected preparation group layout')
del model, params, parameter
with closing(_merged_tensors(sources, groups[group])) as tensors:
converted, local_linears = quantize_group(tensors, eligible, official)
source_tensors_loaded += len(groups[group])
temporary = destination.with_suffix('.partial')
if temporary.exists():
# A previous interrupted group's partial is evidence, not a cache hit.
temporary.rename(temporary.with_name(temporary.name + '.retained-' + str(time.time_ns())))
save_file(converted, temporary)
tensor_count = len(converted)
del tensors, converted
gc.collect()
if destination.exists():
# Interrupted between final rename and receipt commit: keep the
# orphaned file while rebuilding this one group from pinned sources.
destination.rename(destination.with_name(destination.name + '.orphaned-' + str(time.time_ns())))
temporary.replace(destination)
row = {'group': group, 'file': group + '.safetensors', 'bytes': destination.stat().st_size,
'sha256': sha256(destination), 'tensors': tensor_count, 'linears': local_linears}
pending = progress.with_suffix('.json.partial')
pending.write_text(json.dumps(row, indent=2) + '\n', encoding='utf-8')
pending.replace(progress)
records.append(row)
linears.update(local_linears)
torch.cuda.empty_cache()
print(json.dumps({'event': 'prepared_fp8_group', 'preparation': 'streamed', 'group': group,
'index': index, 'total': len(group_names), 'reused': False,
'bytes': row['bytes'], 'linears': len(local_linears),
'seconds': time.monotonic() - started}), flush=True)
if not linears or (eligible is not None and set(linears) != eligible):
raise ValueError('Streamed FP8 cache omitted eligible Linears')
record = {'source_id': key, 'precision': 'fp8', 'scale_granularity': identity['scale_granularity'],
'groups': records, 'linears': linears, 'identity': identity,
'preparation_implementation_sha256': sha256(__file__),
'merge_preparation_implementation_sha256': sha256(weights.__file__),
'reused_groups': reused_groups, 'source_tensors_loaded': source_tensors_loaded,
'preparation_seconds': time.monotonic() - started,
'total_bytes': sum(row['bytes'] for row in records), 'bf16_intermediate_bytes': 0}
pending = output / 'manifest.partial'
pending.write_text(json.dumps(record, indent=2) + '\n', encoding='utf-8')
pending.replace(output / 'manifest.json')
print(json.dumps({'event': 'fp8_cache_ready', 'cache': str(output), 'bytes': record['total_bytes'],
'bf16_intermediate_bytes': 0, 'reused_groups': reused_groups,
'source_tensors_loaded': source_tensors_loaded}), flush=True)
return output
def install_cached_linears(model, linears, *, weight_only=False):
from src.models.ops.fp8_linear import Fp8Linear, FP8_DTYPE
from .weight_only import WeightOnlyLinear
for name, spec in linears.items():
original = model.get_submodule(name)
if not isinstance(original, torch.nn.Linear):
raise ValueError('FP8 cache target is not a Linear: ' + name)
if list(original.weight.shape) != spec['weight_shape']:
raise ValueError('FP8 cache shape differs from the official model: ' + name)
kind = WeightOnlyLinear if weight_only else Fp8Linear
replacement = kind.__new__(kind)
torch.nn.Module.__init__(replacement)
replacement.register_buffer('weight_fp8', torch.empty(spec['weight_shape'], dtype=FP8_DTYPE, device='meta'))
replacement.register_buffer('weight_scale', torch.empty(spec['scale_shape'], dtype=torch.float32, device='meta'))
replacement.original = torch.nn.Module()
replacement.original.register_parameter('bias', original.bias)
replacement.in_features = original.in_features
replacement.out_features = original.out_features
# Newly injected hybrid modules start as FP32 in the meta skeleton;
# official assembly casts them to BF16 before FP8 conversion.
input_name = spec.get('input_dtype', str(original.weight.dtype).removeprefix('torch.'))
if input_name not in ('bfloat16', 'float16', 'float32'):
raise ValueError('Unexpected FP8 input dtype in cache: ' + input_name)
replacement.input_dtype = getattr(torch, input_name)
prefix, _, child = name.rpartition('.')
setattr(model.get_submodule(prefix) if prefix else model, child, replacement)
@torch.no_grad()
def export_compact_model(model, cache_root, *, base, checkpoint):
"""Save the already assembled official FP8 model, one group at a time.
This avoids loading and quantizing the same 70 GB again after a reference
render. No arithmetic or weight conversion is performed during export.
"""
from src.models.ops import fp8_linear as official
if model.training or any(p.requires_grad for p in model.parameters()):
raise ValueError('Export requires a frozen inference model')
state = model.state_dict()
if any(name.endswith('.original.weight') for name in state):
raise ValueError('Export requires compact FP8 without the BF16 backups')
linears = {name: {'weight_shape': list(module.weight_fp8.shape),
'scale_shape': list(module.weight_scale.shape), 'input_dtype': 'bfloat16'}
for name, module in model.named_modules() if isinstance(module, official.Fp8Linear)}
if not linears:
raise ValueError('No official FP8 matrices to export')
sources = Sources(Path(base), Path(checkpoint))
identity = {'origin': 'official assembled FP8; exact registered tensors',
'implementation_sha256': sha256(__file__),
'official_fp8_sha256': sha256(official.__file__),
'base_config_sha256': sha256(Path(base) / 'transformer/config.json'),
'model_spec_sha256': sha256(Path(checkpoint) / 'model_spec.json'),
'torch': torch.__version__, 'scale_granularity': 'per_tensor' if official.per_tensor_gemm() else 'rowwise',
'linears': linears,
'sources': [{'path': str(p), 'bytes': p.stat().st_size, 'mtime_ns': p.stat().st_mtime_ns}
for p in sorted(sources.paths)]}
key = hashlib.sha256(json.dumps(identity, sort_keys=True).encode()).hexdigest()
output = Path(cache_root) / ('vdn-fp8-official-' + key[:16])
output.mkdir(parents=True, exist_ok=True)
marker = output / 'source.json'
if marker.exists() and json.loads(marker.read_text(encoding='utf-8')) != identity:
raise ValueError('Export cache identity mismatch')
marker.write_text(json.dumps(identity, indent=2) + '\n', encoding='utf-8')
groups = {}
for name, tensor in state.items():
groups.setdefault(group_name(name), {})[name] = tensor
records = []
started = time.perf_counter()
for group, tensors in sorted(groups.items()):
destination = output / (group + '.safetensors')
destination.parent.mkdir(parents=True, exist_ok=True)
progress = destination.with_suffix('.json')
record = json.loads(progress.read_text(encoding='utf-8')) if progress.exists() else None
if record and destination.is_file() and destination.stat().st_size == record['bytes'] and sha256(destination) == record['sha256']:
records.append(record)
continue
cpu = {name: tensor.detach().cpu().contiguous() for name, tensor in tensors.items()}
temporary = destination.with_suffix('.partial')
save_file(cpu, temporary)
temporary.replace(destination)
del cpu
record = {'group': group, 'file': group + '.safetensors', 'bytes': destination.stat().st_size,
'sha256': sha256(destination), 'tensors': len(tensors)}
progress.write_text(json.dumps(record) + '\n', encoding='utf-8')
records.append(record)
print(json.dumps({'event': 'exported_official_fp8_group', 'group': group,
'seconds': time.perf_counter() - started}), flush=True)
manifest = {'source_id': key, 'precision': 'fp8', 'scale_granularity': identity['scale_granularity'],
'groups': records, 'linears': linears, 'total_bytes': sum(r['bytes'] for r in records),
'preparation_seconds': time.perf_counter() - started, 'identity': identity}
temporary = output / 'manifest.partial'
temporary.write_text(json.dumps(manifest, indent=2) + '\n', encoding='utf-8')
temporary.replace(output / 'manifest.json')
return output
@torch.no_grad()
def prepare(source, cache_root, *, base, checkpoint):
from .paths import add_vdn
add_vdn()
from src.models.ops import fp8_linear as official
source, cache_root = Path(source), Path(cache_root)
manifest = json.loads((source / 'manifest.json').read_text(encoding='utf-8'))
if manifest.get('precision', 'bf16') != 'bf16':
raise ValueError('FP8 preparation requires the verified BF16 merged cache')
per_tensor = official.per_tensor_gemm()
identity = {'source_id': manifest['source_id'],
'source_manifest_sha256': sha256(source / 'manifest.json'),
'implementation_sha256': sha256(__file__),
'official_fp8_sha256': sha256(official.__file__),
'torch': torch.__version__, 'scale_granularity': 'per_tensor' if per_tensor else 'rowwise',
'min_width': official.MIN_WIDTH, 'skip_end_blocks': 0,
'memory_policy': 'discard BF16 backup after official CUDA quantization; immutable per-group files'}
output, identity, key = cache_location(cache_root, 'vdn-fp8-', identity)
output.mkdir(parents=True, exist_ok=True)
marker = output / 'source.json'
if marker.exists() and json.loads(marker.read_text(encoding='utf-8')) != identity:
raise ValueError('FP8 cache identity conflict')
marker.write_text(json.dumps(identity, indent=2) + '\n', encoding='utf-8')
eligible = None
linears, groups = {}, []
started = time.perf_counter()
if sorted(g['group'] for g in manifest['groups']) != expected_groups(Path(base)):
raise ValueError('Unexpected BF16 preparation group layout')
for index, group in enumerate(manifest['groups'], 1):
if group['file'] != group['group'] + '.safetensors':
raise ValueError('Unexpected source group path')
destination = output / group['file']
destination.parent.mkdir(parents=True, exist_ok=True)
progress = destination.with_suffix('.json')
saved = reusable_group(output, group['group'])
if saved is not None:
groups.append(saved)
linears.update(saved['linears'])
print(json.dumps({'event': 'prepared_fp8_group', 'group': group['group'], 'index': index,
'total': len(manifest['groups']), 'reused': True}), flush=True)
continue
if destination.is_file() and progress.is_file():
rejected = '.hash-rejected-' + str(time.time_ns())
destination.rename(destination.with_name(destination.name + rejected))
progress.rename(progress.with_name(progress.name + rejected))
path = source / group['file']
if path.stat().st_size != group['bytes'] or sha256(path) != group['sha256']:
raise ValueError('Source cache failed integrity check: ' + str(path))
if eligible is None:
model = skeleton(Path(base), Path(checkpoint))
if any(not parameter.is_meta for parameter in model.parameters()):
raise ValueError('Preparation skeleton materialized parameters')
eligible = {name for name, module in model.named_modules()
if isinstance(module, torch.nn.Linear)
and min(module.in_features, module.out_features) >= official.MIN_WIDTH}
del model
with closing(_file_tensors(path)) as values:
converted, local_linears = quantize_group(values, eligible, official)
temporary = destination.with_suffix('.partial')
if temporary.exists():
temporary.rename(temporary.with_name(temporary.name + '.retained-' + str(time.time_ns())))
if not local_linears:
# AdaLN's input is below the official FP8 threshold. Preserve its
# immutable BF16 files without making another 26 GB disk copy.
os.link(path, temporary)
else:
save_file(converted, temporary)
if destination.exists():
destination.rename(destination.with_name(destination.name + '.orphaned-' + str(time.time_ns())))
temporary.replace(destination)
saved = {'group': group['group'], 'file': group['file'],
'bytes': destination.stat().st_size, 'sha256': sha256(destination),
'tensors': len(converted), 'linears': local_linears}
progress.write_text(json.dumps(saved, indent=2) + '\n', encoding='utf-8')
groups.append(saved)
linears.update(local_linears)
del values, converted
gc.collect()
torch.cuda.empty_cache()
print(json.dumps({'event': 'prepared_fp8_group', 'group': group['group'],
'bytes': saved['bytes'], 'linears': len(local_linears),
'seconds': time.perf_counter() - started}), flush=True)
if not linears or eligible is not None and set(linears) != eligible:
raise ValueError('FP8 cache omitted eligible Linears')
record = {'source_id': key, 'bf16_source_id': manifest['source_id'], 'precision': 'fp8',
'scale_granularity': identity['scale_granularity'], 'groups': groups, 'linears': linears,
'preparation_implementation_sha256': sha256(__file__),
'preparation_seconds': time.perf_counter() - started,
'total_bytes': sum(group['bytes'] for group in groups), 'identity': identity}
(output / 'manifest.json').write_text(json.dumps(record, indent=2) + '\n', encoding='utf-8')
print(json.dumps({'event': 'fp8_cache_ready', 'cache': str(output), 'linears': len(linears),
'bytes': record['total_bytes']}), flush=True)
return output
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--source', required=True)
parser.add_argument('--cache-root', required=True)
parser.add_argument('--base', required=True)
parser.add_argument('--checkpoint', required=True)
args = parser.parse_args()
torch.set_num_threads(8)
prepare(args.source, args.cache_root, base=args.base, checkpoint=args.checkpoint)
if __name__ == '__main__':
main()