Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

feat(merge): add --low-memory to merge a LoRA adapter tensor by tensor (#2006)

Kuanysh3ed0d7cc2c94
diff --git a/changelog.d/0.76.0/2006.added.md b/changelog.d/0.76.0/2006.added.md
new file mode 100644
index 00000000..366d7b44
--- /dev/null
+++ b/changelog.d/0.76.0/2006.added.md
@@ -0,0 +1,5 @@
+- Add `soup merge --low-memory`, which merges a plain LoRA adapter into a safetensors base one
+ tensor at a time instead of loading the whole model into RAM. The tensor values are identical to
+ the regular merge for float32, float16 and bfloat16; adapters it cannot reproduce exactly
+ (DoRA, `modules_to_save`, rank or alpha patterns, embedding adapters) are refused with a
+ pointer back to the regular merge. (#2006)
diff --git a/docs/cli-reference.md b/docs/cli-reference.md
index c7f8075f..cf958861 100644
--- a/docs/cli-reference.md
+++ b/docs/cli-reference.md
@@ -2808,6 +2808,7 @@ Usage: `soup merge [OPTIONS]`
| `--trust-remote-code` | flag | off | | Allow loading models that ship custom Python via auto_map. Default deny (v0.36.0). Only enable if you trust the source. |
| `--save-format` | text | `fp16` | | Merged-checkpoint save format. fp16 (default) writes a standard fp16 merge. 4bit / 4bit_forced write a single BNB-4bit-quantized merge without the dequant-merge-requant cycle (v0.53.1 #142). |
| `--double-quant` / `--no-double-quant` | flag | on | | BNB double-quantization for the 4bit / 4bit_forced save formats (#321). Default on, matching the training loaders; pass --no-double-quant to disable. Ignored for the fp16 save format. |
+| `--low-memory` | flag | off | | Merge tensor by tensor straight from the base safetensors files instead of loading the whole model into RAM. Needs the fp16 save format and a plain LoRA adapter. |
| `--hub` | text | `hf` | | Source hub for the base model: hf (default) / modelscope / modelers. Non-HF hubs require the matching SDK (v0.53.10 #152). |
## `soup merge-sharded-fsdp-weights`
diff --git a/docs/commands.md b/docs/commands.md
index 93890b6b..7ef2983b 100644
--- a/docs/commands.md
+++ b/docs/commands.md
@@ -36,6 +36,7 @@ soup infer|chat|diff ... --device cpu|cuda|cuda:N|mps Pick where the model load
soup push --model ./output --repo user/name Upload to HuggingFace
soup push --model ./output --repo user/name --collection user/coll-abc123 Add to HF Collection
soup merge --adapter ./output Merge LoRA with base model
+soup merge --adapter ./output --low-memory Merge tensor by tensor without loading the whole model into RAM
soup merge --adapter ./output --save-format 4bit --no-double-quant Save a BNB-4bit merge with double-quantization disabled (default on; #321)
soup merge-sharded-fsdp-weights ./shards -o merged.safetensors Consolidate FSDP shards into one safetensors (v0.71.14; --plan-only previews)
soup delinearize-llama4 ./src --target ./out [--num-experts N] [--plan-only] Live Llama-4 fused-expert reshape [E*din,dout] -> [E,din,dout] + sidecar copy (v0.71.21)
diff --git a/docs/serving-and-export.md b/docs/serving-and-export.md
index a6693437..fe59751c 100644
--- a/docs/serving-and-export.md
+++ b/docs/serving-and-export.md
@@ -38,8 +38,30 @@ soup merge --adapter ./output --output ./merged
# Specify base model and dtype
soup merge --adapter ./output --base meta-llama/Llama-3.1-8B --dtype bfloat16
+
+# Merge tensor by tensor without loading the model into RAM
+soup merge --adapter ./output --output ./merged --low-memory
```
+`--low-memory` reads the base `.safetensors` files and writes the merged files
+one tensor at a time, so peak RAM is about one shard (the largest `.safetensors`
+file; a single-file model is memory-mapped as a whole, so it saves little). The
+tensor values are the same as the regular merge, but the output keeps the base's
+shard layout instead of a single `model.safetensors`. It needs the default
+`fp16` save format and a plain LoRA adapter; DoRA, `modules_to_save`, rank or
+alpha patterns, adapters on embeddings or `lm_head`, and base weights stored in a
+quantized dtype are refused with a message to run the regular merge. The merge is
+written to a staging directory next to `--output` and moved into place only when
+complete; an existing non-empty `--output` is not overwritten, and base-model
+files that are symlinks pointing outside the model directory (other than a
+Hugging Face cache snapshot) are refused. Only tokenizer and config files
+(`.json`, `.model`, `.txt`, `.tiktoken`, `.jinja`) are copied from the base, plus
+`.py` files when `--trust-remote-code` is given. Adapter values that cannot be
+used safely (a non-positive `r`, a non-finite `lora_alpha`, non-boolean flags,
+mismatched LoRA shapes) are refused before anything is written.
+Measured on a 1.4 GB Llama-shaped model: 1635 MiB of working memory for the
+regular merge against about 190 MiB with `--low-memory`.
+
ReLoRA training outputs are already standalone dense models: Soup merges the
final active adapter into the accumulated in-memory base before writing the
training output. Load or export that directory directly; `soup merge` is only
diff --git a/src/soup_cli/commands/merge.py b/src/soup_cli/commands/merge.py
index 808f017f..04c0ef7c 100644
--- a/src/soup_cli/commands/merge.py
+++ b/src/soup_cli/commands/merge.py
@@ -64,6 +64,15 @@ def merge(
"--no-double-quant to disable. Ignored for the fp16 save format."
),
),
+ low_memory: bool = typer.Option(
+ False,
+ "--low-memory",
+ help=(
+ "Merge tensor by tensor straight from the base safetensors files "
+ "instead of loading the whole model into RAM. Needs the fp16 save "
+ "format and a plain LoRA adapter."
+ ),
+ ),
hub: str = typer.Option(
"hf",
"--hub",
@@ -107,6 +116,10 @@ def merge(
raise typer.Exit(2)
adapter_path = Path(adapter)
+ if low_memory and save_format_canonical != "fp16":
+ console.print("[red]--low-memory only supports --save-format fp16[/]")
+ raise typer.Exit(2)
+
# --- Validate adapter ---
if not adapter_path.exists():
console.print(f"[red]Adapter path not found: {adapter_path}[/]")
@@ -148,6 +161,11 @@ def merge(
)
)
+ if low_memory:
+ _merge_low_memory(adapter_path, base, output_path, dtype, trust_remote_code)
+ _report_complete(output_path)
+ return
+
# --- Merge ---
try:
import torch
@@ -242,7 +260,77 @@ def merge(
console.print(f"[red]Merge failed: {for_terminal(exc)}[/]")
raise typer.Exit(1)
- # Calculate output size
+ _report_complete(output_path)
+
+
+def _merge_low_memory(
+ adapter_path: Path,
+ base: str,
+ output_path: Path,
+ dtype: str,
+ trust_remote_code: bool,
+) -> None:
+ try:
+ from transformers import AutoTokenizer
+
+ from soup_cli.utils.lowmem_merge import LowMemoryMergeError, merge_low_memory
+ from soup_cli.utils.trust_remote import (
+ model_requires_trust_remote_code,
+ resolve_trust_remote_code,
+ )
+
+ trc = resolve_trust_remote_code(
+ base,
+ requested=trust_remote_code,
+ console=console,
+ requires_remote_code=model_requires_trust_remote_code(str(adapter_path)) or False,
+ )
+
+ if Path(base).is_dir():
+ base_dir = Path(base)
+ else:
+ from soup_cli.utils.hubs import snapshot_download
+
+ console.print(f"[dim]Fetching base weights: {for_terminal(base)}...[/]")
+ base_dir = Path(
+ snapshot_download(
+ base,
+ cache_dir=None,
+ allow_patterns=["*.safetensors", "*.json"],
+ )
+ )
+
+ def save_tokenizer(staging: Path) -> None:
+ console.print("[dim]Saving tokenizer...[/]")
+ AutoTokenizer.from_pretrained(
+ str(adapter_path), trust_remote_code=trc
+ ).save_pretrained(str(staging))
+
+ merge_low_memory(
+ base_dir,
+ adapter_path,
+ output_path,
+ dtype,
+ progress=lambda message: console.print(f"[dim]{for_terminal(message)}[/]"),
+ finalize=save_tokenizer,
+ trust_remote_code=trc,
+ )
+ except LowMemoryMergeError as exc:
+ console.print(
+ f"[red]Low-memory merge not possible: {for_terminal(exc)}[/]\n"
+ "Run again without [bold]--low-memory[/]."
+ )
+ raise typer.Exit(1)
+ except ImportError as exc:
+ console.print(f"[red]Missing dependency: {exc}[/]")
+ console.print("Run: [bold]pip install torch transformers safetensors[/]")
+ raise typer.Exit(1)
+ except Exception as exc:
+ console.print(f"[red]Merge failed: {for_terminal(exc)}[/]")
+ raise typer.Exit(1)
+
+
+def _report_complete(output_path: Path) -> None:
total_size = sum(f.stat().st_size for f in output_path.rglob("*") if f.is_file())
size_str = _format_size(total_size)
diff --git a/src/soup_cli/utils/lowmem_merge.py b/src/soup_cli/utils/lowmem_merge.py
new file mode 100644
index 00000000..c0b89d59
--- /dev/null
+++ b/src/soup_cli/utils/lowmem_merge.py
@@ -0,0 +1,455 @@
+"""Merge a LoRA adapter into a base checkpoint one tensor at a time."""
+
+import json
+import math
+import os
+import re
+import shutil
+import struct
+import tempfile
+from pathlib import Path
+from typing import Any, Callable, Dict, List, Optional, Tuple
+
+_ADAPTER_PREFIX = "base_model.model."
+_LORA_A = ".lora_A.weight"
+_LORA_B = ".lora_B.weight"
+_ADAPTER_WEIGHTS = "adapter_model.safetensors"
+_INDEX_NAME = "model.safetensors.index.json"
+_SINGLE_NAME = "model.safetensors"
+_SAFETENSORS_DTYPE = {"float16": "F16", "bfloat16": "BF16", "float32": "F32"}
+_FLOAT_DTYPES = frozenset({"F16", "BF16", "F32", "F64"})
+_ITEMSIZE = {
+ "F64": 8,
+ "F32": 4,
+ "F16": 2,
+ "BF16": 2,
+ "F8_E4M3": 1,
+ "F8_E5M2": 1,
+ "I64": 8,
+ "U64": 8,
+ "I32": 4,
+ "U32": 4,
+ "I16": 2,
+ "U16": 2,
+ "I8": 1,
+ "U8": 1,
+ "BOOL": 1,
+}
+_UNSUPPORTED_CONFIG = (
+ "use_dora",
+ "lora_bias",
+ "modules_to_save",
+ "trainable_token_indices",
+ "rank_pattern",
+ "alpha_pattern",
+ "loftq_config",
+ "layer_replication",
+ "target_parameters",
+ "alora_invocation_tokens",
+ "use_qalora",
+)
+_COPIED_SUFFIXES = (".json", ".model", ".txt", ".tiktoken", ".jinja")
+_SHARD_NAME = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]*\.safetensors")
+_WINDOWS_DEVICES = frozenset(
+ {"CON", "PRN", "AUX", "NUL"}
+ | {f"COM{number}" for number in range(1, 10)}
+ | {f"LPT{number}" for number in range(1, 10)}
+)
+
+Progress = Optional[Callable[[str], None]]
+Finalize = Optional[Callable[[Path], None]]
+
+
+class LowMemoryMergeError(ValueError):
+ """The adapter or checkpoint needs the regular merge path."""
+
+
+def _torch_dtype(name: str) -> Any:
+ import torch
+
+ return {
+ "float16": torch.float16,
+ "bfloat16": torch.bfloat16,
+ "float32": torch.float32,
+ }[name]
+
+
+def load_adapter(adapter_dir: Path) -> Tuple[Dict[str, Any], Dict[str, Tuple[Any, Any]]]:
+ from safetensors.torch import load_file
+
+ config_path = adapter_dir / "adapter_config.json"
+ weights_path = adapter_dir / _ADAPTER_WEIGHTS
+ if not weights_path.is_file():
+ raise LowMemoryMergeError(f"{_ADAPTER_WEIGHTS} not found in {adapter_dir}")
+ with open(config_path, encoding="utf-8") as handle:
+ config = json.load(handle)
+ if config.get("peft_type", "LORA") != "LORA":
+ raise LowMemoryMergeError(
+ f"adapter_config.json has peft_type {config['peft_type']!r}; only LORA is supported"
+ )
+ if config.get("bias", "none") != "none":
+ raise LowMemoryMergeError(
+ f"adapter_config.json sets bias={config['bias']!r}, "
+ "which the low-memory merge does not support"
+ )
+ for field in _UNSUPPORTED_CONFIG:
+ if config.get(field):
+ raise LowMemoryMergeError(
+ f"adapter_config.json sets {field!r}, which the low-memory merge does not support"
+ )
+
+ lora_scaling(config)
+ _flag(config, "fan_in_fan_out")
+
+ half_a: Dict[str, Any] = {}
+ half_b: Dict[str, Any] = {}
+ for key, tensor in load_file(str(weights_path)).items():
+ name = key[len(_ADAPTER_PREFIX):] if key.startswith(_ADAPTER_PREFIX) else key
+ if name.endswith(_LORA_A):
+ half_a[name[: -len(_LORA_A)] + ".weight"] = tensor
+ elif name.endswith(_LORA_B):
+ half_b[name[: -len(_LORA_B)] + ".weight"] = tensor
+ elif "lm_head" in key or "embed" in key:
+ raise LowMemoryMergeError(
+ f"adapter tensor {key!r}: adapters on embeddings or lm_head are not supported"
+ )
+ else:
+ raise LowMemoryMergeError(
+ f"adapter tensor {key!r} is not a plain LoRA A/B weight"
+ )
+ if half_a.keys() != half_b.keys():
+ raise LowMemoryMergeError("adapter has unpaired lora_A / lora_B tensors")
+ return config, {name: (half_a[name], half_b[name]) for name in half_a}
+
+
+def _flag(config: Dict[str, Any], field: str) -> bool:
+ value = config.get(field, False)
+ if not isinstance(value, bool):
+ raise LowMemoryMergeError(
+ f"adapter_config.json field {field!r} must be true or false, got {value!r}"
+ )
+ return value
+
+
+def lora_scaling(config: Dict[str, Any]) -> float:
+ rank = config.get("r")
+ if isinstance(rank, bool) or not isinstance(rank, int) or rank <= 0:
+ raise LowMemoryMergeError(
+ f"adapter_config.json field 'r' must be a positive integer, got {rank!r}"
+ )
+ alpha = config.get("lora_alpha")
+ if isinstance(alpha, bool) or not isinstance(alpha, (int, float)) or not math.isfinite(alpha):
+ raise LowMemoryMergeError(
+ f"adapter_config.json field 'lora_alpha' must be a finite number, got {alpha!r}"
+ )
+ if _flag(config, "use_rslora"):
+ return float(alpha) / math.sqrt(rank)
+ return float(alpha) / rank
+
+
+def _check_delta_shape(
+ lora_a_shape: Any, lora_b_shape: Any, weight_shape: Any, fan_in_fan_out: bool, key: str = ""
+) -> None:
+ """Compare shapes before any matmul, so a hostile adapter cannot force a huge delta."""
+ label = f" for {key!r}" if key else ""
+ if len(lora_a_shape) != 2 or len(lora_b_shape) != 2:
+ raise LowMemoryMergeError(f"LoRA A/B tensors must be 2-D{label}")
+ if lora_b_shape[1] != lora_a_shape[0]:
+ raise LowMemoryMergeError(
+ f"LoRA rank mismatch{label}: lora_B {tuple(lora_b_shape)}, lora_A {tuple(lora_a_shape)}"
+ )
+ delta = (lora_b_shape[0], lora_a_shape[1])
+ if fan_in_fan_out:
+ delta = delta[::-1]
+ if delta != tuple(weight_shape):
+ raise LowMemoryMergeError(
+ f"LoRA delta {delta} does not match weight {tuple(weight_shape)}{label}"
+ )
+
+
+def merged_tensor(
+ weight: Any,
+ lora_a: Any,
+ lora_b: Any,
+ *,
+ scaling: float,
+ fan_in_fan_out: bool,
+ dtype: Any,
+) -> Any:
+ _check_delta_shape(tuple(lora_a.shape), tuple(lora_b.shape), tuple(weight.shape),
+ fan_in_fan_out)
+ delta = lora_b.float() @ lora_a.float()
+ if fan_in_fan_out:
+ delta = delta.T
+ delta = delta * scaling
+ base = weight.to(dtype)
+ return (base.float() + delta).to(dtype)
+
+
+def _inside(path: Any, root: str) -> bool:
+ real_root = os.path.realpath(root)
+ try:
+ return os.path.commonpath([os.path.realpath(path), real_root]) == real_root
+ except ValueError:
+ return False
+
+
+def _source_roots(base_dir: Path) -> List[str]:
+ """Directories a base file may resolve into: the model dir and its HF cache blobs."""
+ roots = [os.path.realpath(base_dir)]
+ absolute = Path(os.path.abspath(base_dir))
+ if absolute.parent.name == "snapshots":
+ roots.append(os.path.realpath(absolute.parent.parent / "blobs"))
+ return roots
+
+
+def _checked_source(base_dir: Path, name: str, roots: List[str]) -> Path:
+ path = base_dir / name
+ if not any(_inside(path, root) for root in roots):
+ raise LowMemoryMergeError(f"{name!r} resolves outside the base model directory")
+ if not path.is_file():
+ raise LowMemoryMergeError(f"{name!r} not found in {base_dir}")
+ return path
+
+
+def _checked_shard_name(name: Any) -> str:
+ if (
+ not isinstance(name, str)
+ or _SHARD_NAME.fullmatch(name) is None
+ or name.split(".")[0].upper() in _WINDOWS_DEVICES
+ ):
+ raise LowMemoryMergeError(
+ f"{_INDEX_NAME} lists {name!r}, which is not a plain .safetensors file name"
+ )
+ return name
+
+
+def _weight_files(
+ base_dir: Path, roots: List[str]
+) -> Tuple[List[str], Optional[Dict[str, Any]]]:
+ index_path = base_dir / _INDEX_NAME
+ if index_path.is_file():
+ _checked_source(base_dir, _INDEX_NAME, roots)
+ with open(index_path, encoding="utf-8") as handle:
+ index = json.load(handle)
+ weight_map = index.get("weight_map") if isinstance(index, dict) else None
+ if not isinstance(weight_map, dict) or not weight_map:
+ raise LowMemoryMergeError(f"{_INDEX_NAME} has no weight_map")
+ names = sorted({_checked_shard_name(name) for name in weight_map.values()})
+ if len({name.lower() for name in names}) != len(names):
+ raise LowMemoryMergeError(f"{_INDEX_NAME} lists shard names that differ only by case")
+ return names, index
+ if (base_dir / _SINGLE_NAME).is_file():
+ return [_SINGLE_NAME], None
+ raise LowMemoryMergeError(f"no .safetensors weights found in {base_dir}")
+
+
+def _preflight(
+ sources: Dict[str, Path],
+ adapter: Dict[str, Tuple[Any, Any]],
+ config: Dict[str, Any],
+) -> None:
+ """Check the adapter against the base tensors before anything is written."""
+ from safetensors import safe_open
+
+ fan_in_fan_out = _flag(config, "fan_in_fan_out")
+ found: Dict[str, Tuple[str, List[int]]] = {}
+ for path in sources.values():
+ with safe_open(str(path), framework="pt") as source:
+ for key in source.keys():
+ piece = source.get_slice(key)
+ found[key] = (piece.get_dtype(), list(piece.get_shape()))
+
+ missing = sorted(set(adapter) - set(found))
+ if missing:
+ raise LowMemoryMergeError(
+ f"{len(missing)} adapter weights match no base tensor, e.g. {missing[0]!r}"
+ )
+ for key, (lora_a, lora_b) in adapter.items():
+ stored, shape = found[key]
+ if stored not in _FLOAT_DTYPES:
+ raise LowMemoryMergeError(
+ f"base tensor {key!r} is stored as {stored}, not a float type "
+ "(quantized weights cannot take a LoRA delta here)"
+ )
+ _check_delta_shape(
+ tuple(lora_a.shape), tuple(lora_b.shape), tuple(shape), fan_in_fan_out, key
+ )
+
+
+def _write_shard(
+ source_path: Path,
+ target_path: Path,
+ out_dtype: str,
+ adapter: Dict[str, Tuple[Any, Any]],
+ config: Dict[str, Any],
+) -> int:
+ import torch
+ from safetensors import safe_open
+
+ scaling = lora_scaling(config)
+ fan_in_fan_out = _flag(config, "fan_in_fan_out")
+ torch_dtype = _torch_dtype(out_dtype)
+ header: Dict[str, Any] = {"__metadata__": {"format": "pt"}}
+ offset = 0
+ order: List[str] = []
+ sizes: Dict[str, int] = {}
+ floats: set = set()
+ with safe_open(str(source_path), framework="pt") as source:
+ for key in sorted(source.keys()):
+ piece = source.get_slice(key)
+ stored = piece.get_dtype()
+ target = _SAFETENSORS_DTYPE[out_dtype] if stored in _FLOAT_DTYPES else stored
+ if target not in _ITEMSIZE:
+ raise LowMemoryMergeError(f"unsupported tensor dtype {target!r}")
+ shape = list(piece.get_shape())
+ size = math.prod(shape) * _ITEMSIZE[target]
+ if stored in _FLOAT_DTYPES:
+ floats.add(key)
+ elif key in adapter:
+ raise LowMemoryMergeError(
+ f"base tensor {key!r} is stored as {stored}, not a float type"
+ )
+ header[key] = {
+ "dtype": target,
+ "shape": shape,
+ "data_offsets": [offset, offset + size],
+ }
+ offset += size
+ sizes[key] = size
+ order.append(key)
+
+ encoded = json.dumps(header, separators=(",", ":")).encode("utf-8")
+ encoded += b" " * (-len(encoded) % 8)
+ with open(target_path, "xb") as sink:
+ sink.write(struct.pack("<Q", len(encoded)))
+ sink.write(encoded)
+ for key in order:
+ tensor = source.get_tensor(key)
+ if key in adapter:
+ lora_a, lora_b = adapter[key]
+ tensor = merged_tensor(
+ tensor,
+ lora_a,
+ lora_b,
+ scaling=scaling,
+ fan_in_fan_out=fan_in_fan_out,
+ dtype=torch_dtype,
+ )
+ elif key in floats:
+ tensor = tensor.to(torch_dtype)
+ flat = tensor.contiguous().reshape(-1).view(torch.uint8)
+ if flat.numel() != sizes[key]:
+ raise LowMemoryMergeError(
+ f"tensor {key!r} is {flat.numel()} bytes, its header declares {sizes[key]}"
+ )
+ sink.write(memoryview(flat.numpy()))
+ return offset
+
+
+def _reject_overlap(directories: Dict[str, Path]) -> None:
+ real = {label: os.path.realpath(path) for label, path in directories.items()}
+ labels = list(real)
+ for position, first in enumerate(labels):
+ for second in labels[position + 1:]:
+ try:
+ shared = os.path.commonpath([real[first], real[second]])
+ except ValueError:
+ continue
+ if shared in (real[first], real[second]):
+ raise ValueError(f"{first} and {second} directories must not overlap")
+
+
+def merge_low_memory(
+ base_dir: Path,
+ adapter_dir: Path,
+ output_dir: Path,
+ dtype: str,
+ progress: Progress = None,
+ finalize: Finalize = None,
+ trust_remote_code: bool = False,
+) -> int:
+ """Write the merged checkpoint to ``output_dir``; return its weight bytes.
+
+ Everything is written to a fresh staging directory next to ``output_dir`` and
+ moved into place only when complete. ``finalize`` runs on the staging
+ directory first, so a failure there leaves no output either. An existing,
+ non-empty ``output_dir`` is never overwritten. ``.py`` files are copied from the
+ base only when ``trust_remote_code`` is set.
+ """
+ if dtype not in _SAFETENSORS_DTYPE:
+ raise ValueError(f"unsupported dtype {dtype!r}")
+ if os.path.realpath(base_dir) == os.path.realpath(output_dir):
+ raise ValueError("output directory must differ from the base model directory")
+ _reject_overlap({"base": base_dir, "adapter": adapter_dir, "output": output_dir})
+ if os.path.islink(output_dir):
+ raise ValueError("output directory must not be a symbolic link")
+ if output_dir.exists() and (not output_dir.is_dir() or any(output_dir.iterdir())):
+ raise FileExistsError(f"output directory {output_dir} already exists and is not empty")
+
+ config, adapter = load_adapter(adapter_dir)
+ roots = _source_roots(base_dir)
+ files, index = _weight_files(base_dir, roots)
+ sources = {name: _checked_source(base_dir, name, roots) for name in files}
+ support = _support_files(base_dir, roots, trust_remote_code)
+ _preflight(sources, adapter, config)
+
+ output_dir.parent.mkdir(parents=True, exist_ok=True)
+ staging = Path(tempfile.mkdtemp(prefix=f".{output_dir.name}.", dir=output_dir.parent))
+ published = False
+ try:
+ total = 0
+ for position, name in enumerate(files, start=1):
+ if progress is not None:
+ progress(f"Merging {name} ({position}/{len(files)})...")
+ total += _write_shard(sources[name], staging / name, dtype, adapter, config)
+
+ if index is not None:
+ index.setdefault("metadata", {})["total_size"] = total
+ with open(staging / _INDEX_NAME, "w", encoding="utf-8") as handle:
+ json.dump(index, handle, indent=2)
+
+ _copy_support_files(support, staging, dtype)
+ if finalize is not None:
+ finalize(staging)
+
+ mask = os.umask(0)
+ os.umask(mask)
+ os.chmod(staging, 0o777 & ~mask)
+ if output_dir.exists():
+ output_dir.rmdir()
+ os.rename(staging, output_dir)
+ published = True
+ return total
+ finally:
+ if not published:
+ shutil.rmtree(staging, ignore_errors=True)
+
+
+def _support_files(base_dir: Path, roots: List[str], trust_remote_code: bool) -> List[Path]:
+ suffixes = _COPIED_SUFFIXES + ((".py",) if trust_remote_code else ())
+ found = []
+ for entry in sorted(base_dir.iterdir()):
+ if entry.name == _INDEX_NAME or not entry.name.endswith(suffixes):
+ continue
+ if not entry.is_file():
+ continue
+ if not any(_inside(entry, root) for root in roots):
+ raise LowMemoryMergeError(f"{entry.name!r} resolves outside the base model directory")
+ found.append(entry)
+ return found
+
+
+def _copy_support_files(files: List[Path], staging: Path, dtype: str) -> None:
+ for entry in files:
+ shutil.copyfile(entry, staging / entry.name)
+
+ config_path = staging / "config.json"
+ if config_path.is_file():
+ with open(config_path, encoding="utf-8") as handle:
+ model_config = json.load(handle)
+ for field in ("torch_dtype", "dtype"):
+ if field in model_config:
+ model_config[field] = dtype
+ with open(config_path, "w", encoding="utf-8") as handle:
+ json.dump(model_config, handle, indent=2)
diff --git a/tests/test_lowmem_merge.py b/tests/test_lowmem_merge.py
new file mode 100644
index 00000000..5ec38278
--- /dev/null
+++ b/tests/test_lowmem_merge.py
@@ -0,0 +1,866 @@
+"""`soup merge --low-memory` must produce exactly what the regular merge produces."""
+
+import json
+import shutil
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+
+from tests.conftest import strip_ansi
+
+torch = pytest.importorskip("torch")
+pytest.importorskip("peft")
+pytest.importorskip("transformers")
+safetensors_torch = pytest.importorskip("safetensors.torch")
+
+
+def _tiny_base(directory: Path, shard_size: str = "40KB", tie: bool = False) -> Path:
+ from transformers import LlamaConfig, LlamaForCausalLM
+
+ torch.manual_seed(0)
+ config = LlamaConfig(
+ vocab_size=128,
+ hidden_size=32,
+ intermediate_size=64,
+ num_hidden_layers=2,
+ num_attention_heads=4,
+ num_key_value_heads=4,
+ max_position_embeddings=64,
+ tie_word_embeddings=tie,
+ )
+ LlamaForCausalLM(config).to(torch.float32).save_pretrained(
+ directory, max_shard_size=shard_size
+ )
+ return directory
+
+
+def _tiny_adapter(base: Path, directory: Path, **lora_kwargs) -> Path:
+ from peft import LoraConfig, get_peft_model
+ from transformers import AutoModelForCausalLM
+
+ model = AutoModelForCausalLM.from_pretrained(base, torch_dtype=torch.float32)
+ options = dict(r=4, lora_alpha=8, target_modules=["q_proj", "v_proj", "down_proj"])
+ options.update(lora_kwargs)
+ peft_model = get_peft_model(model, LoraConfig(**options))
+ torch.manual_seed(1)
+ for name, parameter in peft_model.named_parameters():
+ if "lora_" in name:
+ parameter.data = torch.randn_like(parameter) * 0.05
+ peft_model.save_pretrained(directory)
+ return directory
+
+
+def _regular_merge(base: Path, adapter: Path, dtype: torch.dtype) -> dict:
+ from peft import PeftModel
+ from transformers import AutoModelForCausalLM
+
+ model = AutoModelForCausalLM.from_pretrained(base, torch_dtype=dtype, device_map="cpu")
+ merged = PeftModel.from_pretrained(model, str(adapter)).merge_and_unload()
+ return {key: value.detach().clone() for key, value in merged.state_dict().items()}
+
+
+def _read_all(directory: Path) -> dict:
+ tensors = {}
+ for path in sorted(directory.glob("*.safetensors")):
+ tensors.update(safetensors_torch.load_file(str(path)))
+ return tensors
+
+
+def checkpoints(tmp_path):
+ base = _tiny_base(tmp_path / "base")
+ adapter = _tiny_adapter(base, tmp_path / "adapter")
+ return base, adapter, tmp_path / "merged"
+
+
+ "name, dtype",
+ [("float32", torch.float32), ("float16", torch.float16), ("bfloat16", torch.bfloat16)],
+)
+def test_matches_regular_merge_bit_for_bit(checkpoints, name, dtype):
+ from soup_cli.utils.lowmem_merge import merge_low_memory
+
+ base, adapter, output = checkpoints
+ expected = _regular_merge(base, adapter, dtype)
+
+ merge_low_memory(base, adapter, output, name)
+
+ actual = _read_all(output)
+ assert actual.keys() == expected.keys()
+ for key, tensor in expected.items():
+ assert actual[key].dtype == tensor.dtype, key
+ assert torch.equal(actual[key], tensor), key
+
+
+def test_checkpoint_is_sharded_and_loadable(checkpoints):
+ from transformers import AutoModelForCausalLM
+
+ from soup_cli.utils.lowmem_merge import merge_low_memory
+
+ base, adapter, output = checkpoints
+ assert (base / "model.safetensors.index.json").is_file()
+
+ total = merge_low_memory(base, adapter, output, "float16")
+
+ index = json.loads((output / "model.safetensors.index.json").read_text())
+ assert index["metadata"]["total_size"] == total
+ assert sorted(set(index["weight_map"].values())) == sorted(
+ path.name for path in output.glob("*.safetensors")
+ )
+ model = AutoModelForCausalLM.from_pretrained(output)
+ assert model.config.torch_dtype == torch.float16
+ assert next(model.parameters()).dtype == torch.float16
+
+
+def test_single_file_base(tmp_path):
+ from soup_cli.utils.lowmem_merge import merge_low_memory
+
+ base = _tiny_base(tmp_path / "base", shard_size="10MB")
+ assert (base / "model.safetensors").is_file()
+ adapter = _tiny_adapter(base, tmp_path / "adapter")
+
+ merge_low_memory(base, adapter, tmp_path / "merged", "float32")
+
+ expected = _regular_merge(base, adapter, torch.float32)
+ actual = _read_all(tmp_path / "merged")
+ assert all(torch.equal(actual[key], value) for key, value in expected.items())
+
+
+def test_rslora_scaling_matches(tmp_path):
+ from soup_cli.utils.lowmem_merge import merge_low_memory
+
+ base = _tiny_base(tmp_path / "base")
+ adapter = _tiny_adapter(base, tmp_path / "adapter", use_rslora=True)
+
+ merge_low_memory(base, adapter, tmp_path / "merged", "float32")
+
+ expected = _regular_merge(base, adapter, torch.float32)
+ actual = _read_all(tmp_path / "merged")
+ assert all(torch.equal(actual[key], value) for key, value in expected.items())
+
+
+def test_dora_adapter_is_refused(tmp_path):
+ from soup_cli.utils.lowmem_merge import LowMemoryMergeError, merge_low_memory
+
+ base = _tiny_base(tmp_path / "base")
+ adapter = _tiny_adapter(base, tmp_path / "adapter", use_dora=True)
+
+ with pytest.raises(LowMemoryMergeError, match="use_dora"):
+ merge_low_memory(base, adapter, tmp_path / "merged", "float16")
+
+
+def test_adapter_weights_matching_nothing_are_refused(checkpoints):
+ from soup_cli.utils.lowmem_merge import LowMemoryMergeError, merge_low_memory
+
+ base, adapter, output = checkpoints
+ weights = safetensors_torch.load_file(str(adapter / "adapter_model.safetensors"))
+ renamed = {
+ key.replace("layers.0.", "layers.9."): value for key, value in weights.items()
+ }
+ safetensors_torch.save_file(renamed, str(adapter / "adapter_model.safetensors"))
+
+ with pytest.raises(LowMemoryMergeError, match="match no base tensor"):
+ merge_low_memory(base, adapter, output, "float16")
+
+
+def test_output_must_differ_from_base(checkpoints):
+ from soup_cli.utils.lowmem_merge import merge_low_memory
+
+ base, adapter, _ = checkpoints
+ with pytest.raises(ValueError, match="differ"):
+ merge_low_memory(base, adapter, base, "float16")
+
+
+def test_integer_tensors_keep_their_dtype(tmp_path):
+ from safetensors import safe_open
+
+ from soup_cli.utils.lowmem_merge import merge_low_memory
+
+ base = _tiny_base(tmp_path / "base", shard_size="10MB")
+ weights = safetensors_torch.load_file(str(base / "model.safetensors"))
+ weights["extra.positions"] = torch.arange(6, dtype=torch.int64)
+ safetensors_torch.save_file(
+ weights, str(base / "model.safetensors"), metadata={"format": "pt"}
+ )
+ adapter = _tiny_adapter(_tiny_base(tmp_path / "plain"), tmp_path / "adapter")
+
+ merge_low_memory(base, adapter, tmp_path / "merged", "float16")
+
+ with safe_open(str(tmp_path / "merged" / "model.safetensors"), framework="pt") as merged:
+ assert merged.get_tensor("extra.positions").dtype == torch.int64
+ assert torch.equal(merged.get_tensor("extra.positions"), torch.arange(6))
+
+
+class _StubTokenizer:
+ @staticmethod
+ def from_pretrained(*_args, **_kwargs):
+ return SimpleNamespace(save_pretrained=lambda path: Path(path, "tokenizer.json").touch())
+
+
+class TestCli:
+ @pytest.fixture()
+ def run(self, checkpoints, monkeypatch):
+ import transformers
+ from typer.testing import CliRunner
+
+ from soup_cli.cli import app
+
+ monkeypatch.setattr(transformers, "AutoTokenizer", _StubTokenizer)
+ base, adapter, _ = checkpoints
+ monkeypatch.chdir(base.parent)
+
+ def invoke(*extra):
+ return CliRunner().invoke(
+ app,
+ ["merge", "--adapter", str(adapter), "--base", str(base), *extra],
+ )
+
+ return invoke
+
+ def test_low_memory_merge_writes_checkpoint(self, run, checkpoints):
+ base, adapter, output = checkpoints
+ result = run("--output", "merged_out", "--low-memory")
+ assert result.exit_code == 0, (result.output, repr(result.exception))
+ merged = base.parent / "merged_out"
+ expected = _regular_merge(base, adapter, torch.float16)
+ actual = _read_all(merged)
+ assert all(torch.equal(actual[key], value) for key, value in expected.items())
+ assert (merged / "tokenizer.json").is_file()
+
+ def test_rejects_non_fp16_save_format(self, run):
+ result = run("--output", "merged_out", "--low-memory", "--save-format", "4bit")
+ assert result.exit_code == 2, (result.output, repr(result.exception))
+ assert "--low-memory only supports" in strip_ansi(result.output)
+
+ def test_unsupported_adapter_points_back_to_regular_merge(self, run, checkpoints):
+ _, adapter, _ = checkpoints
+ config_path = adapter / "adapter_config.json"
+ config = json.loads(config_path.read_text())
+ config["use_dora"] = True
+ config_path.write_text(json.dumps(config))
+ result = run("--output", "merged_out", "--low-memory")
+ assert result.exit_code == 1, (result.output, repr(result.exception))
+ output = strip_ansi(result.output)
+ assert "Low-memory merge not possible" in output
+ assert "without" in output and "--low-memory" in output
+
+ def test_hub_base_is_fetched_by_snapshot_download(self, checkpoints, monkeypatch):
+ import transformers
+ from typer.testing import CliRunner
+
+ from soup_cli.cli import app
+
+ base, adapter, _ = checkpoints
+ monkeypatch.setattr(transformers, "AutoTokenizer", _StubTokenizer)
+ monkeypatch.chdir(base.parent)
+ fetched = {}
+
+ def fake_download(repo_id, **kwargs):
+ fetched["repo_id"] = repo_id
+ fetched.update(kwargs)
+ return str(base)
+
+ monkeypatch.setattr("soup_cli.utils.hubs.snapshot_download", fake_download)
+
+ result = CliRunner().invoke(
+ app,
+ [
+ "merge", "--adapter", str(adapter), "--base", "org/tiny-llama",
+ "--output", "merged_out", "--low-memory",
+ ],
+ )
+
+ assert result.exit_code == 0, (result.output, repr(result.exception))
+ assert fetched["repo_id"] == "org/tiny-llama"
+ assert fetched["allow_patterns"] == ["*.safetensors", "*.json"]
+ expected = _regular_merge(base, adapter, torch.float16)
+ actual = _read_all(base.parent / "merged_out")
+ assert all(torch.equal(actual[key], value) for key, value in expected.items())
+
+ def test_tokenizer_failure_leaves_no_output(self, run, checkpoints, monkeypatch):
+ import transformers
+
+ class _Broken:
+ @staticmethod
+ def from_pretrained(*_args, **_kwargs):
+ raise RuntimeError("tokenizer exploded")
+
+ monkeypatch.setattr(transformers, "AutoTokenizer", _Broken)
+ base, _, _ = checkpoints
+ result = run("--output", "merged_out", "--low-memory")
+ assert result.exit_code == 1, (result.output, repr(result.exception))
+ assert sorted(path.name for path in base.parent.iterdir()) == ["adapter", "base"]
+
+ def test_progress_text_is_neutralised_and_trust_flag_is_forwarded(
+ self, checkpoints, monkeypatch
+ ):
+ import transformers
+ from typer.testing import CliRunner
+
+ from soup_cli.cli import app
+
+ base, adapter, _ = checkpoints
+ monkeypatch.setattr(transformers, "AutoTokenizer", _StubTokenizer)
+ monkeypatch.chdir(base.parent)
+ seen = {}
+
+ def fake_merge(base_dir, adapter_dir, output_dir, dtype, **kwargs):
+ seen.update(kwargs)
+ kwargs["progress"]("Merging [red]EVIL[/red]\x1b[31m shard")
+ output_dir.mkdir()
+ return 0
+
+ monkeypatch.setattr("soup_cli.utils.lowmem_merge.merge_low_memory", fake_merge)
+ result = CliRunner().invoke(
+ app,
+ [
+ "merge", "--adapter", str(adapter), "--base", str(base),
+ "--output", "merged_out", "--low-memory", "--trust-remote-code",
+ ],
+ )
+
+ assert result.exit_code == 0, (result.output, repr(result.exception))
+ assert seen["trust_remote_code"] is True
+ output = strip_ansi(result.output)
+ assert "[red]EVIL[/red]" in output
+ assert "\x1b" not in output
+
+
+def _rewrite_adapter_weights(adapter: Path, edit) -> None:
+ path = adapter / "adapter_model.safetensors"
+ weights = safetensors_torch.load_file(str(path))
+ edit(weights)
+ safetensors_torch.save_file(weights, str(path), metadata={"format": "pt"})
+
+
+def _rewrite_adapter_config(adapter: Path, **changes) -> None:
+ path = adapter / "adapter_config.json"
+ config = json.loads(path.read_text())
+ config.update(changes)
+ path.write_text(json.dumps(config))
+
+
+def _first_adapter_target(base: Path, adapter: Path) -> str:
+ weights = safetensors_torch.load_file(str(adapter / "adapter_model.safetensors"))
+ key = sorted(weights)[0]
+ return key.split(".lora_")[0].replace("base_model.model.", "") + ".weight"
+
+
+class TestAdapterCoverage:
+ def test_fan_in_fan_out_matches_regular_merge(self, tmp_path):
+ from transformers import GPT2Config, GPT2LMHeadModel
+
+ from soup_cli.utils.lowmem_merge import merge_low_memory
+
+ torch.manual_seed(0)
+ config = GPT2Config(vocab_size=64, n_positions=32, n_embd=32, n_layer=2, n_head=4)
+ base = tmp_path / "base"
+ GPT2LMHeadModel(config).save_pretrained(base)
+ adapter = _tiny_adapter(
+ base, tmp_path / "adapter",
+ target_modules=["c_attn", "c_proj"], fan_in_fan_out=True,
+ )
+
+ merge_low_memory(base, adapter, tmp_path / "merged", "float32")
+
+ expected = _regular_merge(base, adapter, torch.float32)
+ actual = _read_all(tmp_path / "merged")
+ original = safetensors_torch.load_file(str(base / "model.safetensors"))
+ assert all(torch.equal(value, expected[key]) for key, value in actual.items())
+ assert any(not torch.equal(value, original[key]) for key, value in actual.items())
+
+ def test_tied_embeddings_match_regular_merge(self, tmp_path):
+ from soup_cli.utils.lowmem_merge import merge_low_memory
+
+ base = _tiny_base(tmp_path / "base", tie=True)
+ adapter = _tiny_adapter(base, tmp_path / "adapter")
+
+ merge_low_memory(base, adapter, tmp_path / "merged", "float32")
+
+ expected = _regular_merge(base, adapter, torch.float32)
+ actual = _read_all(tmp_path / "merged")
+ assert set(actual) <= set(expected)
+ assert all(torch.equal(value, expected[key]) for key, value in actual.items())
+
+ @pytest.mark.parametrize(
+ "field, value",
+ [
+ ("modules_to_save", ["lm_head"]),
+ ("rank_pattern", {"q_proj": 2}),
+ ("alpha_pattern", {"q_proj": 2}),
+ ("trainable_token_indices", {"embed_tokens": [1]}),
+ ("loftq_config", {"loftq_bits": 4}),
+ ("lora_bias", True),
+ ("layer_replication", [[0, 1]]),
+ ("target_parameters", ["mlp.experts.gate_up_proj"]),
+ ("alora_invocation_tokens", [1, 2]),
+ ("use_qalora", True),
+ ],
+ )
+ def test_unsupported_config_fields_are_refused(self, checkpoints, field, value):
+ from soup_cli.utils.lowmem_merge import LowMemoryMergeError, merge_low_memory
+
+ base, adapter, output = checkpoints
+ _rewrite_adapter_config(adapter, **{field: value})
+ with pytest.raises(LowMemoryMergeError, match=field):
+ merge_low_memory(base, adapter, output, "float16")
+ assert not output.exists()
+
+ @pytest.mark.parametrize("changes", [{"peft_type": "IA3"}, {"bias": "all"}])
+ def test_other_adapter_types_are_refused(self, checkpoints, changes):
+ from soup_cli.utils.lowmem_merge import LowMemoryMergeError, merge_low_memory
+
+ base, adapter, output = checkpoints
+ _rewrite_adapter_config(adapter, **changes)
+ with pytest.raises(LowMemoryMergeError, match=next(iter(changes))):
+ merge_low_memory(base, adapter, output, "float16")
+
+ def test_unpaired_lora_halves_are_refused(self, checkpoints):
+ from soup_cli.utils.lowmem_merge import LowMemoryMergeError, merge_low_memory
+
+ base, adapter, output = checkpoints
+
+ def drop_one_b(weights):
+ del weights[next(key for key in weights if key.endswith(".lora_B.weight"))]
+
+ _rewrite_adapter_weights(adapter, drop_one_b)
+ with pytest.raises(LowMemoryMergeError, match="unpaired"):
+ merge_low_memory(base, adapter, output, "float16")
+
+ def test_shape_mismatch_is_refused_before_writing(self, checkpoints):
+ from soup_cli.utils.lowmem_merge import LowMemoryMergeError, merge_low_memory
+
+ base, adapter, output = checkpoints
+
+ def shrink_one_a(weights):
+ key = next(key for key in weights if key.endswith(".lora_A.weight"))
+ weights[key] = weights[key][:, :16].contiguous()
+
+ _rewrite_adapter_weights(adapter, shrink_one_a)
+ with pytest.raises(LowMemoryMergeError, match="does not match"):
+ merge_low_memory(base, adapter, output, "float16")
+ assert not output.exists()
+
+ @pytest.mark.parametrize("key", ["lm_head.base_layer.weight", "embed_tokens.weight"])
+ def test_saved_embedding_or_head_weights_get_a_clear_message(self, checkpoints, key):
+ from soup_cli.utils.lowmem_merge import LowMemoryMergeError, merge_low_memory
+
+ base, adapter, output = checkpoints
+ _rewrite_adapter_weights(
+ adapter,
+ lambda weights: weights.update(
+ {f"base_model.model.{key}": torch.zeros(4, 4)}
+ ),
+ )
+ with pytest.raises(LowMemoryMergeError, match="embeddings or lm_head"):
+ merge_low_memory(base, adapter, output, "float16")
+
+ def test_quantized_base_weight_under_an_adapter_is_refused(self, tmp_path):
+ from soup_cli.utils.lowmem_merge import LowMemoryMergeError, merge_low_memory
+
+ base = _tiny_base(tmp_path / "base", shard_size="10MB")
+ adapter = _tiny_adapter(base, tmp_path / "adapter")
+ target = _first_adapter_target(base, adapter)
+ weights = safetensors_torch.load_file(str(base / "model.safetensors"))
+ weights[target] = torch.zeros(weights[target].numel(), dtype=torch.uint8)
+ safetensors_torch.save_file(
+ weights, str(base / "model.safetensors"), metadata={"format": "pt"}
+ )
+
+ with pytest.raises(LowMemoryMergeError, match="not a float type"):
+ merge_low_memory(base, adapter, tmp_path / "merged", "float16")
+ assert not (tmp_path / "merged").exists()
+
+
+class TestHostileAdapterValues:
+ def test_rank_and_shape_are_compared_before_any_matmul(self, monkeypatch):
+ from soup_cli.utils.lowmem_merge import LowMemoryMergeError, merged_tensor
+
+ def boom(*_args, **_kwargs):
+ raise AssertionError("matmul ran before the shape check")
+
+ monkeypatch.setattr(torch.Tensor, "__matmul__", boom)
+ with pytest.raises(LowMemoryMergeError, match="does not match"):
+ merged_tensor(
+ torch.zeros(4, 4), torch.zeros(1, 20000), torch.zeros(20000, 1),
+ scaling=1.0, fan_in_fan_out=False, dtype=torch.float32,
+ )
+ with pytest.raises(LowMemoryMergeError, match="rank mismatch"):
+ merged_tensor(
+ torch.zeros(4, 4), torch.zeros(2, 4), torch.zeros(4, 3),
+ scaling=1.0, fan_in_fan_out=False, dtype=torch.float32,
+ )
+
+ def test_huge_delta_is_refused_before_writing(self, checkpoints):
+ from soup_cli.utils.lowmem_merge import LowMemoryMergeError, merge_low_memory
+
+ base, adapter, output = checkpoints
+
+ def make_huge(weights):
+ key_a = next(key for key in weights if key.endswith(".lora_A.weight"))
+ key_b = key_a.replace("lora_A", "lora_B")
+ weights[key_a] = torch.zeros(1, 20000)
+ weights[key_b] = torch.zeros(20000, 1)
+
+ _rewrite_adapter_weights(adapter, make_huge)
+ with pytest.raises(LowMemoryMergeError, match="does not match"):
+ merge_low_memory(base, adapter, output, "float16")
+ assert not output.exists()
+
+ def test_rank_mismatch_between_a_and_b_is_refused(self, checkpoints):
+ from soup_cli.utils.lowmem_merge import LowMemoryMergeError, merge_low_memory
+
+ base, adapter, output = checkpoints
+
+ def drop_a_row(weights):
+ key = next(key for key in weights if key.endswith(".lora_A.weight"))
+ weights[key] = weights[key][:3].contiguous()
+
+ _rewrite_adapter_weights(adapter, drop_a_row)
+ with pytest.raises(LowMemoryMergeError, match="rank mismatch"):
+ merge_low_memory(base, adapter, output, "float16")
+ assert not output.exists()
+
+ @pytest.mark.parametrize(
+ "field, value",
+ [
+ ("r", 0),
+ ("r", -4),
+ ("r", "4"),
+ ("r", 4.0),
+ ("r", True),
+ ("r", None),
+ ("lora_alpha", float("nan")),
+ ("lora_alpha", float("inf")),
+ ("lora_alpha", "8"),
+ ("lora_alpha", True),
+ ("lora_alpha", None),
+ ("use_rslora", "false"),
+ ("use_rslora", 1),
+ ("fan_in_fan_out", "false"),
+ ("fan_in_fan_out", 0),
+ ],
+ )
+ def test_adapter_config_values_are_validated(self, checkpoints, field, value):
+ from soup_cli.utils.lowmem_merge import LowMemoryMergeError, merge_low_memory
+
+ base, adapter, output = checkpoints
+ path = adapter / "adapter_config.json"
+ config = json.loads(path.read_text())
+ if value is None:
+ config.pop(field)
+ else:
+ config[field] = value
+ path.write_text(json.dumps(config))
+ with pytest.raises(LowMemoryMergeError, match=field):
+ merge_low_memory(base, adapter, output, "float16")
+ assert not output.exists()
+
+
+def _set_weight_map_name(base: Path, name: str) -> None:
+ index_path = base / "model.safetensors.index.json"
+ index = json.loads(index_path.read_text())
+ first = next(iter(index["weight_map"]))
+ index["weight_map"][first] = name
+ index_path.write_text(json.dumps(index))
+
+
+class TestContainment:
+ @pytest.mark.parametrize(
+ "name",
+ [
+ "../escaped.safetensors",
+ "..\\escaped.safetensors",
+ "sub/escaped.safetensors",
+ "/tmp/escaped.safetensors",
+ "C:\\escaped.safetensors",
+ "C:escaped.safetensors",
+ "CON.safetensors",
+ "escaped.bin",
+ ".hidden.safetensors",
+ "",
+ 7,
+ ],
+ )
+ def test_index_shard_names_must_be_plain_file_names(self, checkpoints, name):
+ from soup_cli.utils.lowmem_merge import LowMemoryMergeError, merge_low_memory
+
+ base, adapter, output = checkpoints
+ _set_weight_map_name(base, name)
+ with pytest.raises(LowMemoryMergeError, match="plain .safetensors file name"):
+ merge_low_memory(base, adapter, output, "float16")
+ assert not output.exists()
+ assert not (base.parent / "escaped.safetensors").exists()
+ assert not any(base.parent.glob("*escaped*"))
+
+ def test_case_colliding_shard_names_are_refused(self, checkpoints):
+ from soup_cli.utils.lowmem_merge import LowMemoryMergeError, merge_low_memory
+
+ base, adapter, output = checkpoints
+ index_path = base / "model.safetensors.index.json"
+ index = json.loads(index_path.read_text())
+ shard = next(iter(index["weight_map"].values()))
+ index["weight_map"]["extra.weight"] = shard.upper().replace(".SAFETENSORS", ".safetensors")
+ index_path.write_text(json.dumps(index))
+ with pytest.raises(LowMemoryMergeError, match="differ only by case"):
+ merge_low_memory(base, adapter, output, "float16")
+
+ @pytest.mark.parametrize("where", ["adapter_inside_output", "output_inside_base"])
+ def test_overlapping_directories_are_refused(self, tmp_path, where):
+ from soup_cli.utils.lowmem_merge import merge_low_memory
+
+ base = _tiny_base(tmp_path / "base")
+ if where == "adapter_inside_output":
+ output = tmp_path / "out"
+ adapter = _tiny_adapter(base, output / "adapter")
+ else:
+ output = base / "merged"
+ adapter = _tiny_adapter(base, tmp_path / "adapter")
+ with pytest.raises(ValueError, match="overlap"):
+ merge_low_memory(base, adapter, output, "float16")
+
+ @pytest.mark.requires_symlink
+ def test_symlinked_output_is_refused(self, checkpoints, tmp_path):
+ from soup_cli.utils.lowmem_merge import merge_low_memory
+
+ base, adapter, output = checkpoints
+ elsewhere = tmp_path / "elsewhere"
+ elsewhere.mkdir()
+ output.symlink_to(elsewhere, target_is_directory=True)
+ with pytest.raises(ValueError, match="symbolic link"):
+ merge_low_memory(base, adapter, output, "float16")
+ assert list(elsewhere.iterdir()) == []
+
+ def test_existing_output_is_not_overwritten(self, checkpoints):
+ from soup_cli.utils.lowmem_merge import merge_low_memory
+
+ base, adapter, output = checkpoints
+ output.mkdir()
+ (output / "keep.txt").write_text("mine")
+ with pytest.raises(FileExistsError):
+ merge_low_memory(base, adapter, output, "float16")
+ assert (output / "keep.txt").read_text() == "mine"
+ assert [path.name for path in output.iterdir()] == ["keep.txt"]
+
+ def test_empty_existing_output_directory_is_used(self, checkpoints):
+ from soup_cli.utils.lowmem_merge import merge_low_memory
+
+ base, adapter, output = checkpoints
+ output.mkdir()
+ merge_low_memory(base, adapter, output, "float16")
+ assert (output / "model.safetensors.index.json").is_file()
+
+ @pytest.mark.requires_symlink
+ def test_shard_symlinked_outside_the_model_is_refused(self, checkpoints, tmp_path):
+ from soup_cli.utils.lowmem_merge import LowMemoryMergeError, merge_low_memory
+
+ base, adapter, output = checkpoints
+ shard = next(base.glob("model-*.safetensors"))
+ outside = tmp_path / "outside.safetensors"
+ shard.rename(outside)
+ shard.symlink_to(outside)
+ with pytest.raises(LowMemoryMergeError, match="outside the base model directory"):
+ merge_low_memory(base, adapter, output, "float16")
+ assert not output.exists()
+
+ @pytest.mark.requires_symlink
+ def test_support_file_symlinked_outside_the_model_is_not_copied(self, checkpoints, tmp_path):
+ from soup_cli.utils.lowmem_merge import LowMemoryMergeError, merge_low_memory
+
+ base, adapter, output = checkpoints
+ secret = tmp_path / "secret.json"
+ secret.write_text('{"token": "hunter2"}')
+ (base / "tokenizer_config.json").symlink_to(secret)
+ with pytest.raises(LowMemoryMergeError, match="outside the base model directory"):
+ merge_low_memory(base, adapter, output, "float16")
+ assert not output.exists()
+
+ def test_python_files_are_copied_only_when_remote_code_is_trusted(self, tmp_path):
+ from soup_cli.utils.lowmem_merge import merge_low_memory
+
+ base = _tiny_base(tmp_path / "base")
+ adapter = _tiny_adapter(base, tmp_path / "adapter")
+ (base / "modeling_custom.py").write_text("print('hi')")
+
+ merge_low_memory(base, adapter, tmp_path / "plain", "float16")
+ merge_low_memory(base, adapter, tmp_path / "trusted", "float16", trust_remote_code=True)
+
+ assert not (tmp_path / "plain" / "modeling_custom.py").exists()
+ assert (tmp_path / "plain" / "config.json").is_file()
+ assert (tmp_path / "trusted" / "modeling_custom.py").read_text() == "print('hi')"
+
+ @pytest.mark.requires_symlink
+ def test_trusted_python_symlink_outside_the_model_is_still_refused(self, checkpoints, tmp_path):
+ from soup_cli.utils.lowmem_merge import LowMemoryMergeError, merge_low_memory
+
+ base, adapter, output = checkpoints
+ outside = tmp_path / "outside.py"
+ outside.write_text("print('outside')")
+ (base / "modeling_custom.py").symlink_to(outside)
+ with pytest.raises(LowMemoryMergeError, match="outside the base model directory"):
+ merge_low_memory(base, adapter, output, "float16", trust_remote_code=True)
+ assert not output.exists()
+
+ @pytest.mark.requires_symlink
+ def test_huggingface_cache_snapshot_links_are_followed(self, tmp_path):
+ from soup_cli.utils.lowmem_merge import merge_low_memory
+
+ original = _tiny_base(tmp_path / "original")
+ adapter = _tiny_adapter(original, tmp_path / "adapter")
+ repo = tmp_path / "models--org--tiny"
+ blobs = repo / "blobs"
+ snapshot = repo / "snapshots" / "abc123"
+ blobs.mkdir(parents=True)
+ snapshot.mkdir(parents=True)
+ for position, source in enumerate(sorted(original.iterdir())):
+ blob = blobs / f"{position:040x}"
+ shutil.copyfile(source, blob)
+ (snapshot / source.name).symlink_to(Path("..", "..", "blobs", blob.name))
+
+ merge_low_memory(snapshot, adapter, tmp_path / "merged", "float32")
+
+ expected = _regular_merge(original, adapter, torch.float32)
+ actual = _read_all(tmp_path / "merged")
+ assert all(torch.equal(actual[key], value) for key, value in expected.items())
+ assert (tmp_path / "merged" / "config.json").is_file()
+
+
+class TestAtomicOutput:
+ def test_late_failure_leaves_no_output_and_no_staging(self, checkpoints, tmp_path):
+ from soup_cli.utils.lowmem_merge import merge_low_memory
+
+ base, adapter, output = checkpoints
+ before = sorted(path.name for path in tmp_path.iterdir())
+
+ def fail(staging):
+ assert any(staging.glob("*.safetensors"))
+ raise RuntimeError("late failure")
+
+ with pytest.raises(RuntimeError, match="late failure"):
+ merge_low_memory(base, adapter, output, "float16", finalize=fail)
+
+ assert not output.exists()
+ assert sorted(path.name for path in tmp_path.iterdir()) == before
+
+ def test_failure_while_writing_a_shard_leaves_no_output(
+ self, checkpoints, tmp_path, monkeypatch
+ ):
+ from soup_cli.utils import lowmem_merge
+
+ base, adapter, output = checkpoints
+ before = sorted(path.name for path in tmp_path.iterdir())
+ real_write = lowmem_merge._write_shard
+ calls = []
+
+ def second_shard_fails(*args, **kwargs):
+ calls.append(args[0].name)
+ if len(calls) == 2:
+ raise OSError("disk full")
+ return real_write(*args, **kwargs)
+
+ monkeypatch.setattr(lowmem_merge, "_write_shard", second_shard_fails)
+ with pytest.raises(OSError, match="disk full"):
+ lowmem_merge.merge_low_memory(base, adapter, output, "float16")
+
+ assert len(calls) == 2
+ assert not output.exists()
+ assert sorted(path.name for path in tmp_path.iterdir()) == before
+
+ def test_finalize_output_is_part_of_the_published_directory(self, checkpoints):
+ from soup_cli.utils.lowmem_merge import merge_low_memory
+
+ base, adapter, output = checkpoints
+ merge_low_memory(
+ base, adapter, output, "float16",
+ finalize=lambda staging: (staging / "tokenizer.json").write_text("{}"),
+ )
+ assert (output / "tokenizer.json").is_file()
+ assert not any(path.name.startswith(".merged.") for path in output.parent.iterdir())
+
+
+class TestOrderAndDefenceInDepth:
+ def test_non_2d_lora_tensors_are_refused(self, checkpoints):
+ from soup_cli.utils.lowmem_merge import LowMemoryMergeError, merge_low_memory
+
+ base, adapter, output = checkpoints
+
+ def flatten_one_a(weights):
+ key = next(key for key in weights if key.endswith(".lora_A.weight"))
+ weights[key] = weights[key].reshape(-1)
+
+ _rewrite_adapter_weights(adapter, flatten_one_a)
+ with pytest.raises(LowMemoryMergeError, match="2-D"):
+ merge_low_memory(base, adapter, output, "float16")
+ assert not output.exists()
+
+ def test_a_bad_shape_is_found_before_any_shard_is_written(self, checkpoints, monkeypatch):
+ from soup_cli.utils import lowmem_merge
+
+ base, adapter, output = checkpoints
+
+ def shrink_one_a(weights):
+ key = next(key for key in weights if key.endswith(".lora_A.weight"))
+ weights[key] = weights[key][:, :16].contiguous()
+
+ _rewrite_adapter_weights(adapter, shrink_one_a)
+
+ def no_write(*args, **kwargs):
+ raise AssertionError("a shard was written before the shapes were checked")
+
+ monkeypatch.setattr(lowmem_merge, "_write_shard", no_write)
+ with pytest.raises(lowmem_merge.LowMemoryMergeError, match="does not match"):
+ lowmem_merge.merge_low_memory(base, adapter, output, "float16")
+
+ def test_load_adapter_itself_refuses_bad_config_values(self, checkpoints):
+ from soup_cli.utils.lowmem_merge import LowMemoryMergeError, load_adapter
+
+ _, adapter, _ = checkpoints
+ _rewrite_adapter_config(adapter, r=0)
+ with pytest.raises(LowMemoryMergeError, match="'r'"):
+ load_adapter(adapter)
+
+ def test_write_shard_alone_refuses_a_quantized_tensor_under_an_adapter(self, tmp_path):
+ from soup_cli.utils import lowmem_merge
+
+ base = _tiny_base(tmp_path / "base", shard_size="10MB")
+ adapter = _tiny_adapter(base, tmp_path / "adapter")
+ target = _first_adapter_target(base, adapter)
+ weights = safetensors_torch.load_file(str(base / "model.safetensors"))
+ weights[target] = torch.zeros(weights[target].numel(), dtype=torch.uint8)
+ safetensors_torch.save_file(
+ weights, str(base / "model.safetensors"), metadata={"format": "pt"}
+ )
+ config, paired = lowmem_merge.load_adapter(adapter)
+ with pytest.raises(lowmem_merge.LowMemoryMergeError, match="not a float type"):
+ lowmem_merge._write_shard(
+ base / "model.safetensors", tmp_path / "out.safetensors", "float16", paired, config
+ )
+
+ def test_write_shard_checks_the_bytes_it_writes(self, checkpoints, tmp_path, monkeypatch):
+ from soup_cli.utils import lowmem_merge
+
+ base, adapter, _ = checkpoints
+ config, paired = lowmem_merge.load_adapter(adapter)
+ monkeypatch.setattr(lowmem_merge, "merged_tensor", lambda *args, **kwargs: torch.zeros(3))
+ weight_map = json.loads((base / "model.safetensors.index.json").read_text())["weight_map"]
+ shard = base / weight_map[_first_adapter_target(base, adapter)]
+ with pytest.raises(lowmem_merge.LowMemoryMergeError, match="header declares"):
+ lowmem_merge._write_shard(
+ shard, tmp_path / "out.safetensors", "float16", paired, config
+ )
+
+
+class TestIndexContainment:
+ @pytest.mark.requires_symlink
+ def test_index_symlinked_outside_the_model_is_refused(self, checkpoints, tmp_path):
+ from soup_cli.utils.lowmem_merge import LowMemoryMergeError, merge_low_memory
+
+ base, adapter, output = checkpoints
+ index = base / "model.safetensors.index.json"
+ outside = tmp_path / "outside.index.json"
+ index.rename(outside)
+ index.symlink_to(outside)
+ with pytest.raises(LowMemoryMergeError, match="outside the base model directory"):
+ merge_low_memory(base, adapter, output, "float16")
+ assert not output.exists()