ODS V3 Pre-Release: Public testing and refinement ahead of the official V3 launch. Turn your PC, Mac, or Linux box into a private AI server.
Python ai-agentsamdcomfyuidockerllama-cppllmlocal-ain8nnvidiaopen-webuiragself-hostedspeech-to-textstrix-halotext-to-speechworkflow-automation
#!/usr/bin/env python3
"""
assign_gpus.py — GPU assignment algorithm for ODS
Usage:
python3 assign_gpus.py --topology topo.json --model-size 70000
python3 assign_gpus.py --topology topo.json --model-size 70000 --enabled-services llama_server,whisper
python3 assign_gpus.py --topology topo.json --model-size 132735 --pin-llm-gpu GPU-<uuid>
--pin-llm-gpu gives the LLM role (llama_server) exactly that GPU, as the vLLM
engine needs: its model selector already sized the model for that one GPU, and
a model that fits one GPU is never split across GPUs. The other services take
the remaining GPUs.
Output: gpu_assignment JSON to stdout
Errors: to stderr, exit code 1
"""
import argparse
import json
import math
import sys
from dataclasses import dataclass
from itertools import combinations
from typing import Optional
# Constants
HIGH_BW_THRESHOLD = 80 # min rank for NVLink / XGMI
DEFAULT_SERVICES = ["llama_server", "whisper", "comfyui", "embeddings"]
NON_LLAMA = ["whisper", "comfyui", "embeddings"]
# Data Models
@dataclass
class GPU:
index: int
uuid: str
name: str
memory_mb: float
memory_total_mb: float
memory_type: str = "discrete" # "discrete" or "unified" (APU)
@dataclass
class Link:
gpu_a: int
gpu_b: int
link_type: str
link_label: str
rank: int
@dataclass
class Subset:
gpus: list
min_link_rank: int
total_vram_mb: float
all_pairs_highbw: bool
@dataclass
class LlamaParallelism:
mode: str
tensor_parallel_size: int
pipeline_parallel_size: int
gpu_memory_utilization: float
tensor_split: Optional[list] = None
@dataclass
class ServiceAssignment:
gpus: list
parallelism: Optional[LlamaParallelism] = None
@dataclass
class AssignmentResult:
strategy: str
services: dict
# Phase 1: Topology Analysis
def parse_gpus(topology: dict) -> list:
gpus = []
for g in topology["gpus"]:
total_mb = float(g["memory_gb"]) * 1024
scheduling_mb = total_mb
if g.get("memory_free_gb") is not None:
scheduling_mb = max(float(g["memory_free_gb"]) * 1024, 0.0)
gpus.append(GPU(
index=g["index"],
uuid=g["uuid"],
name=g["name"],
memory_mb=scheduling_mb,
memory_total_mb=total_mb,
memory_type=g.get("memory_type", "discrete"),
))
return gpus
def parse_links(topology: dict) -> list:
links = []
for link in topology.get("links", []):
links.append(Link(
gpu_a=link["gpu_a"],
gpu_b=link["gpu_b"],
link_type=link["link_type"],
link_label=link["link_label"],
rank=link["rank"],
))
return links
def build_rank_matrix(links: list) -> dict:
"""
rank_matrix[(min_idx, max_idx)] = rank
Pairs not in links default to 0.
"""
matrix = {}
for link in links:
key = (min(link.gpu_a, link.gpu_b), max(link.gpu_a, link.gpu_b))
matrix[key] = link.rank
return matrix
def get_rank(rank_matrix: dict, a: int, b: int) -> int:
return rank_matrix.get((min(a, b), max(a, b)), 0)
def compute_subset(gpus: list, rank_matrix: dict) -> Subset:
"""
Compute a Subset from a list of GPUs.
Single GPU: min_link_rank=0, all_pairs_highbw=True (no links needed).
"""
if len(gpus) == 1:
return Subset(
gpus=gpus,
min_link_rank=0,
total_vram_mb=gpus[0].memory_mb,
all_pairs_highbw=True,
)
indices = [g.index for g in gpus]
ranks = [get_rank(rank_matrix, a, b) for a, b in combinations(indices, 2)]
min_rank = min(ranks)
return Subset(
gpus=gpus,
min_link_rank=min_rank,
total_vram_mb=sum(g.memory_mb for g in gpus),
all_pairs_highbw=(min_rank >= HIGH_BW_THRESHOLD),
)
def enumerate_subsets(gpus: list, rank_matrix: dict) -> list:
"""
Generate all non-empty subsets of GPUs, ordered by:
1. min_link_rank DESC (topology quality)
2. subset size ASC (prefer fewer GPUs, leave more for services)
3. total_vram DESC (tiebreaker)
"""
all_subsets = []
for size in range(1, len(gpus) + 1):
for combo in combinations(gpus, size):
all_subsets.append(compute_subset(list(combo), rank_matrix))
return sorted(
all_subsets,
key=lambda s: (s.min_link_rank, -len(s.gpus), s.total_vram_mb),
reverse=True,
)
# Phase 2: GPU Assignment
def find_llama_subset(ordered_subsets: list, model_size_mb: float) -> Subset:
"""
Pick the best-ranked subset whose total VRAM covers model_size_mb.
Returns the first match (best topology, smallest size, most VRAM).
"""
for subset in ordered_subsets:
if subset.total_vram_mb >= model_size_mb and subset_can_host_equal_split(subset, model_size_mb):
return subset
return None
def subset_can_host_equal_split(subset: Subset, model_size_mb: float) -> bool:
"""Conservative fit check for llama.cpp layer/pipeline splits.
ODS emits equal tensor-split weights for non-heterogeneous pipeline splits.
A multi-GPU host with one mostly busy GPU can therefore have enough total
VRAM but still crash when llama.cpp allocates that GPU's share. Treat free
VRAM as the scheduling budget when topology provides it, and require every
selected GPU to be able to carry an equal share of the model file.
"""
if not subset.gpus:
return False
required_per_gpu = model_size_mb / len(subset.gpus)
return all(g.memory_mb >= required_per_gpu for g in subset.gpus)
def span_subsets(all_gpus: list, rank_matrix: dict, model_size_mb: float, ordered_subsets: list) -> Subset:
"""
No single subset covers model_size_mb.
Take the best subset, then greedily add GPUs from the remaining pool
(ordered by memory_mb DESC) until VRAM is covered.
Recomputes min_link_rank on the combined set.
"""
best = ordered_subsets[0]
accumulated = list(best.gpus)
used = {g.index for g in accumulated}
remaining = sorted(
[g for g in all_gpus if g.index not in used],
key=lambda g: g.memory_mb,
reverse=True,
)
for gpu in remaining:
accumulated.append(gpu)
candidate = compute_subset(accumulated, rank_matrix)
if candidate.total_vram_mb >= model_size_mb and subset_can_host_equal_split(candidate, model_size_mb):
return candidate
raise ValueError(
f"Model size {model_size_mb:.0f}MB exceeds assignable free VRAM "
f"({sum(g.memory_mb for g in all_gpus):.0f}MB across all GPUs)."
)
def pinned_llama_subset(gpus: list, rank_matrix: dict, uuid: str) -> Subset:
"""The LLM role on exactly the GPU ``uuid`` names (``--pin-llm-gpu``).
The vLLM model selector already checked that the model fits this GPU's
total memory, and vLLM claims its share of that memory at start, so the
free-VRAM fit used for llama.cpp splits does not apply.
"""
pinned = [g for g in gpus if g.uuid == uuid]
if len(pinned) != 1:
raise ValueError(f"--pin-llm-gpu {uuid} names {len(pinned)} GPUs in the topology, not one")
return compute_subset(pinned, rank_matrix)
def assign_services(all_gpus: list, llama_gpus: list, rank_matrix: dict, enabled_services: list, model_size_mb: float, vendor: str = "nvidia", grow_llama: bool = True) -> tuple:
"""
Assign remaining GPUs to non-llama services.
Returns (service_assignments dict, final_llama_gpus list, strategy str).
Rules:
remaining == 0 → all 3 services share llama's last GPU → colocated
remaining == 1 → all 3 services share remaining[0] → colocated
remaining == 2 → whisper → [0], comfyui+embeddings → [1] → colocated
remaining >= 3 → whisper → [0], comfyui → [1], emb → [2] → dedicated
remaining[3:] → back to llama if the expanded group fits
(never with grow_llama=False, a pinned LLM GPU)
AMD APU+dGPU hybrid: if mixed memory types (unified + discrete), prefer APU
GPUs for auxiliary services (lower bandwidth but sufficient for whisper/embeddings).
"""
llama_indices = {g.index for g in llama_gpus}
remaining = sorted(
[g for g in all_gpus if g.index not in llama_indices],
key=lambda g: g.memory_mb,
reverse=True,
)
# AMD hybrid strategy: sort remaining so APU GPUs come first for auxiliary
# services (they're better suited for lightweight tasks like whisper/embeddings,
# freeing discrete GPUs for LLM inference)
if vendor == "amd" and any(g.memory_type == "unified" for g in remaining):
remaining = sorted(
remaining,
key=lambda g: (0 if g.memory_type == "unified" else 1, -g.memory_mb),
)
active_non_llama = [s for s in NON_LLAMA if s in enabled_services]
assignments = {}
final_llama_gpus = list(llama_gpus)
if len(remaining) == 0:
fallback = llama_gpus[-1]
for s in active_non_llama:
assignments[s] = ServiceAssignment(gpus=[fallback])
strategy = "colocated"
elif len(remaining) == 1:
for s in active_non_llama:
assignments[s] = ServiceAssignment(gpus=[remaining[0]])
strategy = "colocated"
elif len(remaining) == 2:
if "whisper" in enabled_services:
assignments["whisper"] = ServiceAssignment(gpus=[remaining[0]])
if "comfyui" in enabled_services:
assignments["comfyui"] = ServiceAssignment(gpus=[remaining[1]])
if "embeddings" in enabled_services:
assignments["embeddings"] = ServiceAssignment(gpus=[remaining[1]])
strategy = "colocated"
else:
if "whisper" in enabled_services:
assignments["whisper"] = ServiceAssignment(gpus=[remaining[0]])
if "comfyui" in enabled_services:
assignments["comfyui"] = ServiceAssignment(gpus=[remaining[1]])
if "embeddings" in enabled_services:
assignments["embeddings"] = ServiceAssignment(gpus=[remaining[2]])
# Return spare devices only when the expanded LLM group still fits.
if grow_llama and len(remaining) > 3:
expanded = compute_subset(final_llama_gpus + remaining[3:], rank_matrix)
# Adding a mostly occupied device can make an equal layer split fail
# even though the originally selected LLM group comfortably fits.
if subset_can_host_equal_split(expanded, model_size_mb):
final_llama_gpus = expanded.gpus
strategy = "dedicated"
assignments["llama_server"] = ServiceAssignment(gpus=final_llama_gpus)
return assignments, final_llama_gpus, strategy
# Phase 3: Llama Parallelism
def largest_pow2_divisor(n: int) -> int:
"""
Find the largest power of 2 p such that:
- p divides n evenly
- p <= sqrt(n) (keeps tensor_size <= pipeline_size for balance)
Minimum return value is 2 (hybrid requires at least 2 tensor groups).
"""
p = 1
while True:
candidate = p * 2
if candidate > n or n % candidate != 0:
break
if candidate > math.sqrt(n):
break
p = candidate
return max(2, p)
def is_heterogeneous(gpus: list) -> bool:
vrams = [g.memory_mb for g in gpus]
return max(vrams) != min(vrams)
def compute_tensor_split(gpus: list) -> list:
"""Proportional VRAM weights, rounded to 4 decimal places."""
total = sum(g.memory_mb for g in gpus)
return [round(g.memory_mb / total, 4) for g in gpus]
def select_parallelism(subset: Subset) -> LlamaParallelism:
"""
Select parallelism mode based on GPU count and min_link_rank.
Thresholds:
rank >= 80 → NVLink / XGMI → tensor or hybrid
rank 11-79 → same-NUMA PCIe → pipeline, or hybrid if rank >= 40 and >= 4 GPUs
rank <= 10 → cross-NUMA → pipeline only
"""
gpus = subset.gpus
n = len(gpus)
rank = subset.min_link_rank
split = compute_tensor_split(gpus) if is_heterogeneous(gpus) else None
# Single GPU
if n == 1:
return LlamaParallelism(
mode="none",
tensor_parallel_size=1,
pipeline_parallel_size=1,
gpu_memory_utilization=0.95,
)
# High-bandwidth (NVLink / XGMI)
if rank >= HIGH_BW_THRESHOLD:
if n <= 3:
return LlamaParallelism(
mode="tensor",
tensor_parallel_size=n,
pipeline_parallel_size=1,
gpu_memory_utilization=0.92,
tensor_split=split,
)
else:
tp = largest_pow2_divisor(n)
pp = n // tp
return LlamaParallelism(
mode="hybrid",
tensor_parallel_size=tp,
pipeline_parallel_size=pp,
gpu_memory_utilization=0.93,
tensor_split=split,
)
# Cross-NUMA PCIe
if rank <= 10:
return LlamaParallelism(
mode="pipeline",
tensor_parallel_size=1,
pipeline_parallel_size=n,
gpu_memory_utilization=0.95,
)
# Same-NUMA PCIe (rank 11-79)
if n <= 3:
return LlamaParallelism(
mode="pipeline",
tensor_parallel_size=1,
pipeline_parallel_size=n,
gpu_memory_utilization=0.95,
)
else:
if rank >= 40:
tp = largest_pow2_divisor(n)
pp = n // tp
return LlamaParallelism(
mode="hybrid",
tensor_parallel_size=tp,
pipeline_parallel_size=pp,
gpu_memory_utilization=0.93,
tensor_split=split,
)
else:
return LlamaParallelism(
mode="pipeline",
tensor_parallel_size=1,
pipeline_parallel_size=n,
gpu_memory_utilization=0.95,
)
# Phase 4: Build Output JSON
def build_output(result: AssignmentResult) -> dict:
services = {}
for name, assignment in result.services.items():
entry = {
"gpus": [g.uuid for g in assignment.gpus],
"gpu_indices": [g.index for g in assignment.gpus],
}
if assignment.parallelism:
p = assignment.parallelism
para = {
"mode": p.mode,
"tensor_parallel_size": p.tensor_parallel_size,
"pipeline_parallel_size": p.pipeline_parallel_size,
"gpu_memory_utilization": p.gpu_memory_utilization,
}
if p.tensor_split is not None:
para["tensor_split"] = p.tensor_split
entry["parallelism"] = para
services[name] = entry
return {
"gpu_assignment": {
"version": "1.0",
"strategy": result.strategy,
"services": services,
}
}
# Entry Point
def main():
parser = argparse.ArgumentParser(description="GPU assignment algorithm for ODS")
parser.add_argument("--topology", required=True, help="Path to topology JSON file")
parser.add_argument("--model-size", required=True, type=float, help="Model size in MB")
parser.add_argument("--enabled-services", default=",".join(DEFAULT_SERVICES),
help="Comma-separated list of enabled services")
parser.add_argument("--pin-llm-gpu", default="", metavar="UUID",
help="Give the LLM role exactly this GPU (the vLLM engine's VLLM_GPU_UUID); "
"the other services take the remaining GPUs")
args = parser.parse_args()
# Load topology
try:
with open(args.topology) as f:
topology = json.load(f)
except FileNotFoundError:
print(f"ERROR: topology file not found: {args.topology}", file=sys.stderr)
sys.exit(1)
except json.JSONDecodeError as e:
print(f"ERROR: invalid JSON in topology file: {e}", file=sys.stderr)
sys.exit(1)
enabled_services = [s.strip() for s in args.enabled_services.split(",")]
model_size_mb = args.model_size
gpu_count = topology.get("gpu_count", 0)
if not math.isfinite(model_size_mb) or model_size_mb <= 0:
print("ERROR: --model-size must be a finite positive number", file=sys.stderr)
sys.exit(1)
if gpu_count == 0:
print("ERROR: no GPUs found in topology", file=sys.stderr)
sys.exit(1)
# Pinned LLM GPU (the vLLM engine): no subset search, no split
if args.pin_llm_gpu:
gpus = parse_gpus(topology)
rank_matrix = build_rank_matrix(parse_links(topology))
try:
llama_subset = pinned_llama_subset(gpus, rank_matrix, args.pin_llm_gpu)
except ValueError as e:
print(f"ERROR: {e}", file=sys.stderr)
sys.exit(1)
service_assignments, _, strategy = assign_services(
gpus, llama_subset.gpus, rank_matrix, enabled_services, model_size_mb,
vendor=topology.get("vendor", "nvidia"), grow_llama=False,
)
service_assignments["llama_server"].parallelism = select_parallelism(llama_subset)
result = AssignmentResult(strategy="single" if gpu_count == 1 else strategy, services=service_assignments)
print(json.dumps(build_output(result), indent=2))
return
# Early exit: single GPU
if gpu_count == 1:
gpu = parse_gpus(topology)[0]
if model_size_mb > gpu.memory_mb:
print(
f"ERROR: Model size {model_size_mb:.0f}MB exceeds assignable free VRAM "
f"({gpu.memory_mb:.0f}MB across all GPUs).",
file=sys.stderr,
)
sys.exit(1)
parallelism = LlamaParallelism(
mode="none",
tensor_parallel_size=1,
pipeline_parallel_size=1,
gpu_memory_utilization=0.95,
)
services = {}
for s in enabled_services:
services[s] = ServiceAssignment(gpus=[gpu])
# llama_server always runs on the single GPU, even when the caller's
# --enabled-services list omits it. This mirrors the multi-GPU path,
# where assign_services() assigns llama_server unconditionally; without
# it, the line below raised KeyError: 'llama_server'.
services.setdefault("llama_server", ServiceAssignment(gpus=[gpu]))
services["llama_server"].parallelism = parallelism
result = AssignmentResult(strategy="single", services=services)
print(json.dumps(build_output(result), indent=2))
return
# Phase 1: Topology analysis
gpus = parse_gpus(topology)
links = parse_links(topology)
rank_matrix = build_rank_matrix(links)
ordered = enumerate_subsets(gpus, rank_matrix)
# Phase 2: GPU assignment
try:
llama_subset = find_llama_subset(ordered, model_size_mb)
if llama_subset is None:
llama_subset = span_subsets(gpus, rank_matrix, model_size_mb, ordered)
except ValueError as e:
print(f"ERROR: {e}", file=sys.stderr)
sys.exit(1)
vendor = topology.get("vendor", "nvidia")
# AMD APU+dGPU hybrid: if topology has mixed memory types, ensure LLM
# gets discrete GPUs. Re-sort llama subset to prefer discrete over unified.
if vendor == "amd":
has_mixed = (
any(g.memory_type == "unified" for g in gpus)
and any(g.memory_type == "discrete" for g in gpus)
)
if has_mixed:
discrete_gpus = [g for g in llama_subset.gpus if g.memory_type == "discrete"]
if discrete_gpus:
# Rebuild llama subset using only discrete GPUs if they cover model size
discrete_subset = compute_subset(discrete_gpus, rank_matrix)
if discrete_subset.total_vram_mb >= model_size_mb:
llama_subset = discrete_subset
service_assignments, final_llama_gpus, strategy = assign_services(
gpus, llama_subset.gpus, rank_matrix, enabled_services, model_size_mb, vendor=vendor
)
# Phase 3: Llama parallelism
final_subset = compute_subset(final_llama_gpus, rank_matrix)
parallelism = select_parallelism(final_subset)
service_assignments["llama_server"].parallelism = parallelism
# Phase 4: Emit JSON
result = AssignmentResult(strategy=strategy, services=service_assignments)
print(json.dumps(build_output(result), indent=2))
if __name__ == "__main__":
main()