Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.
Python cliconsumer-gpudpofine-tuningggufhuggingfacellmllmopslocal-ailocal-llmloralow-vrammachine-learningollamapeftpythonpytorchqlorasfttransformers
"""Reproducible, local Soup request/decode benchmark; run from the workspace root.
. ./experiments/env.ps1
./.venv/Scripts/python.exe experiments/bench_decode.py --output experiments/baseline.json
The primary measurement invokes the shipped ``infer._generate`` and includes
chat tokenization, input transfers, generation, and output decoding. CUDA is
synchronized at both request boundaries. Forward hooks and profilers run in
SEPARATE requests and never contribute to the primary throughput measurement.
The baseline leaves generation settings alone. Explicit experimental variants
change cache/compilation settings, recorded separately from the workload; no
variant changes EOS, precision, sampling, prompt, or requested output length.
"""
from __future__ import annotations
import argparse
import contextlib
import datetime
import hashlib
import importlib.metadata
import json
import os
import platform
import statistics
import subprocess
import sys
import threading
import time
from pathlib import Path
from typing import Any, Callable
DEFAULT_PROMPT = (
"Explain in detail how database indexes work. Cover B-tree structure, how an "
"index lookup differs from a full table scan, composite indexes and column "
"order, the effect on INSERT and UPDATE operations, and a concrete SQL example. "
"Write a thorough explanation with several paragraphs."
)
EXPERIMENTAL_GRAPH_VARIANTS = ("cudagraphs", "cudagraphs_static_inputs")
GRAPH_VARIANTS = (*EXPERIMENTAL_GRAPH_VARIANTS, "production_cuda_graphs")
def arguments() -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__)
root = Path(__file__).absolute().parent.parent
parser.add_argument("--source", default=str(root / "Soup"), help="Local Soup source checkout")
parser.add_argument(
"--cpu-affinity",
type=int,
help="Pin this benchmark process to one logical CPU before torch import",
)
parser.add_argument("--model", default=str(root / "experiments/models/Qwen2.5-1.5B-Instruct"))
parser.add_argument("--output", required=True)
parser.add_argument("--prompt", default=DEFAULT_PROMPT)
parser.add_argument("--max-tokens", type=int, default=128)
parser.add_argument("--warmups", type=int, default=3)
parser.add_argument("--repeats", type=int, default=7)
parser.add_argument("--phase-repeats", type=int, default=None)
parser.add_argument(
"--prefill-phase-repeats",
type=int,
default=0,
help="Separate graph-safe _prefill/request event samples after one discarded warmup",
)
parser.add_argument(
"--variant",
choices=(
"baseline",
"inference_mode",
"static",
"cudagraphs",
"cudagraphs_static_inputs",
"production_cuda_graphs",
"no_padding_mask",
"mask_callback",
"inference_mode_mask_callback",
),
default="baseline",
)
parser.add_argument(
"--reference", help="Baseline JSON; require identical workload and token IDs"
)
parser.add_argument("--profile", action="store_true", help="Profile separate complete requests")
parser.add_argument(
"--trace-only",
action="store_true",
help="Trace one separate request without shapes/memory/aggregation/cProfile",
)
parser.add_argument(
"--telemetry", action="store_true", help="Sample nvidia-smi once per second"
)
args = parser.parse_args()
if args.cpu_affinity is not None and args.cpu_affinity < 0:
parser.error("--cpu-affinity cannot be negative")
if args.phase_repeats is None:
args.phase_repeats = 0 if args.trace_only else 3
for name in ("max_tokens", "repeats"):
if getattr(args, name) < 1:
parser.error(f"--{name.replace('_', '-')} must be positive")
for name in ("warmups", "phase_repeats", "prefill_phase_repeats"):
if getattr(args, name) < 0:
parser.error(f"--{name.replace('_', '-')} cannot be negative")
return args
def contained_path(path: str | Path) -> Path:
root = os.path.realpath(Path(__file__).absolute().parent.parent)
target = os.path.realpath(path)
if os.path.normcase(os.path.commonpath((root, target))) != os.path.normcase(root):
raise ValueError(f"All benchmark paths must stay inside the workspace: {path}")
return Path(target)
def command(argv: list[str], cwd: Path | None = None) -> dict[str, Any]:
try:
result = subprocess.run(
argv,
cwd=cwd,
capture_output=True,
text=True,
timeout=20,
check=False,
creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0),
)
return {
"returncode": result.returncode,
"stdout": result.stdout.strip(),
"stderr": result.stderr.strip(),
}
except (OSError, subprocess.TimeoutExpired) as exc:
return {"error": str(exc)}
def apply_cpu_affinity(requested: int | None) -> dict[str, Any]:
try:
import psutil
process = psutil.Process()
original = process.cpu_affinity()
if requested is not None:
process.cpu_affinity([requested])
effective = process.cpu_affinity()
if requested is not None and effective != [requested]:
raise RuntimeError(f"Requested CPU {requested}, but actual affinity is {effective}")
return {"requested": requested, "original": original, "effective": effective}
except (ImportError, AttributeError, OSError) as exc:
if requested is not None:
raise RuntimeError("Cannot apply the requested benchmark CPU affinity") from exc
return {"requested": None, "original": None, "effective": None, "error": str(exc)}
def metadata(torch: Any, model_path: Path, affinity: dict[str, Any]) -> dict[str, Any]:
import soup_cli
imported_source = contained_path(Path(soup_cli.__file__).absolute().parents[2])
versions = {}
for name in (
"torch",
"transformers",
"tokenizers",
"accelerate",
"bitsandbytes",
"peft",
"numpy",
"psutil",
"soup-cli",
):
try:
versions[name] = importlib.metadata.version(name)
except importlib.metadata.PackageNotFoundError:
versions[name] = None
properties = torch.cuda.get_device_properties(0)
file_fingerprints = {}
for name in (
"config.json",
"generation_config.json",
"tokenizer_config.json",
"model.safetensors.index.json",
):
path = model_path / name
if path.is_file():
file_fingerprints[name] = hashlib.sha256(path.read_bytes()).hexdigest()
config_path = model_path / "config.json"
config = json.loads(config_path.read_text(encoding="utf-8"))
return {
"python": sys.version,
"executable": sys.executable,
"platform": platform.platform(),
"processor": platform.processor(),
"logical_cpu_count": os.cpu_count(),
"cpu_affinity": affinity,
"versions": versions,
"torch_cuda": torch.version.cuda,
"torch_cudnn": torch.backends.cudnn.version(),
"torch_num_threads": torch.get_num_threads(),
"torch_num_interop_threads": torch.get_num_interop_threads(),
"gpu": {
"name": properties.name,
"total_memory_bytes": properties.total_memory,
"compute_capability": [properties.major, properties.minor],
},
"nvidia_smi": command(
[
"nvidia-smi",
"--query-gpu=name,driver_version,memory.total,power.limit,clocks.max.sm,pstate",
"--format=csv,noheader,nounits",
]
),
"git_sha": command(["git", "rev-parse", "HEAD"], imported_source),
"git_status": command(["git", "status", "--short"], imported_source),
"git_diff": command(["git", "diff", "--no-ext-diff"], imported_source),
"source_checkout": str(imported_source),
"soup_import_path": soup_cli.__file__,
"config_fingerprints": file_fingerprints,
"source_quantization_config": config.get("quantization_config"),
"relevant_environment": {
name: os.environ.get(name)
for name in (
"CUDA_VISIBLE_DEVICES",
"PYTORCH_CUDA_ALLOC_CONF",
"OMP_NUM_THREADS",
"MKL_NUM_THREADS",
"TOKENIZERS_PARALLELISM",
"PYTHONPATH",
)
},
}
class Telemetry:
"""Optional, low-rate external sampler; same setting must be used for A/B."""
def __init__(self, enabled: bool) -> None:
self.enabled = enabled
self.rows: list[dict[str, Any]] = []
self.stop_event = threading.Event()
self.thread: threading.Thread | None = None
def start(self) -> None:
if not self.enabled:
return
self.thread = threading.Thread(target=self._run, daemon=True)
self.thread.start()
def _run(self) -> None:
while not self.stop_event.is_set():
row = command(
[
"nvidia-smi",
"--query-gpu=timestamp,utilization.gpu,utilization.memory,"
"memory.used,power.draw,power.limit,clocks.sm,clocks.mem,temperature.gpu,pstate",
"--format=csv,noheader,nounits",
]
)
row["host_perf_counter"] = time.perf_counter()
self.rows.append(row)
self.stop_event.wait(1.0)
def stop(self) -> None:
self.stop_event.set()
if self.thread is not None:
self.thread.join(timeout=25)
class CaptureGeneration:
"""Keep CUDA tensor references; perform no device-to-host copy during timing."""
def __init__(self, model: Any, kwargs_transform: Callable | None = None) -> None:
self.model = model
self.original = model.generate
self.output: Any = None
self.input_ids: Any = None
self.attention_mask: Any = None
self.kwargs_transform = kwargs_transform
def __enter__(self) -> CaptureGeneration:
def generate(*args: Any, **kwargs: Any) -> Any:
self.input_ids = kwargs["input_ids"]
self.attention_mask = kwargs.get("attention_mask")
if self.kwargs_transform is not None:
kwargs = self.kwargs_transform(kwargs)
self.output = self.original(*args, **kwargs)
return self.output
self.model.generate = generate
return self
def reset(self) -> None:
self.output = self.input_ids = self.attention_mask = None
def snapshot(self) -> dict[str, Any]:
ids = self.input_ids.detach().cpu().tolist()
output = self.output.detach().cpu().tolist()
return {
"input_ids": ids,
"output_ids": [row[len(ids[idx]) :] for idx, row in enumerate(output)],
"attention_mask": self.attention_mask.detach().cpu().tolist()
if self.attention_mask is not None
else None,
}
def __exit__(self, *args: Any) -> None:
self.model.generate = self.original
class ForwardPhases:
"""Record stream intervals, not pure-kernel time, without synchronizing per token."""
def __init__(self, model: Any, torch: Any) -> None:
self.model = model
self.torch = torch
self.rows: list[dict[str, Any]] = []
self.handles: list[Any] = []
def __enter__(self) -> ForwardPhases:
def before(module: Any, args: Any, kwargs: Any) -> None:
start = self.torch.cuda.Event(enable_timing=True)
end = self.torch.cuda.Event(enable_timing=True)
ids = kwargs.get("input_ids")
start.record()
self.rows.append(
{
"cpu_start": time.perf_counter(),
"cuda_start": start,
"cuda_end": end,
"input_shape": list(ids.shape) if ids is not None else None,
}
)
def after(module: Any, args: Any, kwargs: Any, output: Any) -> None:
row = self.rows[-1]
row["cuda_end"].record()
row["cpu_end"] = time.perf_counter()
self.handles = [
self.model.register_forward_pre_hook(before, with_kwargs=True),
self.model.register_forward_hook(after, with_kwargs=True),
]
return self
def summary(self, request_start: float) -> dict[str, Any]:
spans = []
for row in self.rows:
spans.append(
{
"cpu_submit_ms": (row["cpu_end"] - row["cpu_start"]) * 1000,
"cuda_stream_interval_ms": row["cuda_start"].elapsed_time(row["cuda_end"]),
"start_after_request_ms": (row["cpu_start"] - request_start) * 1000,
"input_shape": row["input_shape"],
}
)
if not spans:
raise RuntimeError("No model.forward calls observed; phase timing is invalid")
decode = spans[1:]
return {
"forward_calls": len(spans),
"prefill": spans[0],
"decode_forwards": decode,
"decode_forward_cuda_sum_ms": sum(row["cuda_stream_interval_ms"] for row in decode),
"decode_forward_cpu_submit_sum_ms": sum(row["cpu_submit_ms"] for row in decode),
"first_forward_start_to_last_forward_end_cuda_ms": self.rows[0][
"cuda_start"
].elapsed_time(self.rows[-1]["cuda_end"]),
"decode_phase_cuda_ms": self.rows[0]["cuda_end"].elapsed_time(
self.rows[-1]["cuda_end"]
),
"decode_phase_tokens": len(decode),
"decode_phase_tokens_per_second": len(decode)
/ (self.rows[0]["cuda_end"].elapsed_time(self.rows[-1]["cuda_end"]) / 1000)
if decode
else None,
}
def __exit__(self, *args: Any) -> None:
for handle in self.handles:
handle.remove()
def distribution(values: list[float]) -> dict[str, float]:
return {
"mean": statistics.mean(values),
"median": statistics.median(values),
"min": min(values),
"max": max(values),
"stdev": statistics.stdev(values) if len(values) > 1 else 0.0,
"coefficient_of_variation": statistics.stdev(values) / statistics.mean(values)
if len(values) > 1 and statistics.mean(values)
else 0.0,
}
def save(output: Path, result: dict[str, Any]) -> None:
output.parent.mkdir(parents=True, exist_ok=True)
temporary = output.with_suffix(output.suffix + ".tmp")
temporary.write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding="utf-8")
temporary.replace(output)
def summarize(samples: list[dict[str, Any]]) -> dict[str, Any]:
return {
"aggregate_tokens_per_second": sum(row["tokens"] for row in samples)
/ sum(row["seconds"] for row in samples),
"tokens_per_second": distribution([row["tokens_per_second"] for row in samples]),
"request_seconds": distribution([row["seconds"] for row in samples]),
"latency_per_token_ms": distribution([row["latency_per_token_ms"] for row in samples]),
"cpu_utilization_percent": distribution(
[row["cpu_utilization_percent"] for row in samples]
),
"peak_allocated_bytes": max(row["peak_allocated_bytes"] for row in samples),
"peak_reserved_bytes": max(row["peak_reserved_bytes"] for row in samples),
}
def dynamo_counters() -> dict[str, dict[str, int]]:
from torch._dynamo.utils import counters
return {
str(group): {str(key): value for key, value in entries.items()}
for group, entries in counters.items()
}
def main() -> int:
args = arguments()
output = contained_path(args.output)
model_path = contained_path(args.model)
reference_path = contained_path(args.reference) if args.reference else None
source_path = contained_path(args.source)
if not (source_path / "src/soup_cli/__init__.py").is_file():
raise FileNotFoundError(f"Not a Soup source checkout: {source_path}")
sys.path.insert(0, str(source_path / "src"))
affinity = apply_cpu_affinity(args.cpu_affinity)
import torch
from rich.console import Console
from soup_cli.commands.infer import _generate, _load_model
console = Console()
if not torch.cuda.is_available():
raise RuntimeError("This protocol requires CUDA; refusing to substitute a CPU benchmark")
if not model_path.is_dir():
raise FileNotFoundError(model_path)
messages = [{"role": "user", "content": args.prompt}]
result: dict[str, Any] = {
"schema_version": 1,
"started_at_utc": datetime.datetime.now(datetime.timezone.utc).isoformat(),
"status": "running",
"variant": args.variant,
"environment": metadata(torch, model_path, affinity),
"workload": {
"model_path": str(model_path),
"messages": messages,
"max_new_tokens": args.max_tokens,
"temperature": 0.0,
"batch_size": 1,
"loader": "soup_cli.commands.infer._load_model",
"request": "soup_cli.commands.infer._generate",
"quantization_override": None,
"dtype_override": None,
},
"protocol": {
"warmups": args.warmups,
"repeats": args.repeats,
"phase_repeats": args.phase_repeats,
"prefill_phase_repeats": args.prefill_phase_repeats,
"prefill_phase_warmups": 1 if args.prefill_phase_repeats else 0,
"telemetry": args.telemetry,
"primary_timing": "perf_counter with CUDA synchronize before and after request",
"phases": "Separate requests with top-level forward CUDA events; includes stream gaps",
"cpu_utilization": "process CPU seconds / elapsed seconds * 100; may exceed 100",
},
"samples": [],
"phases": [],
"prefill_phases": [],
"prefill_phase_warmups": [],
"warmups": [],
"profiles": {},
}
save(output, result)
torch.cuda.synchronize()
torch.cuda.reset_peak_memory_stats()
start = time.perf_counter()
model, tokenizer = _load_model(str(model_path), None, "cuda", is_local=True)
torch.cuda.synchronize()
result["load"] = {
"seconds": time.perf_counter() - start,
"peak_allocated_bytes": torch.cuda.max_memory_allocated(),
"peak_reserved_bytes": torch.cuda.max_memory_reserved(),
}
result["model"] = {
"class": type(model).__name__,
"dtype": str(model.dtype),
"device": str(model.device),
"hf_device_map": {
str(key): str(value) for key, value in getattr(model, "hf_device_map", {}).items()
},
"parameter_bytes": sum(
param.numel() * param.element_size() for param in model.parameters()
),
"use_cache": getattr(model.config, "use_cache", None),
"attention_implementation": getattr(model.config, "_attn_implementation", None),
"generation_config": model.generation_config.to_dict(),
}
console.print(f"Loaded {model_path.name} in {result['load']['seconds']:.2f}s; {args.variant}")
expected: dict[str, Any] | None = None
def invoke() -> tuple[str, int]:
scope = (
torch.inference_mode()
if args.variant in ("inference_mode", "inference_mode_mask_callback")
else contextlib.nullcontext()
)
with scope:
production_kwargs = (
{"cuda_graphs": True} if args.variant == "production_cuda_graphs" else {}
)
return _generate(
model,
tokenizer,
messages,
max_tokens=args.max_tokens,
temperature=0.0,
**production_kwargs,
)
def run_request(capture: CaptureGeneration) -> dict[str, Any]:
nonlocal expected
capture.reset()
torch.cuda.synchronize()
torch.cuda.reset_peak_memory_stats()
cpu_start = time.process_time()
wall_start = time.perf_counter()
response, count = invoke()
torch.cuda.synchronize()
seconds = time.perf_counter() - wall_start
cpu_seconds = time.process_time() - cpu_start
sample = {
"seconds": seconds,
"tokens": count,
"tokens_per_second": count / seconds,
"latency_per_token_ms": seconds * 1000 / count if count else None,
"cpu_seconds": cpu_seconds,
"cpu_utilization_percent": cpu_seconds / seconds * 100,
"peak_allocated_bytes": torch.cuda.max_memory_allocated(),
"peak_reserved_bytes": torch.cuda.max_memory_reserved(),
"request_start_perf_counter": wall_start,
}
# Copies and comparisons are deliberately AFTER the synchronized timing boundary.
snapshot = capture.snapshot()
if count != len(snapshot["output_ids"][0]):
raise AssertionError("Soup's reported count differs from captured generated token IDs")
if expected is None:
expected = snapshot
result["correctness"] = {
**snapshot,
"response_text": response,
"prompt_tokens": len(snapshot["input_ids"][0]),
"generated_tokens": count,
"all_requests_identical": True,
}
elif snapshot != expected:
result["correctness"]["all_requests_identical"] = False
result["correctness"]["mismatching_snapshot"] = snapshot
save(output, result)
raise AssertionError("Repeated requests produced different prompt/output token IDs")
return sample
result["experimental_settings"] = {}
if args.variant == "production_cuda_graphs":
result["experimental_settings"]["production_generate_cuda_graphs"] = True
if args.variant == "static" or args.variant in EXPERIMENTAL_GRAPH_VARIANTS:
model.generation_config.cache_implementation = "static"
# Static cache alone should isolate allocation/layout, not implicitly
# opt into Transformers' default compile backend.
model.generation_config.disable_compile = args.variant == "static"
result["experimental_settings"].update(
cache_implementation="static", disable_compile=args.variant == "static"
)
if args.variant in EXPERIMENTAL_GRAPH_VARIANTS:
from transformers import CompileConfig
backend = "cudagraphs"
if args.variant == "cudagraphs_static_inputs":
from static_cudagraph_backend import static_cudagraphs
from torch._dynamo.backends.registry import register_backend
backend = "soup_experiment_static_inputs"
register_backend(static_cudagraphs, name=backend)
model.generation_config.compile_config = CompileConfig(
backend=backend, fullgraph=True, dynamic=False
)
result["experimental_settings"]["compile_config"] = {
"backend": backend,
"fullgraph": True,
"dynamic": False,
}
extra_context = contextlib.nullcontext()
kwargs_transform = None
callback_counts = {"requests": 0}
if args.variant in ("no_padding_mask", "mask_callback", "inference_mode_mask_callback"):
from soup_cli.utils.vllm import encode_chat_prompt
validated_inputs = encode_chat_prompt(
messages, tokenizer, fallback_on_error=False, return_tensors="pt"
)
if args.variant == "no_padding_mask":
from mask_experiment import omit_redundant_qwen2_padding_mask
extra_context = omit_redundant_qwen2_padding_mask(model, validated_inputs["attention_mask"])
result["experimental_settings"]["omit_redundant_qwen2_padding_mask"] = True
elif args.variant in ("mask_callback", "inference_mode_mask_callback"):
from mask_callback_experiment import no_padding_generation_kwargs
def kwargs_transform(kwargs: dict[str, Any]) -> dict[str, Any]:
prepared = no_padding_generation_kwargs(
model, validated_inputs["attention_mask"], kwargs
)
if "custom_generate" not in prepared:
raise RuntimeError("The experimental mask callback eligibility gate did not pass")
callback_counts["requests"] += 1
return prepared
result["experimental_settings"]["no_padding_custom_generate"] = True
if args.variant in GRAPH_VARIANTS and args.phase_repeats:
console.print("Skipping forward-hook phase instrumentation for the compiled variant.")
result["protocol"]["phase_repeats_requested"] = args.phase_repeats
result["protocol"]["phase_repeats"] = 0
result["protocol"]["phases_skip_reason"] = (
"Python hooks and CUDA event construction can invalidate compiled graph capture"
)
args.phase_repeats = 0
with extra_context as experiment_stats, CaptureGeneration(model, kwargs_transform) as capture:
for index in range(args.warmups):
sample = run_request(capture)
result["warmups"].append(sample)
console.print(
f"Warmup {index + 1}/{args.warmups}: {sample['tokens_per_second']:.2f} tok/s"
)
telemetry = Telemetry(args.telemetry)
telemetry.start()
try:
for index in range(args.repeats):
sample = run_request(capture)
result["samples"].append(sample)
save(output, result)
console.print(
f"Measured {index + 1}/{args.repeats}: "
f"{sample['tokens_per_second']:.2f} tok/s, {sample['tokens']} tokens"
)
finally:
telemetry.stop()
result["telemetry"] = {
"enabled": args.telemetry,
"rows": telemetry.rows,
"fields": [
"timestamp",
"gpu_util_percent",
"memory_util_percent",
"memory_used_mib",
"power_w",
"power_limit_w",
"sm_clock_mhz",
"memory_clock_mhz",
"temperature_c",
"pstate",
],
}
result["summary"] = summarize(result["samples"])
for kind in ("allocated", "reserved"):
key = f"peak_{kind}_bytes"
result["summary"][f"{key}_including_load_and_warmup"] = max(
row[key] for row in [result["load"], *result["warmups"], *result["samples"]]
)
result["status"] = "measurements_complete"
if args.variant in GRAPH_VARIANTS:
result["dynamo_counters_after_measured"] = dynamo_counters()
if args.variant == "cudagraphs_static_inputs":
from static_cudagraph_backend import COMPILE_RECORDS
result["backend_compile_records_after_measured"] = list(COMPILE_RECORDS)
save(output, result)
for _ in range(args.phase_repeats):
with ForwardPhases(model, torch) as phases:
sample = run_request(capture)
sample.update(phases.summary(sample["request_start_perf_counter"]))
result["phases"].append(sample)
save(output, result)
if args.prefill_phase_repeats:
from validate_cuda_graphs import _request_phases
for index in range(args.prefill_phase_repeats + 1):
capture.reset()
sample = _request_phases(model, invoke)
# The helper has synchronized and closed its timing boundary.
# Capture evidence afterwards, as in the uninstrumented requests.
snapshot = capture.snapshot()
if snapshot != expected:
result["correctness"]["all_requests_identical"] = False
result["correctness"]["mismatching_prefill_phase_snapshot"] = snapshot
save(output, result)
raise AssertionError("Prefill-instrumented request changed prompt/output IDs")
count = sample["generated_tokens"]
if count != len(snapshot["output_ids"][0]):
raise AssertionError("Prefill diagnostic token count mismatch")
sample.update(
input_ids=snapshot["input_ids"],
output_ids=snapshot["output_ids"],
attention_mask=snapshot["attention_mask"],
prompt_tokens=len(snapshot["input_ids"][0]),
identical_to_measured_requests=True,
decode_forward_tokens=max(0, count - 1),
pre_prefill_cuda_event_span_ms=sample["request_cuda_event_span_ms"]
- sample["prefill_cuda_event_span_ms"]
- sample["post_prefill_cuda_event_span_ms"],
)
sample["post_prefill_decode_forward_tokens_per_second"] = sample[
"decode_forward_tokens"
] / (sample["post_prefill_cuda_event_span_ms"] / 1000)
bucket = "prefill_phase_warmups" if index == 0 else "prefill_phases"
result[bucket].append(sample)
save(output, result)
console.print(
f"Prefill phase {'warmup' if index == 0 else index}: "
f"{sample['prefill_cuda_event_span_ms']:.3f} ms prefill; "
f"{sample['post_prefill_cuda_event_span_ms']:.3f} ms post-prefill"
)
result["prefill_phase_summary"] = {
key: distribution([sample[key] for sample in result["prefill_phases"]])
for key in (
"synchronized_request_wall_ms",
"request_cuda_event_span_ms",
"pre_prefill_cuda_event_span_ms",
"prefill_cuda_event_span_ms",
"post_prefill_cuda_event_span_ms",
"post_prefill_decode_forward_tokens_per_second",
)
}
result["prefill_phase_summary"]["aggregate_post_prefill_decode_tokens_per_second"] = (
sum(sample["decode_forward_tokens"] for sample in result["prefill_phases"])
/ (
sum(
sample["post_prefill_cuda_event_span_ms"]
for sample in result["prefill_phases"]
)
/ 1000
)
)
save(output, result)
if args.trace_only:
console.print("Tracing a separate complete request without shape/memory aggregation...")
with torch.profiler.profile(
activities=[
torch.profiler.ProfilerActivity.CPU,
torch.profiler.ProfilerActivity.CUDA,
],
record_shapes=False,
profile_memory=False,
with_stack=False,
) as profiler:
run_request(capture)
trace_path = Path(str(output.with_suffix("")) + ".trace.json")
profiler.export_chrome_trace(str(trace_path))
result["profiles"]["torch"] = {
"trace": str(trace_path),
"trace_only": True,
"record_shapes": False,
"profile_memory": False,
"key_averages_processed": False,
"cprofile_run": False,
}
save(output, result)
elif args.profile:
import cProfile
import io
import pstats
prefix = output.with_suffix("")
console.print("Profiling a separate complete request with torch.profiler...")
with torch.profiler.profile(
activities=[
torch.profiler.ProfilerActivity.CPU,
torch.profiler.ProfilerActivity.CUDA,
],
record_shapes=True,
profile_memory=True,
with_stack=False,
) as profiler:
run_request(capture)
trace_path = Path(str(prefix) + ".trace.json")
profiler.export_chrome_trace(str(trace_path))
table_path = Path(str(prefix) + ".torch-profile.txt")
averages = profiler.key_averages()
table_path.write_text(
averages.table(sort_by="self_cpu_time_total", row_limit=60)
+ "\n\n"
+ averages.table(sort_by="self_device_time_total", row_limit=60),
encoding="utf-8",
)
profiler_rows = []
for event in averages:
profiler_rows.append(
{
"key": event.key,
"count": event.count,
"self_cpu_time_total_us": event.self_cpu_time_total,
"cpu_time_total_us": event.cpu_time_total,
"self_device_time_total_us": getattr(event, "self_device_time_total", None),
"device_time_total_us": getattr(event, "device_time_total", None),
}
)
result["profiles"]["torch"] = {
"trace": str(trace_path),
"table": str(table_path),
"operators": profiler_rows,
}
console.print("Profiling another separate complete request with cProfile...")
cpu_profiler = cProfile.Profile()
cpu_profiler.enable()
run_request(capture)
cpu_profiler.disable()
profile_path = Path(str(prefix) + ".pstats")
cpu_profiler.dump_stats(str(profile_path))
buffer = io.StringIO()
pstats.Stats(cpu_profiler, stream=buffer).sort_stats("cumulative").print_stats(70)
text_path = Path(str(prefix) + ".cprofile.txt")
text_path.write_text(buffer.getvalue(), encoding="utf-8")
result["profiles"]["cprofile"] = {
"stats": str(profile_path),
"table": str(text_path),
}
if experiment_stats is not None:
from dataclasses import asdict
result["experimental_call_counts"] = asdict(experiment_stats)
if kwargs_transform is not None:
result["experimental_call_counts"] = callback_counts
if args.variant in GRAPH_VARIANTS:
result["dynamo_counters_after_diagnostics"] = dynamo_counters()
if args.variant == "cudagraphs_static_inputs":
result["backend_compile_records_after_diagnostics"] = list(COMPILE_RECORDS)
if reference_path is not None:
reference = json.loads(reference_path.read_text(encoding="utf-8"))
identical_workload = reference["workload"] == result["workload"]
identical_tokens = all(
reference["correctness"][key] == result["correctness"][key]
for key in ("input_ids", "output_ids", "attention_mask")
)
same_config_files = (
reference["environment"]["config_fingerprints"]
== result["environment"]["config_fingerprints"]
)
same_versions = reference["environment"]["versions"] == result["environment"]["versions"]
reference_affinity = reference["environment"].get("cpu_affinity", {}).get("effective")
current_affinity = result["environment"]["cpu_affinity"]["effective"]
same_affinity = (
reference_affinity == current_affinity
if reference_affinity is not None and current_affinity is not None
else None
)
result["reference_comparison"] = {
"path": str(reference_path),
"identical_workload": identical_workload,
"identical_tokens": identical_tokens,
"same_config_files": same_config_files,
"same_versions": same_versions,
"same_effective_cpu_affinity": same_affinity,
"improvement_percent": (
result["summary"]["aggregate_tokens_per_second"]
/ reference["summary"]["aggregate_tokens_per_second"]
- 1
)
* 100,
}
if not all(
(
identical_workload,
identical_tokens,
same_config_files,
same_versions,
same_affinity is not False,
)
):
result["status"] = "reference_mismatch"
save(output, result)
raise AssertionError("Baseline comparison is invalid; see reference_comparison")
result["status"] = "complete"
result["finished_at_utc"] = datetime.datetime.now(datetime.timezone.utc).isoformat()
save(output, result)
console.print(
f"Aggregate: {result['summary']['aggregate_tokens_per_second']:.3f} tok/s; "
f"peak allocated: {result['summary']['peak_allocated_bytes'] / 1024**3:.3f} GiB"
)
console.print(f"Saved {output}")
return 0
if __name__ == "__main__":
raise SystemExit(main())