A high-performance inference engine for AI models
add gpu trace tool
Eugene Bokhane6a4e1c1df04
diff --git a/.gitignore b/.gitignoreindex 69b6e8ae6..28763c5f9 100644--- a/.gitignore+++ b/.gitignore@@ -1,38 +1,28 @@-# Generated by Cargo-# will have compiled files and executables-debug/-target/--# These are backup files generated by rustfmt-**/*.rs.bk--# MSVC Windows builds of rustc generate these, which store debugging information*.pdb--# Generated by cargo mutants-# Contains mutation testing data**/mutants.out*/--# RustRover-# JetBrains specific template is maintained in a separate JetBrains.gitignore that can-# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore-# and can be added to the global gitignore or merged into this file. For a more nuclear-# option (not recommended) you can uncomment the following to ignore the entire idea folder.-#.idea/*.DS_Store# Metal traces*.gputrace+*.trace-# Models-models/-external/+# Rust+/debug/+/target/+**/*.rs.bk-# GH-.github/*.md+# Python+*__pycache__-# Local-only dev examples-crates/uzu/examples/quick_chat.rs+# AI tools+.github/*.md.cursor-tools.claude++# Models+/models/++# Local-only dev tools+/crates/uzu/examples/quick_chat.rs+/external+diff --git a/scripts/download_test_model.sh b/scripts/download_test_model.shindex ea1425f1d..cab4a5875 100755--- a/scripts/download_test_model.sh+++ b/scripts/download_test_model.sh@@ -7,9 +7,9 @@ if [[ "$(uname)" != "Darwin" ]]; thenexit 0fi-SCRIPTS_PATH="$(cd "$(dirname "$0")" && pwd)"-TOOLS_PATH="$SCRIPTS_PATH/tools"+ROOT_PATH="$(cd "$(dirname "$0")/.." && pwd)"+HELPERS_PATH="$ROOT_PATH/tools/helpers"-cd $TOOLS_PATH+cd "$HELPERS_PATH"uv sync-uv run main.py download-model meta-llama/Llama-3.2-1B-Instruct\ No newline at end of file+uv run python main.py download-model meta-llama/Llama-3.2-1B-Instruct\ No newline at end of filediff --git a/scripts/tools/.python-version b/tools/gpu_trace/.python-versionsimilarity index 100%rename from scripts/tools/.python-versionrename to tools/gpu_trace/.python-versiondiff --git a/tools/gpu_trace/pyproject.toml b/tools/gpu_trace/pyproject.tomlnew file mode 100644index 000000000..8b0248067--- /dev/null+++ b/tools/gpu_trace/pyproject.toml@@ -0,0 +1,13 @@+[project]+name = "gpu-trace"+version = "0.1.0"+description = "Record, analyze, and export Metal GPU traces"+requires-python = ">=3.12"+dependencies = ["cattrs>=24.1.0"]++[build-system]+requires = ["hatchling"]+build-backend = "hatchling.build"++[tool.hatch.build.targets.wheel]+packages = ["src/gpu_trace"]diff --git a/tools/gpu_trace/src/gpu_trace/__init__.py b/tools/gpu_trace/src/gpu_trace/__init__.pynew file mode 100644index 000000000..60ae86a2e--- /dev/null+++ b/tools/gpu_trace/src/gpu_trace/__init__.py@@ -0,0 +1,29 @@+from .models import (+ CounterType,+ GpuCounterDefinition,+ GpuPerformanceStateInterval,+ GpuState,+ GpuUtilization,+ KernelDispatch,+ KernelStats,+ PerformanceState,+ ShaderInfo,+ ShaderType,+ TraceExport,+ TraceMetadata,+)++__all__ = [+ "CounterType",+ "GpuCounterDefinition",+ "GpuPerformanceStateInterval",+ "GpuState",+ "GpuUtilization",+ "KernelDispatch",+ "KernelStats",+ "PerformanceState",+ "ShaderInfo",+ "ShaderType",+ "TraceExport",+ "TraceMetadata",+]diff --git a/tools/gpu_trace/src/gpu_trace/__main__.py b/tools/gpu_trace/src/gpu_trace/__main__.pynew file mode 100644index 000000000..3629204a4--- /dev/null+++ b/tools/gpu_trace/src/gpu_trace/__main__.py@@ -0,0 +1,5 @@+from .main import main+import sys++if __name__ == "__main__":+ raise SystemExit(main(sys.argv[1:]))diff --git a/tools/gpu_trace/src/gpu_trace/extractors.py b/tools/gpu_trace/src/gpu_trace/extractors.pynew file mode 100644index 000000000..b3f46f5c1--- /dev/null+++ b/tools/gpu_trace/src/gpu_trace/extractors.py@@ -0,0 +1,441 @@+from __future__ import annotations++import subprocess+import xml.etree.ElementTree as ET+from pathlib import Path++from .models import (+ CounterType,+ GpuCounterDefinition,+ GpuPerformanceStateInterval,+ GpuState,+ GpuUtilization,+ KernelDispatch,+ KernelStats,+ PerformanceState,+ ShaderInfo,+ ShaderType,+ TraceMetadata,+)+++def _check_output(cmd: list[str]) -> str:+ return subprocess.check_output(cmd, text=True)+++def _xctrace_cmd(*args: str) -> list[str]:+ return ["xcrun", "xctrace", *args]+++def _build_id_maps(+ root: ET.Element,+) -> tuple[dict[str, ET.Element], dict[str, str]]:+ id_to_element: dict[str, ET.Element] = {}+ id_to_text: dict[str, str] = {}+ for el in root.iter():+ el_id = el.attrib.get("id")+ if el_id is None:+ continue+ id_to_element[el_id] = el+ if el.text is not None:+ id_to_text[el_id] = el.text+ return id_to_element, id_to_text+++def _deref(+ el: ET.Element | None, id_to_element: dict[str, ET.Element]+) -> ET.Element | None:+ if el is None:+ return None+ ref = el.attrib.get("ref")+ return id_to_element.get(ref) if ref else el+++def _resolve_text(el: ET.Element | None, id_to_text: dict[str, str]) -> str | None:+ if el is None:+ return None+ if el.text is not None:+ return el.text+ ref = el.attrib.get("ref")+ return id_to_text.get(ref) if ref else None+++def _toc_root(trace: Path) -> ET.Element:+ xml = _check_output(_xctrace_cmd("export", "--input", str(trace), "--toc"))+ return ET.fromstring(xml)+++def _export_table(trace: Path, run_number: int, schema: str) -> ET.Element:+ xpath = f'/trace-toc/run[@number="{run_number}"]/data/table[@schema="{schema}"]'+ xml = _check_output(_xctrace_cmd("export", "--input", str(trace), "--xpath", xpath))+ return ET.fromstring(xml)+++def _union_ns(intervals: list[tuple[int, int]]) -> int:+ if not intervals:+ return 0+ sorted_intervals = sorted(intervals)+ total = 0+ cur_s, cur_e = sorted_intervals[0]+ for s, e in sorted_intervals[1:]:+ if s > cur_e:+ total += cur_e - cur_s+ cur_s, cur_e = s, e+ else:+ cur_e = max(cur_e, e)+ total += cur_e - cur_s+ return total+++def extract_metadata(trace: Path, run_number: int) -> TraceMetadata:+ toc = _toc_root(trace)+ run = toc.find(f".//run[@number='{run_number}']")+ if run is None:+ raise ValueError(f"Run {run_number} not found in trace TOC")++ device = run.find("./info/target/device")+ device_name = device.attrib.get("name", "") if device is not None else ""+ device_model = device.attrib.get("model", "") if device is not None else ""+ os_version = device.attrib.get("os-version", "") if device is not None else ""++ process = run.find("./info/target/process")+ target_process = process.attrib.get("name", "") if process is not None else ""+ target_pid_str = process.attrib.get("pid", "0") if process is not None else "0"+ target_pid = int(target_pid_str) if target_pid_str else 0++ summary = run.find("./info/summary")+ duration_el = summary.find("./duration") if summary is not None else None+ duration_ns = (+ int(float(duration_el.text) * 1e9)+ if duration_el is not None and duration_el.text+ else 0+ )++ start_date_el = summary.find("./start-date") if summary is not None else None+ start_date = start_date_el.text if start_date_el is not None else ""++ end_date_el = summary.find("./end-date") if summary is not None else None+ end_date = end_date_el.text if end_date_el is not None else ""++ template_el = summary.find("./template-name") if summary is not None else None+ template_name = template_el.text if template_el is not None else ""++ return TraceMetadata(+ device_name=device_name or "",+ device_model=device_model or "",+ os_version=os_version or "",+ target_process=target_process or "",+ target_pid=target_pid,+ duration_ns=duration_ns,+ start_date=start_date or "",+ end_date=end_date or "",+ template_name=template_name or "",+ )+++def extract_gpu_utilization(trace: Path, run_number: int) -> GpuUtilization:+ toc = _toc_root(trace)+ run = toc.find(f".//run[@number='{run_number}']")+ if run is None:+ raise ValueError(f"Run {run_number} not found")++ duration_el = run.find("./info/summary/duration")+ run_duration_ns = (+ int(float(duration_el.text) * 1e9)+ if duration_el is not None and duration_el.text+ else 0+ )++ pid_el = run.find("./info/target/process")+ pid = int(pid_el.attrib.get("pid", "0")) if pid_el is not None else 0++ root = _export_table(trace, run_number, "metal-gpu-state-intervals")+ _, id_to_text = _build_id_maps(root)++ active_ns = 0+ idle_ns = 0+ for row in root.findall(".//row"):+ state = _resolve_text(row.find("./gpu-state"), id_to_text)+ dur_s = _resolve_text(row.find("./duration"), id_to_text)+ if state is None or dur_s is None:+ continue+ dur = int(dur_s)+ if state == GpuState.ACTIVE.value:+ active_ns += dur+ elif state == GpuState.IDLE.value:+ idle_ns += dur++ target_active_ns = 0+ window_start_ns = 0+ window_end_ns = 0+ command_buffer_count = 0+ encoder_count = 0++ try:+ encoders = _export_table(trace, run_number, "metal-application-encoders-list")+ gpu_intervals = _export_table(trace, run_number, "metal-gpu-intervals")++ enc_id_to_el, enc_id_to_text = _build_id_maps(encoders)+ command_buffers: set[int] = set()+ for row in encoders.findall(".//row"):+ proc = _deref(row.find("./process"), enc_id_to_el)+ proc_pid_s = _resolve_text(+ proc.find("./pid") if proc is not None else None, enc_id_to_text+ )+ if proc_pid_s is None or int(proc_pid_s) != pid:+ continue+ ids = row.findall("./metal-command-buffer-id")+ if not ids:+ continue+ cb_s = _resolve_text(ids[0], enc_id_to_text)+ if cb_s:+ command_buffers.add(int(cb_s))+ encoder_count += 1++ command_buffer_count = len(command_buffers)++ gpu_id_to_el, gpu_id_to_text = _build_id_maps(gpu_intervals)+ intervals: list[tuple[int, int]] = []+ for row in gpu_intervals.findall(".//row"):+ ids = row.findall("./metal-command-buffer-id")+ if not ids:+ continue+ cb_s = _resolve_text(ids[0], gpu_id_to_text)+ if cb_s is None:+ continue+ cb = int(cb_s)+ if cb == 0 or cb not in command_buffers:+ continue+ start_s = _resolve_text(row.find("./start-time"), gpu_id_to_text)+ dur_s = _resolve_text(row.find("./duration"), gpu_id_to_text)+ if start_s is None or dur_s is None:+ continue+ start_ns = int(start_s)+ dur_ns = int(dur_s)+ if dur_ns <= 0:+ continue+ intervals.append((start_ns, start_ns + dur_ns))++ if intervals:+ window_start_ns = min(start for start, _ in intervals)+ window_end_ns = max(end for _, end in intervals)+ target_active_ns = _union_ns(intervals)+ except subprocess.CalledProcessError:+ pass++ return GpuUtilization(+ run_duration_ns=run_duration_ns,+ global_active_ns=active_ns,+ global_idle_ns=idle_ns,+ global_coverage_ns=active_ns + idle_ns,+ target_active_ns=target_active_ns,+ window_start_ns=window_start_ns,+ window_end_ns=window_end_ns,+ command_buffer_count=command_buffer_count,+ encoder_count=encoder_count,+ )+++def extract_performance_states(+ trace: Path, run_number: int+) -> tuple[GpuPerformanceStateInterval, ...]:+ try:+ root = _export_table(trace, run_number, "gpu-performance-state-intervals")+ except subprocess.CalledProcessError:+ return ()++ _, id_to_text = _build_id_maps(root)+ states: list[GpuPerformanceStateInterval] = []++ for row in root.findall(".//row"):+ start_s = _resolve_text(row.find("./start-time"), id_to_text)+ dur_s = _resolve_text(row.find("./duration"), id_to_text)+ state_s = _resolve_text(row.find("./gpu-performance-state"), id_to_text)+ gpu_s = _resolve_text(row.find("./track-label"), id_to_text)+ induced_s = _resolve_text(row.find("./is-induced"), id_to_text)++ if start_s is None or dur_s is None or state_s is None:+ continue++ try:+ state = PerformanceState(state_s)+ except ValueError:+ state = PerformanceState.UNKNOWN++ states.append(+ GpuPerformanceStateInterval(+ start_ns=int(start_s),+ duration_ns=int(dur_s),+ state=state,+ is_induced=induced_s == "1",+ gpu_name=gpu_s or "",+ )+ )++ return tuple(states)+++def extract_counter_definitions(+ trace: Path, run_number: int+) -> tuple[GpuCounterDefinition, ...]:+ try:+ root = _export_table(trace, run_number, "gpu-counter-info")+ except subprocess.CalledProcessError:+ return ()++ _, id_to_text = _build_id_maps(root)+ counters: list[GpuCounterDefinition] = []++ for row in root.findall(".//row"):+ counter_id_s = _resolve_text(row.find("./counter-id"), id_to_text)+ name = _resolve_text(row.find("./name"), id_to_text)+ description = _resolve_text(row.find("./description"), id_to_text)+ max_value_s = _resolve_text(row.find("./max-value"), id_to_text)+ counter_type_s = _resolve_text(row.find("./type"), id_to_text)+ sample_interval_s = _resolve_text(row.find("./sample-interval"), id_to_text)++ if counter_id_s is None or name is None:+ continue++ try:+ counter_type = CounterType(counter_type_s) if counter_type_s else CounterType.VALUE+ except ValueError:+ counter_type = CounterType.VALUE++ counters.append(+ GpuCounterDefinition(+ counter_id=int(counter_id_s),+ name=name,+ description=description or "",+ max_value=int(max_value_s) if max_value_s else 0,+ counter_type=counter_type,+ sample_interval_us=int(sample_interval_s) if sample_interval_s else 0,+ )+ )++ return tuple(counters)+++def extract_kernel_dispatches(+ trace: Path, run_number: int+) -> tuple[KernelDispatch, ...]:+ toc = _toc_root(trace)+ pid_el = toc.find(f".//run[@number='{run_number}']/info/target/process")+ pid = int(pid_el.attrib.get("pid", "0")) if pid_el is not None else 0++ try:+ root = _export_table(trace, run_number, "metal-shader-profiler-intervals")+ except subprocess.CalledProcessError:+ return ()++ id_to_el, id_to_text = _build_id_maps(root)+ dispatches: list[KernelDispatch] = []++ for row in root.findall(".//row"):+ proc = _deref(row.find("./process"), id_to_el)+ pid_s = _resolve_text(+ proc.find("./pid") if proc is not None else None, id_to_text+ )+ if pid_s is None or int(pid_s) != pid:+ continue++ name = _resolve_text(row.find("./metal-object-label"), id_to_text)+ start_s = _resolve_text(row.find("./start-time"), id_to_text)+ dur_s = _resolve_text(row.find("./duration"), id_to_text)+ channel = _resolve_text(row.find("./gpu-channel-name"), id_to_text)++ cb_ids = row.findall("./metal-command-buffer-id")+ cb_s = _resolve_text(cb_ids[0], id_to_text) if cb_ids else None++ if name is None or start_s is None or dur_s is None:+ continue++ dur_ns = int(dur_s)+ if dur_ns <= 0:+ continue++ dispatches.append(+ KernelDispatch(+ name=name,+ start_ns=int(start_s),+ duration_ns=dur_ns,+ command_buffer_id=int(cb_s) if cb_s else 0,+ channel=channel or "",+ )+ )++ dispatches.sort(key=lambda d: d.start_ns)+ return tuple(dispatches)+++def extract_shaders(trace: Path, run_number: int) -> tuple[ShaderInfo, ...]:+ toc = _toc_root(trace)+ pid_el = toc.find(f".//run[@number='{run_number}']/info/target/process")+ pid = int(pid_el.attrib.get("pid", "0")) if pid_el is not None else 0++ try:+ root = _export_table(trace, run_number, "metal-shader-profiler-shader-list")+ except subprocess.CalledProcessError:+ return ()++ id_to_el, id_to_text = _build_id_maps(root)+ shaders: list[ShaderInfo] = []++ for row in root.findall(".//row"):+ proc = _deref(row.find("./process"), id_to_el)+ pid_s = _resolve_text(+ proc.find("./pid") if proc is not None else None, id_to_text+ )+ if pid_s is None or int(pid_s) != pid:+ continue++ shader_id_s = _resolve_text(row.find("./id"), id_to_text)+ name = _resolve_text(row.find("./name"), id_to_text)+ shader_type_s = _resolve_text(row.find("./shader-type"), id_to_text)+ pc_start_s = _resolve_text(row.find("./pc-start"), id_to_text)+ pc_end_s = _resolve_text(row.find("./pc-end"), id_to_text)++ if shader_id_s is None or name is None:+ continue++ try:+ shader_type = ShaderType(shader_type_s) if shader_type_s else ShaderType.UNKNOWN+ except ValueError:+ shader_type = ShaderType.UNKNOWN++ shaders.append(+ ShaderInfo(+ shader_id=int(shader_id_s),+ name=name,+ shader_type=shader_type,+ pc_start=int(pc_start_s) if pc_start_s else 0,+ pc_end=int(pc_end_s) if pc_end_s else 0,+ )+ )++ return tuple(shaders)+++def compute_kernel_summary(+ dispatches: tuple[KernelDispatch, ...]+) -> tuple[KernelStats, ...]:+ stats: dict[str, list[int]] = {}+ for d in dispatches:+ if d.name not in stats:+ stats[d.name] = []+ stats[d.name].append(d.duration_ns)++ result: list[KernelStats] = []+ for name, durations in stats.items():+ result.append(+ KernelStats(+ name=name,+ total_ns=sum(durations),+ count=len(durations),+ min_ns=min(durations),+ max_ns=max(durations),+ )+ )++ result.sort(key=lambda s: s.total_ns, reverse=True)+ return tuple(result)diff --git a/tools/gpu_trace/src/gpu_trace/formatting.py b/tools/gpu_trace/src/gpu_trace/formatting.pynew file mode 100644index 000000000..2600d5bcb--- /dev/null+++ b/tools/gpu_trace/src/gpu_trace/formatting.py@@ -0,0 +1,221 @@+from __future__ import annotations++from .models import TraceExport+++def _fmt_ns(ns: int) -> str:+ if ns >= 1_000_000_000:+ return f"{ns / 1e9:.2f} s"+ if ns >= 1_000_000:+ return f"{ns / 1e6:.2f} ms"+ if ns >= 1_000:+ return f"{ns / 1e3:.1f} Β΅s"+ return f"{ns} ns"+++def _fmt_percent(value: float) -> str:+ return f"{value:.2f}%"+++def _bar(value: float, width: int = 20) -> str:+ filled = int(value / 100 * width)+ return "β" * filled + "β" * (width - filled)+++def print_summary(export: TraceExport) -> None:+ meta = export.metadata+ util = export.utilization++ print()+ print("β" + "β" * 78 + "β")+ print("β" + " GPU TRACE SUMMARY ".center(78) + "β")+ print("β" + "β" * 78 + "β€")++ print("β" + f" Device: {meta.device_model} ({meta.device_name})".ljust(78) + "β")+ print("β" + f" OS: {meta.os_version}".ljust(78) + "β")+ print(+ "β"+ + f" Process: {meta.target_process} (PID {meta.target_pid})".ljust(78)+ + "β"+ )+ print("β" + f" Template: {meta.template_name}".ljust(78) + "β")+ print("β" + f" Duration: {_fmt_ns(util.run_duration_ns)}".ljust(78) + "β")++ print("β" + "β" * 78 + "β€")+ print("β" + " UTILIZATION ".center(78) + "β")+ print("β" + "β" * 78 + "β€")++ global_pct = util.global_utilization_percent+ target_pct = util.target_utilization_percent+ window_pct = util.window_utilization_percent++ print(+ "β"+ + f" Global GPU Active: {_bar(global_pct)} {_fmt_percent(global_pct):>8} ({_fmt_ns(util.global_active_ns)})".ljust(+ 78+ )+ + "β"+ )+ print(+ "β"+ + f" Target GPU Active: {_bar(target_pct)} {_fmt_percent(target_pct):>8} ({_fmt_ns(util.target_active_ns)})".ljust(+ 78+ )+ + "β"+ )+ print(+ "β"+ + f" Window Utilization: {_bar(window_pct)} {_fmt_percent(window_pct):>8} (when active)".ljust(+ 78+ )+ + "β"+ )+ print("β" + f" Command Buffers: {util.command_buffer_count}".ljust(78) + "β")++ if export.performance_states:+ states = {}+ for ps in export.performance_states:+ key = ps.state.value+ states[key] = states.get(key, 0) + ps.duration_ns+ induced_count = sum(1 for ps in export.performance_states if ps.is_induced)+ print("β" + "β" * 78 + "β€")+ print("β" + " GPU PERFORMANCE STATES ".center(78) + "β")+ print("β" + "β" * 78 + "β€")+ for state_name, duration in sorted(states.items(), key=lambda x: -x[1]):+ print("β" + f" {state_name}: {_fmt_ns(duration)}".ljust(78) + "β")+ if induced_count:+ print(+ "β" + f" β Induced states: {induced_count} intervals".ljust(78) + "β"+ )++ if export.counter_definitions:+ print("β" + "β" * 78 + "β€")+ print("β" + " AVAILABLE COUNTERS ".center(78) + "β")+ print("β" + "β" * 78 + "β€")+ print(+ "β"+ + f" {len(export.counter_definitions)} GPU hardware counters available".ljust(+ 78+ )+ + "β"+ )++ print("β" + "β" * 78 + "β€")+ print("β" + " KERNEL DISPATCHES ".center(78) + "β")+ print("β" + "β" * 78 + "β€")+ print("β" + f" Total dispatches: {len(export.dispatches):,}".ljust(78) + "β")+ print("β" + f" Unique kernels: {len(export.kernel_summary)}".ljust(78) + "β")+ print("β" + f" Shaders compiled: {len(export.shaders)}".ljust(78) + "β")++ if export.kernel_summary:+ print("β" + "β" * 78 + "β€")+ print("β" + " TOP 15 KERNELS BY TIME ".center(78) + "β")+ print("β" + "β" * 78 + "β€")+ header = f" {'Kernel':<30} {'Total':>10} {'Avg':>10} {'Count':>7} {'%':>6}"+ print("β" + header.ljust(78) + "β")+ print("β" + "β" * 78 + "β€")++ total_ns = sum(k.total_ns for k in export.kernel_summary)+ for k in export.kernel_summary[:15]:+ pct = (k.total_ns / total_ns * 100) if total_ns else 0+ name = k.name[:30] if len(k.name) <= 30 else k.name[:27] + "..."+ row = f" {name:<30} {_fmt_ns(k.total_ns):>10} {_fmt_ns(int(k.avg_ns)):>10} {k.count:>7} {pct:>5.1f}%"+ print("β" + row.ljust(78) + "β")++ print("β" + "β" * 78 + "β")+ print()+++def print_comparison(+ export1: TraceExport,+ export2: TraceExport,+ label1: str,+ label2: str,+) -> None:+ print()+ print("β" + "β" * 78 + "β")+ print("β" + " GPU TRACE COMPARISON ".center(78) + "β")+ print("β" + "β" * 78 + "β€")++ print("β" + f" {label1}: {export1.metadata.target_process}".ljust(78) + "β")+ print("β" + f" {label2}: {export2.metadata.target_process}".ljust(78) + "β")++ print("β" + "β" * 78 + "β€")+ print("β" + " UTILIZATION ".center(78) + "β")+ print("β" + "β" * 78 + "β€")++ header = f" {'Metric':<22} {label1:>14} {label2:>14} {'Diff':>14}"+ print("β" + header.ljust(78) + "β")+ print("β" + "β" * 78 + "β€")++ def _diff_str(v1: float, v2: float) -> str:+ if v1 == 0:+ return "n/a"+ diff = (v2 - v1) / v1 * 100+ sign = "+" if diff >= 0 else ""+ return f"{sign}{diff:.1f}%"++ u1, u2 = export1.utilization, export2.utilization++ rows = [+ (+ "Duration",+ _fmt_ns(u1.run_duration_ns),+ _fmt_ns(u2.run_duration_ns),+ _diff_str(u1.run_duration_ns, u2.run_duration_ns),+ ),+ (+ "GPU Active",+ _fmt_ns(u1.target_active_ns),+ _fmt_ns(u2.target_active_ns),+ _diff_str(u1.target_active_ns, u2.target_active_ns),+ ),+ (+ "Window Util",+ _fmt_percent(u1.window_utilization_percent),+ _fmt_percent(u2.window_utilization_percent),+ _diff_str(u1.window_utilization_percent, u2.window_utilization_percent),+ ),+ ]++ for metric, v1, v2, diff in rows:+ row = f" {metric:<22} {v1:>14} {v2:>14} {diff:>14}"+ print("β" + row.ljust(78) + "β")++ print("β" + "β" * 78 + "β€")+ print("β" + " TOP KERNELS COMPARISON ".center(78) + "β")+ print("β" + "β" * 78 + "β€")++ header = f" {'Kernel':<32} {label1:>12} {label2:>12} {'Diff':>10}"+ print("β" + header.ljust(78) + "β")+ print("β" + "β" * 78 + "β€")++ kernel_map1 = {k.name: k for k in export1.kernel_summary}+ kernel_map2 = {k.name: k for k in export2.kernel_summary}+ all_names = set(kernel_map1.keys()) | set(kernel_map2.keys())++ combined: list[tuple[str, int, int, str]] = []+ for name in all_names:+ k1 = kernel_map1.get(name)+ k2 = kernel_map2.get(name)+ ns1 = k1.total_ns if k1 else 0+ ns2 = k2.total_ns if k2 else 0+ diff = _diff_str(ns1, ns2)+ combined.append((name, ns1, ns2, diff))++ combined.sort(key=lambda x: x[1] + x[2], reverse=True)++ for name, ns1, ns2, diff in combined[:15]:+ short_name = name[:32] if len(name) <= 32 else name[:29] + "..."+ row = f" {short_name:<32} {_fmt_ns(ns1):>12} {_fmt_ns(ns2):>12} {diff:>10}"+ print("β" + row.ljust(78) + "β")++ total1 = sum(k.total_ns for k in export1.kernel_summary)+ total2 = sum(k.total_ns for k in export2.kernel_summary)++ print("β" + "β" * 78 + "β€")+ row = f" {'TOTAL':<32} {_fmt_ns(total1):>12} {_fmt_ns(total2):>12} {_diff_str(total1, total2):>10}"+ print("β" + row.ljust(78) + "β")++ print("β" + "β" * 78 + "β")+ print()diff --git a/tools/gpu_trace/src/gpu_trace/main.py b/tools/gpu_trace/src/gpu_trace/main.pynew file mode 100644index 000000000..2473eee5c--- /dev/null+++ b/tools/gpu_trace/src/gpu_trace/main.py@@ -0,0 +1,270 @@+from __future__ import annotations++import argparse+import sys+from pathlib import Path++from .extractors import (+ compute_kernel_summary,+ extract_counter_definitions,+ extract_gpu_utilization,+ extract_kernel_dispatches,+ extract_metadata,+ extract_performance_states,+ extract_shaders,+)+from .formatting import print_comparison, print_summary+from .models import TraceExport+from .recorder import default_output_path, record_trace+from .serialization import to_json+++def export_trace(trace: Path, run_number: int) -> TraceExport:+ metadata = extract_metadata(trace, run_number)+ utilization = extract_gpu_utilization(trace, run_number)+ performance_states = extract_performance_states(trace, run_number)+ counter_definitions = extract_counter_definitions(trace, run_number)+ dispatches = extract_kernel_dispatches(trace, run_number)+ shaders = extract_shaders(trace, run_number)+ kernel_summary = compute_kernel_summary(dispatches)++ return TraceExport(+ metadata=metadata,+ utilization=utilization,+ performance_states=performance_states,+ counter_definitions=counter_definitions,+ dispatches=dispatches,+ shaders=shaders,+ kernel_summary=kernel_summary,+ )+++def cmd_run(args: argparse.Namespace) -> int:+ command = list(args.command)+ if command and command[0] == "--":+ command = command[1:]++ if not command and args.attach is None:+ print("Error: Either command or --attach must be specified", file=sys.stderr)+ return 1++ output = args.output or default_output_path()++ record_rc = record_trace(+ output=output,+ command=command,+ template=args.template,+ gpu_counters=args.gpu_counters,+ time_limit=args.time_limit,+ env=args.env,+ attach_pid=args.attach,+ )++ if record_rc != 0:+ print(f"xctrace record failed (exit={record_rc})", file=sys.stderr)+ return record_rc++ print(f"Trace saved: {output}", file=sys.stderr)++ export = export_trace(output, 1)+ print_summary(export)++ if args.json:+ json_path = args.json+ json_path.write_text(to_json(export, indent=2))+ print(f"JSON exported: {json_path}", file=sys.stderr)++ return 0+++def cmd_analyze(args: argparse.Namespace) -> int:+ if not args.trace.exists():+ print(f"Error: Trace not found: {args.trace}", file=sys.stderr)+ return 1++ export = export_trace(args.trace, args.run)+ print_summary(export)++ if args.json:+ indent = None if args.compact else 2+ json_output = to_json(export, indent=indent)+ if args.json == Path("-"):+ print(json_output)+ else:+ args.json.write_text(json_output)+ print(f"JSON exported: {args.json}", file=sys.stderr)++ return 0+++def cmd_compare(args: argparse.Namespace) -> int:+ if not args.trace1.exists():+ print(f"Error: Trace not found: {args.trace1}", file=sys.stderr)+ return 1+ if not args.trace2.exists():+ print(f"Error: Trace not found: {args.trace2}", file=sys.stderr)+ return 1++ export1 = export_trace(args.trace1, args.run)+ export2 = export_trace(args.trace2, args.run)++ print_comparison(export1, export2, args.label1, args.label2)++ return 0+++def main(argv: list[str]) -> int:+ parser = argparse.ArgumentParser(+ prog="gpu_trace",+ description="Record, analyze, and export Metal GPU traces",+ formatter_class=argparse.RawDescriptionHelpFormatter,+ )+ sub = parser.add_subparsers(dest="cmd", required=True)++ # --- run subcommand ---+ run_p = sub.add_parser(+ "run",+ help="Record a GPU trace while running a command",+ formatter_class=argparse.RawDescriptionHelpFormatter,+ epilog="""+Examples:+ gpu_trace run -- ./my_app arg1 arg2+ gpu_trace run --gpu-counters -- ./my_app+ gpu_trace run --attach 12345 --time-limit 5s+ gpu_trace run --json trace.json -- ./my_app+ """,+ )+ run_p.add_argument(+ "--output",+ "-o",+ type=Path,+ default=None,+ help="Output .trace path (default: /tmp/gpu_trace_<timestamp>.trace)",+ )+ run_p.add_argument(+ "--template",+ default="Metal System Trace",+ help="xctrace template (default: Metal System Trace)",+ )+ run_p.add_argument(+ "--gpu-counters",+ action="store_true",+ help="Enable Metal GPU Counters for per-kernel timing",+ )+ run_p.add_argument(+ "--time-limit",+ default=None,+ help="Recording time limit (e.g., 5s, 500ms)",+ )+ run_p.add_argument(+ "--env",+ action="append",+ default=[],+ help="Environment variable (VAR=value)",+ )+ run_p.add_argument(+ "--attach",+ type=int,+ default=None,+ metavar="PID",+ help="Attach to running process instead of launching",+ )+ run_p.add_argument(+ "--json",+ type=Path,+ default=None,+ help="Also export JSON to file",+ )+ run_p.add_argument(+ "command",+ nargs=argparse.REMAINDER,+ help="Command to run (use -- before command)",+ )++ # --- analyze subcommand ---+ analyze_p = sub.add_parser(+ "analyze",+ help="Analyze an existing .trace bundle",+ formatter_class=argparse.RawDescriptionHelpFormatter,+ epilog="""+Examples:+ gpu_trace analyze /tmp/my.trace+ gpu_trace analyze /tmp/my.trace --json output.json+ gpu_trace analyze /tmp/my.trace --json - # JSON to stdout+ """,+ )+ analyze_p.add_argument(+ "trace",+ type=Path,+ help="Path to .trace bundle",+ )+ analyze_p.add_argument(+ "--run",+ type=int,+ default=1,+ help="Run number to analyze (default: 1)",+ )+ analyze_p.add_argument(+ "--json",+ type=Path,+ default=None,+ help="Export JSON to file (use - for stdout)",+ )+ analyze_p.add_argument(+ "--compact",+ action="store_true",+ help="Compact JSON output",+ )++ # --- compare subcommand ---+ compare_p = sub.add_parser(+ "compare",+ help="Compare two traces side-by-side",+ formatter_class=argparse.RawDescriptionHelpFormatter,+ epilog="""+Examples:+ gpu_trace compare baseline.trace optimized.trace+ gpu_trace compare a.trace b.trace --label1 before --label2 after+ """,+ )+ compare_p.add_argument(+ "trace1",+ type=Path,+ help="First trace (baseline)",+ )+ compare_p.add_argument(+ "trace2",+ type=Path,+ help="Second trace (comparison)",+ )+ compare_p.add_argument(+ "--run",+ type=int,+ default=1,+ help="Run number to compare (default: 1)",+ )+ compare_p.add_argument(+ "--label1",+ default="baseline",+ help="Label for first trace",+ )+ compare_p.add_argument(+ "--label2",+ default="comparison",+ help="Label for second trace",+ )++ args = parser.parse_args(argv)++ if args.cmd == "run":+ return cmd_run(args)+ elif args.cmd == "analyze":+ return cmd_analyze(args)+ elif args.cmd == "compare":+ return cmd_compare(args)++ return 1+++if __name__ == "__main__":+ raise SystemExit(main(sys.argv[1:]))diff --git a/tools/gpu_trace/src/gpu_trace/models.py b/tools/gpu_trace/src/gpu_trace/models.pynew file mode 100644index 000000000..07e4b6a2f--- /dev/null+++ b/tools/gpu_trace/src/gpu_trace/models.py@@ -0,0 +1,141 @@+from __future__ import annotations++from dataclasses import dataclass+from enum import Enum+++class GpuState(Enum):+ ACTIVE = "Active"+ IDLE = "Idle"+ UNKNOWN = "Unknown"+++class PerformanceState(Enum):+ MINIMUM = "Minimum"+ NOMINAL = "Nominal"+ MAXIMUM = "Maximum"+ UNKNOWN = "Unknown"+++class ShaderType(Enum):+ COMPUTE = "Compute"+ VERTEX = "Vertex"+ FRAGMENT = "Fragment"+ UNKNOWN = "Unknown"+++class CounterType(Enum):+ PERCENTAGE = "Percentage"+ VALUE = "Value"+++@dataclass(frozen=True)+class TraceMetadata:+ device_name: str+ device_model: str+ os_version: str+ target_process: str+ target_pid: int+ duration_ns: int+ start_date: str+ end_date: str+ template_name: str+++@dataclass(frozen=True)+class GpuUtilization:+ run_duration_ns: int+ global_active_ns: int+ global_idle_ns: int+ global_coverage_ns: int+ target_active_ns: int+ window_start_ns: int+ window_end_ns: int+ command_buffer_count: int+ encoder_count: int++ @property+ def global_utilization_percent(self) -> float:+ if self.run_duration_ns == 0:+ return 0.0+ return self.global_active_ns / self.run_duration_ns * 100.0++ @property+ def target_utilization_percent(self) -> float:+ if self.run_duration_ns == 0:+ return 0.0+ return self.target_active_ns / self.run_duration_ns * 100.0++ @property+ def window_duration_ns(self) -> int:+ return max(0, self.window_end_ns - self.window_start_ns)++ @property+ def window_utilization_percent(self) -> float:+ window_ns = self.window_duration_ns+ if window_ns == 0:+ return 0.0+ return self.target_active_ns / window_ns * 100.0+++@dataclass(frozen=True)+class GpuCounterDefinition:+ counter_id: int+ name: str+ description: str+ max_value: int+ counter_type: CounterType+ sample_interval_us: int+++@dataclass(frozen=True)+class GpuPerformanceStateInterval:+ start_ns: int+ duration_ns: int+ state: PerformanceState+ is_induced: bool+ gpu_name: str+++@dataclass(frozen=True)+class KernelDispatch:+ name: str+ start_ns: int+ duration_ns: int+ command_buffer_id: int+ channel: str+++@dataclass(frozen=True)+class ShaderInfo:+ shader_id: int+ name: str+ shader_type: ShaderType+ pc_start: int+ pc_end: int+++@dataclass(frozen=True)+class KernelStats:+ name: str+ total_ns: int+ count: int+ min_ns: int+ max_ns: int++ @property+ def avg_ns(self) -> float:+ if self.count == 0:+ return 0.0+ return self.total_ns / self.count+++@dataclass(frozen=True)+class TraceExport:+ metadata: TraceMetadata+ utilization: GpuUtilization+ performance_states: tuple[GpuPerformanceStateInterval, ...]+ counter_definitions: tuple[GpuCounterDefinition, ...]+ dispatches: tuple[KernelDispatch, ...]+ shaders: tuple[ShaderInfo, ...]+ kernel_summary: tuple[KernelStats, ...]diff --git a/tools/gpu_trace/src/gpu_trace/recorder.py b/tools/gpu_trace/src/gpu_trace/recorder.pynew file mode 100644index 000000000..6cbe08d36--- /dev/null+++ b/tools/gpu_trace/src/gpu_trace/recorder.py@@ -0,0 +1,60 @@+from __future__ import annotations++import subprocess+import time+from pathlib import Path+++def _xctrace_cmd(*args: str) -> list[str]:+ return ["xcrun", "xctrace", *args]+++def record_trace(+ *,+ output: Path,+ command: list[str],+ template: str = "Metal System Trace",+ gpu_counters: bool = False,+ time_limit: str | None = None,+ env: list[str] | None = None,+ attach_pid: int | None = None,+) -> int:+ output.parent.mkdir(parents=True, exist_ok=True)+ if output.exists():+ raise SystemExit(f"Output trace already exists: {output}")++ cmd = _xctrace_cmd(+ "record",+ "--template",+ template,+ "--output",+ str(output),+ "--no-prompt",+ "--target-stdout",+ "-",+ )++ if gpu_counters:+ cmd += ["--instrument", "Metal GPU Counters"]++ if time_limit:+ cmd += ["--time-limit", time_limit]++ for item in env or []:+ cmd += ["--env", item]++ if attach_pid is not None:+ cmd += ["--attach", str(attach_pid)]+ elif command:+ cmd += ["--launch", "--", *command]+ else:+ raise SystemExit("Either command or --attach must be specified")++ proc = subprocess.run(cmd)+ if not output.exists():+ return proc.returncode+ return 0+++def default_output_path() -> Path:+ return Path(f"/tmp/gpu_trace_{int(time.time())}.trace")diff --git a/tools/gpu_trace/src/gpu_trace/serialization.py b/tools/gpu_trace/src/gpu_trace/serialization.pynew file mode 100644index 000000000..618261568--- /dev/null+++ b/tools/gpu_trace/src/gpu_trace/serialization.py@@ -0,0 +1,62 @@+from __future__ import annotations++import json+from enum import Enum+from typing import Any++import cattrs+from cattrs.strategies import configure_tagged_union++from .models import (+ CounterType,+ GpuCounterDefinition,+ GpuPerformanceStateInterval,+ GpuUtilization,+ KernelDispatch,+ KernelStats,+ PerformanceState,+ ShaderInfo,+ ShaderType,+ TraceExport,+ TraceMetadata,+)+++def _make_converter() -> cattrs.Converter:+ converter = cattrs.Converter()++ def enum_unstructure(val: Enum) -> str:+ return val.value++ def enum_structure(val: str, cls: type[Enum]) -> Enum:+ return cls(val)++ for enum_cls in (CounterType, PerformanceState, ShaderType):+ converter.register_unstructure_hook(enum_cls, enum_unstructure)+ converter.register_structure_hook(enum_cls, enum_structure)++ def tuple_unstructure(val: tuple[Any, ...]) -> list[Any]:+ return [converter.unstructure(item) for item in val]++ converter.register_unstructure_hook(tuple, tuple_unstructure)++ return converter+++_converter = _make_converter()+++def to_dict(export: TraceExport) -> dict[str, Any]:+ return _converter.unstructure(export)+++def to_json(export: TraceExport, indent: int | None = 2) -> str:+ return json.dumps(to_dict(export), indent=indent)+++def from_dict(data: dict[str, Any]) -> TraceExport:+ return _converter.structure(data, TraceExport)+++def from_json(json_str: str) -> TraceExport:+ return from_dict(json.loads(json_str))diff --git a/tools/gpu_trace/uv.lock b/tools/gpu_trace/uv.locknew file mode 100644index 000000000..ab3521ade--- /dev/null+++ b/tools/gpu_trace/uv.lock@@ -0,0 +1,45 @@+version = 1+revision = 3+requires-python = ">=3.12"++[[package]]+name = "attrs"+version = "25.4.0"+source = { registry = "https://pypi.org/simple" }+sdist = { url = "https://files.pythonhosted.org/packages/6b/5c/685e6633917e101e5dcb62b9dd76946cbb57c26e133bae9e0cd36033c0a9/attrs-25.4.0.tar.gz", hash = "sha256:16d5969b87f0859ef33a48b35d55ac1be6e42ae49d5e853b597db70c35c57e11", size = 934251, upload-time = "2025-10-06T13:54:44.725Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/3a/2a/7cc015f5b9f5db42b7d48157e23356022889fc354a2813c15934b7cb5c0e/attrs-25.4.0-py3-none-any.whl", hash = "sha256:adcf7e2a1fb3b36ac48d97835bb6d8ade15b8dcce26aba8bf1d14847b57a3373", size = 67615, upload-time = "2025-10-06T13:54:43.17Z" },+]++[[package]]+name = "cattrs"+version = "25.3.0"+source = { registry = "https://pypi.org/simple" }+dependencies = [+ { name = "attrs" },+ { name = "typing-extensions" },+]+sdist = { url = "https://files.pythonhosted.org/packages/6e/00/2432bb2d445b39b5407f0a90e01b9a271475eea7caf913d7a86bcb956385/cattrs-25.3.0.tar.gz", hash = "sha256:1ac88d9e5eda10436c4517e390a4142d88638fe682c436c93db7ce4a277b884a", size = 509321, upload-time = "2025-10-07T12:26:08.737Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/d8/2b/a40e1488fdfa02d3f9a653a61a5935ea08b3c2225ee818db6a76c7ba9695/cattrs-25.3.0-py3-none-any.whl", hash = "sha256:9896e84e0a5bf723bc7b4b68f4481785367ce07a8a02e7e9ee6eb2819bc306ff", size = 70738, upload-time = "2025-10-07T12:26:06.603Z" },+]++[[package]]+name = "gpu-trace"+version = "0.1.0"+source = { editable = "." }+dependencies = [+ { name = "cattrs" },+]++[package.metadata]+requires-dist = [{ name = "cattrs", specifier = ">=24.1.0" }]++[[package]]+name = "typing-extensions"+version = "4.15.0"+source = { registry = "https://pypi.org/simple" }+sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac84269196e94cf00f187f7ed21c242792a923cdb1c61f/typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", size = 109391, upload-time = "2025-08-25T13:49:26.313Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" },+]diff --git a/scripts/tools/.gitignore b/tools/helpers/.gitignoresimilarity index 100%rename from scripts/tools/.gitignorerename to tools/helpers/.gitignorediff --git a/tools/helpers/.python-version b/tools/helpers/.python-versionnew file mode 100644index 000000000..e4fba2183--- /dev/null+++ b/tools/helpers/.python-version@@ -0,0 +1 @@+3.12diff --git a/scripts/tools/main.py b/tools/helpers/main.pysimilarity index 100%rename from scripts/tools/main.pyrename to tools/helpers/main.pydiff --git a/scripts/tools/model.py b/tools/helpers/model.pysimilarity index 100%rename from scripts/tools/model.pyrename to tools/helpers/model.pydiff --git a/scripts/tools/pyproject.toml b/tools/helpers/pyproject.tomlsimilarity index 100%rename from scripts/tools/pyproject.tomlrename to tools/helpers/pyproject.tomldiff --git a/scripts/tools/utils.py b/tools/helpers/utils.pysimilarity index 100%rename from scripts/tools/utils.pyrename to tools/helpers/utils.pydiff --git a/scripts/tools/uv.lock b/tools/helpers/uv.locksimilarity index 100%rename from scripts/tools/uv.lockrename to tools/helpers/uv.lock