A high-performance inference engine for AI models
update gpu trace
Eugene Bokhan598ae41a2649
diff --git a/tools/gpu_trace/pyproject.toml b/tools/gpu_trace/pyproject.tomlindex 8b0248067..8907af9b8 100644--- a/tools/gpu_trace/pyproject.toml+++ b/tools/gpu_trace/pyproject.toml@@ -5,6 +5,9 @@ description = "Record, analyze, and export Metal GPU traces"requires-python = ">=3.12"dependencies = ["cattrs>=24.1.0"]+[project.scripts]+gpu_trace = "gpu_trace.main:cli"+[build-system]requires = ["hatchling"]build-backend = "hatchling.build"diff --git a/tools/gpu_trace/src/gpu_trace/extractors.py b/tools/gpu_trace/src/gpu_trace/extractors.pyindex b3f46f5c1..b84eaba5c 100644--- a/tools/gpu_trace/src/gpu_trace/extractors.py+++ b/tools/gpu_trace/src/gpu_trace/extractors.py@@ -1,15 +1,18 @@from __future__ import annotations+import bisectimport subprocessimport xml.etree.ElementTree as ETfrom pathlib import Pathfrom .models import (+ CounterSample,CounterType,GpuCounterDefinition,GpuPerformanceStateInterval,GpuState,GpuUtilization,+ KernelCounterStats,KernelDispatch,KernelStats,PerformanceState,@@ -287,18 +290,24 @@ def extract_counter_definitions(counters: list[GpuCounterDefinition] = []for row in root.findall(".//row"):- counter_id_s = _resolve_text(row.find("./counter-id"), id_to_text)- name = _resolve_text(row.find("./name"), id_to_text)- description = _resolve_text(row.find("./description"), id_to_text)- max_value_s = _resolve_text(row.find("./max-value"), id_to_text)- counter_type_s = _resolve_text(row.find("./type"), id_to_text)- sample_interval_s = _resolve_text(row.find("./sample-interval"), id_to_text)+ children = list(row)+ if len(children) < 11:+ continue++ counter_id_s = _resolve_text(children[1], id_to_text)+ name = _resolve_text(children[2], id_to_text)+ max_value_s = _resolve_text(children[3], id_to_text)+ description = _resolve_text(children[5], id_to_text)+ counter_type_s = _resolve_text(children[7], id_to_text)+ sample_interval_s = _resolve_text(children[10], id_to_text)if counter_id_s is None or name is None:continuetry:- counter_type = CounterType(counter_type_s) if counter_type_s else CounterType.VALUE+ counter_type = (+ CounterType(counter_type_s) if counter_type_s else CounterType.VALUE+ )except ValueError:counter_type = CounterType.VALUE@@ -399,7 +408,9 @@ def extract_shaders(trace: Path, run_number: int) -> tuple[ShaderInfo, ...]:continuetry:- shader_type = ShaderType(shader_type_s) if shader_type_s else ShaderType.UNKNOWN+ shader_type = (+ ShaderType(shader_type_s) if shader_type_s else ShaderType.UNKNOWN+ )except ValueError:shader_type = ShaderType.UNKNOWN@@ -417,7 +428,7 @@ def extract_shaders(trace: Path, run_number: int) -> tuple[ShaderInfo, ...]:def compute_kernel_summary(- dispatches: tuple[KernelDispatch, ...]+ dispatches: tuple[KernelDispatch, ...],) -> tuple[KernelStats, ...]:stats: dict[str, list[int]] = {}for d in dispatches:@@ -439,3 +450,95 @@ def compute_kernel_summary(result.sort(key=lambda s: s.total_ns, reverse=True)return tuple(result)+++def extract_counter_samples(trace: Path, run_number: int) -> tuple[CounterSample, ...]:+ try:+ root = _export_table(trace, run_number, "gpu-counter-value")+ except subprocess.CalledProcessError:+ return ()++ _, id_to_text = _build_id_maps(root)+ samples: list[CounterSample] = []++ for row in root.findall(".//row"):+ children = list(row)+ if len(children) < 3:+ continue++ timestamp_s = _resolve_text(children[0], id_to_text)+ counter_id_s = _resolve_text(children[1], id_to_text)+ value_s = _resolve_text(children[2], id_to_text)++ if timestamp_s is None or counter_id_s is None or value_s is None:+ continue++ samples.append(+ CounterSample(+ timestamp_ns=int(timestamp_s),+ counter_id=int(counter_id_s),+ value=float(value_s),+ )+ )++ samples.sort(key=lambda s: s.timestamp_ns)+ return tuple(samples)+++def compute_kernel_counter_stats(+ dispatches: tuple[KernelDispatch, ...],+ counter_samples: tuple[CounterSample, ...],+ counter_definitions: tuple[GpuCounterDefinition, ...],+) -> tuple[KernelCounterStats, ...]:+ if not dispatches or not counter_samples or not counter_definitions:+ return ()++ counter_id_to_name = {c.counter_id: c.name for c in counter_definitions}+ samples_list = sorted(counter_samples, key=lambda s: s.timestamp_ns)+ sample_times = [s.timestamp_ns for s in samples_list]++ kernel_data: dict[str, dict[str, list[float]]] = {}+ kernel_timing: dict[str, tuple[int, int]] = {}++ for d in dispatches:+ if d.name not in kernel_data:+ kernel_data[d.name] = {}+ kernel_timing[d.name] = (0, 0)++ start_ns, end_ns = d.start_ns, d.start_ns + d.duration_ns+ prev_total, prev_count = kernel_timing[d.name]+ kernel_timing[d.name] = (prev_total + d.duration_ns, prev_count + 1)++ start_idx = bisect.bisect_left(sample_times, start_ns)+ end_idx = bisect.bisect_right(sample_times, end_ns)++ for i in range(start_idx, end_idx):+ sample = samples_list[i]+ counter_name = counter_id_to_name.get(sample.counter_id)+ if counter_name is None:+ continue++ if counter_name not in kernel_data[d.name]:+ kernel_data[d.name][counter_name] = []+ kernel_data[d.name][counter_name].append(sample.value)++ result: list[KernelCounterStats] = []+ for name, counter_values in kernel_data.items():+ total_ns, count = kernel_timing[name]+ avg_counters: list[tuple[str, float]] = []++ for counter_name, values in sorted(counter_values.items()):+ if values:+ avg_counters.append((counter_name, sum(values) / len(values)))++ result.append(+ KernelCounterStats(+ name=name,+ total_ns=total_ns,+ count=count,+ counters=tuple(avg_counters),+ )+ )++ result.sort(key=lambda s: s.total_ns, reverse=True)+ return tuple(result)diff --git a/tools/gpu_trace/src/gpu_trace/formatting.py b/tools/gpu_trace/src/gpu_trace/formatting.pyindex 2600d5bcb..ba7f65358 100644--- a/tools/gpu_trace/src/gpu_trace/formatting.py+++ b/tools/gpu_trace/src/gpu_trace/formatting.py@@ -1,6 +1,6 @@from __future__ import annotations-from .models import TraceExport+from .models import KernelCounterStats, TraceExportdef _fmt_ns(ns: int) -> str:@@ -22,7 +22,89 @@ def _bar(value: float, width: int = 20) -> str:return "█" * filled + "░" * (width - filled)-def print_summary(export: TraceExport) -> None:+def _find_bottleneck(counter_dict: dict[str, float]) -> tuple[str, float] | None:+ limiters = [+ ("ALU Limiter", "ALU"),+ ("Buffer Read Limiter", "Buffer Read"),+ ("Buffer Write Limiter", "Buffer Write"),+ ("Texture Sample Limiter", "Texture Sample"),+ ("Texture Filtering Limiter", "Texture Filter"),+ ("GPU Last Level Cache Limiter", "LLC"),+ ("Threadgroup/Imageblock Load Limiter", "TG Load"),+ ("Threadgroup/Imageblock Store Limiter", "TG Store"),+ ("MMU Limiter", "MMU"),+ ]+ max_limiter = None+ max_value = 0.0+ for counter_name, short_name in limiters:+ value = counter_dict.get(counter_name, 0.0)+ if value > max_value:+ max_value = value+ max_limiter = short_name+ if max_limiter and max_value > 0:+ return (max_limiter, max_value)+ return None+++def _print_kernel_detail(kcs: KernelCounterStats) -> None:+ counter_dict = dict(kcs.counters)++ name = kcs.name[:50] if len(kcs.name) <= 50 else kcs.name[:47] + "..."+ header = f" {name} ({kcs.count} calls, {_fmt_ns(kcs.total_ns)} total)"++ print("┌" + "─" * 78 + "┐")+ print("│" + header.ljust(78) + "│")+ print("├" + "─" * 78 + "┤")++ alu_lim = counter_dict.get("ALU Limiter", 0)+ alu_util = counter_dict.get("ALU Utilization", 0)+ f32 = counter_dict.get("F32 Utilization", 0)+ f16 = counter_dict.get("F16 Utilization", 0)+ row = f" ALU: Limiter {alu_lim:>5.1f}% Util {alu_util:>5.1f}% F32 {f32:>5.1f}% F16 {f16:>5.1f}%"+ print("│" + row.ljust(78) + "│")++ buf_rd_lim = counter_dict.get("Buffer Read Limiter", 0)+ buf_ld = counter_dict.get("Buffer Load Utilization", 0)+ buf_wr_lim = counter_dict.get("Buffer Write Limiter", 0)+ buf_st = counter_dict.get("Buffer Store Utilization", 0)+ row = f" Buffer: RdLim {buf_rd_lim:>5.1f}% Load {buf_ld:>5.1f}% WrLim {buf_wr_lim:>5.1f}% Store {buf_st:>5.1f}%"+ print("│" + row.ljust(78) + "│")++ gpu_rd = counter_dict.get("GPU Read Bandwidth", 0)+ gpu_wr = counter_dict.get("GPU Write Bandwidth", 0)+ row = f" Memory: Read {gpu_rd:>6.1f} GB/s Write {gpu_wr:>6.1f} GB/s"+ print("│" + row.ljust(78) + "│")++ llc_lim = counter_dict.get("GPU Last Level Cache Limiter", 0)+ llc_util = counter_dict.get("GPU Last Level Cache Utilization", 0)+ tex_lim = counter_dict.get("Texture Sample Limiter", 0)+ tex_util = counter_dict.get("Texture Sample Utilization", 0)+ row = f" Cache: LLC Lim {llc_lim:>5.1f}% Util {llc_util:>5.1f}% Tex {tex_lim:>5.1f}%/{tex_util:>5.1f}%"+ print("│" + row.ljust(78) + "│")++ tg_ld_lim = counter_dict.get("Threadgroup/Imageblock Load Limiter", 0)+ tg_ld = counter_dict.get("Threadgroup/Imageblock Load Utilization", 0)+ tg_st_lim = counter_dict.get("Threadgroup/Imageblock Store Limiter", 0)+ tg_st = counter_dict.get("Threadgroup/Imageblock Store Utilization", 0)+ row = f" TGroup: LdLim {tg_ld_lim:>5.1f}% Load {tg_ld:>5.1f}% StLim {tg_st_lim:>5.1f}% Store {tg_st:>5.1f}%"+ print("│" + row.ljust(78) + "│")++ compute_occ = counter_dict.get("Compute Occupancy", 0)+ mmu_lim = counter_dict.get("MMU Limiter", 0)+ mmu_util = counter_dict.get("MMU Utilization", 0)+ row = f" Occup: Compute {compute_occ:>5.1f}% MMU Lim {mmu_lim:>5.1f}% Util {mmu_util:>5.1f}%"+ print("│" + row.ljust(78) + "│")++ bottleneck = _find_bottleneck(counter_dict)+ if bottleneck:+ bn_name, bn_value = bottleneck+ row = f" >>> Bottleneck: {bn_name} ({bn_value:.1f}%)"+ print("│" + row.ljust(78) + "│")++ print("└" + "─" * 78 + "┘")+++def print_summary(export: TraceExport, verbose: bool = False) -> None:meta = export.metadatautil = export.utilization@@ -73,7 +155,7 @@ def print_summary(export: TraceExport) -> None:print("│" + f" Command Buffers: {util.command_buffer_count}".ljust(78) + "│")if export.performance_states:- states = {}+ states: dict[str, int] = {}for ps in export.performance_states:key = ps.state.valuestates[key] = states.get(key, 0) + ps.duration_ns@@ -122,6 +204,43 @@ def print_summary(export: TraceExport) -> None:row = f" {name:<30} {_fmt_ns(k.total_ns):>10} {_fmt_ns(int(k.avg_ns)):>10} {k.count:>7} {pct:>5.1f}%"print("│" + row.ljust(78) + "│")+ if export.kernel_counter_stats:+ if verbose:+ print("├" + "─" * 78 + "┤")+ print("│" + " TOP 10 KERNELS - DETAILED COUNTERS ".center(78) + "│")+ print("└" + "─" * 78 + "┘")+ print()+ for kcs in export.kernel_counter_stats[:10]:+ if kcs.counters:+ _print_kernel_detail(kcs)+ print()+ return+ else:+ print("├" + "─" * 78 + "┤")+ print("│" + " TOP 10 KERNELS WITH COUNTERS ".center(78) + "│")+ print("├" + "─" * 78 + "┤")+ header = f" {'Kernel':<26} {'ALU':>8} {'BufLd':>8} {'BufSt':>8} {'Occup':>8} {'Cache':>8}"+ print("│" + header.ljust(78) + "│")+ print("├" + "─" * 78 + "┤")++ for kcs in export.kernel_counter_stats[:10]:+ if not kcs.counters:+ continue+ counter_dict = dict(kcs.counters)+ alu = counter_dict.get("ALU Utilization", 0)+ buf_ld = counter_dict.get("Buffer Load Utilization", 0)+ buf_st = counter_dict.get("Buffer Store Utilization", 0)+ occup = counter_dict.get("Compute Occupancy", 0)+ cache = counter_dict.get("GPU Last Level Cache Utilization", 0)++ name = kcs.name[:26] if len(kcs.name) <= 26 else kcs.name[:23] + "..."+ row = f" {name:<26} {alu:>7.1f}% {buf_ld:>7.1f}% {buf_st:>7.1f}% {occup:>7.1f}% {cache:>7.1f}%"+ print("│" + row.ljust(78) + "│")++ print("└" + "─" * 78 + "┘")+ print()+ return+print("└" + "─" * 78 + "┘")print()diff --git a/tools/gpu_trace/src/gpu_trace/main.py b/tools/gpu_trace/src/gpu_trace/main.pyindex 2473eee5c..be3ba1bd0 100644--- a/tools/gpu_trace/src/gpu_trace/main.py+++ b/tools/gpu_trace/src/gpu_trace/main.py@@ -5,8 +5,10 @@ import sysfrom pathlib import Pathfrom .extractors import (+ compute_kernel_counter_stats,compute_kernel_summary,extract_counter_definitions,+ extract_counter_samples,extract_gpu_utilization,extract_kernel_dispatches,extract_metadata,@@ -15,18 +17,40 @@ from .extractors import ()from .formatting import print_comparison, print_summaryfrom .models import TraceExport+from .progress import statusfrom .recorder import default_output_path, record_tracefrom .serialization import to_jsondef export_trace(trace: Path, run_number: int) -> TraceExport:- metadata = extract_metadata(trace, run_number)- utilization = extract_gpu_utilization(trace, run_number)- performance_states = extract_performance_states(trace, run_number)- counter_definitions = extract_counter_definitions(trace, run_number)- dispatches = extract_kernel_dispatches(trace, run_number)- shaders = extract_shaders(trace, run_number)- kernel_summary = compute_kernel_summary(dispatches)+ with status("Extracting trace metadata..."):+ metadata = extract_metadata(trace, run_number)++ with status("Analyzing GPU utilization..."):+ utilization = extract_gpu_utilization(trace, run_number)++ with status("Reading performance states..."):+ performance_states = extract_performance_states(trace, run_number)++ with status("Loading counter definitions..."):+ counter_definitions = extract_counter_definitions(trace, run_number)++ with status("Extracting kernel dispatches..."):+ dispatches = extract_kernel_dispatches(trace, run_number)++ with status("Loading shader info..."):+ shaders = extract_shaders(trace, run_number)++ with status("Computing kernel statistics..."):+ kernel_summary = compute_kernel_summary(dispatches)++ with status("Extracting counter samples..."):+ counter_samples = extract_counter_samples(trace, run_number)++ with status("Correlating counters with kernels..."):+ kernel_counter_stats = compute_kernel_counter_stats(+ dispatches, counter_samples, counter_definitions+ )return TraceExport(metadata=metadata,@@ -36,6 +60,8 @@ def export_trace(trace: Path, run_number: int) -> TraceExport:dispatches=dispatches,shaders=shaders,kernel_summary=kernel_summary,+ counter_samples=counter_samples,+ kernel_counter_stats=kernel_counter_stats,)@@ -67,7 +93,7 @@ def cmd_run(args: argparse.Namespace) -> int:print(f"Trace saved: {output}", file=sys.stderr)export = export_trace(output, 1)- print_summary(export)+ print_summary(export, verbose=args.verbose)if args.json:json_path = args.json@@ -83,7 +109,7 @@ def cmd_analyze(args: argparse.Namespace) -> int:return 1export = export_trace(args.trace, args.run)- print_summary(export)+ print_summary(export, verbose=args.verbose)if args.json:indent = None if args.compact else 2@@ -113,54 +139,149 @@ def cmd_compare(args: argparse.Namespace) -> int:return 0+MAIN_DESCRIPTION = """+Metal GPU Trace Tool - Profile GPU performance on macOS.++Record, analyze, and compare Metal GPU traces with per-kernel timing,+hardware counter data (ALU, memory bandwidth, cache utilization),+and performance bottleneck analysis.++Requires Xcode and xctrace to be installed.+""".strip()++MAIN_EPILOG = """+Quick Start:+ 1. Record a trace: gpu_trace run -- ./my_metal_app+ 2. Analyze results: gpu_trace analyze /tmp/gpu_trace_*.trace+ 3. Compare traces: gpu_trace compare before.trace after.trace++For detailed per-kernel hardware counters, use --gpu-counters:+ gpu_trace run --gpu-counters -- ./my_metal_app++Export to JSON for further analysis:+ gpu_trace analyze /tmp/my.trace --json output.json++Use -v/--verbose to show all hardware counters per kernel.+"""++RUN_DESCRIPTION = """+Record a Metal GPU trace using Xcode Instruments.++Launches the specified command and records GPU activity until the process+exits or Ctrl-C is pressed. Automatically displays a summary of kernel+timing and GPU utilization.+""".strip()++RUN_EPILOG = """+Examples:+ # Basic recording+ gpu_trace run -- ./my_app arg1 arg2++ # With hardware performance counters (ALU, memory, cache)+ gpu_trace run --gpu-counters -- ./my_app++ # Attach to running process with time limit+ gpu_trace run --attach 12345 --time-limit 5s++ # Save JSON export alongside trace+ gpu_trace run --json results.json -- ./my_app++ # Custom output path+ gpu_trace run -o /path/to/my.trace -- ./my_app++ # Verbose output with all counters+ gpu_trace run -v --gpu-counters -- ./my_app+"""++ANALYZE_DESCRIPTION = """+Analyze an existing .trace bundle created by gpu_trace or Xcode Instruments.++Displays GPU utilization, per-kernel timing, and hardware counter data+if available. Use --json to export full data for further analysis.+""".strip()++ANALYZE_EPILOG = """+Examples:+ # Basic analysis+ gpu_trace analyze /tmp/my.trace++ # Export to JSON file+ gpu_trace analyze /tmp/my.trace --json output.json++ # JSON to stdout (for piping)+ gpu_trace analyze /tmp/my.trace --json -++ # Compact JSON (no indentation)+ gpu_trace analyze /tmp/my.trace --json output.json --compact++ # Verbose output with all counters+ gpu_trace analyze /tmp/my.trace -v+"""++COMPARE_DESCRIPTION = """+Compare two GPU traces side-by-side.++Shows differences in GPU utilization, kernel timing, and performance+metrics. Useful for measuring the impact of optimizations.+""".strip()++COMPARE_EPILOG = """+Examples:+ # Basic comparison+ gpu_trace compare baseline.trace optimized.trace++ # With custom labels+ gpu_trace compare a.trace b.trace --label1 before --label2 after+"""++def main(argv: list[str]) -> int:parser = argparse.ArgumentParser(prog="gpu_trace",- description="Record, analyze, and export Metal GPU traces",+ description=MAIN_DESCRIPTION,formatter_class=argparse.RawDescriptionHelpFormatter,+ epilog=MAIN_EPILOG,)- sub = parser.add_subparsers(dest="cmd", required=True)+ sub = parser.add_subparsers(dest="cmd", required=True, metavar="COMMAND")# --- run subcommand ---run_p = sub.add_parser("run",help="Record a GPU trace while running a command",+ description=RUN_DESCRIPTION,formatter_class=argparse.RawDescriptionHelpFormatter,- epilog="""-Examples:- gpu_trace run -- ./my_app arg1 arg2- gpu_trace run --gpu-counters -- ./my_app- gpu_trace run --attach 12345 --time-limit 5s- gpu_trace run --json trace.json -- ./my_app- """,+ epilog=RUN_EPILOG,)run_p.add_argument(- "--output",- "-o",+ "-o", "--output",type=Path,default=None,+ metavar="PATH",help="Output .trace path (default: /tmp/gpu_trace_<timestamp>.trace)",)run_p.add_argument("--template",default="Metal System Trace",- help="xctrace template (default: Metal System Trace)",+ metavar="NAME",+ help="Xcode Instruments template (default: Metal System Trace)",)run_p.add_argument("--gpu-counters",action="store_true",- help="Enable Metal GPU Counters for per-kernel timing",+ help="Enable Metal GPU Counters for detailed hardware metrics",)run_p.add_argument("--time-limit",default=None,- help="Recording time limit (e.g., 5s, 500ms)",+ metavar="DURATION",+ help="Recording time limit, e.g. 5s, 500ms (default: until exit)",)run_p.add_argument("--env",action="append",default=[],- help="Environment variable (VAR=value)",+ metavar="VAR=VALUE",+ help="Set environment variable for the launched process",)run_p.add_argument("--attach",@@ -173,7 +294,13 @@ Examples:"--json",type=Path,default=None,- help="Also export JSON to file",+ metavar="PATH",+ help="Export trace data to JSON file",+ )+ run_p.add_argument(+ "-v", "--verbose",+ action="store_true",+ help="Show all hardware counters per kernel",)run_p.add_argument("command",@@ -185,13 +312,9 @@ Examples:analyze_p = sub.add_parser("analyze",help="Analyze an existing .trace bundle",+ description=ANALYZE_DESCRIPTION,formatter_class=argparse.RawDescriptionHelpFormatter,- epilog="""-Examples:- gpu_trace analyze /tmp/my.trace- gpu_trace analyze /tmp/my.trace --json output.json- gpu_trace analyze /tmp/my.trace --json - # JSON to stdout- """,+ epilog=ANALYZE_EPILOG,)analyze_p.add_argument("trace",@@ -202,30 +325,34 @@ Examples:"--run",type=int,default=1,+ metavar="N",help="Run number to analyze (default: 1)",)analyze_p.add_argument("--json",type=Path,default=None,- help="Export JSON to file (use - for stdout)",+ metavar="PATH",+ help="Export trace data to JSON file (use - for stdout)",)analyze_p.add_argument("--compact",action="store_true",- help="Compact JSON output",+ help="Compact JSON output (no indentation)",+ )+ analyze_p.add_argument(+ "-v", "--verbose",+ action="store_true",+ help="Show all hardware counters per kernel",)# --- compare subcommand ---compare_p = sub.add_parser("compare",help="Compare two traces side-by-side",+ description=COMPARE_DESCRIPTION,formatter_class=argparse.RawDescriptionHelpFormatter,- epilog="""-Examples:- gpu_trace compare baseline.trace optimized.trace- gpu_trace compare a.trace b.trace --label1 before --label2 after- """,+ epilog=COMPARE_EPILOG,)compare_p.add_argument("trace1",@@ -241,17 +368,20 @@ Examples:"--run",type=int,default=1,+ metavar="N",help="Run number to compare (default: 1)",)compare_p.add_argument("--label1",default="baseline",- help="Label for first trace",+ metavar="LABEL",+ help="Label for first trace (default: baseline)",)compare_p.add_argument("--label2",default="comparison",- help="Label for second trace",+ metavar="LABEL",+ help="Label for second trace (default: comparison)",)args = parser.parse_args(argv)@@ -266,5 +396,9 @@ Examples:return 1-if __name__ == "__main__":+def cli() -> None:raise SystemExit(main(sys.argv[1:]))+++if __name__ == "__main__":+ cli()diff --git a/tools/gpu_trace/src/gpu_trace/models.py b/tools/gpu_trace/src/gpu_trace/models.pyindex 07e4b6a2f..1b3ba9a57 100644--- a/tools/gpu_trace/src/gpu_trace/models.py+++ b/tools/gpu_trace/src/gpu_trace/models.py@@ -130,6 +130,21 @@ class KernelStats:return self.total_ns / self.count+@dataclass(frozen=True)+class CounterSample:+ timestamp_ns: int+ counter_id: int+ value: float+++@dataclass(frozen=True)+class KernelCounterStats:+ name: str+ total_ns: int+ count: int+ counters: tuple[tuple[str, float], ...] # (counter_name, avg_value)++@dataclass(frozen=True)class TraceExport:metadata: TraceMetadata@@ -139,3 +154,5 @@ class TraceExport:dispatches: tuple[KernelDispatch, ...]shaders: tuple[ShaderInfo, ...]kernel_summary: tuple[KernelStats, ...]+ counter_samples: tuple[CounterSample, ...]+ kernel_counter_stats: tuple[KernelCounterStats, ...]diff --git a/tools/gpu_trace/src/gpu_trace/progress.py b/tools/gpu_trace/src/gpu_trace/progress.pynew file mode 100644index 000000000..2588e1ec6--- /dev/null+++ b/tools/gpu_trace/src/gpu_trace/progress.py@@ -0,0 +1,57 @@+from __future__ import annotations++import sys+import threading+import time+from contextlib import contextmanager+from typing import Iterator++SPINNER_FRAMES = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"]+GREEN = "\033[32m"+CYAN = "\033[36m"+RESET = "\033[0m"+CLEAR_LINE = "\033[2K\r"+++class Spinner:+ def __init__(self, message: str) -> None:+ self._message = message+ self._running = False+ self._thread: threading.Thread | None = None+ self._frame_idx = 0++ def _spin(self) -> None:+ while self._running:+ frame = SPINNER_FRAMES[self._frame_idx % len(SPINNER_FRAMES)]+ sys.stderr.write(f"{CLEAR_LINE}{CYAN}{frame}{RESET} {self._message}")+ sys.stderr.flush()+ self._frame_idx += 1+ time.sleep(0.08)++ def start(self) -> None:+ self._running = True+ self._thread = threading.Thread(target=self._spin, daemon=True)+ self._thread.start()++ def stop(self, success: bool = True) -> None:+ self._running = False+ if self._thread:+ self._thread.join(timeout=0.2)+ icon = f"{GREEN}✓{RESET}" if success else "✗"+ sys.stderr.write(f"{CLEAR_LINE}{icon} {self._message}\n")+ sys.stderr.flush()+++@contextmanager+def status(message: str) -> Iterator[None]:+ spinner = Spinner(message)+ spinner.start()+ try:+ yield+ finally:+ spinner.stop(success=True)+++def done(message: str) -> None:+ sys.stderr.write(f"{GREEN}✓{RESET} {message}\n")+ sys.stderr.flush()