Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

update gpu trace

Eugene Bokhan598ae41a2649
diff --git a/tools/gpu_trace/pyproject.toml b/tools/gpu_trace/pyproject.toml
index 8b0248067..8907af9b8 100644
--- a/tools/gpu_trace/pyproject.toml
+++ b/tools/gpu_trace/pyproject.toml
@@ -5,6 +5,9 @@ description = "Record, analyze, and export Metal GPU traces"
requires-python = ">=3.12"
dependencies = ["cattrs>=24.1.0"]
+[project.scripts]
+gpu_trace = "gpu_trace.main:cli"
+
[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"
diff --git a/tools/gpu_trace/src/gpu_trace/extractors.py b/tools/gpu_trace/src/gpu_trace/extractors.py
index b3f46f5c1..b84eaba5c 100644
--- a/tools/gpu_trace/src/gpu_trace/extractors.py
+++ b/tools/gpu_trace/src/gpu_trace/extractors.py
@@ -1,15 +1,18 @@
from __future__ import annotations
+import bisect
import subprocess
import xml.etree.ElementTree as ET
from pathlib import Path
from .models import (
+ CounterSample,
CounterType,
GpuCounterDefinition,
GpuPerformanceStateInterval,
GpuState,
GpuUtilization,
+ KernelCounterStats,
KernelDispatch,
KernelStats,
PerformanceState,
@@ -287,18 +290,24 @@ def extract_counter_definitions(
counters: list[GpuCounterDefinition] = []
for row in root.findall(".//row"):
- counter_id_s = _resolve_text(row.find("./counter-id"), id_to_text)
- name = _resolve_text(row.find("./name"), id_to_text)
- description = _resolve_text(row.find("./description"), id_to_text)
- max_value_s = _resolve_text(row.find("./max-value"), id_to_text)
- counter_type_s = _resolve_text(row.find("./type"), id_to_text)
- sample_interval_s = _resolve_text(row.find("./sample-interval"), id_to_text)
+ children = list(row)
+ if len(children) < 11:
+ continue
+
+ counter_id_s = _resolve_text(children[1], id_to_text)
+ name = _resolve_text(children[2], id_to_text)
+ max_value_s = _resolve_text(children[3], id_to_text)
+ description = _resolve_text(children[5], id_to_text)
+ counter_type_s = _resolve_text(children[7], id_to_text)
+ sample_interval_s = _resolve_text(children[10], id_to_text)
if counter_id_s is None or name is None:
continue
try:
- counter_type = CounterType(counter_type_s) if counter_type_s else CounterType.VALUE
+ counter_type = (
+ CounterType(counter_type_s) if counter_type_s else CounterType.VALUE
+ )
except ValueError:
counter_type = CounterType.VALUE
@@ -399,7 +408,9 @@ def extract_shaders(trace: Path, run_number: int) -> tuple[ShaderInfo, ...]:
continue
try:
- shader_type = ShaderType(shader_type_s) if shader_type_s else ShaderType.UNKNOWN
+ shader_type = (
+ ShaderType(shader_type_s) if shader_type_s else ShaderType.UNKNOWN
+ )
except ValueError:
shader_type = ShaderType.UNKNOWN
@@ -417,7 +428,7 @@ def extract_shaders(trace: Path, run_number: int) -> tuple[ShaderInfo, ...]:
def compute_kernel_summary(
- dispatches: tuple[KernelDispatch, ...]
+ dispatches: tuple[KernelDispatch, ...],
) -> tuple[KernelStats, ...]:
stats: dict[str, list[int]] = {}
for d in dispatches:
@@ -439,3 +450,95 @@ def compute_kernel_summary(
result.sort(key=lambda s: s.total_ns, reverse=True)
return tuple(result)
+
+
+def extract_counter_samples(trace: Path, run_number: int) -> tuple[CounterSample, ...]:
+ try:
+ root = _export_table(trace, run_number, "gpu-counter-value")
+ except subprocess.CalledProcessError:
+ return ()
+
+ _, id_to_text = _build_id_maps(root)
+ samples: list[CounterSample] = []
+
+ for row in root.findall(".//row"):
+ children = list(row)
+ if len(children) < 3:
+ continue
+
+ timestamp_s = _resolve_text(children[0], id_to_text)
+ counter_id_s = _resolve_text(children[1], id_to_text)
+ value_s = _resolve_text(children[2], id_to_text)
+
+ if timestamp_s is None or counter_id_s is None or value_s is None:
+ continue
+
+ samples.append(
+ CounterSample(
+ timestamp_ns=int(timestamp_s),
+ counter_id=int(counter_id_s),
+ value=float(value_s),
+ )
+ )
+
+ samples.sort(key=lambda s: s.timestamp_ns)
+ return tuple(samples)
+
+
+def compute_kernel_counter_stats(
+ dispatches: tuple[KernelDispatch, ...],
+ counter_samples: tuple[CounterSample, ...],
+ counter_definitions: tuple[GpuCounterDefinition, ...],
+) -> tuple[KernelCounterStats, ...]:
+ if not dispatches or not counter_samples or not counter_definitions:
+ return ()
+
+ counter_id_to_name = {c.counter_id: c.name for c in counter_definitions}
+ samples_list = sorted(counter_samples, key=lambda s: s.timestamp_ns)
+ sample_times = [s.timestamp_ns for s in samples_list]
+
+ kernel_data: dict[str, dict[str, list[float]]] = {}
+ kernel_timing: dict[str, tuple[int, int]] = {}
+
+ for d in dispatches:
+ if d.name not in kernel_data:
+ kernel_data[d.name] = {}
+ kernel_timing[d.name] = (0, 0)
+
+ start_ns, end_ns = d.start_ns, d.start_ns + d.duration_ns
+ prev_total, prev_count = kernel_timing[d.name]
+ kernel_timing[d.name] = (prev_total + d.duration_ns, prev_count + 1)
+
+ start_idx = bisect.bisect_left(sample_times, start_ns)
+ end_idx = bisect.bisect_right(sample_times, end_ns)
+
+ for i in range(start_idx, end_idx):
+ sample = samples_list[i]
+ counter_name = counter_id_to_name.get(sample.counter_id)
+ if counter_name is None:
+ continue
+
+ if counter_name not in kernel_data[d.name]:
+ kernel_data[d.name][counter_name] = []
+ kernel_data[d.name][counter_name].append(sample.value)
+
+ result: list[KernelCounterStats] = []
+ for name, counter_values in kernel_data.items():
+ total_ns, count = kernel_timing[name]
+ avg_counters: list[tuple[str, float]] = []
+
+ for counter_name, values in sorted(counter_values.items()):
+ if values:
+ avg_counters.append((counter_name, sum(values) / len(values)))
+
+ result.append(
+ KernelCounterStats(
+ name=name,
+ total_ns=total_ns,
+ count=count,
+ counters=tuple(avg_counters),
+ )
+ )
+
+ result.sort(key=lambda s: s.total_ns, reverse=True)
+ return tuple(result)
diff --git a/tools/gpu_trace/src/gpu_trace/formatting.py b/tools/gpu_trace/src/gpu_trace/formatting.py
index 2600d5bcb..ba7f65358 100644
--- a/tools/gpu_trace/src/gpu_trace/formatting.py
+++ b/tools/gpu_trace/src/gpu_trace/formatting.py
@@ -1,6 +1,6 @@
from __future__ import annotations
-from .models import TraceExport
+from .models import KernelCounterStats, TraceExport
def _fmt_ns(ns: int) -> str:
@@ -22,7 +22,89 @@ def _bar(value: float, width: int = 20) -> str:
return "█" * filled + "░" * (width - filled)
-def print_summary(export: TraceExport) -> None:
+def _find_bottleneck(counter_dict: dict[str, float]) -> tuple[str, float] | None:
+ limiters = [
+ ("ALU Limiter", "ALU"),
+ ("Buffer Read Limiter", "Buffer Read"),
+ ("Buffer Write Limiter", "Buffer Write"),
+ ("Texture Sample Limiter", "Texture Sample"),
+ ("Texture Filtering Limiter", "Texture Filter"),
+ ("GPU Last Level Cache Limiter", "LLC"),
+ ("Threadgroup/Imageblock Load Limiter", "TG Load"),
+ ("Threadgroup/Imageblock Store Limiter", "TG Store"),
+ ("MMU Limiter", "MMU"),
+ ]
+ max_limiter = None
+ max_value = 0.0
+ for counter_name, short_name in limiters:
+ value = counter_dict.get(counter_name, 0.0)
+ if value > max_value:
+ max_value = value
+ max_limiter = short_name
+ if max_limiter and max_value > 0:
+ return (max_limiter, max_value)
+ return None
+
+
+def _print_kernel_detail(kcs: KernelCounterStats) -> None:
+ counter_dict = dict(kcs.counters)
+
+ name = kcs.name[:50] if len(kcs.name) <= 50 else kcs.name[:47] + "..."
+ header = f" {name} ({kcs.count} calls, {_fmt_ns(kcs.total_ns)} total)"
+
+ print("┌" + "─" * 78 + "┐")
+ print("│" + header.ljust(78) + "│")
+ print("├" + "─" * 78 + "┤")
+
+ alu_lim = counter_dict.get("ALU Limiter", 0)
+ alu_util = counter_dict.get("ALU Utilization", 0)
+ f32 = counter_dict.get("F32 Utilization", 0)
+ f16 = counter_dict.get("F16 Utilization", 0)
+ row = f" ALU: Limiter {alu_lim:>5.1f}% Util {alu_util:>5.1f}% F32 {f32:>5.1f}% F16 {f16:>5.1f}%"
+ print("│" + row.ljust(78) + "│")
+
+ buf_rd_lim = counter_dict.get("Buffer Read Limiter", 0)
+ buf_ld = counter_dict.get("Buffer Load Utilization", 0)
+ buf_wr_lim = counter_dict.get("Buffer Write Limiter", 0)
+ buf_st = counter_dict.get("Buffer Store Utilization", 0)
+ row = f" Buffer: RdLim {buf_rd_lim:>5.1f}% Load {buf_ld:>5.1f}% WrLim {buf_wr_lim:>5.1f}% Store {buf_st:>5.1f}%"
+ print("│" + row.ljust(78) + "│")
+
+ gpu_rd = counter_dict.get("GPU Read Bandwidth", 0)
+ gpu_wr = counter_dict.get("GPU Write Bandwidth", 0)
+ row = f" Memory: Read {gpu_rd:>6.1f} GB/s Write {gpu_wr:>6.1f} GB/s"
+ print("│" + row.ljust(78) + "│")
+
+ llc_lim = counter_dict.get("GPU Last Level Cache Limiter", 0)
+ llc_util = counter_dict.get("GPU Last Level Cache Utilization", 0)
+ tex_lim = counter_dict.get("Texture Sample Limiter", 0)
+ tex_util = counter_dict.get("Texture Sample Utilization", 0)
+ row = f" Cache: LLC Lim {llc_lim:>5.1f}% Util {llc_util:>5.1f}% Tex {tex_lim:>5.1f}%/{tex_util:>5.1f}%"
+ print("│" + row.ljust(78) + "│")
+
+ tg_ld_lim = counter_dict.get("Threadgroup/Imageblock Load Limiter", 0)
+ tg_ld = counter_dict.get("Threadgroup/Imageblock Load Utilization", 0)
+ tg_st_lim = counter_dict.get("Threadgroup/Imageblock Store Limiter", 0)
+ tg_st = counter_dict.get("Threadgroup/Imageblock Store Utilization", 0)
+ row = f" TGroup: LdLim {tg_ld_lim:>5.1f}% Load {tg_ld:>5.1f}% StLim {tg_st_lim:>5.1f}% Store {tg_st:>5.1f}%"
+ print("│" + row.ljust(78) + "│")
+
+ compute_occ = counter_dict.get("Compute Occupancy", 0)
+ mmu_lim = counter_dict.get("MMU Limiter", 0)
+ mmu_util = counter_dict.get("MMU Utilization", 0)
+ row = f" Occup: Compute {compute_occ:>5.1f}% MMU Lim {mmu_lim:>5.1f}% Util {mmu_util:>5.1f}%"
+ print("│" + row.ljust(78) + "│")
+
+ bottleneck = _find_bottleneck(counter_dict)
+ if bottleneck:
+ bn_name, bn_value = bottleneck
+ row = f" >>> Bottleneck: {bn_name} ({bn_value:.1f}%)"
+ print("│" + row.ljust(78) + "│")
+
+ print("└" + "─" * 78 + "┘")
+
+
+def print_summary(export: TraceExport, verbose: bool = False) -> None:
meta = export.metadata
util = export.utilization
@@ -73,7 +155,7 @@ def print_summary(export: TraceExport) -> None:
print("│" + f" Command Buffers: {util.command_buffer_count}".ljust(78) + "│")
if export.performance_states:
- states = {}
+ states: dict[str, int] = {}
for ps in export.performance_states:
key = ps.state.value
states[key] = states.get(key, 0) + ps.duration_ns
@@ -122,6 +204,43 @@ def print_summary(export: TraceExport) -> None:
row = f" {name:<30} {_fmt_ns(k.total_ns):>10} {_fmt_ns(int(k.avg_ns)):>10} {k.count:>7} {pct:>5.1f}%"
print("│" + row.ljust(78) + "│")
+ if export.kernel_counter_stats:
+ if verbose:
+ print("├" + "─" * 78 + "┤")
+ print("│" + " TOP 10 KERNELS - DETAILED COUNTERS ".center(78) + "│")
+ print("└" + "─" * 78 + "┘")
+ print()
+ for kcs in export.kernel_counter_stats[:10]:
+ if kcs.counters:
+ _print_kernel_detail(kcs)
+ print()
+ return
+ else:
+ print("├" + "─" * 78 + "┤")
+ print("│" + " TOP 10 KERNELS WITH COUNTERS ".center(78) + "│")
+ print("├" + "─" * 78 + "┤")
+ header = f" {'Kernel':<26} {'ALU':>8} {'BufLd':>8} {'BufSt':>8} {'Occup':>8} {'Cache':>8}"
+ print("│" + header.ljust(78) + "│")
+ print("├" + "─" * 78 + "┤")
+
+ for kcs in export.kernel_counter_stats[:10]:
+ if not kcs.counters:
+ continue
+ counter_dict = dict(kcs.counters)
+ alu = counter_dict.get("ALU Utilization", 0)
+ buf_ld = counter_dict.get("Buffer Load Utilization", 0)
+ buf_st = counter_dict.get("Buffer Store Utilization", 0)
+ occup = counter_dict.get("Compute Occupancy", 0)
+ cache = counter_dict.get("GPU Last Level Cache Utilization", 0)
+
+ name = kcs.name[:26] if len(kcs.name) <= 26 else kcs.name[:23] + "..."
+ row = f" {name:<26} {alu:>7.1f}% {buf_ld:>7.1f}% {buf_st:>7.1f}% {occup:>7.1f}% {cache:>7.1f}%"
+ print("│" + row.ljust(78) + "│")
+
+ print("└" + "─" * 78 + "┘")
+ print()
+ return
+
print("└" + "─" * 78 + "┘")
print()
diff --git a/tools/gpu_trace/src/gpu_trace/main.py b/tools/gpu_trace/src/gpu_trace/main.py
index 2473eee5c..be3ba1bd0 100644
--- a/tools/gpu_trace/src/gpu_trace/main.py
+++ b/tools/gpu_trace/src/gpu_trace/main.py
@@ -5,8 +5,10 @@ import sys
from pathlib import Path
from .extractors import (
+ compute_kernel_counter_stats,
compute_kernel_summary,
extract_counter_definitions,
+ extract_counter_samples,
extract_gpu_utilization,
extract_kernel_dispatches,
extract_metadata,
@@ -15,18 +17,40 @@ from .extractors import (
)
from .formatting import print_comparison, print_summary
from .models import TraceExport
+from .progress import status
from .recorder import default_output_path, record_trace
from .serialization import to_json
def export_trace(trace: Path, run_number: int) -> TraceExport:
- metadata = extract_metadata(trace, run_number)
- utilization = extract_gpu_utilization(trace, run_number)
- performance_states = extract_performance_states(trace, run_number)
- counter_definitions = extract_counter_definitions(trace, run_number)
- dispatches = extract_kernel_dispatches(trace, run_number)
- shaders = extract_shaders(trace, run_number)
- kernel_summary = compute_kernel_summary(dispatches)
+ with status("Extracting trace metadata..."):
+ metadata = extract_metadata(trace, run_number)
+
+ with status("Analyzing GPU utilization..."):
+ utilization = extract_gpu_utilization(trace, run_number)
+
+ with status("Reading performance states..."):
+ performance_states = extract_performance_states(trace, run_number)
+
+ with status("Loading counter definitions..."):
+ counter_definitions = extract_counter_definitions(trace, run_number)
+
+ with status("Extracting kernel dispatches..."):
+ dispatches = extract_kernel_dispatches(trace, run_number)
+
+ with status("Loading shader info..."):
+ shaders = extract_shaders(trace, run_number)
+
+ with status("Computing kernel statistics..."):
+ kernel_summary = compute_kernel_summary(dispatches)
+
+ with status("Extracting counter samples..."):
+ counter_samples = extract_counter_samples(trace, run_number)
+
+ with status("Correlating counters with kernels..."):
+ kernel_counter_stats = compute_kernel_counter_stats(
+ dispatches, counter_samples, counter_definitions
+ )
return TraceExport(
metadata=metadata,
@@ -36,6 +60,8 @@ def export_trace(trace: Path, run_number: int) -> TraceExport:
dispatches=dispatches,
shaders=shaders,
kernel_summary=kernel_summary,
+ counter_samples=counter_samples,
+ kernel_counter_stats=kernel_counter_stats,
)
@@ -67,7 +93,7 @@ def cmd_run(args: argparse.Namespace) -> int:
print(f"Trace saved: {output}", file=sys.stderr)
export = export_trace(output, 1)
- print_summary(export)
+ print_summary(export, verbose=args.verbose)
if args.json:
json_path = args.json
@@ -83,7 +109,7 @@ def cmd_analyze(args: argparse.Namespace) -> int:
return 1
export = export_trace(args.trace, args.run)
- print_summary(export)
+ print_summary(export, verbose=args.verbose)
if args.json:
indent = None if args.compact else 2
@@ -113,54 +139,149 @@ def cmd_compare(args: argparse.Namespace) -> int:
return 0
+MAIN_DESCRIPTION = """
+Metal GPU Trace Tool - Profile GPU performance on macOS.
+
+Record, analyze, and compare Metal GPU traces with per-kernel timing,
+hardware counter data (ALU, memory bandwidth, cache utilization),
+and performance bottleneck analysis.
+
+Requires Xcode and xctrace to be installed.
+""".strip()
+
+MAIN_EPILOG = """
+Quick Start:
+ 1. Record a trace: gpu_trace run -- ./my_metal_app
+ 2. Analyze results: gpu_trace analyze /tmp/gpu_trace_*.trace
+ 3. Compare traces: gpu_trace compare before.trace after.trace
+
+For detailed per-kernel hardware counters, use --gpu-counters:
+ gpu_trace run --gpu-counters -- ./my_metal_app
+
+Export to JSON for further analysis:
+ gpu_trace analyze /tmp/my.trace --json output.json
+
+Use -v/--verbose to show all hardware counters per kernel.
+"""
+
+RUN_DESCRIPTION = """
+Record a Metal GPU trace using Xcode Instruments.
+
+Launches the specified command and records GPU activity until the process
+exits or Ctrl-C is pressed. Automatically displays a summary of kernel
+timing and GPU utilization.
+""".strip()
+
+RUN_EPILOG = """
+Examples:
+ # Basic recording
+ gpu_trace run -- ./my_app arg1 arg2
+
+ # With hardware performance counters (ALU, memory, cache)
+ gpu_trace run --gpu-counters -- ./my_app
+
+ # Attach to running process with time limit
+ gpu_trace run --attach 12345 --time-limit 5s
+
+ # Save JSON export alongside trace
+ gpu_trace run --json results.json -- ./my_app
+
+ # Custom output path
+ gpu_trace run -o /path/to/my.trace -- ./my_app
+
+ # Verbose output with all counters
+ gpu_trace run -v --gpu-counters -- ./my_app
+"""
+
+ANALYZE_DESCRIPTION = """
+Analyze an existing .trace bundle created by gpu_trace or Xcode Instruments.
+
+Displays GPU utilization, per-kernel timing, and hardware counter data
+if available. Use --json to export full data for further analysis.
+""".strip()
+
+ANALYZE_EPILOG = """
+Examples:
+ # Basic analysis
+ gpu_trace analyze /tmp/my.trace
+
+ # Export to JSON file
+ gpu_trace analyze /tmp/my.trace --json output.json
+
+ # JSON to stdout (for piping)
+ gpu_trace analyze /tmp/my.trace --json -
+
+ # Compact JSON (no indentation)
+ gpu_trace analyze /tmp/my.trace --json output.json --compact
+
+ # Verbose output with all counters
+ gpu_trace analyze /tmp/my.trace -v
+"""
+
+COMPARE_DESCRIPTION = """
+Compare two GPU traces side-by-side.
+
+Shows differences in GPU utilization, kernel timing, and performance
+metrics. Useful for measuring the impact of optimizations.
+""".strip()
+
+COMPARE_EPILOG = """
+Examples:
+ # Basic comparison
+ gpu_trace compare baseline.trace optimized.trace
+
+ # With custom labels
+ gpu_trace compare a.trace b.trace --label1 before --label2 after
+"""
+
+
def main(argv: list[str]) -> int:
parser = argparse.ArgumentParser(
prog="gpu_trace",
- description="Record, analyze, and export Metal GPU traces",
+ description=MAIN_DESCRIPTION,
formatter_class=argparse.RawDescriptionHelpFormatter,
+ epilog=MAIN_EPILOG,
)
- sub = parser.add_subparsers(dest="cmd", required=True)
+ sub = parser.add_subparsers(dest="cmd", required=True, metavar="COMMAND")
# --- run subcommand ---
run_p = sub.add_parser(
"run",
help="Record a GPU trace while running a command",
+ description=RUN_DESCRIPTION,
formatter_class=argparse.RawDescriptionHelpFormatter,
- epilog="""
-Examples:
- gpu_trace run -- ./my_app arg1 arg2
- gpu_trace run --gpu-counters -- ./my_app
- gpu_trace run --attach 12345 --time-limit 5s
- gpu_trace run --json trace.json -- ./my_app
- """,
+ epilog=RUN_EPILOG,
)
run_p.add_argument(
- "--output",
- "-o",
+ "-o", "--output",
type=Path,
default=None,
+ metavar="PATH",
help="Output .trace path (default: /tmp/gpu_trace_<timestamp>.trace)",
)
run_p.add_argument(
"--template",
default="Metal System Trace",
- help="xctrace template (default: Metal System Trace)",
+ metavar="NAME",
+ help="Xcode Instruments template (default: Metal System Trace)",
)
run_p.add_argument(
"--gpu-counters",
action="store_true",
- help="Enable Metal GPU Counters for per-kernel timing",
+ help="Enable Metal GPU Counters for detailed hardware metrics",
)
run_p.add_argument(
"--time-limit",
default=None,
- help="Recording time limit (e.g., 5s, 500ms)",
+ metavar="DURATION",
+ help="Recording time limit, e.g. 5s, 500ms (default: until exit)",
)
run_p.add_argument(
"--env",
action="append",
default=[],
- help="Environment variable (VAR=value)",
+ metavar="VAR=VALUE",
+ help="Set environment variable for the launched process",
)
run_p.add_argument(
"--attach",
@@ -173,7 +294,13 @@ Examples:
"--json",
type=Path,
default=None,
- help="Also export JSON to file",
+ metavar="PATH",
+ help="Export trace data to JSON file",
+ )
+ run_p.add_argument(
+ "-v", "--verbose",
+ action="store_true",
+ help="Show all hardware counters per kernel",
)
run_p.add_argument(
"command",
@@ -185,13 +312,9 @@ Examples:
analyze_p = sub.add_parser(
"analyze",
help="Analyze an existing .trace bundle",
+ description=ANALYZE_DESCRIPTION,
formatter_class=argparse.RawDescriptionHelpFormatter,
- epilog="""
-Examples:
- gpu_trace analyze /tmp/my.trace
- gpu_trace analyze /tmp/my.trace --json output.json
- gpu_trace analyze /tmp/my.trace --json - # JSON to stdout
- """,
+ epilog=ANALYZE_EPILOG,
)
analyze_p.add_argument(
"trace",
@@ -202,30 +325,34 @@ Examples:
"--run",
type=int,
default=1,
+ metavar="N",
help="Run number to analyze (default: 1)",
)
analyze_p.add_argument(
"--json",
type=Path,
default=None,
- help="Export JSON to file (use - for stdout)",
+ metavar="PATH",
+ help="Export trace data to JSON file (use - for stdout)",
)
analyze_p.add_argument(
"--compact",
action="store_true",
- help="Compact JSON output",
+ help="Compact JSON output (no indentation)",
+ )
+ analyze_p.add_argument(
+ "-v", "--verbose",
+ action="store_true",
+ help="Show all hardware counters per kernel",
)
# --- compare subcommand ---
compare_p = sub.add_parser(
"compare",
help="Compare two traces side-by-side",
+ description=COMPARE_DESCRIPTION,
formatter_class=argparse.RawDescriptionHelpFormatter,
- epilog="""
-Examples:
- gpu_trace compare baseline.trace optimized.trace
- gpu_trace compare a.trace b.trace --label1 before --label2 after
- """,
+ epilog=COMPARE_EPILOG,
)
compare_p.add_argument(
"trace1",
@@ -241,17 +368,20 @@ Examples:
"--run",
type=int,
default=1,
+ metavar="N",
help="Run number to compare (default: 1)",
)
compare_p.add_argument(
"--label1",
default="baseline",
- help="Label for first trace",
+ metavar="LABEL",
+ help="Label for first trace (default: baseline)",
)
compare_p.add_argument(
"--label2",
default="comparison",
- help="Label for second trace",
+ metavar="LABEL",
+ help="Label for second trace (default: comparison)",
)
args = parser.parse_args(argv)
@@ -266,5 +396,9 @@ Examples:
return 1
-if __name__ == "__main__":
+def cli() -> None:
raise SystemExit(main(sys.argv[1:]))
+
+
+if __name__ == "__main__":
+ cli()
diff --git a/tools/gpu_trace/src/gpu_trace/models.py b/tools/gpu_trace/src/gpu_trace/models.py
index 07e4b6a2f..1b3ba9a57 100644
--- a/tools/gpu_trace/src/gpu_trace/models.py
+++ b/tools/gpu_trace/src/gpu_trace/models.py
@@ -130,6 +130,21 @@ class KernelStats:
return self.total_ns / self.count
+@dataclass(frozen=True)
+class CounterSample:
+ timestamp_ns: int
+ counter_id: int
+ value: float
+
+
+@dataclass(frozen=True)
+class KernelCounterStats:
+ name: str
+ total_ns: int
+ count: int
+ counters: tuple[tuple[str, float], ...] # (counter_name, avg_value)
+
+
@dataclass(frozen=True)
class TraceExport:
metadata: TraceMetadata
@@ -139,3 +154,5 @@ class TraceExport:
dispatches: tuple[KernelDispatch, ...]
shaders: tuple[ShaderInfo, ...]
kernel_summary: tuple[KernelStats, ...]
+ counter_samples: tuple[CounterSample, ...]
+ kernel_counter_stats: tuple[KernelCounterStats, ...]
diff --git a/tools/gpu_trace/src/gpu_trace/progress.py b/tools/gpu_trace/src/gpu_trace/progress.py
new file mode 100644
index 000000000..2588e1ec6
--- /dev/null
+++ b/tools/gpu_trace/src/gpu_trace/progress.py
@@ -0,0 +1,57 @@
+from __future__ import annotations
+
+import sys
+import threading
+import time
+from contextlib import contextmanager
+from typing import Iterator
+
+SPINNER_FRAMES = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"]
+GREEN = "\033[32m"
+CYAN = "\033[36m"
+RESET = "\033[0m"
+CLEAR_LINE = "\033[2K\r"
+
+
+class Spinner:
+ def __init__(self, message: str) -> None:
+ self._message = message
+ self._running = False
+ self._thread: threading.Thread | None = None
+ self._frame_idx = 0
+
+ def _spin(self) -> None:
+ while self._running:
+ frame = SPINNER_FRAMES[self._frame_idx % len(SPINNER_FRAMES)]
+ sys.stderr.write(f"{CLEAR_LINE}{CYAN}{frame}{RESET} {self._message}")
+ sys.stderr.flush()
+ self._frame_idx += 1
+ time.sleep(0.08)
+
+ def start(self) -> None:
+ self._running = True
+ self._thread = threading.Thread(target=self._spin, daemon=True)
+ self._thread.start()
+
+ def stop(self, success: bool = True) -> None:
+ self._running = False
+ if self._thread:
+ self._thread.join(timeout=0.2)
+ icon = f"{GREEN}✓{RESET}" if success else "✗"
+ sys.stderr.write(f"{CLEAR_LINE}{icon} {self._message}\n")
+ sys.stderr.flush()
+
+
+@contextmanager
+def status(message: str) -> Iterator[None]:
+ spinner = Spinner(message)
+ spinner.start()
+ try:
+ yield
+ finally:
+ spinner.stop(success=True)
+
+
+def done(message: str) -> None:
+ sys.stderr.write(f"{GREEN}✓{RESET} {message}\n")
+ sys.stderr.flush()