Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

add gpu trace tool

Eugene Bokhane6a4e1c1df04
diff --git a/.gitignore b/.gitignore
index 69b6e8ae6..28763c5f9 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,38 +1,28 @@
-# Generated by Cargo
-# will have compiled files and executables
-debug/
-target/
-
-# These are backup files generated by rustfmt
-**/*.rs.bk
-
-# MSVC Windows builds of rustc generate these, which store debugging information
*.pdb
-
-# Generated by cargo mutants
-# Contains mutation testing data
**/mutants.out*/
-
-# RustRover
-# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
-# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
-# and can be added to the global gitignore or merged into this file. For a more nuclear
-# option (not recommended) you can uncomment the following to ignore the entire idea folder.
-#.idea/
*.DS_Store
# Metal traces
*.gputrace
+*.trace
-# Models
-models/
-external/
+# Rust
+/debug/
+/target/
+**/*.rs.bk
-# GH
-.github/*.md
+# Python
+*__pycache__
-# Local-only dev examples
-crates/uzu/examples/quick_chat.rs
+# AI tools
+.github/*.md
.cursor
-tools
.claude
+
+# Models
+/models/
+
+# Local-only dev tools
+/crates/uzu/examples/quick_chat.rs
+/external
+
diff --git a/scripts/download_test_model.sh b/scripts/download_test_model.sh
index ea1425f1d..cab4a5875 100755
--- a/scripts/download_test_model.sh
+++ b/scripts/download_test_model.sh
@@ -7,9 +7,9 @@ if [[ "$(uname)" != "Darwin" ]]; then
exit 0
fi
-SCRIPTS_PATH="$(cd "$(dirname "$0")" && pwd)"
-TOOLS_PATH="$SCRIPTS_PATH/tools"
+ROOT_PATH="$(cd "$(dirname "$0")/.." && pwd)"
+HELPERS_PATH="$ROOT_PATH/tools/helpers"
-cd $TOOLS_PATH
+cd "$HELPERS_PATH"
uv sync
-uv run main.py download-model meta-llama/Llama-3.2-1B-Instruct
\ No newline at end of file
+uv run python main.py download-model meta-llama/Llama-3.2-1B-Instruct
\ No newline at end of file
diff --git a/scripts/tools/.python-version b/tools/gpu_trace/.python-version
similarity index 100%
rename from scripts/tools/.python-version
rename to tools/gpu_trace/.python-version
diff --git a/tools/gpu_trace/pyproject.toml b/tools/gpu_trace/pyproject.toml
new file mode 100644
index 000000000..8b0248067
--- /dev/null
+++ b/tools/gpu_trace/pyproject.toml
@@ -0,0 +1,13 @@
+[project]
+name = "gpu-trace"
+version = "0.1.0"
+description = "Record, analyze, and export Metal GPU traces"
+requires-python = ">=3.12"
+dependencies = ["cattrs>=24.1.0"]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/gpu_trace"]
diff --git a/tools/gpu_trace/src/gpu_trace/__init__.py b/tools/gpu_trace/src/gpu_trace/__init__.py
new file mode 100644
index 000000000..60ae86a2e
--- /dev/null
+++ b/tools/gpu_trace/src/gpu_trace/__init__.py
@@ -0,0 +1,29 @@
+from .models import (
+ CounterType,
+ GpuCounterDefinition,
+ GpuPerformanceStateInterval,
+ GpuState,
+ GpuUtilization,
+ KernelDispatch,
+ KernelStats,
+ PerformanceState,
+ ShaderInfo,
+ ShaderType,
+ TraceExport,
+ TraceMetadata,
+)
+
+__all__ = [
+ "CounterType",
+ "GpuCounterDefinition",
+ "GpuPerformanceStateInterval",
+ "GpuState",
+ "GpuUtilization",
+ "KernelDispatch",
+ "KernelStats",
+ "PerformanceState",
+ "ShaderInfo",
+ "ShaderType",
+ "TraceExport",
+ "TraceMetadata",
+]
diff --git a/tools/gpu_trace/src/gpu_trace/__main__.py b/tools/gpu_trace/src/gpu_trace/__main__.py
new file mode 100644
index 000000000..3629204a4
--- /dev/null
+++ b/tools/gpu_trace/src/gpu_trace/__main__.py
@@ -0,0 +1,5 @@
+from .main import main
+import sys
+
+if __name__ == "__main__":
+ raise SystemExit(main(sys.argv[1:]))
diff --git a/tools/gpu_trace/src/gpu_trace/extractors.py b/tools/gpu_trace/src/gpu_trace/extractors.py
new file mode 100644
index 000000000..b3f46f5c1
--- /dev/null
+++ b/tools/gpu_trace/src/gpu_trace/extractors.py
@@ -0,0 +1,441 @@
+from __future__ import annotations
+
+import subprocess
+import xml.etree.ElementTree as ET
+from pathlib import Path
+
+from .models import (
+ CounterType,
+ GpuCounterDefinition,
+ GpuPerformanceStateInterval,
+ GpuState,
+ GpuUtilization,
+ KernelDispatch,
+ KernelStats,
+ PerformanceState,
+ ShaderInfo,
+ ShaderType,
+ TraceMetadata,
+)
+
+
+def _check_output(cmd: list[str]) -> str:
+ return subprocess.check_output(cmd, text=True)
+
+
+def _xctrace_cmd(*args: str) -> list[str]:
+ return ["xcrun", "xctrace", *args]
+
+
+def _build_id_maps(
+ root: ET.Element,
+) -> tuple[dict[str, ET.Element], dict[str, str]]:
+ id_to_element: dict[str, ET.Element] = {}
+ id_to_text: dict[str, str] = {}
+ for el in root.iter():
+ el_id = el.attrib.get("id")
+ if el_id is None:
+ continue
+ id_to_element[el_id] = el
+ if el.text is not None:
+ id_to_text[el_id] = el.text
+ return id_to_element, id_to_text
+
+
+def _deref(
+ el: ET.Element | None, id_to_element: dict[str, ET.Element]
+) -> ET.Element | None:
+ if el is None:
+ return None
+ ref = el.attrib.get("ref")
+ return id_to_element.get(ref) if ref else el
+
+
+def _resolve_text(el: ET.Element | None, id_to_text: dict[str, str]) -> str | None:
+ if el is None:
+ return None
+ if el.text is not None:
+ return el.text
+ ref = el.attrib.get("ref")
+ return id_to_text.get(ref) if ref else None
+
+
+def _toc_root(trace: Path) -> ET.Element:
+ xml = _check_output(_xctrace_cmd("export", "--input", str(trace), "--toc"))
+ return ET.fromstring(xml)
+
+
+def _export_table(trace: Path, run_number: int, schema: str) -> ET.Element:
+ xpath = f'/trace-toc/run[@number="{run_number}"]/data/table[@schema="{schema}"]'
+ xml = _check_output(_xctrace_cmd("export", "--input", str(trace), "--xpath", xpath))
+ return ET.fromstring(xml)
+
+
+def _union_ns(intervals: list[tuple[int, int]]) -> int:
+ if not intervals:
+ return 0
+ sorted_intervals = sorted(intervals)
+ total = 0
+ cur_s, cur_e = sorted_intervals[0]
+ for s, e in sorted_intervals[1:]:
+ if s > cur_e:
+ total += cur_e - cur_s
+ cur_s, cur_e = s, e
+ else:
+ cur_e = max(cur_e, e)
+ total += cur_e - cur_s
+ return total
+
+
+def extract_metadata(trace: Path, run_number: int) -> TraceMetadata:
+ toc = _toc_root(trace)
+ run = toc.find(f".//run[@number='{run_number}']")
+ if run is None:
+ raise ValueError(f"Run {run_number} not found in trace TOC")
+
+ device = run.find("./info/target/device")
+ device_name = device.attrib.get("name", "") if device is not None else ""
+ device_model = device.attrib.get("model", "") if device is not None else ""
+ os_version = device.attrib.get("os-version", "") if device is not None else ""
+
+ process = run.find("./info/target/process")
+ target_process = process.attrib.get("name", "") if process is not None else ""
+ target_pid_str = process.attrib.get("pid", "0") if process is not None else "0"
+ target_pid = int(target_pid_str) if target_pid_str else 0
+
+ summary = run.find("./info/summary")
+ duration_el = summary.find("./duration") if summary is not None else None
+ duration_ns = (
+ int(float(duration_el.text) * 1e9)
+ if duration_el is not None and duration_el.text
+ else 0
+ )
+
+ start_date_el = summary.find("./start-date") if summary is not None else None
+ start_date = start_date_el.text if start_date_el is not None else ""
+
+ end_date_el = summary.find("./end-date") if summary is not None else None
+ end_date = end_date_el.text if end_date_el is not None else ""
+
+ template_el = summary.find("./template-name") if summary is not None else None
+ template_name = template_el.text if template_el is not None else ""
+
+ return TraceMetadata(
+ device_name=device_name or "",
+ device_model=device_model or "",
+ os_version=os_version or "",
+ target_process=target_process or "",
+ target_pid=target_pid,
+ duration_ns=duration_ns,
+ start_date=start_date or "",
+ end_date=end_date or "",
+ template_name=template_name or "",
+ )
+
+
+def extract_gpu_utilization(trace: Path, run_number: int) -> GpuUtilization:
+ toc = _toc_root(trace)
+ run = toc.find(f".//run[@number='{run_number}']")
+ if run is None:
+ raise ValueError(f"Run {run_number} not found")
+
+ duration_el = run.find("./info/summary/duration")
+ run_duration_ns = (
+ int(float(duration_el.text) * 1e9)
+ if duration_el is not None and duration_el.text
+ else 0
+ )
+
+ pid_el = run.find("./info/target/process")
+ pid = int(pid_el.attrib.get("pid", "0")) if pid_el is not None else 0
+
+ root = _export_table(trace, run_number, "metal-gpu-state-intervals")
+ _, id_to_text = _build_id_maps(root)
+
+ active_ns = 0
+ idle_ns = 0
+ for row in root.findall(".//row"):
+ state = _resolve_text(row.find("./gpu-state"), id_to_text)
+ dur_s = _resolve_text(row.find("./duration"), id_to_text)
+ if state is None or dur_s is None:
+ continue
+ dur = int(dur_s)
+ if state == GpuState.ACTIVE.value:
+ active_ns += dur
+ elif state == GpuState.IDLE.value:
+ idle_ns += dur
+
+ target_active_ns = 0
+ window_start_ns = 0
+ window_end_ns = 0
+ command_buffer_count = 0
+ encoder_count = 0
+
+ try:
+ encoders = _export_table(trace, run_number, "metal-application-encoders-list")
+ gpu_intervals = _export_table(trace, run_number, "metal-gpu-intervals")
+
+ enc_id_to_el, enc_id_to_text = _build_id_maps(encoders)
+ command_buffers: set[int] = set()
+ for row in encoders.findall(".//row"):
+ proc = _deref(row.find("./process"), enc_id_to_el)
+ proc_pid_s = _resolve_text(
+ proc.find("./pid") if proc is not None else None, enc_id_to_text
+ )
+ if proc_pid_s is None or int(proc_pid_s) != pid:
+ continue
+ ids = row.findall("./metal-command-buffer-id")
+ if not ids:
+ continue
+ cb_s = _resolve_text(ids[0], enc_id_to_text)
+ if cb_s:
+ command_buffers.add(int(cb_s))
+ encoder_count += 1
+
+ command_buffer_count = len(command_buffers)
+
+ gpu_id_to_el, gpu_id_to_text = _build_id_maps(gpu_intervals)
+ intervals: list[tuple[int, int]] = []
+ for row in gpu_intervals.findall(".//row"):
+ ids = row.findall("./metal-command-buffer-id")
+ if not ids:
+ continue
+ cb_s = _resolve_text(ids[0], gpu_id_to_text)
+ if cb_s is None:
+ continue
+ cb = int(cb_s)
+ if cb == 0 or cb not in command_buffers:
+ continue
+ start_s = _resolve_text(row.find("./start-time"), gpu_id_to_text)
+ dur_s = _resolve_text(row.find("./duration"), gpu_id_to_text)
+ if start_s is None or dur_s is None:
+ continue
+ start_ns = int(start_s)
+ dur_ns = int(dur_s)
+ if dur_ns <= 0:
+ continue
+ intervals.append((start_ns, start_ns + dur_ns))
+
+ if intervals:
+ window_start_ns = min(start for start, _ in intervals)
+ window_end_ns = max(end for _, end in intervals)
+ target_active_ns = _union_ns(intervals)
+ except subprocess.CalledProcessError:
+ pass
+
+ return GpuUtilization(
+ run_duration_ns=run_duration_ns,
+ global_active_ns=active_ns,
+ global_idle_ns=idle_ns,
+ global_coverage_ns=active_ns + idle_ns,
+ target_active_ns=target_active_ns,
+ window_start_ns=window_start_ns,
+ window_end_ns=window_end_ns,
+ command_buffer_count=command_buffer_count,
+ encoder_count=encoder_count,
+ )
+
+
+def extract_performance_states(
+ trace: Path, run_number: int
+) -> tuple[GpuPerformanceStateInterval, ...]:
+ try:
+ root = _export_table(trace, run_number, "gpu-performance-state-intervals")
+ except subprocess.CalledProcessError:
+ return ()
+
+ _, id_to_text = _build_id_maps(root)
+ states: list[GpuPerformanceStateInterval] = []
+
+ for row in root.findall(".//row"):
+ start_s = _resolve_text(row.find("./start-time"), id_to_text)
+ dur_s = _resolve_text(row.find("./duration"), id_to_text)
+ state_s = _resolve_text(row.find("./gpu-performance-state"), id_to_text)
+ gpu_s = _resolve_text(row.find("./track-label"), id_to_text)
+ induced_s = _resolve_text(row.find("./is-induced"), id_to_text)
+
+ if start_s is None or dur_s is None or state_s is None:
+ continue
+
+ try:
+ state = PerformanceState(state_s)
+ except ValueError:
+ state = PerformanceState.UNKNOWN
+
+ states.append(
+ GpuPerformanceStateInterval(
+ start_ns=int(start_s),
+ duration_ns=int(dur_s),
+ state=state,
+ is_induced=induced_s == "1",
+ gpu_name=gpu_s or "",
+ )
+ )
+
+ return tuple(states)
+
+
+def extract_counter_definitions(
+ trace: Path, run_number: int
+) -> tuple[GpuCounterDefinition, ...]:
+ try:
+ root = _export_table(trace, run_number, "gpu-counter-info")
+ except subprocess.CalledProcessError:
+ return ()
+
+ _, id_to_text = _build_id_maps(root)
+ counters: list[GpuCounterDefinition] = []
+
+ for row in root.findall(".//row"):
+ counter_id_s = _resolve_text(row.find("./counter-id"), id_to_text)
+ name = _resolve_text(row.find("./name"), id_to_text)
+ description = _resolve_text(row.find("./description"), id_to_text)
+ max_value_s = _resolve_text(row.find("./max-value"), id_to_text)
+ counter_type_s = _resolve_text(row.find("./type"), id_to_text)
+ sample_interval_s = _resolve_text(row.find("./sample-interval"), id_to_text)
+
+ if counter_id_s is None or name is None:
+ continue
+
+ try:
+ counter_type = CounterType(counter_type_s) if counter_type_s else CounterType.VALUE
+ except ValueError:
+ counter_type = CounterType.VALUE
+
+ counters.append(
+ GpuCounterDefinition(
+ counter_id=int(counter_id_s),
+ name=name,
+ description=description or "",
+ max_value=int(max_value_s) if max_value_s else 0,
+ counter_type=counter_type,
+ sample_interval_us=int(sample_interval_s) if sample_interval_s else 0,
+ )
+ )
+
+ return tuple(counters)
+
+
+def extract_kernel_dispatches(
+ trace: Path, run_number: int
+) -> tuple[KernelDispatch, ...]:
+ toc = _toc_root(trace)
+ pid_el = toc.find(f".//run[@number='{run_number}']/info/target/process")
+ pid = int(pid_el.attrib.get("pid", "0")) if pid_el is not None else 0
+
+ try:
+ root = _export_table(trace, run_number, "metal-shader-profiler-intervals")
+ except subprocess.CalledProcessError:
+ return ()
+
+ id_to_el, id_to_text = _build_id_maps(root)
+ dispatches: list[KernelDispatch] = []
+
+ for row in root.findall(".//row"):
+ proc = _deref(row.find("./process"), id_to_el)
+ pid_s = _resolve_text(
+ proc.find("./pid") if proc is not None else None, id_to_text
+ )
+ if pid_s is None or int(pid_s) != pid:
+ continue
+
+ name = _resolve_text(row.find("./metal-object-label"), id_to_text)
+ start_s = _resolve_text(row.find("./start-time"), id_to_text)
+ dur_s = _resolve_text(row.find("./duration"), id_to_text)
+ channel = _resolve_text(row.find("./gpu-channel-name"), id_to_text)
+
+ cb_ids = row.findall("./metal-command-buffer-id")
+ cb_s = _resolve_text(cb_ids[0], id_to_text) if cb_ids else None
+
+ if name is None or start_s is None or dur_s is None:
+ continue
+
+ dur_ns = int(dur_s)
+ if dur_ns <= 0:
+ continue
+
+ dispatches.append(
+ KernelDispatch(
+ name=name,
+ start_ns=int(start_s),
+ duration_ns=dur_ns,
+ command_buffer_id=int(cb_s) if cb_s else 0,
+ channel=channel or "",
+ )
+ )
+
+ dispatches.sort(key=lambda d: d.start_ns)
+ return tuple(dispatches)
+
+
+def extract_shaders(trace: Path, run_number: int) -> tuple[ShaderInfo, ...]:
+ toc = _toc_root(trace)
+ pid_el = toc.find(f".//run[@number='{run_number}']/info/target/process")
+ pid = int(pid_el.attrib.get("pid", "0")) if pid_el is not None else 0
+
+ try:
+ root = _export_table(trace, run_number, "metal-shader-profiler-shader-list")
+ except subprocess.CalledProcessError:
+ return ()
+
+ id_to_el, id_to_text = _build_id_maps(root)
+ shaders: list[ShaderInfo] = []
+
+ for row in root.findall(".//row"):
+ proc = _deref(row.find("./process"), id_to_el)
+ pid_s = _resolve_text(
+ proc.find("./pid") if proc is not None else None, id_to_text
+ )
+ if pid_s is None or int(pid_s) != pid:
+ continue
+
+ shader_id_s = _resolve_text(row.find("./id"), id_to_text)
+ name = _resolve_text(row.find("./name"), id_to_text)
+ shader_type_s = _resolve_text(row.find("./shader-type"), id_to_text)
+ pc_start_s = _resolve_text(row.find("./pc-start"), id_to_text)
+ pc_end_s = _resolve_text(row.find("./pc-end"), id_to_text)
+
+ if shader_id_s is None or name is None:
+ continue
+
+ try:
+ shader_type = ShaderType(shader_type_s) if shader_type_s else ShaderType.UNKNOWN
+ except ValueError:
+ shader_type = ShaderType.UNKNOWN
+
+ shaders.append(
+ ShaderInfo(
+ shader_id=int(shader_id_s),
+ name=name,
+ shader_type=shader_type,
+ pc_start=int(pc_start_s) if pc_start_s else 0,
+ pc_end=int(pc_end_s) if pc_end_s else 0,
+ )
+ )
+
+ return tuple(shaders)
+
+
+def compute_kernel_summary(
+ dispatches: tuple[KernelDispatch, ...]
+) -> tuple[KernelStats, ...]:
+ stats: dict[str, list[int]] = {}
+ for d in dispatches:
+ if d.name not in stats:
+ stats[d.name] = []
+ stats[d.name].append(d.duration_ns)
+
+ result: list[KernelStats] = []
+ for name, durations in stats.items():
+ result.append(
+ KernelStats(
+ name=name,
+ total_ns=sum(durations),
+ count=len(durations),
+ min_ns=min(durations),
+ max_ns=max(durations),
+ )
+ )
+
+ result.sort(key=lambda s: s.total_ns, reverse=True)
+ return tuple(result)
diff --git a/tools/gpu_trace/src/gpu_trace/formatting.py b/tools/gpu_trace/src/gpu_trace/formatting.py
new file mode 100644
index 000000000..2600d5bcb
--- /dev/null
+++ b/tools/gpu_trace/src/gpu_trace/formatting.py
@@ -0,0 +1,221 @@
+from __future__ import annotations
+
+from .models import TraceExport
+
+
+def _fmt_ns(ns: int) -> str:
+ if ns >= 1_000_000_000:
+ return f"{ns / 1e9:.2f} s"
+ if ns >= 1_000_000:
+ return f"{ns / 1e6:.2f} ms"
+ if ns >= 1_000:
+ return f"{ns / 1e3:.1f} Β΅s"
+ return f"{ns} ns"
+
+
+def _fmt_percent(value: float) -> str:
+ return f"{value:.2f}%"
+
+
+def _bar(value: float, width: int = 20) -> str:
+ filled = int(value / 100 * width)
+ return "β–ˆ" * filled + "β–‘" * (width - filled)
+
+
+def print_summary(export: TraceExport) -> None:
+ meta = export.metadata
+ util = export.utilization
+
+ print()
+ print("β”Œ" + "─" * 78 + "┐")
+ print("β”‚" + " GPU TRACE SUMMARY ".center(78) + "β”‚")
+ print("β”œ" + "─" * 78 + "─")
+
+ print("β”‚" + f" Device: {meta.device_model} ({meta.device_name})".ljust(78) + "β”‚")
+ print("β”‚" + f" OS: {meta.os_version}".ljust(78) + "β”‚")
+ print(
+ "β”‚"
+ + f" Process: {meta.target_process} (PID {meta.target_pid})".ljust(78)
+ + "β”‚"
+ )
+ print("β”‚" + f" Template: {meta.template_name}".ljust(78) + "β”‚")
+ print("β”‚" + f" Duration: {_fmt_ns(util.run_duration_ns)}".ljust(78) + "β”‚")
+
+ print("β”œ" + "─" * 78 + "─")
+ print("β”‚" + " UTILIZATION ".center(78) + "β”‚")
+ print("β”œ" + "─" * 78 + "─")
+
+ global_pct = util.global_utilization_percent
+ target_pct = util.target_utilization_percent
+ window_pct = util.window_utilization_percent
+
+ print(
+ "β”‚"
+ + f" Global GPU Active: {_bar(global_pct)} {_fmt_percent(global_pct):>8} ({_fmt_ns(util.global_active_ns)})".ljust(
+ 78
+ )
+ + "β”‚"
+ )
+ print(
+ "β”‚"
+ + f" Target GPU Active: {_bar(target_pct)} {_fmt_percent(target_pct):>8} ({_fmt_ns(util.target_active_ns)})".ljust(
+ 78
+ )
+ + "β”‚"
+ )
+ print(
+ "β”‚"
+ + f" Window Utilization: {_bar(window_pct)} {_fmt_percent(window_pct):>8} (when active)".ljust(
+ 78
+ )
+ + "β”‚"
+ )
+ print("β”‚" + f" Command Buffers: {util.command_buffer_count}".ljust(78) + "β”‚")
+
+ if export.performance_states:
+ states = {}
+ for ps in export.performance_states:
+ key = ps.state.value
+ states[key] = states.get(key, 0) + ps.duration_ns
+ induced_count = sum(1 for ps in export.performance_states if ps.is_induced)
+ print("β”œ" + "─" * 78 + "─")
+ print("β”‚" + " GPU PERFORMANCE STATES ".center(78) + "β”‚")
+ print("β”œ" + "─" * 78 + "─")
+ for state_name, duration in sorted(states.items(), key=lambda x: -x[1]):
+ print("β”‚" + f" {state_name}: {_fmt_ns(duration)}".ljust(78) + "β”‚")
+ if induced_count:
+ print(
+ "β”‚" + f" ⚠ Induced states: {induced_count} intervals".ljust(78) + "β”‚"
+ )
+
+ if export.counter_definitions:
+ print("β”œ" + "─" * 78 + "─")
+ print("β”‚" + " AVAILABLE COUNTERS ".center(78) + "β”‚")
+ print("β”œ" + "─" * 78 + "─")
+ print(
+ "β”‚"
+ + f" {len(export.counter_definitions)} GPU hardware counters available".ljust(
+ 78
+ )
+ + "β”‚"
+ )
+
+ print("β”œ" + "─" * 78 + "─")
+ print("β”‚" + " KERNEL DISPATCHES ".center(78) + "β”‚")
+ print("β”œ" + "─" * 78 + "─")
+ print("β”‚" + f" Total dispatches: {len(export.dispatches):,}".ljust(78) + "β”‚")
+ print("β”‚" + f" Unique kernels: {len(export.kernel_summary)}".ljust(78) + "β”‚")
+ print("β”‚" + f" Shaders compiled: {len(export.shaders)}".ljust(78) + "β”‚")
+
+ if export.kernel_summary:
+ print("β”œ" + "─" * 78 + "─")
+ print("β”‚" + " TOP 15 KERNELS BY TIME ".center(78) + "β”‚")
+ print("β”œ" + "─" * 78 + "─")
+ header = f" {'Kernel':<30} {'Total':>10} {'Avg':>10} {'Count':>7} {'%':>6}"
+ print("β”‚" + header.ljust(78) + "β”‚")
+ print("β”œ" + "─" * 78 + "─")
+
+ total_ns = sum(k.total_ns for k in export.kernel_summary)
+ for k in export.kernel_summary[:15]:
+ pct = (k.total_ns / total_ns * 100) if total_ns else 0
+ name = k.name[:30] if len(k.name) <= 30 else k.name[:27] + "..."
+ row = f" {name:<30} {_fmt_ns(k.total_ns):>10} {_fmt_ns(int(k.avg_ns)):>10} {k.count:>7} {pct:>5.1f}%"
+ print("β”‚" + row.ljust(78) + "β”‚")
+
+ print("β””" + "─" * 78 + "β”˜")
+ print()
+
+
+def print_comparison(
+ export1: TraceExport,
+ export2: TraceExport,
+ label1: str,
+ label2: str,
+) -> None:
+ print()
+ print("β”Œ" + "─" * 78 + "┐")
+ print("β”‚" + " GPU TRACE COMPARISON ".center(78) + "β”‚")
+ print("β”œ" + "─" * 78 + "─")
+
+ print("β”‚" + f" {label1}: {export1.metadata.target_process}".ljust(78) + "β”‚")
+ print("β”‚" + f" {label2}: {export2.metadata.target_process}".ljust(78) + "β”‚")
+
+ print("β”œ" + "─" * 78 + "─")
+ print("β”‚" + " UTILIZATION ".center(78) + "β”‚")
+ print("β”œ" + "─" * 78 + "─")
+
+ header = f" {'Metric':<22} {label1:>14} {label2:>14} {'Diff':>14}"
+ print("β”‚" + header.ljust(78) + "β”‚")
+ print("β”œ" + "─" * 78 + "─")
+
+ def _diff_str(v1: float, v2: float) -> str:
+ if v1 == 0:
+ return "n/a"
+ diff = (v2 - v1) / v1 * 100
+ sign = "+" if diff >= 0 else ""
+ return f"{sign}{diff:.1f}%"
+
+ u1, u2 = export1.utilization, export2.utilization
+
+ rows = [
+ (
+ "Duration",
+ _fmt_ns(u1.run_duration_ns),
+ _fmt_ns(u2.run_duration_ns),
+ _diff_str(u1.run_duration_ns, u2.run_duration_ns),
+ ),
+ (
+ "GPU Active",
+ _fmt_ns(u1.target_active_ns),
+ _fmt_ns(u2.target_active_ns),
+ _diff_str(u1.target_active_ns, u2.target_active_ns),
+ ),
+ (
+ "Window Util",
+ _fmt_percent(u1.window_utilization_percent),
+ _fmt_percent(u2.window_utilization_percent),
+ _diff_str(u1.window_utilization_percent, u2.window_utilization_percent),
+ ),
+ ]
+
+ for metric, v1, v2, diff in rows:
+ row = f" {metric:<22} {v1:>14} {v2:>14} {diff:>14}"
+ print("β”‚" + row.ljust(78) + "β”‚")
+
+ print("β”œ" + "─" * 78 + "─")
+ print("β”‚" + " TOP KERNELS COMPARISON ".center(78) + "β”‚")
+ print("β”œ" + "─" * 78 + "─")
+
+ header = f" {'Kernel':<32} {label1:>12} {label2:>12} {'Diff':>10}"
+ print("β”‚" + header.ljust(78) + "β”‚")
+ print("β”œ" + "─" * 78 + "─")
+
+ kernel_map1 = {k.name: k for k in export1.kernel_summary}
+ kernel_map2 = {k.name: k for k in export2.kernel_summary}
+ all_names = set(kernel_map1.keys()) | set(kernel_map2.keys())
+
+ combined: list[tuple[str, int, int, str]] = []
+ for name in all_names:
+ k1 = kernel_map1.get(name)
+ k2 = kernel_map2.get(name)
+ ns1 = k1.total_ns if k1 else 0
+ ns2 = k2.total_ns if k2 else 0
+ diff = _diff_str(ns1, ns2)
+ combined.append((name, ns1, ns2, diff))
+
+ combined.sort(key=lambda x: x[1] + x[2], reverse=True)
+
+ for name, ns1, ns2, diff in combined[:15]:
+ short_name = name[:32] if len(name) <= 32 else name[:29] + "..."
+ row = f" {short_name:<32} {_fmt_ns(ns1):>12} {_fmt_ns(ns2):>12} {diff:>10}"
+ print("β”‚" + row.ljust(78) + "β”‚")
+
+ total1 = sum(k.total_ns for k in export1.kernel_summary)
+ total2 = sum(k.total_ns for k in export2.kernel_summary)
+
+ print("β”œ" + "─" * 78 + "─")
+ row = f" {'TOTAL':<32} {_fmt_ns(total1):>12} {_fmt_ns(total2):>12} {_diff_str(total1, total2):>10}"
+ print("β”‚" + row.ljust(78) + "β”‚")
+
+ print("β””" + "─" * 78 + "β”˜")
+ print()
diff --git a/tools/gpu_trace/src/gpu_trace/main.py b/tools/gpu_trace/src/gpu_trace/main.py
new file mode 100644
index 000000000..2473eee5c
--- /dev/null
+++ b/tools/gpu_trace/src/gpu_trace/main.py
@@ -0,0 +1,270 @@
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+
+from .extractors import (
+ compute_kernel_summary,
+ extract_counter_definitions,
+ extract_gpu_utilization,
+ extract_kernel_dispatches,
+ extract_metadata,
+ extract_performance_states,
+ extract_shaders,
+)
+from .formatting import print_comparison, print_summary
+from .models import TraceExport
+from .recorder import default_output_path, record_trace
+from .serialization import to_json
+
+
+def export_trace(trace: Path, run_number: int) -> TraceExport:
+ metadata = extract_metadata(trace, run_number)
+ utilization = extract_gpu_utilization(trace, run_number)
+ performance_states = extract_performance_states(trace, run_number)
+ counter_definitions = extract_counter_definitions(trace, run_number)
+ dispatches = extract_kernel_dispatches(trace, run_number)
+ shaders = extract_shaders(trace, run_number)
+ kernel_summary = compute_kernel_summary(dispatches)
+
+ return TraceExport(
+ metadata=metadata,
+ utilization=utilization,
+ performance_states=performance_states,
+ counter_definitions=counter_definitions,
+ dispatches=dispatches,
+ shaders=shaders,
+ kernel_summary=kernel_summary,
+ )
+
+
+def cmd_run(args: argparse.Namespace) -> int:
+ command = list(args.command)
+ if command and command[0] == "--":
+ command = command[1:]
+
+ if not command and args.attach is None:
+ print("Error: Either command or --attach must be specified", file=sys.stderr)
+ return 1
+
+ output = args.output or default_output_path()
+
+ record_rc = record_trace(
+ output=output,
+ command=command,
+ template=args.template,
+ gpu_counters=args.gpu_counters,
+ time_limit=args.time_limit,
+ env=args.env,
+ attach_pid=args.attach,
+ )
+
+ if record_rc != 0:
+ print(f"xctrace record failed (exit={record_rc})", file=sys.stderr)
+ return record_rc
+
+ print(f"Trace saved: {output}", file=sys.stderr)
+
+ export = export_trace(output, 1)
+ print_summary(export)
+
+ if args.json:
+ json_path = args.json
+ json_path.write_text(to_json(export, indent=2))
+ print(f"JSON exported: {json_path}", file=sys.stderr)
+
+ return 0
+
+
+def cmd_analyze(args: argparse.Namespace) -> int:
+ if not args.trace.exists():
+ print(f"Error: Trace not found: {args.trace}", file=sys.stderr)
+ return 1
+
+ export = export_trace(args.trace, args.run)
+ print_summary(export)
+
+ if args.json:
+ indent = None if args.compact else 2
+ json_output = to_json(export, indent=indent)
+ if args.json == Path("-"):
+ print(json_output)
+ else:
+ args.json.write_text(json_output)
+ print(f"JSON exported: {args.json}", file=sys.stderr)
+
+ return 0
+
+
+def cmd_compare(args: argparse.Namespace) -> int:
+ if not args.trace1.exists():
+ print(f"Error: Trace not found: {args.trace1}", file=sys.stderr)
+ return 1
+ if not args.trace2.exists():
+ print(f"Error: Trace not found: {args.trace2}", file=sys.stderr)
+ return 1
+
+ export1 = export_trace(args.trace1, args.run)
+ export2 = export_trace(args.trace2, args.run)
+
+ print_comparison(export1, export2, args.label1, args.label2)
+
+ return 0
+
+
+def main(argv: list[str]) -> int:
+ parser = argparse.ArgumentParser(
+ prog="gpu_trace",
+ description="Record, analyze, and export Metal GPU traces",
+ formatter_class=argparse.RawDescriptionHelpFormatter,
+ )
+ sub = parser.add_subparsers(dest="cmd", required=True)
+
+ # --- run subcommand ---
+ run_p = sub.add_parser(
+ "run",
+ help="Record a GPU trace while running a command",
+ formatter_class=argparse.RawDescriptionHelpFormatter,
+ epilog="""
+Examples:
+ gpu_trace run -- ./my_app arg1 arg2
+ gpu_trace run --gpu-counters -- ./my_app
+ gpu_trace run --attach 12345 --time-limit 5s
+ gpu_trace run --json trace.json -- ./my_app
+ """,
+ )
+ run_p.add_argument(
+ "--output",
+ "-o",
+ type=Path,
+ default=None,
+ help="Output .trace path (default: /tmp/gpu_trace_<timestamp>.trace)",
+ )
+ run_p.add_argument(
+ "--template",
+ default="Metal System Trace",
+ help="xctrace template (default: Metal System Trace)",
+ )
+ run_p.add_argument(
+ "--gpu-counters",
+ action="store_true",
+ help="Enable Metal GPU Counters for per-kernel timing",
+ )
+ run_p.add_argument(
+ "--time-limit",
+ default=None,
+ help="Recording time limit (e.g., 5s, 500ms)",
+ )
+ run_p.add_argument(
+ "--env",
+ action="append",
+ default=[],
+ help="Environment variable (VAR=value)",
+ )
+ run_p.add_argument(
+ "--attach",
+ type=int,
+ default=None,
+ metavar="PID",
+ help="Attach to running process instead of launching",
+ )
+ run_p.add_argument(
+ "--json",
+ type=Path,
+ default=None,
+ help="Also export JSON to file",
+ )
+ run_p.add_argument(
+ "command",
+ nargs=argparse.REMAINDER,
+ help="Command to run (use -- before command)",
+ )
+
+ # --- analyze subcommand ---
+ analyze_p = sub.add_parser(
+ "analyze",
+ help="Analyze an existing .trace bundle",
+ formatter_class=argparse.RawDescriptionHelpFormatter,
+ epilog="""
+Examples:
+ gpu_trace analyze /tmp/my.trace
+ gpu_trace analyze /tmp/my.trace --json output.json
+ gpu_trace analyze /tmp/my.trace --json - # JSON to stdout
+ """,
+ )
+ analyze_p.add_argument(
+ "trace",
+ type=Path,
+ help="Path to .trace bundle",
+ )
+ analyze_p.add_argument(
+ "--run",
+ type=int,
+ default=1,
+ help="Run number to analyze (default: 1)",
+ )
+ analyze_p.add_argument(
+ "--json",
+ type=Path,
+ default=None,
+ help="Export JSON to file (use - for stdout)",
+ )
+ analyze_p.add_argument(
+ "--compact",
+ action="store_true",
+ help="Compact JSON output",
+ )
+
+ # --- compare subcommand ---
+ compare_p = sub.add_parser(
+ "compare",
+ help="Compare two traces side-by-side",
+ formatter_class=argparse.RawDescriptionHelpFormatter,
+ epilog="""
+Examples:
+ gpu_trace compare baseline.trace optimized.trace
+ gpu_trace compare a.trace b.trace --label1 before --label2 after
+ """,
+ )
+ compare_p.add_argument(
+ "trace1",
+ type=Path,
+ help="First trace (baseline)",
+ )
+ compare_p.add_argument(
+ "trace2",
+ type=Path,
+ help="Second trace (comparison)",
+ )
+ compare_p.add_argument(
+ "--run",
+ type=int,
+ default=1,
+ help="Run number to compare (default: 1)",
+ )
+ compare_p.add_argument(
+ "--label1",
+ default="baseline",
+ help="Label for first trace",
+ )
+ compare_p.add_argument(
+ "--label2",
+ default="comparison",
+ help="Label for second trace",
+ )
+
+ args = parser.parse_args(argv)
+
+ if args.cmd == "run":
+ return cmd_run(args)
+ elif args.cmd == "analyze":
+ return cmd_analyze(args)
+ elif args.cmd == "compare":
+ return cmd_compare(args)
+
+ return 1
+
+
+if __name__ == "__main__":
+ raise SystemExit(main(sys.argv[1:]))
diff --git a/tools/gpu_trace/src/gpu_trace/models.py b/tools/gpu_trace/src/gpu_trace/models.py
new file mode 100644
index 000000000..07e4b6a2f
--- /dev/null
+++ b/tools/gpu_trace/src/gpu_trace/models.py
@@ -0,0 +1,141 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import Enum
+
+
+class GpuState(Enum):
+ ACTIVE = "Active"
+ IDLE = "Idle"
+ UNKNOWN = "Unknown"
+
+
+class PerformanceState(Enum):
+ MINIMUM = "Minimum"
+ NOMINAL = "Nominal"
+ MAXIMUM = "Maximum"
+ UNKNOWN = "Unknown"
+
+
+class ShaderType(Enum):
+ COMPUTE = "Compute"
+ VERTEX = "Vertex"
+ FRAGMENT = "Fragment"
+ UNKNOWN = "Unknown"
+
+
+class CounterType(Enum):
+ PERCENTAGE = "Percentage"
+ VALUE = "Value"
+
+
+@dataclass(frozen=True)
+class TraceMetadata:
+ device_name: str
+ device_model: str
+ os_version: str
+ target_process: str
+ target_pid: int
+ duration_ns: int
+ start_date: str
+ end_date: str
+ template_name: str
+
+
+@dataclass(frozen=True)
+class GpuUtilization:
+ run_duration_ns: int
+ global_active_ns: int
+ global_idle_ns: int
+ global_coverage_ns: int
+ target_active_ns: int
+ window_start_ns: int
+ window_end_ns: int
+ command_buffer_count: int
+ encoder_count: int
+
+ @property
+ def global_utilization_percent(self) -> float:
+ if self.run_duration_ns == 0:
+ return 0.0
+ return self.global_active_ns / self.run_duration_ns * 100.0
+
+ @property
+ def target_utilization_percent(self) -> float:
+ if self.run_duration_ns == 0:
+ return 0.0
+ return self.target_active_ns / self.run_duration_ns * 100.0
+
+ @property
+ def window_duration_ns(self) -> int:
+ return max(0, self.window_end_ns - self.window_start_ns)
+
+ @property
+ def window_utilization_percent(self) -> float:
+ window_ns = self.window_duration_ns
+ if window_ns == 0:
+ return 0.0
+ return self.target_active_ns / window_ns * 100.0
+
+
+@dataclass(frozen=True)
+class GpuCounterDefinition:
+ counter_id: int
+ name: str
+ description: str
+ max_value: int
+ counter_type: CounterType
+ sample_interval_us: int
+
+
+@dataclass(frozen=True)
+class GpuPerformanceStateInterval:
+ start_ns: int
+ duration_ns: int
+ state: PerformanceState
+ is_induced: bool
+ gpu_name: str
+
+
+@dataclass(frozen=True)
+class KernelDispatch:
+ name: str
+ start_ns: int
+ duration_ns: int
+ command_buffer_id: int
+ channel: str
+
+
+@dataclass(frozen=True)
+class ShaderInfo:
+ shader_id: int
+ name: str
+ shader_type: ShaderType
+ pc_start: int
+ pc_end: int
+
+
+@dataclass(frozen=True)
+class KernelStats:
+ name: str
+ total_ns: int
+ count: int
+ min_ns: int
+ max_ns: int
+
+ @property
+ def avg_ns(self) -> float:
+ if self.count == 0:
+ return 0.0
+ return self.total_ns / self.count
+
+
+@dataclass(frozen=True)
+class TraceExport:
+ metadata: TraceMetadata
+ utilization: GpuUtilization
+ performance_states: tuple[GpuPerformanceStateInterval, ...]
+ counter_definitions: tuple[GpuCounterDefinition, ...]
+ dispatches: tuple[KernelDispatch, ...]
+ shaders: tuple[ShaderInfo, ...]
+ kernel_summary: tuple[KernelStats, ...]
diff --git a/tools/gpu_trace/src/gpu_trace/recorder.py b/tools/gpu_trace/src/gpu_trace/recorder.py
new file mode 100644
index 000000000..6cbe08d36
--- /dev/null
+++ b/tools/gpu_trace/src/gpu_trace/recorder.py
@@ -0,0 +1,60 @@
+from __future__ import annotations
+
+import subprocess
+import time
+from pathlib import Path
+
+
+def _xctrace_cmd(*args: str) -> list[str]:
+ return ["xcrun", "xctrace", *args]
+
+
+def record_trace(
+ *,
+ output: Path,
+ command: list[str],
+ template: str = "Metal System Trace",
+ gpu_counters: bool = False,
+ time_limit: str | None = None,
+ env: list[str] | None = None,
+ attach_pid: int | None = None,
+) -> int:
+ output.parent.mkdir(parents=True, exist_ok=True)
+ if output.exists():
+ raise SystemExit(f"Output trace already exists: {output}")
+
+ cmd = _xctrace_cmd(
+ "record",
+ "--template",
+ template,
+ "--output",
+ str(output),
+ "--no-prompt",
+ "--target-stdout",
+ "-",
+ )
+
+ if gpu_counters:
+ cmd += ["--instrument", "Metal GPU Counters"]
+
+ if time_limit:
+ cmd += ["--time-limit", time_limit]
+
+ for item in env or []:
+ cmd += ["--env", item]
+
+ if attach_pid is not None:
+ cmd += ["--attach", str(attach_pid)]
+ elif command:
+ cmd += ["--launch", "--", *command]
+ else:
+ raise SystemExit("Either command or --attach must be specified")
+
+ proc = subprocess.run(cmd)
+ if not output.exists():
+ return proc.returncode
+ return 0
+
+
+def default_output_path() -> Path:
+ return Path(f"/tmp/gpu_trace_{int(time.time())}.trace")
diff --git a/tools/gpu_trace/src/gpu_trace/serialization.py b/tools/gpu_trace/src/gpu_trace/serialization.py
new file mode 100644
index 000000000..618261568
--- /dev/null
+++ b/tools/gpu_trace/src/gpu_trace/serialization.py
@@ -0,0 +1,62 @@
+from __future__ import annotations
+
+import json
+from enum import Enum
+from typing import Any
+
+import cattrs
+from cattrs.strategies import configure_tagged_union
+
+from .models import (
+ CounterType,
+ GpuCounterDefinition,
+ GpuPerformanceStateInterval,
+ GpuUtilization,
+ KernelDispatch,
+ KernelStats,
+ PerformanceState,
+ ShaderInfo,
+ ShaderType,
+ TraceExport,
+ TraceMetadata,
+)
+
+
+def _make_converter() -> cattrs.Converter:
+ converter = cattrs.Converter()
+
+ def enum_unstructure(val: Enum) -> str:
+ return val.value
+
+ def enum_structure(val: str, cls: type[Enum]) -> Enum:
+ return cls(val)
+
+ for enum_cls in (CounterType, PerformanceState, ShaderType):
+ converter.register_unstructure_hook(enum_cls, enum_unstructure)
+ converter.register_structure_hook(enum_cls, enum_structure)
+
+ def tuple_unstructure(val: tuple[Any, ...]) -> list[Any]:
+ return [converter.unstructure(item) for item in val]
+
+ converter.register_unstructure_hook(tuple, tuple_unstructure)
+
+ return converter
+
+
+_converter = _make_converter()
+
+
+def to_dict(export: TraceExport) -> dict[str, Any]:
+ return _converter.unstructure(export)
+
+
+def to_json(export: TraceExport, indent: int | None = 2) -> str:
+ return json.dumps(to_dict(export), indent=indent)
+
+
+def from_dict(data: dict[str, Any]) -> TraceExport:
+ return _converter.structure(data, TraceExport)
+
+
+def from_json(json_str: str) -> TraceExport:
+ return from_dict(json.loads(json_str))
diff --git a/tools/gpu_trace/uv.lock b/tools/gpu_trace/uv.lock
new file mode 100644
index 000000000..ab3521ade
--- /dev/null
+++ b/tools/gpu_trace/uv.lock
@@ -0,0 +1,45 @@
+version = 1
+revision = 3
+requires-python = ">=3.12"
+
+[[package]]
+name = "attrs"
+version = "25.4.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/6b/5c/685e6633917e101e5dcb62b9dd76946cbb57c26e133bae9e0cd36033c0a9/attrs-25.4.0.tar.gz", hash = "sha256:16d5969b87f0859ef33a48b35d55ac1be6e42ae49d5e853b597db70c35c57e11", size = 934251, upload-time = "2025-10-06T13:54:44.725Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/3a/2a/7cc015f5b9f5db42b7d48157e23356022889fc354a2813c15934b7cb5c0e/attrs-25.4.0-py3-none-any.whl", hash = "sha256:adcf7e2a1fb3b36ac48d97835bb6d8ade15b8dcce26aba8bf1d14847b57a3373", size = 67615, upload-time = "2025-10-06T13:54:43.17Z" },
+]
+
+[[package]]
+name = "cattrs"
+version = "25.3.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "attrs" },
+ { name = "typing-extensions" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/6e/00/2432bb2d445b39b5407f0a90e01b9a271475eea7caf913d7a86bcb956385/cattrs-25.3.0.tar.gz", hash = "sha256:1ac88d9e5eda10436c4517e390a4142d88638fe682c436c93db7ce4a277b884a", size = 509321, upload-time = "2025-10-07T12:26:08.737Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/d8/2b/a40e1488fdfa02d3f9a653a61a5935ea08b3c2225ee818db6a76c7ba9695/cattrs-25.3.0-py3-none-any.whl", hash = "sha256:9896e84e0a5bf723bc7b4b68f4481785367ce07a8a02e7e9ee6eb2819bc306ff", size = 70738, upload-time = "2025-10-07T12:26:06.603Z" },
+]
+
+[[package]]
+name = "gpu-trace"
+version = "0.1.0"
+source = { editable = "." }
+dependencies = [
+ { name = "cattrs" },
+]
+
+[package.metadata]
+requires-dist = [{ name = "cattrs", specifier = ">=24.1.0" }]
+
+[[package]]
+name = "typing-extensions"
+version = "4.15.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac84269196e94cf00f187f7ed21c242792a923cdb1c61f/typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", size = 109391, upload-time = "2025-08-25T13:49:26.313Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" },
+]
diff --git a/scripts/tools/.gitignore b/tools/helpers/.gitignore
similarity index 100%
rename from scripts/tools/.gitignore
rename to tools/helpers/.gitignore
diff --git a/tools/helpers/.python-version b/tools/helpers/.python-version
new file mode 100644
index 000000000..e4fba2183
--- /dev/null
+++ b/tools/helpers/.python-version
@@ -0,0 +1 @@
+3.12
diff --git a/scripts/tools/main.py b/tools/helpers/main.py
similarity index 100%
rename from scripts/tools/main.py
rename to tools/helpers/main.py
diff --git a/scripts/tools/model.py b/tools/helpers/model.py
similarity index 100%
rename from scripts/tools/model.py
rename to tools/helpers/model.py
diff --git a/scripts/tools/pyproject.toml b/tools/helpers/pyproject.toml
similarity index 100%
rename from scripts/tools/pyproject.toml
rename to tools/helpers/pyproject.toml
diff --git a/scripts/tools/utils.py b/tools/helpers/utils.py
similarity index 100%
rename from scripts/tools/utils.py
rename to tools/helpers/utils.py
diff --git a/scripts/tools/uv.lock b/tools/helpers/uv.lock
similarity index 100%
rename from scripts/tools/uv.lock
rename to tools/helpers/uv.lock