Skip to content

Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.

fix: generate arm64 eBPF bindings (#1213)

Dogxia6a2ac2f4359
diff --git a/.github/workflows/otel-gpu-collector-tests.yml b/.github/workflows/otel-gpu-collector-tests.yml
index c30b7871..d30838b3 100644
--- a/.github/workflows/otel-gpu-collector-tests.yml
+++ b/.github/workflows/otel-gpu-collector-tests.yml
@@ -45,3 +45,56 @@ jobs:
- name: Run tests
working-directory: opentelemetry-gpu-collector
run: go test ./...
+
+ # Cross-compile matrix: confirm the collector builds for every supported
+ # GOOS/GOARCH after `make generate` has produced the per-arch eBPF
+ # bindings. CGO_ENABLED=0 keeps the cross-builds reproducible without
+ # installing arch-specific C toolchains; NVML/eBPF paths fall back to
+ # their build-tagged stubs on unsupported targets.
+ - name: Cross-compile linux/arm64 (eBPF path)
+ working-directory: opentelemetry-gpu-collector
+ env:
+ CGO_ENABLED: "0"
+ GOOS: linux
+ GOARCH: arm64
+ run: go build ./...
+
+ - name: Cross-compile linux/riscv64 (stub fallback)
+ working-directory: opentelemetry-gpu-collector
+ env:
+ CGO_ENABLED: "0"
+ GOOS: linux
+ GOARCH: riscv64
+ run: go build ./...
+
+ - name: Cross-compile linux/ppc64le (stub fallback)
+ working-directory: opentelemetry-gpu-collector
+ env:
+ CGO_ENABLED: "0"
+ GOOS: linux
+ GOARCH: ppc64le
+ run: go build ./...
+
+ - name: Cross-compile darwin/amd64 (stub fallback)
+ working-directory: opentelemetry-gpu-collector
+ env:
+ CGO_ENABLED: "0"
+ GOOS: darwin
+ GOARCH: amd64
+ run: go build ./...
+
+ - name: Cross-compile darwin/arm64 (stub fallback)
+ working-directory: opentelemetry-gpu-collector
+ env:
+ CGO_ENABLED: "0"
+ GOOS: darwin
+ GOARCH: arm64
+ run: go build ./...
+
+ - name: Cross-compile windows/amd64 (stub fallback)
+ working-directory: opentelemetry-gpu-collector
+ env:
+ CGO_ENABLED: "0"
+ GOOS: windows
+ GOARCH: amd64
+ run: go build ./...
diff --git a/opentelemetry-gpu-collector/.gitignore b/opentelemetry-gpu-collector/.gitignore
index 6a4503bb..4e972504 100644
--- a/opentelemetry-gpu-collector/.gitignore
+++ b/opentelemetry-gpu-collector/.gitignore
@@ -1,9 +1,12 @@
# Binary
openlit-collector
-# Generated eBPF Go bindings (output of go generate / bpf2go)
-internal/ebpf/gpuevent_bpfel*.go
-internal/ebpf/gpuevent_bpfel*.o
+# Generated eBPF Go bindings (output of go generate / bpf2go).
+# bpf2go output stems are always "<linux>_<clang>" (clang = bpfel|bpfeb).
+internal/ebpf/gpuevent_*bpfel.go
+internal/ebpf/gpuevent_*bpfel.o
+internal/ebpf/gpuevent_*bpfeb.go
+internal/ebpf/gpuevent_*bpfeb.o
# Copied/generated eBPF build headers (from setup-bpf)
internal/ebpf/bpf/vmlinux.h
diff --git a/opentelemetry-gpu-collector/Makefile b/opentelemetry-gpu-collector/Makefile
index 56f12722..b7eae497 100644
--- a/opentelemetry-gpu-collector/Makefile
+++ b/opentelemetry-gpu-collector/Makefile
@@ -18,6 +18,16 @@ setup-bpf:
@dpkg -s libbpf-dev >/dev/null 2>&1 || (echo "Installing libbpf-dev..." && sudo apt-get update && sudo apt-get install -y libbpf-dev)
@echo "==> Generating vmlinux.h from kernel BTF..."
bpftool btf dump file /sys/kernel/btf/vmlinux format c > $(BPF_DIR)/bpf/vmlinux.h
+ @# Cross-arch shim: vmlinux.h only contains the build host's kernel types,
+ @# but bpf2go runs clang once per -target arch (amd64 + arm64). When the
+ @# arm64 pass runs on a non-arm64 host, libbpf's bpf_tracing.h references
+ @# struct user_pt_regs which the host BTF lacks. Append the canonical arm64
+ @# layout (arch/arm64/include/uapi/asm/ptrace.h) if vmlinux.h doesn't
+ @# already define it. Idempotent: the grep guard prevents redefinition
+ @# when this host's BTF (e.g. native arm64) already provides the struct.
+ @if ! grep -q 'struct user_pt_regs' $(BPF_DIR)/bpf/vmlinux.h; then \
+ printf '\n/* openlit cross-arch BPF shim: arm64 user_pt_regs missing from host BTF */\nstruct user_pt_regs {\n\t__u64 regs[31];\n\t__u64 sp;\n\t__u64 pc;\n\t__u64 pstate;\n};\n' >> $(BPF_DIR)/bpf/vmlinux.h; \
+ fi
@echo "==> Copying libbpf headers..."
cp /usr/include/bpf/bpf_helpers.h $(BPF_DIR)/bpf/
cp /usr/include/bpf/bpf_helper_defs.h $(BPF_DIR)/bpf/
@@ -37,7 +47,11 @@ lint:
clean:
rm -f $(BINARY)
- rm -f $(BPF_DIR)/gpuevent_bpfel*.go $(BPF_DIR)/gpuevent_bpfel*.o
+ # bpf2go output stems are always "<linux>_<clang>" (clang = bpfel|bpfeb),
+ # so restricting to those endings avoids deleting any hand-written file
+ # that might one day live alongside the generated ones.
+ rm -f $(BPF_DIR)/gpuevent_*bpfel.go $(BPF_DIR)/gpuevent_*bpfel.o
+ rm -f $(BPF_DIR)/gpuevent_*bpfeb.go $(BPF_DIR)/gpuevent_*bpfeb.o
rm -f $(BPF_DIR)/bpf/vmlinux.h
rm -f $(BPF_DIR)/bpf/bpf_helpers.h $(BPF_DIR)/bpf/bpf_helper_defs.h
rm -f $(BPF_DIR)/bpf/bpf_tracing.h $(BPF_DIR)/bpf/bpf_core_read.h $(BPF_DIR)/bpf/bpf_endian.h
diff --git a/opentelemetry-gpu-collector/README.md b/opentelemetry-gpu-collector/README.md
index 928deeac..c1ae366f 100644
--- a/opentelemetry-gpu-collector/README.md
+++ b/opentelemetry-gpu-collector/README.md
@@ -110,7 +110,7 @@ services:
Download a pre-built binary from the [Releases](https://github.com/openlit/openlit/releases) page:
```sh
-# Linux amd64
+# Pick the asset matching your platform: linux-amd64 or linux-arm64.
curl -L https://github.com/openlit/openlit/releases/latest/download/opentelemetry-gpu-collector-<version>-linux-amd64 \
-o opentelemetry-gpu-collector
chmod +x opentelemetry-gpu-collector
@@ -123,7 +123,7 @@ OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317 ./opentelemetry-gpu-collector
```sh
git clone https://github.com/openlit/openlit.git
cd openlit/opentelemetry-gpu-collector
-make build
+make all
./opentelemetry-gpu-collector
```
diff --git a/opentelemetry-gpu-collector/internal/ebpf/bpf/gpuevent.c b/opentelemetry-gpu-collector/internal/ebpf/bpf/gpuevent.c
index 6a2cc01c..3d4c368a 100644
--- a/opentelemetry-gpu-collector/internal/ebpf/bpf/gpuevent.c
+++ b/opentelemetry-gpu-collector/internal/ebpf/bpf/gpuevent.c
@@ -50,16 +50,15 @@ static __always_inline struct pid_info_t get_pid_info() {
// Signature: cudaError_t cudaLaunchKernel(const void *func, dim3 gridDim,
// dim3 blockDim, void **args, size_t sharedMem, cudaStream_t stream)
//
-// ABI NOTE (x86_64 System V only):
-// dim3 is a struct of three uint32 values (12 bytes). Structs <= 16 bytes that
-// are "integer class" are split across consecutive argument registers. With func
-// in rdi (PARM1), the two dim3 structs consume rsi/rdx/rcx/r8 as follows:
-// gridDim.x, gridDim.y -> rsi (PARM2) low/high 32 bits
-// gridDim.z, blockDim.x -> rdx (PARM3) low/high 32 bits
-// blockDim.y, blockDim.z -> rcx (PARM4) low/high 32 bits
-// This packing was validated against CUDA 11.x–12.x on x86_64 gcc/clang.
-// It will NOT be correct on aarch64 (ARM64) or for future CUDA ABI changes.
-// TODO: replace with BTF CO-RE struct access once CUDA BTF info is available.
+// dim3 is a 12-byte struct of three uint32 values. The way it is passed in
+// registers at the libcudart entry point differs by target architecture, so
+// the decoder is split per __TARGET_ARCH_* macro (auto-defined by bpf2go).
+//
+// `stream` is intentionally not decoded here — it is the 6th argument and on
+// both ABIs it spills past the registers covered by libbpf's PT_REGS_PARMn
+// macros (x86_64: stack; arm64: x7). Userspace does not consume the field
+// today, so we keep the wire layout but zero the value to avoid reading
+// arbitrary stack bytes inside the BPF program.
SEC("uprobe/cudaLaunchKernel")
int handle_cuda_launch(struct pt_regs *ctx) {
struct gpu_kernel_launch_t *ev;
@@ -69,25 +68,46 @@ int handle_cuda_launch(struct pt_regs *ctx) {
ev->flags = EVENT_GPU_KERNEL_LAUNCH;
ev->pid_info = get_pid_info();
+ ev->stream = 0;
// arg0: const void *func (kernel function pointer)
ev->kern_func_off = PT_REGS_PARM1(ctx);
- // gridDim.x, gridDim.y packed into rsi; gridDim.z in low 32 bits of rdx
+#if defined(__TARGET_ARCH_x86)
+ // x86_64 System V: dim3 (12 bytes) is decomposed into 3 uint32 fields and
+ // packed two-per-register across rsi/rdx/rcx, as observed in libcudart
+ // CUDA 11.x–12.x (gcc/clang). With func consuming rdi:
+ // rsi (PARM2): gridDim.x | gridDim.y
+ // rdx (PARM3): gridDim.z | blockDim.x
+ // rcx (PARM4): blockDim.y | blockDim.z
+ __u64 grid_xy = PT_REGS_PARM2(ctx);
+ ev->grid_x = grid_xy & 0xFFFFFFFF;
+ ev->grid_y = grid_xy >> 32;
+ __u64 grid_z_block_x = PT_REGS_PARM3(ctx);
+ ev->grid_z = grid_z_block_x & 0xFFFFFFFF;
+ ev->block_x = grid_z_block_x >> 32;
+ __u64 block_yz = PT_REGS_PARM4(ctx);
+ ev->block_y = block_yz & 0xFFFFFFFF;
+ ev->block_z = block_yz >> 32;
+#elif defined(__TARGET_ARCH_arm64)
+ // AArch64 AAPCS64 (§6.4.2): each dim3 (12 bytes, rounded to 16) occupies
+ // two consecutive 64-bit GPRs and tail padding is NOT merged with the
+ // next argument. With func consuming x0:
+ // x1 (PARM2): gridDim.x | gridDim.y
+ // x2 (PARM3): gridDim.z | (pad)
+ // x3 (PARM4): blockDim.x | blockDim.y
+ // x4 (PARM5): blockDim.z | (pad)
__u64 grid_xy = PT_REGS_PARM2(ctx);
ev->grid_x = grid_xy & 0xFFFFFFFF;
ev->grid_y = grid_xy >> 32;
ev->grid_z = PT_REGS_PARM3(ctx) & 0xFFFFFFFF;
-
- // blockDim.x in high 32 bits of rdx; blockDim.y, blockDim.z in rcx
- __u64 block_xy = PT_REGS_PARM3(ctx) >> 32;
+ __u64 block_xy = PT_REGS_PARM4(ctx);
ev->block_x = block_xy & 0xFFFFFFFF;
- __u64 parm4 = PT_REGS_PARM4(ctx);
- ev->block_y = parm4 & 0xFFFFFFFF;
- ev->block_z = parm4 >> 32;
-
- // arg4: cudaStream_t stream (6th parameter)
- ev->stream = PT_REGS_PARM6(ctx);
+ ev->block_y = block_xy >> 32;
+ ev->block_z = PT_REGS_PARM5(ctx) & 0xFFFFFFFF;
+#else
+#error "cudaLaunchKernel argument decoding not implemented for this architecture"
+#endif
// Capture user-space stack trace
bpf_get_stack(ctx, ev->ustack, sizeof(ev->ustack), BPF_F_USER_STACK);
diff --git a/opentelemetry-gpu-collector/internal/ebpf/bpf/gpuevent.h b/opentelemetry-gpu-collector/internal/ebpf/bpf/gpuevent.h
index 786885fe..6e4da5a0 100644
--- a/opentelemetry-gpu-collector/internal/ebpf/bpf/gpuevent.h
+++ b/opentelemetry-gpu-collector/internal/ebpf/bpf/gpuevent.h
@@ -52,15 +52,4 @@ struct gpu_memcpy_t {
__u64 size;
};
-// libbpf 0.3 (Debian Bullseye) only defines PT_REGS_PARM1–5.
-#ifndef PT_REGS_PARM6
-#if defined(__TARGET_ARCH_x86)
-#define PT_REGS_PARM6(x) ((__u64)(x)->r9)
-#elif defined(__TARGET_ARCH_arm64)
-#define PT_REGS_PARM6(x) ((__u64)(x)->regs[5])
-#else
-#error "PT_REGS_PARM6 not defined for this architecture"
-#endif
-#endif
-
#endif /* __GPUEVENT_H__ */
diff --git a/opentelemetry-gpu-collector/internal/ebpf/demangle.go b/opentelemetry-gpu-collector/internal/ebpf/demangle.go
index e095ce9e..22fcd5a9 100644
--- a/opentelemetry-gpu-collector/internal/ebpf/demangle.go
+++ b/opentelemetry-gpu-collector/internal/ebpf/demangle.go
@@ -1,4 +1,4 @@
-//go:build linux
+//go:build linux && (amd64 || arm64)
package ebpf
diff --git a/opentelemetry-gpu-collector/internal/ebpf/symbols.go b/opentelemetry-gpu-collector/internal/ebpf/symbols.go
index 04c54081..9e67e1e6 100644
--- a/opentelemetry-gpu-collector/internal/ebpf/symbols.go
+++ b/opentelemetry-gpu-collector/internal/ebpf/symbols.go
@@ -1,4 +1,4 @@
-//go:build linux
+//go:build linux && (amd64 || arm64)
package ebpf
diff --git a/opentelemetry-gpu-collector/internal/ebpf/tracer.go b/opentelemetry-gpu-collector/internal/ebpf/tracer.go
index 4c19d238..1396ef3d 100644
--- a/opentelemetry-gpu-collector/internal/ebpf/tracer.go
+++ b/opentelemetry-gpu-collector/internal/ebpf/tracer.go
@@ -1,4 +1,4 @@
-//go:build linux
+//go:build linux && (amd64 || arm64)
package ebpf
@@ -19,7 +19,7 @@ import (
"github.com/cilium/ebpf/ringbuf"
)
-//go:generate go run github.com/cilium/ebpf/cmd/bpf2go -cc clang -target bpfel gpuevent ./bpf/gpuevent.c -- -I./bpf -D__TARGET_ARCH_x86
+//go:generate go run github.com/cilium/ebpf/cmd/bpf2go -cc clang -target amd64,arm64 gpuevent ./bpf/gpuevent.c -- -I./bpf
// Tracer manages eBPF programs for CUDA runtime interception.
type Tracer struct {
@@ -207,6 +207,7 @@ func findCudaLib() string {
candidates := []string{
"/usr/local/cuda/lib64/libcudart.so",
"/usr/lib/x86_64-linux-gnu/libcudart.so",
+ "/usr/lib/aarch64-linux-gnu/libcudart.so",
"/usr/lib64/libcudart.so",
"/usr/lib/libcudart.so",
}
diff --git a/opentelemetry-gpu-collector/internal/ebpf/tracer_stub.go b/opentelemetry-gpu-collector/internal/ebpf/tracer_stub.go
index 6a8f3e8e..82c4560b 100644
--- a/opentelemetry-gpu-collector/internal/ebpf/tracer_stub.go
+++ b/opentelemetry-gpu-collector/internal/ebpf/tracer_stub.go
@@ -1,4 +1,4 @@
-//go:build !linux
+//go:build !(linux && (amd64 || arm64))
package ebpf
@@ -6,12 +6,13 @@ import (
"context"
"fmt"
"log/slog"
+ "runtime"
)
type Tracer struct{}
func NewTracer(_ *slog.Logger, _ EventHandler) (*Tracer, error) {
- return nil, fmt.Errorf("eBPF CUDA tracing is only supported on Linux")
+ return nil, fmt.Errorf("eBPF CUDA tracing is not supported on %s/%s (supported: linux/amd64, linux/arm64)", runtime.GOOS, runtime.GOARCH)
}
func (t *Tracer) Run(_ context.Context) {}
diff --git a/opentelemetry-gpu-collector/internal/gpu/nvidia/nvidia.go b/opentelemetry-gpu-collector/internal/gpu/nvidia/nvidia.go
index df813792..c0f96777 100644
--- a/opentelemetry-gpu-collector/internal/gpu/nvidia/nvidia.go
+++ b/opentelemetry-gpu-collector/internal/gpu/nvidia/nvidia.go
@@ -1,4 +1,4 @@
-//go:build linux
+//go:build linux && (amd64 || arm64) && cgo
package nvidia
diff --git a/opentelemetry-gpu-collector/internal/gpu/nvidia/nvidia_stub.go b/opentelemetry-gpu-collector/internal/gpu/nvidia/nvidia_stub.go
index c61500e6..73f1e938 100644
--- a/opentelemetry-gpu-collector/internal/gpu/nvidia/nvidia_stub.go
+++ b/opentelemetry-gpu-collector/internal/gpu/nvidia/nvidia_stub.go
@@ -1,25 +1,35 @@
-//go:build !linux
+//go:build !(linux && (amd64 || arm64) && cgo)
package nvidia
import (
"fmt"
"log/slog"
+ "runtime"
"github.com/openlit/openlit/opentelemetry-gpu-collector/internal/gpu"
)
-// Device is a stub for non-Linux platforms.
+// Device is a stub for platforms/builds where NVML is unavailable
+// (non-Linux, unsupported arch, or CGO disabled).
type Device struct {
info gpu.DeviceInfo
}
-func (d *Device) Info() gpu.DeviceInfo { return d.info }
-func (d *Device) Collect() (*gpu.Snapshot, error) { return nil, fmt.Errorf("not supported on this OS") }
-func (d *Device) Close() {}
+func (d *Device) Info() gpu.DeviceInfo { return d.info }
+func (d *Device) Collect() (*gpu.Snapshot, error) { return nil, errUnsupported() }
+func (d *Device) Close() {}
func DiscoverDevices(_ *slog.Logger) ([]*Device, error) {
- return nil, fmt.Errorf("NVIDIA GPU support requires Linux")
+ return nil, errUnsupported()
}
+// InitNVML is a no-op on unsupported builds; included so callers don't need
+// build tags to invoke it.
+func InitNVML() error { return errUnsupported() }
+
func ShutdownNVML() {}
+
+func errUnsupported() error {
+ return fmt.Errorf("NVIDIA NVML support requires linux/amd64 or linux/arm64 with cgo enabled (current: %s/%s)", runtime.GOOS, runtime.GOARCH)
+}