Skip to content

Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.

fix(collector): add hw.id (#1541)

Aman Agarwal203ce1323143
diff --git a/.so/.gitignore b/.so/.gitignore
new file mode 100644
index 00000000..c26a1aca
--- /dev/null
+++ b/.so/.gitignore
@@ -0,0 +1,4 @@
+# Superopen machine-local data (do not commit).
+sessions/
+db/
+harvest/
diff --git a/opentelemetry-gpu-collector/cmd/collector/main.go b/opentelemetry-gpu-collector/cmd/collector/main.go
index 89de5a6f..5ba191b0 100644
--- a/opentelemetry-gpu-collector/cmd/collector/main.go
+++ b/opentelemetry-gpu-collector/cmd/collector/main.go
@@ -502,6 +502,7 @@ func logEffectiveAvailability(logger *slog.Logger, cfg *config.Config, a effecti
type gpuCollectors struct {
mc *export.MetricsCollector
ebpfTracer *gpuebpf.Tracer
+ ebpfMetrics *export.EBPFMetrics
occMetrics *export.OccupancyMetrics
dcgmMetrics *export.DCGMMetrics
dcgmClient dcgm.Client
@@ -516,6 +517,9 @@ func (c *gpuCollectors) close() {
if c.ebpfTracer != nil {
c.ebpfTracer.Close()
}
+ if c.ebpfMetrics != nil {
+ c.ebpfMetrics.Close()
+ }
if c.occMetrics != nil {
c.occMetrics.Close()
}
@@ -567,6 +571,7 @@ func setupCollectors(
report.fault(logger, config.FeatureEBPF, err.Error())
}
} else {
+ out.ebpfMetrics = ebpfMetrics
occ, err := export.NewOccupancyMetrics(provider, devices, resolver, logger)
if err != nil {
if report != nil {
diff --git a/opentelemetry-gpu-collector/internal/cudaoccupancy/engine.go b/opentelemetry-gpu-collector/internal/cudaoccupancy/engine.go
index cda64bbf..f119120c 100644
--- a/opentelemetry-gpu-collector/internal/cudaoccupancy/engine.go
+++ b/opentelemetry-gpu-collector/internal/cudaoccupancy/engine.go
@@ -25,6 +25,7 @@ type KernelLaunch struct {
TID uint32
StreamID uint64
DeviceUUID string
+ DeviceIdx int // CUDA index when known; -1 unknown. Ignored when DeviceUUID is set.
KtimeNs uint64
Name string
Kind cudaspans.LaunchKind
@@ -38,6 +39,7 @@ type SyncEvent struct {
TID uint32
StreamID uint64
DeviceUUID string
+ DeviceIdx int // CUDA index when known; -1 unknown. Ignored when DeviceUUID is set.
KtimeNs uint64
DeviceWide bool
}
@@ -187,11 +189,16 @@ func (e *Engine) HandleSetDevice(ev SetDeviceEvent) {
e.devices.NoteSetDevice(ev.PID, ev.TID, ev.DeviceIdx)
}
-func (e *Engine) resolveUUID(pid, tid uint32, explicit string) string {
+func (e *Engine) resolveUUID(pid, tid uint32, explicit string, explicitIdx int) string {
if explicit != "" {
return explicit
}
if e.devices != nil {
+ if explicitIdx >= 0 {
+ if u := e.devices.UUIDForIndex(explicitIdx); u != "" {
+ return u
+ }
+ }
if u := e.devices.ResolveUUID(pid, tid); u != "" {
return u
}
@@ -204,7 +211,7 @@ func (e *Engine) HandleLaunch(ev KernelLaunch) {
e.mu.Lock()
defer e.mu.Unlock()
- uuid := e.resolveUUID(ev.PID, ev.TID, ev.DeviceUUID)
+ uuid := e.resolveUUID(ev.PID, ev.TID, ev.DeviceUUID, ev.DeviceIdx)
if uuid == "" {
uuid = "unknown"
}
@@ -231,7 +238,7 @@ func (e *Engine) HandleSync(ev SyncEvent) {
e.mu.Lock()
defer e.mu.Unlock()
- uuid := e.resolveUUID(ev.PID, ev.TID, ev.DeviceUUID)
+ uuid := e.resolveUUID(ev.PID, ev.TID, ev.DeviceUUID, ev.DeviceIdx)
if ev.DeviceWide || ev.StreamID == 0 {
e.fanOutDeviceSync(ev.PID, uuid, ev.KtimeNs)
@@ -343,20 +350,21 @@ func (e *Engine) markSynchronization(h *streamHandler, ts uint64) {
kind = cudaspans.LaunchKindKernel
}
e.closedSinceTake = append(e.closedSinceTake, cudaspans.ClosedSpan{
- StartNs: l.KtimeNs,
- EndNs: ts,
- PID: l.PID,
- TID: l.TID,
- StreamID: l.StreamID,
- DeviceUUID: l.DeviceUUID,
- KernelName: l.Name,
- Kind: kind,
- GridX: l.GridX,
- GridY: l.GridY,
- GridZ: l.GridZ,
- BlockX: l.BlockX,
- BlockY: l.BlockY,
- BlockZ: l.BlockZ,
+ StartNs: l.KtimeNs,
+ EndNs: ts,
+ PID: l.PID,
+ TID: l.TID,
+ StreamID: l.StreamID,
+ DeviceUUID: l.DeviceUUID,
+ DeviceIndex: l.DeviceIdx,
+ KernelName: l.Name,
+ Kind: kind,
+ GridX: l.GridX,
+ GridY: l.GridY,
+ GridZ: l.GridZ,
+ BlockX: l.BlockX,
+ BlockY: l.BlockY,
+ BlockZ: l.BlockZ,
})
}
// Keep launches at/after ts
diff --git a/opentelemetry-gpu-collector/internal/cudaoccupancy/engine_test.go b/opentelemetry-gpu-collector/internal/cudaoccupancy/engine_test.go
index 5f315d36..d221d875 100644
--- a/opentelemetry-gpu-collector/internal/cudaoccupancy/engine_test.go
+++ b/opentelemetry-gpu-collector/internal/cudaoccupancy/engine_test.go
@@ -1,6 +1,42 @@
package cudaoccupancy
-import "testing"
+import (
+ "testing"
+
+ "github.com/openlit/openlit/opentelemetry-gpu-collector/internal/cudaspans"
+)
+
+func TestHandleLaunchMultiGPUDefaultsToIndexZero(t *testing.T) {
+ r := cudaspans.NewDeviceResolver(nil)
+ r.SetDeviceIndexUUID(0, "gpu-a")
+ r.SetDeviceIndexUUID(1, "gpu-b")
+ e := NewEngine(map[string]uint64{"gpu-a": 64, "gpu-b": 64}, r)
+ e.HandleLaunch(KernelLaunch{
+ PID: 1, TID: 1, StreamID: 9, DeviceIdx: -1, KtimeNs: 100,
+ Name: "k", GridX: 1, GridY: 1, GridZ: 1, BlockX: 32, BlockY: 1, BlockZ: 1,
+ })
+ e.HandleSync(SyncEvent{PID: 1, TID: 1, StreamID: 9, DeviceIdx: -1, KtimeNs: 200})
+ closed := e.TakeClosedSpans()
+ if len(closed) != 1 {
+ t.Fatalf("closed=%d", len(closed))
+ }
+ if closed[0].DeviceUUID != "gpu-a" {
+ t.Fatalf("DeviceUUID=%q, want gpu-a (CUDA default device 0)", closed[0].DeviceUUID)
+ }
+
+ e.HandleLaunch(KernelLaunch{
+ PID: 1, TID: 1, StreamID: 9, DeviceIdx: 1, KtimeNs: 300,
+ Name: "k2", GridX: 1, GridY: 1, GridZ: 1, BlockX: 32, BlockY: 1, BlockZ: 1,
+ })
+ e.HandleSync(SyncEvent{PID: 1, TID: 1, StreamID: 9, DeviceIdx: 1, KtimeNs: 400})
+ closed = e.TakeClosedSpans()
+ if len(closed) != 1 {
+ t.Fatalf("closed=%d", len(closed))
+ }
+ if closed[0].DeviceUUID != "gpu-b" {
+ t.Fatalf("DeviceUUID=%q, want gpu-b from event DeviceIdx", closed[0].DeviceUUID)
+ }
+}
func TestTakeClosedSpansOnSync(t *testing.T) {
e := NewEngine(map[string]uint64{"gpu-a": 64}, nil)
diff --git a/opentelemetry-gpu-collector/internal/cudaspans/resolver.go b/opentelemetry-gpu-collector/internal/cudaspans/resolver.go
index 7ef4e53b..73948415 100644
--- a/opentelemetry-gpu-collector/internal/cudaspans/resolver.go
+++ b/opentelemetry-gpu-collector/internal/cudaspans/resolver.go
@@ -10,7 +10,14 @@ import (
const threadDevMaxSize = 8192
// DeviceResolver maps cudaSetDevice(index) onto NVML device identity.
-// Fallback when unset: sole NVIDIA GPU only (not an arbitrary map entry).
+//
+// Resolution order for a (pid, tid):
+// 1. Last successful cudaSetDevice for that thread
+// 2. Sole NVIDIA GPU on the host (unambiguous)
+// 3. CUDA runtime default current device 0, when index 0 is a known NVIDIA GPU
+//
+// CUDA_VISIBLE_DEVICES remapping is not applied: CUDA index is treated as the
+// NVML index. That assumption already applies to NoteSetDevice.
type DeviceResolver struct {
mu sync.Mutex
@@ -82,17 +89,14 @@ func (r *DeviceResolver) NoteSetDevice(pid, tid uint32, deviceIdx int) {
r.threadDev[threadKey(pid, tid)] = deviceIdx
}
-// ResolveIndex returns the CUDA device index for a thread, or sole GPU, or -1.
+// ResolveIndex returns the CUDA device index for a thread, or -1 if unknown.
func (r *DeviceResolver) ResolveIndex(pid, tid uint32) int {
if r == nil {
return -1
}
r.mu.Lock()
defer r.mu.Unlock()
- if idx, ok := r.threadDev[threadKey(pid, tid)]; ok {
- return idx
- }
- return r.soleIndex
+ return r.resolveIndexLocked(pid, tid)
}
// ResolveUUID returns the GPU UUID for a thread (via index), or "".
@@ -102,16 +106,27 @@ func (r *DeviceResolver) ResolveUUID(pid, tid uint32) string {
}
r.mu.Lock()
defer r.mu.Unlock()
- idx, ok := r.threadDev[threadKey(pid, tid)]
- if !ok {
- if r.soleIndex < 0 {
- return ""
- }
- idx = r.soleIndex
+ idx := r.resolveIndexLocked(pid, tid)
+ if idx < 0 {
+ return ""
}
return r.indexUUID[idx]
}
+func (r *DeviceResolver) resolveIndexLocked(pid, tid uint32) int {
+ if idx, ok := r.threadDev[threadKey(pid, tid)]; ok {
+ return idx
+ }
+ if r.soleIndex >= 0 {
+ return r.soleIndex
+ }
+ // CUDA runtime: current device is 0 until cudaSetDevice.
+ if _, ok := r.indexUUID[0]; ok {
+ return 0
+ }
+ return -1
+}
+
// UUIDForIndex returns the UUID for a CUDA device index.
func (r *DeviceResolver) UUIDForIndex(idx int) string {
if r == nil {
@@ -136,6 +151,21 @@ func (r *DeviceResolver) IndexInfo(idx int) (uuid, name, pci string, ok bool) {
return uuid, r.indexName[idx], r.indexPCI[idx], true
}
+// IndexForUUID returns the CUDA/NVML index for a GPU UUID.
+func (r *DeviceResolver) IndexForUUID(uuid string) (int, bool) {
+ if r == nil || uuid == "" {
+ return -1, false
+ }
+ r.mu.Lock()
+ defer r.mu.Unlock()
+ for idx, u := range r.indexUUID {
+ if u == uuid {
+ return idx, true
+ }
+ }
+ return -1, false
+}
+
// ForgetPID drops thread mappings for a PID (inactive cleanup).
func (r *DeviceResolver) ForgetPID(pid uint32) {
if r == nil {
diff --git a/opentelemetry-gpu-collector/internal/cudaspans/spans_test.go b/opentelemetry-gpu-collector/internal/cudaspans/spans_test.go
index 1838070d..59dddff0 100644
--- a/opentelemetry-gpu-collector/internal/cudaspans/spans_test.go
+++ b/opentelemetry-gpu-collector/internal/cudaspans/spans_test.go
@@ -39,3 +39,25 @@ func TestDeviceResolverSoleGPU(t *testing.T) {
t.Fatalf("got %q", got)
}
}
+
+func TestDeviceResolverMultiGPUDefaultsToCUDAZero(t *testing.T) {
+ r := NewDeviceResolver(nil)
+ r.SetDeviceIndexUUID(0, "gpu-a")
+ r.SetDeviceIndexUUID(1, "gpu-b")
+ if got := r.ResolveIndex(1, 1); got != 0 {
+ t.Fatalf("ResolveIndex = %d, want 0 (CUDA default current device)", got)
+ }
+ if got := r.ResolveUUID(1, 1); got != "gpu-a" {
+ t.Fatalf("ResolveUUID = %q, want gpu-a", got)
+ }
+ r.NoteSetDevice(1, 1, 1)
+ if got := r.ResolveIndex(1, 1); got != 1 {
+ t.Fatalf("after SetDevice ResolveIndex = %d, want 1", got)
+ }
+ if got := r.ResolveUUID(1, 1); got != "gpu-b" {
+ t.Fatalf("after SetDevice ResolveUUID = %q, want gpu-b", got)
+ }
+ if idx, ok := r.IndexForUUID("gpu-b"); !ok || idx != 1 {
+ t.Fatalf("IndexForUUID(gpu-b) = %d ok=%v", idx, ok)
+ }
+}
diff --git a/opentelemetry-gpu-collector/internal/ebpf/bpf/gpuevent.c b/opentelemetry-gpu-collector/internal/ebpf/bpf/gpuevent.c
index ea3008b5..d90f947a 100644
--- a/opentelemetry-gpu-collector/internal/ebpf/bpf/gpuevent.c
+++ b/opentelemetry-gpu-collector/internal/ebpf/bpf/gpuevent.c
@@ -37,6 +37,17 @@ struct {
__type(value, __s32);
} cuda_set_device_cache SEC(".maps");
+// Last successful cudaSetDevice per thread (pid_tgid). Stamped onto later
+// events so userspace can attribute hw.id on multi-GPU hosts without waiting
+// for the SetDevice ringbuf event to be processed. Userspace drops keys for
+// dead PIDs so a reused pid_tgid cannot inherit the previous process's index.
+struct {
+ __uint(type, BPF_MAP_TYPE_LRU_HASH);
+ __uint(max_entries, 65536);
+ __type(key, __u64);
+ __type(value, __s32);
+} cuda_current_device SEC(".maps");
+
struct {
__uint(type, BPF_MAP_TYPE_LRU_HASH);
__uint(max_entries, 1024);
@@ -44,6 +55,19 @@ struct {
__type(value, __u64);
} cuda_malloc_size_cache SEC(".maps");
+struct cuda_memcpy_args_t {
+ __u64 size;
+ __u8 kind;
+ __u8 pad[7];
+};
+
+struct {
+ __uint(type, BPF_MAP_TYPE_LRU_HASH);
+ __uint(max_entries, 1024);
+ __type(key, __u64);
+ __type(value, struct cuda_memcpy_args_t);
+} cuda_memcpy_cache SEC(".maps");
+
// CUDA 13 lazily converts the ELF host-function pointer into an opaque
// cudaKernel_t via __cudaGetKernel, then passes that handle to
// cudaLaunchKernel. Retain the relationship so userspace can still resolve a
@@ -151,6 +175,13 @@ static __always_inline void fill_header(struct cuda_event_header_t *hdr, __u8 ty
hdr->pad1 = 0;
hdr->stream_id = stream_id;
hdr->ktime_ns = bpf_ktime_get_ns();
+
+ __s32 *cur = bpf_map_lookup_elem(&cuda_current_device, &pid_tgid);
+ if (cur) {
+ __s32 d = *cur;
+ if (d >= 0 && d < 0xffff)
+ hdr->device_idx = (__u16)d;
+ }
}
// cudaLaunchKernel(const void *func, dim3 gridDim, dim3 blockDim,
@@ -300,13 +331,46 @@ int handle_cuda_free(struct pt_regs *ctx) {
return 0;
}
-// Synchronous cudaMemcpy → treat completion as device-wide sync (Datadog parity).
+// Synchronous cudaMemcpy → copy bytes plus device-wide sync (Datadog parity
+// for occupancy). Size/kind come from the entry probe; if that missses we
+// still emit the sync so spans close.
+SEC("uprobe/cudaMemcpy")
+int handle_cuda_memcpy_enter(struct pt_regs *ctx) {
+ __u64 pid_tgid = bpf_get_current_pid_tgid();
+ struct cuda_memcpy_args_t args = {};
+ args.size = PT_REGS_PARM3(ctx);
+ args.kind = (__u8)PT_REGS_PARM4(ctx);
+ bpf_map_update_elem(&cuda_memcpy_cache, &pid_tgid, &args, BPF_ANY);
+ return 0;
+}
+
SEC("uretprobe/cudaMemcpy")
int handle_cuda_memcpy(struct pt_regs *ctx) {
+ __u64 pid_tgid = bpf_get_current_pid_tgid();
+ struct cuda_memcpy_args_t *argp = bpf_map_lookup_elem(&cuda_memcpy_cache, &pid_tgid);
+ struct cuda_memcpy_args_t local = {};
+ int have_args = 0;
+ if (argp) {
+ local = *argp;
+ have_args = 1;
+ bpf_map_delete_elem(&cuda_memcpy_cache, &pid_tgid);
+ }
+
long ret = (long)PT_REGS_RC(ctx);
if (ret != 0)
return 0;
+ if (have_args) {
+ struct gpu_memcpy_t *cp;
+ cp = bpf_ringbuf_reserve(&gpu_events, sizeof(*cp), 0);
+ if (cp) {
+ fill_header(&cp->hdr, EVENT_GPU_MEMCPY, 0);
+ cp->size = local.size;
+ cp->kind = local.kind;
+ bpf_ringbuf_submit(cp, 0);
+ }
+ }
+
struct gpu_sync_t *ev;
ev = bpf_ringbuf_reserve(&gpu_events, sizeof(*ev), 0);
if (!ev)
@@ -415,6 +479,7 @@ int handle_cuda_set_device(struct pt_regs *ctx) {
ev->hdr.device_idx = (__u16)device;
ev->device = device;
ev->pad = 0;
+ bpf_map_update_elem(&cuda_current_device, &pid_tgid, &device, BPF_ANY);
bpf_ringbuf_submit(ev, 0);
return 0;
}
diff --git a/opentelemetry-gpu-collector/internal/ebpf/pidtgid.go b/opentelemetry-gpu-collector/internal/ebpf/pidtgid.go
new file mode 100644
index 00000000..fd0b0ac6
--- /dev/null
+++ b/opentelemetry-gpu-collector/internal/ebpf/pidtgid.go
@@ -0,0 +1,28 @@
+package ebpf
+
+// pidFromPidTgid extracts the kernel tgid (userspace PID) from a BPF pid_tgid key.
+func pidFromPidTgid(key uint64) uint32 {
+ return uint32(key >> 32)
+}
+
+// stalePidTgidKeys returns pid_tgid keys whose process is no longer alive.
+// alive is called at most once per distinct PID.
+func stalePidTgidKeys(keys []uint64, alive func(uint32) bool) []uint64 {
+ if alive == nil || len(keys) == 0 {
+ return nil
+ }
+ checked := make(map[uint32]bool)
+ var stale []uint64
+ for _, key := range keys {
+ pid := pidFromPidTgid(key)
+ ok, seen := checked[pid]
+ if !seen {
+ ok = alive(pid)
+ checked[pid] = ok
+ }
+ if !ok {
+ stale = append(stale, key)
+ }
+ }
+ return stale
+}
diff --git a/opentelemetry-gpu-collector/internal/ebpf/pidtgid_test.go b/opentelemetry-gpu-collector/internal/ebpf/pidtgid_test.go
new file mode 100644
index 00000000..a3e36ca6
--- /dev/null
+++ b/opentelemetry-gpu-collector/internal/ebpf/pidtgid_test.go
@@ -0,0 +1,33 @@
+package ebpf
+
+import "testing"
+
+func TestPidFromPidTgid(t *testing.T) {
+ key := uint64(42)<<32 | uint64(7)
+ if got := pidFromPidTgid(key); got != 42 {
+ t.Fatalf("pidFromPidTgid(%#x) = %d, want 42", key, got)
+ }
+}
+
+func TestStalePidTgidKeysDropsDeadPID(t *testing.T) {
+ live := uint64(10)<<32 | uint64(11)
+ deadMain := uint64(20)<<32 | uint64(20)
+ deadThread := uint64(20)<<32 | uint64(21)
+ aliveCalls := map[uint32]int{}
+ stale := stalePidTgidKeys([]uint64{live, deadMain, deadThread}, func(pid uint32) bool {
+ aliveCalls[pid]++
+ return pid == 10
+ })
+ if len(stale) != 2 || stale[0] != deadMain || stale[1] != deadThread {
+ t.Fatalf("stale = %v, want [%d %d]", stale, deadMain, deadThread)
+ }
+ if aliveCalls[10] != 1 || aliveCalls[20] != 1 {
+ t.Fatalf("alive calls = %v, want one per PID", aliveCalls)
+ }
+}
+
+func TestStalePidTgidKeysNilAlive(t *testing.T) {
+ if stalePidTgidKeys([]uint64{1}, nil) != nil {
+ t.Fatal("nil alive should skip cleanup")
+ }
+}
diff --git a/opentelemetry-gpu-collector/internal/ebpf/tracer.go b/opentelemetry-gpu-collector/internal/ebpf/tracer.go
index 293d33e3..c1d05ec7 100644
--- a/opentelemetry-gpu-collector/internal/ebpf/tracer.go
+++ b/opentelemetry-gpu-collector/internal/ebpf/tracer.go
@@ -9,6 +9,7 @@ import (
"fmt"
"log/slog"
"os"
+ "strconv"
"sync"
"sync/atomic"
"syscall"
@@ -82,7 +83,9 @@ func NewTracer(logger *slog.Logger, handler EventHandler) (*Tracer, error) {
"cudaGraphLaunch": objs.HandleCudaGraphLaunch,
"__cudaGetKernel": objs.HandleCudaGetKernelEnter,
"cudaMalloc": objs.HandleCudaMallocEnter,
+ "cudaMallocAsync": objs.HandleCudaMallocEnter,
"cudaFree": objs.HandleCudaFree,
+ "cudaMemcpy": objs.HandleCudaMemcpyEnter,
"cudaMemcpyAsync": objs.HandleCudaMemcpyAsync,
"cudaStreamSynchronize": objs.HandleCudaStreamSyncEnter,
"cudaEventRecord": objs.HandleCudaEventRecordEnter,
@@ -92,6 +95,7 @@ func NewTracer(logger *slog.Logger, handler EventHandler) (*Tracer, error) {
uretprobes: map[string]*ebpf.Program{
"__cudaGetKernel": objs.HandleCudaGetKernelExit,
"cudaMalloc": objs.HandleCudaMalloc,
+ "cudaMallocAsync": objs.HandleCudaMalloc,
"cudaMemcpy": objs.HandleCudaMemcpy,
"cudaStreamSynchronize": objs.HandleCudaStreamSync,
"cudaDeviceSynchronize": objs.HandleCudaDeviceSync,
@@ -161,6 +165,7 @@ func (t *Tracer) attachNewCudaLibs() int {
}
}
t.refreshCudartPIDs()
+ t.dropDeadCurrentDevices()
return attached
}
@@ -190,6 +195,31 @@ func (t *Tracer) refreshCudartPIDs() {
}
}
+func procExists(pid uint32) bool {
+ _, err := os.Stat("/proc/" + strconv.FormatUint(uint64(pid), 10))
+ return err == nil
+}
+
+// dropDeadCurrentDevices removes cuda_current_device entries for exited
+// processes. Without this, a reused pid_tgid inherits the previous process's
+// CUDA index and fill_header stamps the wrong hw.id.
+func (t *Tracer) dropDeadCurrentDevices() {
+ if t.closed.Load() || t.objs == nil || t.objs.CudaCurrentDevice == nil {
+ return
+ }
+ var key uint64
+ var val int32
+ var keys []uint64
+ iter := t.objs.CudaCurrentDevice.Iterate()
+ for iter.Next(&key, &val) {
+ keys = append(keys, key)
+ }
+ for _, k := range stalePidTgidKeys(keys, procExists) {
+ key := k
+ _ = t.objs.CudaCurrentDevice.Delete(&key)
+ }
+}
+
func (t *Tracer) attachCudaLib(cudaLib string, uprobes, uretprobes map[string]*ebpf.Program) bool {
if t.closed.Load() {
return false
@@ -308,11 +338,12 @@ func (t *Tracer) Run(ctx context.Context) {
go func() {
defer t.wg.Done()
- // Fast while waiting for the first libcudart/libcuda; back off once
- // attached so steady-state /proc scans stay cheap on busy nodes.
- interval := 30 * time.Second
- ticker := time.NewTicker(interval)
+ // Device-map hygiene stays at 30s so a reused pid_tgid cannot keep a
+ // dead process's CUDA index. Full libcudart/libcuda /proc scans still
+ // back off after the first attach.
+ ticker := time.NewTicker(30 * time.Second)
defer ticker.Stop()
+ var lastAttach time.Time
for {
select {
case <-ctx.Done():
@@ -320,15 +351,16 @@ func (t *Tracer) Run(ctx context.Context) {
case <-t.stop:
return
case <-ticker.C:
+ t.dropDeadCurrentDevices()
+ next := cudaRescanInterval(t.hasAttachedProbes())
+ if !lastAttach.IsZero() && time.Since(lastAttach) < next {
+ continue
+ }
n := t.attachNewCudaLibs()
+ lastAttach = time.Now()
if n > 0 {
t.logger.Info("attached CUDA probes after /proc rescan", "libraries", n)
}
- next := cudaRescanInterval(t.hasAttachedProbes())
- if next != interval {
- interval = next
- ticker.Reset(interval)
- }
}
}
}()
diff --git a/opentelemetry-gpu-collector/internal/ebpf/tracer_bpf_test.go b/opentelemetry-gpu-collector/internal/ebpf/tracer_bpf_test.go
index b2b67064..796393f0 100644
--- a/opentelemetry-gpu-collector/internal/ebpf/tracer_bpf_test.go
+++ b/opentelemetry-gpu-collector/internal/ebpf/tracer_bpf_test.go
@@ -13,6 +13,7 @@ func TestGeneratedSpecIncludesCUDA13KernelHandleProbes(t *testing.T) {
for _, name := range []string{
"handle_cuda_get_kernel_enter",
"handle_cuda_get_kernel_exit",
+ "handle_cuda_memcpy_enter",
"handle_cu_launch_kernel",
"handle_cu_launch_kernel_exit",
"handle_cu_launch_kernel_ex",
@@ -30,6 +31,8 @@ func TestGeneratedSpecIncludesCUDA13KernelHandleProbes(t *testing.T) {
for _, name := range []string{
"cuda_get_kernel_cache",
"cuda_kernel_handles",
+ "cuda_current_device",
+ "cuda_memcpy_cache",
"cudart_pids",
"cu_func_names",
} {
diff --git a/opentelemetry-gpu-collector/internal/ebpf/types.go b/opentelemetry-gpu-collector/internal/ebpf/types.go
index af95b611..dc60e86c 100644
--- a/opentelemetry-gpu-collector/internal/ebpf/types.go
+++ b/opentelemetry-gpu-collector/internal/ebpf/types.go
@@ -18,12 +18,23 @@ type CUDAEvent interface {
ProcessPID() uint32
}
+// DeviceIdxUnknown is the BPF header sentinel when CUDA device index is unset.
+const DeviceIdxUnknown = uint16(0xffff)
+
type eventMeta struct {
PID uint32
TID uint32
StreamID uint64
KtimeNs uint64
- DeviceIdx uint16 // 0xffff = unknown
+ DeviceIdx uint16 // DeviceIdxUnknown when unset
+}
+
+// DeviceIndex converts a BPF header device_idx to a CUDA index, or -1 if unknown.
+func DeviceIndex(idx uint16) int {
+ if idx == DeviceIdxUnknown {
+ return -1
+ }
+ return int(idx)
}
type KernelLaunchEvent struct {
diff --git a/opentelemetry-gpu-collector/internal/ebpf/types_test.go b/opentelemetry-gpu-collector/internal/ebpf/types_test.go
index b63c8326..1e553a6f 100644
--- a/opentelemetry-gpu-collector/internal/ebpf/types_test.go
+++ b/opentelemetry-gpu-collector/internal/ebpf/types_test.go
@@ -69,6 +69,15 @@ func TestMemcpyKindString(t *testing.T) {
}
}
+func TestDeviceIndexUnknown(t *testing.T) {
+ if DeviceIndex(DeviceIdxUnknown) != -1 {
+ t.Fatalf("unknown should be -1")
+ }
+ if DeviceIndex(0) != 0 || DeviceIndex(1) != 1 {
+ t.Fatal("known indices should pass through")
+ }
+}
+
func TestCUDAEventInterface(t *testing.T) {
// Verify all event types satisfy the CUDAEvent interface at compile time.
var _ CUDAEvent = (*KernelLaunchEvent)(nil)
diff --git a/opentelemetry-gpu-collector/internal/export/ebpfmetrics.go b/opentelemetry-gpu-collector/internal/export/ebpfmetrics.go
index 8f546df9..ddb2374a 100644
--- a/opentelemetry-gpu-collector/internal/export/ebpfmetrics.go
+++ b/opentelemetry-gpu-collector/internal/export/ebpfmetrics.go
@@ -25,15 +25,21 @@ import (
type EBPFMetrics struct {
logger *slog.Logger
- kernelLaunchCalls metric.Int64Counter
+ kernelLaunchCalls metric.Int64ObservableCounter
kernelGridSize metric.Float64Histogram
kernelBlockSize metric.Float64Histogram
kernelSharedMemory metric.Float64Histogram
kernelDuration metric.Float64Histogram
- graphLaunchCalls metric.Int64Counter
- memoryAllocations metric.Int64Counter
+ graphLaunchCalls metric.Int64ObservableCounter
+ memoryAllocations metric.Int64ObservableCounter
memoryCopies metric.Float64Histogram
+ launchCounts *liveCounter
+ graphCounts *liveCounter
+ allocCounts *liveCounter
+ pidAlive func(uint32) bool
+ reg metric.Registration
+
devices *cudaspans.DeviceResolver
kernelNamesMu sync.Mutex
@@ -49,7 +55,7 @@ func NewEBPFMetrics(provider *sdkmetric.MeterProvider, devices *cudaspans.Device
metric.WithInstrumentationVersion("1.0.0"),
)
- kernelCalls, err := meter.Int64Counter("gpu.kernel.launch.calls",
+ kernelCalls, err := meter.Int64ObservableCounter("gpu.kernel.launch.calls",
metric.WithDescription("Number of CUDA kernel launches"),
metric.WithUnit("{call}"),
)
@@ -89,7 +95,7 @@ func NewEBPFMetrics(provider *sdkmetric.MeterProvider, devices *cudaspans.Device
return nil, fmt.Errorf("creating gpu.kernel.duration: %w", err)
}
- graphCalls, err := meter.Int64Counter("gpu.graph.launch.calls",
+ graphCalls, err := meter.Int64ObservableCounter("gpu.graph.launch.calls",
metric.WithDescription("Number of CUDA graph replay invocations (cudaGraphLaunch/cuGraphLaunch). Counts replays, not the kernels executed inside each replay; that number is not observable at this API-tracing layer."),
metric.WithUnit("{call}"),
)
@@ -97,8 +103,8 @@ func NewEBPFMetrics(provider *sdkmetric.MeterProvider, devices *cudaspans.Device
return nil, fmt.Errorf("creating gpu.graph.launch.calls: %w", err)
}
- memAlloc, err := meter.Int64Counter("gpu.memory.allocations",
- metric.WithDescription("Total bytes allocated via cudaMalloc"),
+ memAlloc, err := meter.Int64ObservableCounter("gpu.memory.allocations",
+ metric.WithDescription("Total bytes allocated via cudaMalloc/cudaMallocAsync"),
metric.WithUnit("By"),
)
if err != nil {
@@ -113,7 +119,7 @@ func NewEBPFMetrics(provider *sdkmetric.MeterProvider, devices *cudaspans.Device
return nil, fmt.Errorf("creating gpu.memory.copies: %w", err)
}
- return &EBPFMetrics{
+ em := &EBPFMetrics{
logger: logger,
kernelLaunchCalls: kernelCalls,
kernelGridSize: gridSize,
@@ -123,9 +129,50 @@ func NewEBPFMetrics(provider *sdkmetric.MeterProvider, devices *cudaspans.Device
graphLaunchCalls: graphCalls,
memoryAllocations: memAlloc,
memoryCopies: memCopies,
+ launchCounts: newLiveCounter(),
+ graphCounts: newLiveCounter(),
+ allocCounts: newLiveCounter(),
+ pidAlive: pidExists,
devices: devices,
kernelNames: make(map[string]struct{}),
- }, nil
+ }
+
+ reg, err := meter.RegisterCallback(func(ctx context.Context, o metric.Observer) error {
+ alive := em.pidAlive
+ dead := map[uint32]struct{}{}
+ observe := func(inst metric.Int64ObservableCounter, c *liveCounter) {
+ series, dropped := c.collectLive(alive)
+ for _, s := range series {
+ o.ObserveInt64(inst, s.value, metric.WithAttributes(s.attrs...))
+ }
+ for _, pid := range dropped {
+ dead[pid] = struct{}{}
+ }
+ }
+ observe(em.kernelLaunchCalls, em.launchCounts)
+ observe(em.graphLaunchCalls, em.graphCounts)
+ observe(em.memoryAllocations, em.allocCounts)
+ if em.devices != nil {
+ for pid := range dead {
+ em.devices.ForgetPID(pid)
+ }
+ }
+ return nil
+ }, kernelCalls, graphCalls, memAlloc)
+ if err != nil {
+ return nil, fmt.Errorf("registering eBPF counter callback: %w", err)
+ }
+ em.reg = reg
+ return em, nil
+}
+
+// Close unregisters observable callbacks.
+func (em *EBPFMetrics) Close() {
+ if em == nil || em.reg == nil {
+ return
+ }
+ _ = em.reg.Unregister()
+ em.reg = nil
}
type pidAttrCacheEntry struct {
@@ -195,11 +242,23 @@ func cachedPIDAttrs(pid uint32) []attribute.KeyValue {
return attrs
}
-func (em *EBPFMetrics) deviceAttrs(pid, tid uint32) []attribute.KeyValue {
+func (em *EBPFMetrics) deviceAttrs(pid, tid uint32, explicitIdx int, explicitUUID string) []attribute.KeyValue {
if em.devices == nil {
return nil
}
- idx := em.devices.ResolveIndex(pid, tid)
+ idx := explicitIdx
+ if explicitUUID != "" && explicitUUID != "unknown" {
+ if uidx, ok := em.devices.IndexForUUID(explicitUUID); ok {
+ idx = uidx
+ }
+ }
+ if idx < 0 {
+ idx = em.devices.ResolveIndex(pid, tid)
+ }
+ return em.attrsForIndex(idx)
+}
+
+func (em *EBPFMetrics) attrsForIndex(idx int) []attribute.KeyValue {
uuid, name, pci, ok := em.devices.IndexInfo(idx)
if !ok {
return nil
@@ -222,15 +281,19 @@ func (em *EBPFMetrics) deviceAttrs(pid, tid uint32) []attribute.KeyValue {
return attrs
}
-func (em *EBPFMetrics) activityAttrs(pid, tid uint32, extra ...attribute.KeyValue) metric.MeasurementOption {
+func (em *EBPFMetrics) activityAttrList(pid, tid uint32, deviceIdx int, deviceUUID string, extra ...attribute.KeyValue) []attribute.KeyValue {
base := cachedPIDAttrs(pid)
- dev := em.deviceAttrs(pid, tid)
+ dev := em.deviceAttrs(pid, tid, deviceIdx, deviceUUID)
attrs := make([]attribute.KeyValue, 0, len(base)+len(dev)+len(extra)+1)
attrs = append(attrs, base...)
attrs = append(attrs, attribute.String("gpu.measurement.source", "ebpf"))
attrs = append(attrs, dev...)
attrs = append(attrs, extra...)
- return metric.WithAttributes(attrs...)
+ return attrs
+}
+
+func (em *EBPFMetrics) activityAttrs(pid, tid uint32, deviceIdx int, deviceUUID string, extra ...attribute.KeyValue) metric.MeasurementOption {
+ return metric.WithAttributes(em.activityAttrList(pid, tid, deviceIdx, deviceUUID, extra...)...)
}
func kernelMetricName(e *gpuebpf.KernelLaunchEvent) string {
@@ -271,7 +334,7 @@ func (em *EBPFMetrics) RecordClosedSpans(ctx context.Context, spans []cudaspans.
if s.Kind == cudaspans.LaunchKindGraph {
extra = append(extra, attribute.String("cuda.launch.kind", "graph"))
}
- attrs := em.activityAttrs(s.PID, s.TID, extra...)
+ attrs := em.activityAttrs(s.PID, s.TID, s.DeviceIndex, s.DeviceUUID, extra...)
em.kernelDuration.Record(ctx, durSec, attrs)
}
}
@@ -280,9 +343,10 @@ func (em *EBPFMetrics) RecordClosedSpans(ctx context.Context, spans []cudaspans.
func (em *EBPFMetrics) RecordLaunchActivity(ctx context.Context, e *gpuebpf.KernelLaunchEvent) {
kernelName := kernelMetricName(e)
extra := []attribute.KeyValue{attribute.String("cuda.kernel.name", kernelName)}
- attrs := em.activityAttrs(e.PID, e.TID, extra...)
+ attrList := em.activityAttrList(e.PID, e.TID, gpuebpf.DeviceIndex(e.DeviceIdx), "", extra...)
+ attrs := metric.WithAttributes(attrList...)
- em.kernelLaunchCalls.Add(ctx, 1, attrs)
+ em.launchCounts.add(e.PID, 1, attrList)
gridTotal := float64(e.GridX) * float64(e.GridY) * float64(e.GridZ)
em.kernelGridSize.Record(ctx, gridTotal, attrs)
@@ -293,8 +357,8 @@ func (em *EBPFMetrics) RecordLaunchActivity(ctx context.Context, e *gpuebpf.Kern
em.kernelSharedMemory.Record(ctx, float64(e.SharedMemBytes), attrs)
}
-func (em *EBPFMetrics) RecordGraphLaunch(ctx context.Context, e *gpuebpf.GraphLaunchEvent) {
- em.graphLaunchCalls.Add(ctx, 1, em.activityAttrs(e.PID, e.TID))
+func (em *EBPFMetrics) RecordGraphLaunch(_ context.Context, e *gpuebpf.GraphLaunchEvent) {
+ em.graphCounts.add(e.PID, 1, em.activityAttrList(e.PID, e.TID, gpuebpf.DeviceIndex(e.DeviceIdx), ""))
}
// HandleEvent processes activity-only events when used without SpanFanout.
@@ -315,10 +379,10 @@ func (em *EBPFMetrics) HandleEvent(ev gpuebpf.CUDAEvent) {
em.RecordGraphLaunch(ctx, e)
case *gpuebpf.MallocEvent:
- em.memoryAllocations.Add(ctx, int64(e.Size), em.activityAttrs(e.PID, e.TID))
+ em.allocCounts.add(e.PID, int64(e.Size), em.activityAttrList(e.PID, e.TID, gpuebpf.DeviceIndex(e.DeviceIdx), ""))
case *gpuebpf.MemcpyEvent:
- em.memoryCopies.Record(ctx, float64(e.Size), em.activityAttrs(e.PID, e.TID,
+ em.memoryCopies.Record(ctx, float64(e.Size), em.activityAttrs(e.PID, e.TID, gpuebpf.DeviceIndex(e.DeviceIdx), "",
attribute.String("cuda.memcpy.kind", gpuebpf.MemcpyKindString(e.Kind)),
))
}
diff --git a/opentelemetry-gpu-collector/internal/export/ebpfmetrics_duration_test.go b/opentelemetry-gpu-collector/internal/export/ebpfmetrics_duration_test.go
index c14aefdc..24d571ee 100644
--- a/opentelemetry-gpu-collector/internal/export/ebpfmetrics_duration_test.go
+++ b/opentelemetry-gpu-collector/internal/export/ebpfmetrics_duration_test.go
@@ -3,6 +3,7 @@ package export
import (
"context"
"fmt"
+ "os"
"testing"
"go.opentelemetry.io/otel/sdk/metric"
@@ -11,6 +12,8 @@ import (
gpuebpf "github.com/openlit/openlit/opentelemetry-gpu-collector/internal/ebpf"
)
+func testPID() uint32 { return uint32(os.Getpid()) }
+
func newTestSpanFanout(t *testing.T) (*SpanFanout, *metric.ManualReader, func()) {
t.Helper()
reader := metric.NewManualReader()
@@ -25,6 +28,7 @@ func newTestSpanFanout(t *testing.T) (*SpanFanout, *metric.ManualReader, func())
}
return NewSpanFanout(em, occ), reader, func() {
occ.Close()
+ em.Close()
_ = provider.Shutdown(context.Background())
}
}
@@ -34,14 +38,14 @@ func TestKernelDurationLaunchToSync(t *testing.T) {
defer cleanup()
launch := &gpuebpf.KernelLaunchEvent{KernelName: "matmul"}
- launch.PID = 10
+ launch.PID = testPID()
launch.TID = 11
launch.StreamID = 7
launch.KtimeNs = 1_000_000_000
fanout.HandleEvent(launch)
sync := &gpuebpf.SyncEvent{}
- sync.PID = 10
+ sync.PID = testPID()
sync.TID = 11
sync.StreamID = 7
sync.KtimeNs = 1_500_000_000
@@ -82,14 +86,14 @@ func TestGraphDurationLaunchToSync(t *testing.T) {
defer cleanup()
launch := &gpuebpf.GraphLaunchEvent{}
- launch.PID = 10
+ launch.PID = testPID()
launch.TID = 11
launch.StreamID = 7
launch.KtimeNs = 1_000_000_000
fanout.HandleEvent(launch)
sync := &gpuebpf.SyncEvent{}
- sync.PID = 10
+ sync.PID = testPID()
sync.TID = 11
sync.StreamID = 7
sync.KtimeNs = 1_500_000_000
@@ -160,14 +164,14 @@ func TestKernelDurationSkipsInvertedWindow(t *testing.T) {
defer cleanup()
launch := &gpuebpf.KernelLaunchEvent{KernelName: "late"}
- launch.PID = 1
+ launch.PID = testPID()
launch.TID = 1
launch.StreamID = 1
launch.KtimeNs = 2_000_000_000
fanout.HandleEvent(launch)
sync := &gpuebpf.SyncEvent{}
- sync.PID = 1
+ sync.PID = testPID()
sync.TID = 1
sync.StreamID = 1
sync.KtimeNs = 1_000_000_000
diff --git a/opentelemetry-gpu-collector/internal/export/ebpfmetrics_test.go b/opentelemetry-gpu-collector/internal/export/ebpfmetrics_test.go
index 53bd26e8..b01558b2 100644
--- a/opentelemetry-gpu-collector/internal/export/ebpfmetrics_test.go
+++ b/opentelemetry-gpu-collector/internal/export/ebpfmetrics_test.go
@@ -2,6 +2,7 @@ package export
import (
"context"
+ "os"
"testing"
"go.opentelemetry.io/otel/sdk/metric"
@@ -21,14 +22,15 @@ func TestGraphLaunchDoesNotIncrementKernelCalls(t *testing.T) {
if err != nil {
t.Fatal(err)
}
+ defer em.Close()
kernel := &gpuebpf.KernelLaunchEvent{KernelName: "vector_add", GridX: 2, GridY: 1, GridZ: 1, BlockX: 32, BlockY: 1, BlockZ: 1}
- kernel.PID = 42
+ kernel.PID = uint32(os.Getpid())
kernel.TID = 43
em.HandleEvent(kernel)
graph := &gpuebpf.GraphLaunchEvent{}
- graph.PID = 42
+ graph.PID = uint32(os.Getpid())
graph.TID = 43
em.HandleEvent(graph)
@@ -85,7 +87,7 @@ func TestDeviceAttrsSoleGPU(t *testing.T) {
&mockDevice{info: gpu.DeviceInfo{Vendor: gpu.VendorNVIDIA, UUID: "gpu-a", Index: 0, Name: "A100", PCIAddress: "0000:01:00.0"}},
}
em := &EBPFMetrics{devices: cudaspans.NewDeviceResolver(devs)}
- attrs := em.deviceAttrs(1, 2)
+ attrs := em.deviceAttrs(1, 2, -1, "")
if len(attrs) < 2 {
t.Fatalf("expected sole-GPU attrs, got %#v", attrs)
}
@@ -104,18 +106,23 @@ func TestDeviceAttrsSoleGPU(t *testing.T) {
}
}
-func TestDeviceAttrsMultiGPUNeedsSetDevice(t *testing.T) {
+func TestDeviceAttrsMultiGPUDefaultsToCUDAZero(t *testing.T) {
devs := []gpu.Device{
&mockDevice{info: gpu.DeviceInfo{Vendor: gpu.VendorNVIDIA, UUID: "gpu-a", Index: 0, Name: "A100"}},
&mockDevice{info: gpu.DeviceInfo{Vendor: gpu.VendorNVIDIA, UUID: "gpu-b", Index: 1, Name: "A100"}},
}
em := &EBPFMetrics{devices: cudaspans.NewDeviceResolver(devs)}
- if attrs := em.deviceAttrs(1, 2); len(attrs) != 0 {
- t.Fatalf("expected no attrs before SetDevice, got %#v", attrs)
+ attrs := em.deviceAttrs(1, 2, -1, "")
+ m := map[string]string{}
+ for _, a := range attrs {
+ m[string(a.Key)] = a.Value.Emit()
+ }
+ if m["hw.id"] != "gpu-a" {
+ t.Fatalf("hw.id = %q, want gpu-a (CUDA default device 0)", m["hw.id"])
}
em.devices.NoteSetDevice(1, 2, 1)
- attrs := em.deviceAttrs(1, 2)
- m := map[string]string{}
+ attrs = em.deviceAttrs(1, 2, -1, "")
+ m = map[string]string{}
for _, a := range attrs {
m[string(a.Key)] = a.Value.Emit()
}
@@ -127,14 +134,135 @@ func TestDeviceAttrsMultiGPUNeedsSetDevice(t *testing.T) {
}
}
+func TestDeadPIDCountersStopExporting(t *testing.T) {
+ reader := metric.NewManualReader()
+ provider := metric.NewMeterProvider(metric.WithReader(reader))
+ defer func() { _ = provider.Shutdown(context.Background()) }()
+
+ em, err := NewEBPFMetrics(provider, nil, nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer em.Close()
+ em.pidAlive = func(pid uint32) bool { return pid == 1 }
+
+ live := &gpuebpf.KernelLaunchEvent{KernelName: "live"}
+ live.PID = 1
+ em.HandleEvent(live)
+ dead := &gpuebpf.KernelLaunchEvent{KernelName: "dead"}
+ dead.PID = 2
+ em.HandleEvent(dead)
+
+ var rm metricdata.ResourceMetrics
+ if err := reader.Collect(context.Background(), &rm); err != nil {
+ t.Fatal(err)
+ }
+ var pids []string
+ for _, sm := range rm.ScopeMetrics {
+ for _, m := range sm.Metrics {
+ if m.Name != "gpu.kernel.launch.calls" {
+ continue
+ }
+ sum := m.Data.(metricdata.Sum[int64])
+ for _, dp := range sum.DataPoints {
+ v, _ := dp.Attributes.Value("process.pid")
+ pids = append(pids, v.AsString())
+ }
+ }
+ }
+ if len(pids) != 1 || pids[0] != "1" {
+ t.Fatalf("exported pids = %v, want [1]", pids)
+ }
+ if n := sumInt64Counter(t, rm, "gpu.kernel.launch.calls"); n != 1 {
+ t.Fatalf("calls = %d, want 1", n)
+ }
+}
+
+func TestDeviceAttrsPrefersEventDeviceIndex(t *testing.T) {
+ devs := []gpu.Device{
+ &mockDevice{info: gpu.DeviceInfo{Vendor: gpu.VendorNVIDIA, UUID: "gpu-a", Index: 0, Name: "A100"}},
+ &mockDevice{info: gpu.DeviceInfo{Vendor: gpu.VendorNVIDIA, UUID: "gpu-b", Index: 1, Name: "A100"}},
+ }
+ em := &EBPFMetrics{devices: cudaspans.NewDeviceResolver(devs)}
+ attrs := em.deviceAttrs(1, 2, 1, "")
+ m := map[string]string{}
+ for _, a := range attrs {
+ m[string(a.Key)] = a.Value.Emit()
+ }
+ if m["hw.id"] != "gpu-b" {
+ t.Fatalf("hw.id = %q, want gpu-b from event DeviceIdx", m["hw.id"])
+ }
+}
+
+func TestDeviceAttrsPrefersSpanUUID(t *testing.T) {
+ devs := []gpu.Device{
+ &mockDevice{info: gpu.DeviceInfo{Vendor: gpu.VendorNVIDIA, UUID: "gpu-a", Index: 0}},
+ &mockDevice{info: gpu.DeviceInfo{Vendor: gpu.VendorNVIDIA, UUID: "gpu-b", Index: 1}},
+ }
+ em := &EBPFMetrics{devices: cudaspans.NewDeviceResolver(devs)}
+ attrs := em.deviceAttrs(1, 2, 0, "gpu-b")
+ m := map[string]string{}
+ for _, a := range attrs {
+ m[string(a.Key)] = a.Value.Emit()
+ }
+ if m["hw.id"] != "gpu-b" {
+ t.Fatalf("hw.id = %q, want gpu-b from span UUID", m["hw.id"])
+ }
+}
+
func TestDeviceAttrsIgnoresUnknownIndex(t *testing.T) {
devs := []gpu.Device{
&mockDevice{info: gpu.DeviceInfo{Vendor: gpu.VendorNVIDIA, UUID: "gpu-a", Index: 0}},
}
em := &EBPFMetrics{devices: cudaspans.NewDeviceResolver(devs)}
em.devices.NoteSetDevice(1, 2, 9) // unknown CUDA index — ignored
- attrs := em.deviceAttrs(1, 2)
+ attrs := em.deviceAttrs(1, 2, -1, "")
if len(attrs) == 0 {
t.Fatal("sole GPU should still attribute")
}
}
+
+func TestLaunchCounterUsesEventDeviceIndex(t *testing.T) {
+ reader := metric.NewManualReader()
+ provider := metric.NewMeterProvider(metric.WithReader(reader))
+ defer func() { _ = provider.Shutdown(context.Background()) }()
+
+ devs := []gpu.Device{
+ &mockDevice{info: gpu.DeviceInfo{Vendor: gpu.VendorNVIDIA, UUID: "gpu-a", Index: 0, Name: "A100"}},
+ &mockDevice{info: gpu.DeviceInfo{Vendor: gpu.VendorNVIDIA, UUID: "gpu-b", Index: 1, Name: "A100"}},
+ }
+ em, err := NewEBPFMetrics(provider, cudaspans.NewDeviceResolver(devs), nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer em.Close()
+
+ kernel := &gpuebpf.KernelLaunchEvent{KernelName: "matmul"}
+ kernel.PID = uint32(os.Getpid())
+ kernel.DeviceIdx = 1
+ em.HandleEvent(kernel)
+
+ var rm metricdata.ResourceMetrics
+ if err := reader.Collect(context.Background(), &rm); err != nil {
+ t.Fatal(err)
+ }
+ found := false
+ for _, sm := range rm.ScopeMetrics {
+ for _, m := range sm.Metrics {
+ if m.Name != "gpu.kernel.launch.calls" {
+ continue
+ }
+ sum := m.Data.(metricdata.Sum[int64])
+ for _, dp := range sum.DataPoints {
+ id, _ := dp.Attributes.Value("hw.id")
+ if id.AsString() != "gpu-b" {
+ t.Fatalf("hw.id = %q, want gpu-b", id.AsString())
+ }
+ found = true
+ }
+ }
+ }
+ if !found {
+ t.Fatal("gpu.kernel.launch.calls missing hw.id")
+ }
+}
diff --git a/opentelemetry-gpu-collector/internal/export/livecounter.go b/opentelemetry-gpu-collector/internal/export/livecounter.go
new file mode 100644
index 00000000..c48c398c
--- /dev/null
+++ b/opentelemetry-gpu-collector/internal/export/livecounter.go
@@ -0,0 +1,87 @@
+package export
+
+import (
+ "os"
+ "runtime"
+ "strconv"
+ "sync"
+
+ "go.opentelemetry.io/otel/attribute"
+)
+
+const maxLiveCounterSeries = 16384
+
+// liveCounter is a cumulative counter whose series can be dropped when the
+// producing PID is gone. The OTel Go SDK keeps sync Int64Counter attribute
+// sets forever; Prom remote-write then re-exports a frozen value and rate()
+// stays 0. Observable counters plus this map omit dead PIDs so the series
+// can go stale instead.
+type liveCounter struct {
+ mu sync.Mutex
+ series map[string]*liveCounterSeries
+}
+
+type liveCounterSeries struct {
+ pid uint32
+ attrs []attribute.KeyValue
+ value int64
+}
+
+func newLiveCounter() *liveCounter {
+ return &liveCounter{series: make(map[string]*liveCounterSeries)}
+}
+
+func (c *liveCounter) add(pid uint32, n int64, attrs []attribute.KeyValue) {
+ if c == nil || n == 0 {
+ return
+ }
+ set := attribute.NewSet(attrs...)
+ key := (&set).Encoded(attribute.DefaultEncoder())
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ s, ok := c.series[key]
+ if !ok {
+ if len(c.series) >= maxLiveCounterSeries {
+ return
+ }
+ cloned := make([]attribute.KeyValue, len(attrs))
+ copy(cloned, attrs)
+ s = &liveCounterSeries{pid: pid, attrs: cloned}
+ c.series[key] = s
+ }
+ s.value += n
+}
+
+func (c *liveCounter) collectLive(alive func(uint32) bool) (live []liveCounterSeries, deadPIDs []uint32) {
+ if c == nil {
+ return nil, nil
+ }
+ if alive == nil {
+ alive = func(uint32) bool { return true }
+ }
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ dead := map[uint32]struct{}{}
+ live = make([]liveCounterSeries, 0, len(c.series))
+ for k, s := range c.series {
+ if alive(s.pid) {
+ live = append(live, liveCounterSeries{pid: s.pid, attrs: s.attrs, value: s.value})
+ continue
+ }
+ delete(c.series, k)
+ dead[s.pid] = struct{}{}
+ }
+ for pid := range dead {
+ deadPIDs = append(deadPIDs, pid)
+ }
+ return live, deadPIDs
+}
+
+func pidExists(pid uint32) bool {
+ if runtime.GOOS != "linux" {
+ // eBPF metrics are Linux-only; keep series alive in unit tests elsewhere.
+ return true
+ }
+ _, err := os.Stat("/proc/" + strconv.FormatUint(uint64(pid), 10))
+ return err == nil
+}
diff --git a/opentelemetry-gpu-collector/internal/export/occupancy.go b/opentelemetry-gpu-collector/internal/export/occupancy.go
index aef0206e..f078a0f6 100644
--- a/opentelemetry-gpu-collector/internal/export/occupancy.go
+++ b/opentelemetry-gpu-collector/internal/export/occupancy.go
@@ -230,20 +230,23 @@ func (om *OccupancyMetrics) HandleEvent(ev gpuebpf.CUDAEvent) {
om.engine.HandleLaunch(cudaoccupancy.KernelLaunch{
PID: e.PID, TID: e.TID, StreamID: e.StreamID, KtimeNs: e.KtimeNs,
Name: name, Kind: cudaspans.LaunchKindKernel,
- GridX: e.GridX, GridY: e.GridY, GridZ: e.GridZ,
+ DeviceIdx: gpuebpf.DeviceIndex(e.DeviceIdx),
+ GridX: e.GridX, GridY: e.GridY, GridZ: e.GridZ,
BlockX: e.BlockX, BlockY: e.BlockY, BlockZ: e.BlockZ,
})
case *gpuebpf.GraphLaunchEvent:
om.engine.HandleLaunch(cudaoccupancy.KernelLaunch{
PID: e.PID, TID: e.TID, StreamID: e.StreamID, KtimeNs: e.KtimeNs,
Name: "graph", Kind: cudaspans.LaunchKindGraph,
- GridX: 1, GridY: 1, GridZ: 1,
+ DeviceIdx: gpuebpf.DeviceIndex(e.DeviceIdx),
+ GridX: 1, GridY: 1, GridZ: 1,
BlockX: 1, BlockY: 1, BlockZ: 1,
})
case *gpuebpf.SyncEvent:
om.engine.HandleSync(cudaoccupancy.SyncEvent{
PID: e.PID, TID: e.TID, StreamID: e.StreamID, KtimeNs: e.KtimeNs,
DeviceWide: e.DeviceWide,
+ DeviceIdx: gpuebpf.DeviceIndex(e.DeviceIdx),
})
case *gpuebpf.SetDeviceEvent:
om.engine.HandleSetDevice(cudaoccupancy.SetDeviceEvent{
diff --git a/opentelemetry-gpu-collector/internal/export/spanfanout.go b/opentelemetry-gpu-collector/internal/export/spanfanout.go
index c73949b5..ca90c060 100644
--- a/opentelemetry-gpu-collector/internal/export/spanfanout.go
+++ b/opentelemetry-gpu-collector/internal/export/spanfanout.go
@@ -24,10 +24,11 @@ func (f *SpanFanout) HandleEvent(ev gpuebpf.CUDAEvent) {
ctx := context.Background()
switch e := ev.(type) {
case *gpuebpf.SetDeviceEvent:
+ if f.ebpf != nil && f.ebpf.devices != nil {
+ f.ebpf.devices.NoteSetDevice(e.PID, e.TID, int(e.Device))
+ }
if f.occ != nil {
f.occ.HandleEvent(ev)
- } else if f.ebpf != nil && f.ebpf.devices != nil {
- f.ebpf.devices.NoteSetDevice(e.PID, e.TID, int(e.Device))
}
case *gpuebpf.KernelLaunchEvent: