Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.
TypeScript agent-harnessagent-harness-engineeringai-engineeringai-observabilityamd-gpuclaude-codeclickhousecodexcoding-agentcoding-agentscursorgenaigpu-monitoringllm-observabilityllmopsnvidia-smiobservabilityopen-sourceopenaiopentelemetry
fix(collector): add hw.id (#1541)
Aman Agarwal203ce1323143
diff --git a/.so/.gitignore b/.so/.gitignorenew file mode 100644index 00000000..c26a1aca--- /dev/null+++ b/.so/.gitignore@@ -0,0 +1,4 @@+# Superopen machine-local data (do not commit).+sessions/+db/+harvest/diff --git a/opentelemetry-gpu-collector/cmd/collector/main.go b/opentelemetry-gpu-collector/cmd/collector/main.goindex 89de5a6f..5ba191b0 100644--- a/opentelemetry-gpu-collector/cmd/collector/main.go+++ b/opentelemetry-gpu-collector/cmd/collector/main.go@@ -502,6 +502,7 @@ func logEffectiveAvailability(logger *slog.Logger, cfg *config.Config, a effectitype gpuCollectors struct {mc *export.MetricsCollectorebpfTracer *gpuebpf.Tracer+ ebpfMetrics *export.EBPFMetricsoccMetrics *export.OccupancyMetricsdcgmMetrics *export.DCGMMetricsdcgmClient dcgm.Client@@ -516,6 +517,9 @@ func (c *gpuCollectors) close() {if c.ebpfTracer != nil {c.ebpfTracer.Close()}+ if c.ebpfMetrics != nil {+ c.ebpfMetrics.Close()+ }if c.occMetrics != nil {c.occMetrics.Close()}@@ -567,6 +571,7 @@ func setupCollectors(report.fault(logger, config.FeatureEBPF, err.Error())}} else {+ out.ebpfMetrics = ebpfMetricsocc, err := export.NewOccupancyMetrics(provider, devices, resolver, logger)if err != nil {if report != nil {diff --git a/opentelemetry-gpu-collector/internal/cudaoccupancy/engine.go b/opentelemetry-gpu-collector/internal/cudaoccupancy/engine.goindex cda64bbf..f119120c 100644--- a/opentelemetry-gpu-collector/internal/cudaoccupancy/engine.go+++ b/opentelemetry-gpu-collector/internal/cudaoccupancy/engine.go@@ -25,6 +25,7 @@ type KernelLaunch struct {TID uint32StreamID uint64DeviceUUID string+ DeviceIdx int // CUDA index when known; -1 unknown. Ignored when DeviceUUID is set.KtimeNs uint64Name stringKind cudaspans.LaunchKind@@ -38,6 +39,7 @@ type SyncEvent struct {TID uint32StreamID uint64DeviceUUID string+ DeviceIdx int // CUDA index when known; -1 unknown. Ignored when DeviceUUID is set.KtimeNs uint64DeviceWide bool}@@ -187,11 +189,16 @@ func (e *Engine) HandleSetDevice(ev SetDeviceEvent) {e.devices.NoteSetDevice(ev.PID, ev.TID, ev.DeviceIdx)}-func (e *Engine) resolveUUID(pid, tid uint32, explicit string) string {+func (e *Engine) resolveUUID(pid, tid uint32, explicit string, explicitIdx int) string {if explicit != "" {return explicit}if e.devices != nil {+ if explicitIdx >= 0 {+ if u := e.devices.UUIDForIndex(explicitIdx); u != "" {+ return u+ }+ }if u := e.devices.ResolveUUID(pid, tid); u != "" {return u}@@ -204,7 +211,7 @@ func (e *Engine) HandleLaunch(ev KernelLaunch) {e.mu.Lock()defer e.mu.Unlock()- uuid := e.resolveUUID(ev.PID, ev.TID, ev.DeviceUUID)+ uuid := e.resolveUUID(ev.PID, ev.TID, ev.DeviceUUID, ev.DeviceIdx)if uuid == "" {uuid = "unknown"}@@ -231,7 +238,7 @@ func (e *Engine) HandleSync(ev SyncEvent) {e.mu.Lock()defer e.mu.Unlock()- uuid := e.resolveUUID(ev.PID, ev.TID, ev.DeviceUUID)+ uuid := e.resolveUUID(ev.PID, ev.TID, ev.DeviceUUID, ev.DeviceIdx)if ev.DeviceWide || ev.StreamID == 0 {e.fanOutDeviceSync(ev.PID, uuid, ev.KtimeNs)@@ -343,20 +350,21 @@ func (e *Engine) markSynchronization(h *streamHandler, ts uint64) {kind = cudaspans.LaunchKindKernel}e.closedSinceTake = append(e.closedSinceTake, cudaspans.ClosedSpan{- StartNs: l.KtimeNs,- EndNs: ts,- PID: l.PID,- TID: l.TID,- StreamID: l.StreamID,- DeviceUUID: l.DeviceUUID,- KernelName: l.Name,- Kind: kind,- GridX: l.GridX,- GridY: l.GridY,- GridZ: l.GridZ,- BlockX: l.BlockX,- BlockY: l.BlockY,- BlockZ: l.BlockZ,+ StartNs: l.KtimeNs,+ EndNs: ts,+ PID: l.PID,+ TID: l.TID,+ StreamID: l.StreamID,+ DeviceUUID: l.DeviceUUID,+ DeviceIndex: l.DeviceIdx,+ KernelName: l.Name,+ Kind: kind,+ GridX: l.GridX,+ GridY: l.GridY,+ GridZ: l.GridZ,+ BlockX: l.BlockX,+ BlockY: l.BlockY,+ BlockZ: l.BlockZ,})}// Keep launches at/after tsdiff --git a/opentelemetry-gpu-collector/internal/cudaoccupancy/engine_test.go b/opentelemetry-gpu-collector/internal/cudaoccupancy/engine_test.goindex 5f315d36..d221d875 100644--- a/opentelemetry-gpu-collector/internal/cudaoccupancy/engine_test.go+++ b/opentelemetry-gpu-collector/internal/cudaoccupancy/engine_test.go@@ -1,6 +1,42 @@package cudaoccupancy-import "testing"+import (+ "testing"++ "github.com/openlit/openlit/opentelemetry-gpu-collector/internal/cudaspans"+)++func TestHandleLaunchMultiGPUDefaultsToIndexZero(t *testing.T) {+ r := cudaspans.NewDeviceResolver(nil)+ r.SetDeviceIndexUUID(0, "gpu-a")+ r.SetDeviceIndexUUID(1, "gpu-b")+ e := NewEngine(map[string]uint64{"gpu-a": 64, "gpu-b": 64}, r)+ e.HandleLaunch(KernelLaunch{+ PID: 1, TID: 1, StreamID: 9, DeviceIdx: -1, KtimeNs: 100,+ Name: "k", GridX: 1, GridY: 1, GridZ: 1, BlockX: 32, BlockY: 1, BlockZ: 1,+ })+ e.HandleSync(SyncEvent{PID: 1, TID: 1, StreamID: 9, DeviceIdx: -1, KtimeNs: 200})+ closed := e.TakeClosedSpans()+ if len(closed) != 1 {+ t.Fatalf("closed=%d", len(closed))+ }+ if closed[0].DeviceUUID != "gpu-a" {+ t.Fatalf("DeviceUUID=%q, want gpu-a (CUDA default device 0)", closed[0].DeviceUUID)+ }++ e.HandleLaunch(KernelLaunch{+ PID: 1, TID: 1, StreamID: 9, DeviceIdx: 1, KtimeNs: 300,+ Name: "k2", GridX: 1, GridY: 1, GridZ: 1, BlockX: 32, BlockY: 1, BlockZ: 1,+ })+ e.HandleSync(SyncEvent{PID: 1, TID: 1, StreamID: 9, DeviceIdx: 1, KtimeNs: 400})+ closed = e.TakeClosedSpans()+ if len(closed) != 1 {+ t.Fatalf("closed=%d", len(closed))+ }+ if closed[0].DeviceUUID != "gpu-b" {+ t.Fatalf("DeviceUUID=%q, want gpu-b from event DeviceIdx", closed[0].DeviceUUID)+ }+}func TestTakeClosedSpansOnSync(t *testing.T) {e := NewEngine(map[string]uint64{"gpu-a": 64}, nil)diff --git a/opentelemetry-gpu-collector/internal/cudaspans/resolver.go b/opentelemetry-gpu-collector/internal/cudaspans/resolver.goindex 7ef4e53b..73948415 100644--- a/opentelemetry-gpu-collector/internal/cudaspans/resolver.go+++ b/opentelemetry-gpu-collector/internal/cudaspans/resolver.go@@ -10,7 +10,14 @@ import (const threadDevMaxSize = 8192// DeviceResolver maps cudaSetDevice(index) onto NVML device identity.-// Fallback when unset: sole NVIDIA GPU only (not an arbitrary map entry).+//+// Resolution order for a (pid, tid):+// 1. Last successful cudaSetDevice for that thread+// 2. Sole NVIDIA GPU on the host (unambiguous)+// 3. CUDA runtime default current device 0, when index 0 is a known NVIDIA GPU+//+// CUDA_VISIBLE_DEVICES remapping is not applied: CUDA index is treated as the+// NVML index. That assumption already applies to NoteSetDevice.type DeviceResolver struct {mu sync.Mutex@@ -82,17 +89,14 @@ func (r *DeviceResolver) NoteSetDevice(pid, tid uint32, deviceIdx int) {r.threadDev[threadKey(pid, tid)] = deviceIdx}-// ResolveIndex returns the CUDA device index for a thread, or sole GPU, or -1.+// ResolveIndex returns the CUDA device index for a thread, or -1 if unknown.func (r *DeviceResolver) ResolveIndex(pid, tid uint32) int {if r == nil {return -1}r.mu.Lock()defer r.mu.Unlock()- if idx, ok := r.threadDev[threadKey(pid, tid)]; ok {- return idx- }- return r.soleIndex+ return r.resolveIndexLocked(pid, tid)}// ResolveUUID returns the GPU UUID for a thread (via index), or "".@@ -102,16 +106,27 @@ func (r *DeviceResolver) ResolveUUID(pid, tid uint32) string {}r.mu.Lock()defer r.mu.Unlock()- idx, ok := r.threadDev[threadKey(pid, tid)]- if !ok {- if r.soleIndex < 0 {- return ""- }- idx = r.soleIndex+ idx := r.resolveIndexLocked(pid, tid)+ if idx < 0 {+ return ""}return r.indexUUID[idx]}+func (r *DeviceResolver) resolveIndexLocked(pid, tid uint32) int {+ if idx, ok := r.threadDev[threadKey(pid, tid)]; ok {+ return idx+ }+ if r.soleIndex >= 0 {+ return r.soleIndex+ }+ // CUDA runtime: current device is 0 until cudaSetDevice.+ if _, ok := r.indexUUID[0]; ok {+ return 0+ }+ return -1+}+// UUIDForIndex returns the UUID for a CUDA device index.func (r *DeviceResolver) UUIDForIndex(idx int) string {if r == nil {@@ -136,6 +151,21 @@ func (r *DeviceResolver) IndexInfo(idx int) (uuid, name, pci string, ok bool) {return uuid, r.indexName[idx], r.indexPCI[idx], true}+// IndexForUUID returns the CUDA/NVML index for a GPU UUID.+func (r *DeviceResolver) IndexForUUID(uuid string) (int, bool) {+ if r == nil || uuid == "" {+ return -1, false+ }+ r.mu.Lock()+ defer r.mu.Unlock()+ for idx, u := range r.indexUUID {+ if u == uuid {+ return idx, true+ }+ }+ return -1, false+}+// ForgetPID drops thread mappings for a PID (inactive cleanup).func (r *DeviceResolver) ForgetPID(pid uint32) {if r == nil {diff --git a/opentelemetry-gpu-collector/internal/cudaspans/spans_test.go b/opentelemetry-gpu-collector/internal/cudaspans/spans_test.goindex 1838070d..59dddff0 100644--- a/opentelemetry-gpu-collector/internal/cudaspans/spans_test.go+++ b/opentelemetry-gpu-collector/internal/cudaspans/spans_test.go@@ -39,3 +39,25 @@ func TestDeviceResolverSoleGPU(t *testing.T) {t.Fatalf("got %q", got)}}++func TestDeviceResolverMultiGPUDefaultsToCUDAZero(t *testing.T) {+ r := NewDeviceResolver(nil)+ r.SetDeviceIndexUUID(0, "gpu-a")+ r.SetDeviceIndexUUID(1, "gpu-b")+ if got := r.ResolveIndex(1, 1); got != 0 {+ t.Fatalf("ResolveIndex = %d, want 0 (CUDA default current device)", got)+ }+ if got := r.ResolveUUID(1, 1); got != "gpu-a" {+ t.Fatalf("ResolveUUID = %q, want gpu-a", got)+ }+ r.NoteSetDevice(1, 1, 1)+ if got := r.ResolveIndex(1, 1); got != 1 {+ t.Fatalf("after SetDevice ResolveIndex = %d, want 1", got)+ }+ if got := r.ResolveUUID(1, 1); got != "gpu-b" {+ t.Fatalf("after SetDevice ResolveUUID = %q, want gpu-b", got)+ }+ if idx, ok := r.IndexForUUID("gpu-b"); !ok || idx != 1 {+ t.Fatalf("IndexForUUID(gpu-b) = %d ok=%v", idx, ok)+ }+}diff --git a/opentelemetry-gpu-collector/internal/ebpf/bpf/gpuevent.c b/opentelemetry-gpu-collector/internal/ebpf/bpf/gpuevent.cindex ea3008b5..d90f947a 100644--- a/opentelemetry-gpu-collector/internal/ebpf/bpf/gpuevent.c+++ b/opentelemetry-gpu-collector/internal/ebpf/bpf/gpuevent.c@@ -37,6 +37,17 @@ struct {__type(value, __s32);} cuda_set_device_cache SEC(".maps");+// Last successful cudaSetDevice per thread (pid_tgid). Stamped onto later+// events so userspace can attribute hw.id on multi-GPU hosts without waiting+// for the SetDevice ringbuf event to be processed. Userspace drops keys for+// dead PIDs so a reused pid_tgid cannot inherit the previous process's index.+struct {+ __uint(type, BPF_MAP_TYPE_LRU_HASH);+ __uint(max_entries, 65536);+ __type(key, __u64);+ __type(value, __s32);+} cuda_current_device SEC(".maps");+struct {__uint(type, BPF_MAP_TYPE_LRU_HASH);__uint(max_entries, 1024);@@ -44,6 +55,19 @@ struct {__type(value, __u64);} cuda_malloc_size_cache SEC(".maps");+struct cuda_memcpy_args_t {+ __u64 size;+ __u8 kind;+ __u8 pad[7];+};++struct {+ __uint(type, BPF_MAP_TYPE_LRU_HASH);+ __uint(max_entries, 1024);+ __type(key, __u64);+ __type(value, struct cuda_memcpy_args_t);+} cuda_memcpy_cache SEC(".maps");+// CUDA 13 lazily converts the ELF host-function pointer into an opaque// cudaKernel_t via __cudaGetKernel, then passes that handle to// cudaLaunchKernel. Retain the relationship so userspace can still resolve a@@ -151,6 +175,13 @@ static __always_inline void fill_header(struct cuda_event_header_t *hdr, __u8 tyhdr->pad1 = 0;hdr->stream_id = stream_id;hdr->ktime_ns = bpf_ktime_get_ns();++ __s32 *cur = bpf_map_lookup_elem(&cuda_current_device, &pid_tgid);+ if (cur) {+ __s32 d = *cur;+ if (d >= 0 && d < 0xffff)+ hdr->device_idx = (__u16)d;+ }}// cudaLaunchKernel(const void *func, dim3 gridDim, dim3 blockDim,@@ -300,13 +331,46 @@ int handle_cuda_free(struct pt_regs *ctx) {return 0;}-// Synchronous cudaMemcpy → treat completion as device-wide sync (Datadog parity).+// Synchronous cudaMemcpy → copy bytes plus device-wide sync (Datadog parity+// for occupancy). Size/kind come from the entry probe; if that missses we+// still emit the sync so spans close.+SEC("uprobe/cudaMemcpy")+int handle_cuda_memcpy_enter(struct pt_regs *ctx) {+ __u64 pid_tgid = bpf_get_current_pid_tgid();+ struct cuda_memcpy_args_t args = {};+ args.size = PT_REGS_PARM3(ctx);+ args.kind = (__u8)PT_REGS_PARM4(ctx);+ bpf_map_update_elem(&cuda_memcpy_cache, &pid_tgid, &args, BPF_ANY);+ return 0;+}+SEC("uretprobe/cudaMemcpy")int handle_cuda_memcpy(struct pt_regs *ctx) {+ __u64 pid_tgid = bpf_get_current_pid_tgid();+ struct cuda_memcpy_args_t *argp = bpf_map_lookup_elem(&cuda_memcpy_cache, &pid_tgid);+ struct cuda_memcpy_args_t local = {};+ int have_args = 0;+ if (argp) {+ local = *argp;+ have_args = 1;+ bpf_map_delete_elem(&cuda_memcpy_cache, &pid_tgid);+ }+long ret = (long)PT_REGS_RC(ctx);if (ret != 0)return 0;+ if (have_args) {+ struct gpu_memcpy_t *cp;+ cp = bpf_ringbuf_reserve(&gpu_events, sizeof(*cp), 0);+ if (cp) {+ fill_header(&cp->hdr, EVENT_GPU_MEMCPY, 0);+ cp->size = local.size;+ cp->kind = local.kind;+ bpf_ringbuf_submit(cp, 0);+ }+ }+struct gpu_sync_t *ev;ev = bpf_ringbuf_reserve(&gpu_events, sizeof(*ev), 0);if (!ev)@@ -415,6 +479,7 @@ int handle_cuda_set_device(struct pt_regs *ctx) {ev->hdr.device_idx = (__u16)device;ev->device = device;ev->pad = 0;+ bpf_map_update_elem(&cuda_current_device, &pid_tgid, &device, BPF_ANY);bpf_ringbuf_submit(ev, 0);return 0;}diff --git a/opentelemetry-gpu-collector/internal/ebpf/pidtgid.go b/opentelemetry-gpu-collector/internal/ebpf/pidtgid.gonew file mode 100644index 00000000..fd0b0ac6--- /dev/null+++ b/opentelemetry-gpu-collector/internal/ebpf/pidtgid.go@@ -0,0 +1,28 @@+package ebpf++// pidFromPidTgid extracts the kernel tgid (userspace PID) from a BPF pid_tgid key.+func pidFromPidTgid(key uint64) uint32 {+ return uint32(key >> 32)+}++// stalePidTgidKeys returns pid_tgid keys whose process is no longer alive.+// alive is called at most once per distinct PID.+func stalePidTgidKeys(keys []uint64, alive func(uint32) bool) []uint64 {+ if alive == nil || len(keys) == 0 {+ return nil+ }+ checked := make(map[uint32]bool)+ var stale []uint64+ for _, key := range keys {+ pid := pidFromPidTgid(key)+ ok, seen := checked[pid]+ if !seen {+ ok = alive(pid)+ checked[pid] = ok+ }+ if !ok {+ stale = append(stale, key)+ }+ }+ return stale+}diff --git a/opentelemetry-gpu-collector/internal/ebpf/pidtgid_test.go b/opentelemetry-gpu-collector/internal/ebpf/pidtgid_test.gonew file mode 100644index 00000000..a3e36ca6--- /dev/null+++ b/opentelemetry-gpu-collector/internal/ebpf/pidtgid_test.go@@ -0,0 +1,33 @@+package ebpf++import "testing"++func TestPidFromPidTgid(t *testing.T) {+ key := uint64(42)<<32 | uint64(7)+ if got := pidFromPidTgid(key); got != 42 {+ t.Fatalf("pidFromPidTgid(%#x) = %d, want 42", key, got)+ }+}++func TestStalePidTgidKeysDropsDeadPID(t *testing.T) {+ live := uint64(10)<<32 | uint64(11)+ deadMain := uint64(20)<<32 | uint64(20)+ deadThread := uint64(20)<<32 | uint64(21)+ aliveCalls := map[uint32]int{}+ stale := stalePidTgidKeys([]uint64{live, deadMain, deadThread}, func(pid uint32) bool {+ aliveCalls[pid]+++ return pid == 10+ })+ if len(stale) != 2 || stale[0] != deadMain || stale[1] != deadThread {+ t.Fatalf("stale = %v, want [%d %d]", stale, deadMain, deadThread)+ }+ if aliveCalls[10] != 1 || aliveCalls[20] != 1 {+ t.Fatalf("alive calls = %v, want one per PID", aliveCalls)+ }+}++func TestStalePidTgidKeysNilAlive(t *testing.T) {+ if stalePidTgidKeys([]uint64{1}, nil) != nil {+ t.Fatal("nil alive should skip cleanup")+ }+}diff --git a/opentelemetry-gpu-collector/internal/ebpf/tracer.go b/opentelemetry-gpu-collector/internal/ebpf/tracer.goindex 293d33e3..c1d05ec7 100644--- a/opentelemetry-gpu-collector/internal/ebpf/tracer.go+++ b/opentelemetry-gpu-collector/internal/ebpf/tracer.go@@ -9,6 +9,7 @@ import ("fmt""log/slog""os"+ "strconv""sync""sync/atomic""syscall"@@ -82,7 +83,9 @@ func NewTracer(logger *slog.Logger, handler EventHandler) (*Tracer, error) {"cudaGraphLaunch": objs.HandleCudaGraphLaunch,"__cudaGetKernel": objs.HandleCudaGetKernelEnter,"cudaMalloc": objs.HandleCudaMallocEnter,+ "cudaMallocAsync": objs.HandleCudaMallocEnter,"cudaFree": objs.HandleCudaFree,+ "cudaMemcpy": objs.HandleCudaMemcpyEnter,"cudaMemcpyAsync": objs.HandleCudaMemcpyAsync,"cudaStreamSynchronize": objs.HandleCudaStreamSyncEnter,"cudaEventRecord": objs.HandleCudaEventRecordEnter,@@ -92,6 +95,7 @@ func NewTracer(logger *slog.Logger, handler EventHandler) (*Tracer, error) {uretprobes: map[string]*ebpf.Program{"__cudaGetKernel": objs.HandleCudaGetKernelExit,"cudaMalloc": objs.HandleCudaMalloc,+ "cudaMallocAsync": objs.HandleCudaMalloc,"cudaMemcpy": objs.HandleCudaMemcpy,"cudaStreamSynchronize": objs.HandleCudaStreamSync,"cudaDeviceSynchronize": objs.HandleCudaDeviceSync,@@ -161,6 +165,7 @@ func (t *Tracer) attachNewCudaLibs() int {}}t.refreshCudartPIDs()+ t.dropDeadCurrentDevices()return attached}@@ -190,6 +195,31 @@ func (t *Tracer) refreshCudartPIDs() {}}+func procExists(pid uint32) bool {+ _, err := os.Stat("/proc/" + strconv.FormatUint(uint64(pid), 10))+ return err == nil+}++// dropDeadCurrentDevices removes cuda_current_device entries for exited+// processes. Without this, a reused pid_tgid inherits the previous process's+// CUDA index and fill_header stamps the wrong hw.id.+func (t *Tracer) dropDeadCurrentDevices() {+ if t.closed.Load() || t.objs == nil || t.objs.CudaCurrentDevice == nil {+ return+ }+ var key uint64+ var val int32+ var keys []uint64+ iter := t.objs.CudaCurrentDevice.Iterate()+ for iter.Next(&key, &val) {+ keys = append(keys, key)+ }+ for _, k := range stalePidTgidKeys(keys, procExists) {+ key := k+ _ = t.objs.CudaCurrentDevice.Delete(&key)+ }+}+func (t *Tracer) attachCudaLib(cudaLib string, uprobes, uretprobes map[string]*ebpf.Program) bool {if t.closed.Load() {return false@@ -308,11 +338,12 @@ func (t *Tracer) Run(ctx context.Context) {go func() {defer t.wg.Done()- // Fast while waiting for the first libcudart/libcuda; back off once- // attached so steady-state /proc scans stay cheap on busy nodes.- interval := 30 * time.Second- ticker := time.NewTicker(interval)+ // Device-map hygiene stays at 30s so a reused pid_tgid cannot keep a+ // dead process's CUDA index. Full libcudart/libcuda /proc scans still+ // back off after the first attach.+ ticker := time.NewTicker(30 * time.Second)defer ticker.Stop()+ var lastAttach time.Timefor {select {case <-ctx.Done():@@ -320,15 +351,16 @@ func (t *Tracer) Run(ctx context.Context) {case <-t.stop:returncase <-ticker.C:+ t.dropDeadCurrentDevices()+ next := cudaRescanInterval(t.hasAttachedProbes())+ if !lastAttach.IsZero() && time.Since(lastAttach) < next {+ continue+ }n := t.attachNewCudaLibs()+ lastAttach = time.Now()if n > 0 {t.logger.Info("attached CUDA probes after /proc rescan", "libraries", n)}- next := cudaRescanInterval(t.hasAttachedProbes())- if next != interval {- interval = next- ticker.Reset(interval)- }}}}()diff --git a/opentelemetry-gpu-collector/internal/ebpf/tracer_bpf_test.go b/opentelemetry-gpu-collector/internal/ebpf/tracer_bpf_test.goindex b2b67064..796393f0 100644--- a/opentelemetry-gpu-collector/internal/ebpf/tracer_bpf_test.go+++ b/opentelemetry-gpu-collector/internal/ebpf/tracer_bpf_test.go@@ -13,6 +13,7 @@ func TestGeneratedSpecIncludesCUDA13KernelHandleProbes(t *testing.T) {for _, name := range []string{"handle_cuda_get_kernel_enter","handle_cuda_get_kernel_exit",+ "handle_cuda_memcpy_enter","handle_cu_launch_kernel","handle_cu_launch_kernel_exit","handle_cu_launch_kernel_ex",@@ -30,6 +31,8 @@ func TestGeneratedSpecIncludesCUDA13KernelHandleProbes(t *testing.T) {for _, name := range []string{"cuda_get_kernel_cache","cuda_kernel_handles",+ "cuda_current_device",+ "cuda_memcpy_cache","cudart_pids","cu_func_names",} {diff --git a/opentelemetry-gpu-collector/internal/ebpf/types.go b/opentelemetry-gpu-collector/internal/ebpf/types.goindex af95b611..dc60e86c 100644--- a/opentelemetry-gpu-collector/internal/ebpf/types.go+++ b/opentelemetry-gpu-collector/internal/ebpf/types.go@@ -18,12 +18,23 @@ type CUDAEvent interface {ProcessPID() uint32}+// DeviceIdxUnknown is the BPF header sentinel when CUDA device index is unset.+const DeviceIdxUnknown = uint16(0xffff)+type eventMeta struct {PID uint32TID uint32StreamID uint64KtimeNs uint64- DeviceIdx uint16 // 0xffff = unknown+ DeviceIdx uint16 // DeviceIdxUnknown when unset+}++// DeviceIndex converts a BPF header device_idx to a CUDA index, or -1 if unknown.+func DeviceIndex(idx uint16) int {+ if idx == DeviceIdxUnknown {+ return -1+ }+ return int(idx)}type KernelLaunchEvent struct {diff --git a/opentelemetry-gpu-collector/internal/ebpf/types_test.go b/opentelemetry-gpu-collector/internal/ebpf/types_test.goindex b63c8326..1e553a6f 100644--- a/opentelemetry-gpu-collector/internal/ebpf/types_test.go+++ b/opentelemetry-gpu-collector/internal/ebpf/types_test.go@@ -69,6 +69,15 @@ func TestMemcpyKindString(t *testing.T) {}}+func TestDeviceIndexUnknown(t *testing.T) {+ if DeviceIndex(DeviceIdxUnknown) != -1 {+ t.Fatalf("unknown should be -1")+ }+ if DeviceIndex(0) != 0 || DeviceIndex(1) != 1 {+ t.Fatal("known indices should pass through")+ }+}+func TestCUDAEventInterface(t *testing.T) {// Verify all event types satisfy the CUDAEvent interface at compile time.var _ CUDAEvent = (*KernelLaunchEvent)(nil)diff --git a/opentelemetry-gpu-collector/internal/export/ebpfmetrics.go b/opentelemetry-gpu-collector/internal/export/ebpfmetrics.goindex 8f546df9..ddb2374a 100644--- a/opentelemetry-gpu-collector/internal/export/ebpfmetrics.go+++ b/opentelemetry-gpu-collector/internal/export/ebpfmetrics.go@@ -25,15 +25,21 @@ import (type EBPFMetrics struct {logger *slog.Logger- kernelLaunchCalls metric.Int64Counter+ kernelLaunchCalls metric.Int64ObservableCounterkernelGridSize metric.Float64HistogramkernelBlockSize metric.Float64HistogramkernelSharedMemory metric.Float64HistogramkernelDuration metric.Float64Histogram- graphLaunchCalls metric.Int64Counter- memoryAllocations metric.Int64Counter+ graphLaunchCalls metric.Int64ObservableCounter+ memoryAllocations metric.Int64ObservableCountermemoryCopies metric.Float64Histogram+ launchCounts *liveCounter+ graphCounts *liveCounter+ allocCounts *liveCounter+ pidAlive func(uint32) bool+ reg metric.Registration+devices *cudaspans.DeviceResolverkernelNamesMu sync.Mutex@@ -49,7 +55,7 @@ func NewEBPFMetrics(provider *sdkmetric.MeterProvider, devices *cudaspans.Devicemetric.WithInstrumentationVersion("1.0.0"),)- kernelCalls, err := meter.Int64Counter("gpu.kernel.launch.calls",+ kernelCalls, err := meter.Int64ObservableCounter("gpu.kernel.launch.calls",metric.WithDescription("Number of CUDA kernel launches"),metric.WithUnit("{call}"),)@@ -89,7 +95,7 @@ func NewEBPFMetrics(provider *sdkmetric.MeterProvider, devices *cudaspans.Devicereturn nil, fmt.Errorf("creating gpu.kernel.duration: %w", err)}- graphCalls, err := meter.Int64Counter("gpu.graph.launch.calls",+ graphCalls, err := meter.Int64ObservableCounter("gpu.graph.launch.calls",metric.WithDescription("Number of CUDA graph replay invocations (cudaGraphLaunch/cuGraphLaunch). Counts replays, not the kernels executed inside each replay; that number is not observable at this API-tracing layer."),metric.WithUnit("{call}"),)@@ -97,8 +103,8 @@ func NewEBPFMetrics(provider *sdkmetric.MeterProvider, devices *cudaspans.Devicereturn nil, fmt.Errorf("creating gpu.graph.launch.calls: %w", err)}- memAlloc, err := meter.Int64Counter("gpu.memory.allocations",- metric.WithDescription("Total bytes allocated via cudaMalloc"),+ memAlloc, err := meter.Int64ObservableCounter("gpu.memory.allocations",+ metric.WithDescription("Total bytes allocated via cudaMalloc/cudaMallocAsync"),metric.WithUnit("By"),)if err != nil {@@ -113,7 +119,7 @@ func NewEBPFMetrics(provider *sdkmetric.MeterProvider, devices *cudaspans.Devicereturn nil, fmt.Errorf("creating gpu.memory.copies: %w", err)}- return &EBPFMetrics{+ em := &EBPFMetrics{logger: logger,kernelLaunchCalls: kernelCalls,kernelGridSize: gridSize,@@ -123,9 +129,50 @@ func NewEBPFMetrics(provider *sdkmetric.MeterProvider, devices *cudaspans.DevicegraphLaunchCalls: graphCalls,memoryAllocations: memAlloc,memoryCopies: memCopies,+ launchCounts: newLiveCounter(),+ graphCounts: newLiveCounter(),+ allocCounts: newLiveCounter(),+ pidAlive: pidExists,devices: devices,kernelNames: make(map[string]struct{}),- }, nil+ }++ reg, err := meter.RegisterCallback(func(ctx context.Context, o metric.Observer) error {+ alive := em.pidAlive+ dead := map[uint32]struct{}{}+ observe := func(inst metric.Int64ObservableCounter, c *liveCounter) {+ series, dropped := c.collectLive(alive)+ for _, s := range series {+ o.ObserveInt64(inst, s.value, metric.WithAttributes(s.attrs...))+ }+ for _, pid := range dropped {+ dead[pid] = struct{}{}+ }+ }+ observe(em.kernelLaunchCalls, em.launchCounts)+ observe(em.graphLaunchCalls, em.graphCounts)+ observe(em.memoryAllocations, em.allocCounts)+ if em.devices != nil {+ for pid := range dead {+ em.devices.ForgetPID(pid)+ }+ }+ return nil+ }, kernelCalls, graphCalls, memAlloc)+ if err != nil {+ return nil, fmt.Errorf("registering eBPF counter callback: %w", err)+ }+ em.reg = reg+ return em, nil+}++// Close unregisters observable callbacks.+func (em *EBPFMetrics) Close() {+ if em == nil || em.reg == nil {+ return+ }+ _ = em.reg.Unregister()+ em.reg = nil}type pidAttrCacheEntry struct {@@ -195,11 +242,23 @@ func cachedPIDAttrs(pid uint32) []attribute.KeyValue {return attrs}-func (em *EBPFMetrics) deviceAttrs(pid, tid uint32) []attribute.KeyValue {+func (em *EBPFMetrics) deviceAttrs(pid, tid uint32, explicitIdx int, explicitUUID string) []attribute.KeyValue {if em.devices == nil {return nil}- idx := em.devices.ResolveIndex(pid, tid)+ idx := explicitIdx+ if explicitUUID != "" && explicitUUID != "unknown" {+ if uidx, ok := em.devices.IndexForUUID(explicitUUID); ok {+ idx = uidx+ }+ }+ if idx < 0 {+ idx = em.devices.ResolveIndex(pid, tid)+ }+ return em.attrsForIndex(idx)+}++func (em *EBPFMetrics) attrsForIndex(idx int) []attribute.KeyValue {uuid, name, pci, ok := em.devices.IndexInfo(idx)if !ok {return nil@@ -222,15 +281,19 @@ func (em *EBPFMetrics) deviceAttrs(pid, tid uint32) []attribute.KeyValue {return attrs}-func (em *EBPFMetrics) activityAttrs(pid, tid uint32, extra ...attribute.KeyValue) metric.MeasurementOption {+func (em *EBPFMetrics) activityAttrList(pid, tid uint32, deviceIdx int, deviceUUID string, extra ...attribute.KeyValue) []attribute.KeyValue {base := cachedPIDAttrs(pid)- dev := em.deviceAttrs(pid, tid)+ dev := em.deviceAttrs(pid, tid, deviceIdx, deviceUUID)attrs := make([]attribute.KeyValue, 0, len(base)+len(dev)+len(extra)+1)attrs = append(attrs, base...)attrs = append(attrs, attribute.String("gpu.measurement.source", "ebpf"))attrs = append(attrs, dev...)attrs = append(attrs, extra...)- return metric.WithAttributes(attrs...)+ return attrs+}++func (em *EBPFMetrics) activityAttrs(pid, tid uint32, deviceIdx int, deviceUUID string, extra ...attribute.KeyValue) metric.MeasurementOption {+ return metric.WithAttributes(em.activityAttrList(pid, tid, deviceIdx, deviceUUID, extra...)...)}func kernelMetricName(e *gpuebpf.KernelLaunchEvent) string {@@ -271,7 +334,7 @@ func (em *EBPFMetrics) RecordClosedSpans(ctx context.Context, spans []cudaspans.if s.Kind == cudaspans.LaunchKindGraph {extra = append(extra, attribute.String("cuda.launch.kind", "graph"))}- attrs := em.activityAttrs(s.PID, s.TID, extra...)+ attrs := em.activityAttrs(s.PID, s.TID, s.DeviceIndex, s.DeviceUUID, extra...)em.kernelDuration.Record(ctx, durSec, attrs)}}@@ -280,9 +343,10 @@ func (em *EBPFMetrics) RecordClosedSpans(ctx context.Context, spans []cudaspans.func (em *EBPFMetrics) RecordLaunchActivity(ctx context.Context, e *gpuebpf.KernelLaunchEvent) {kernelName := kernelMetricName(e)extra := []attribute.KeyValue{attribute.String("cuda.kernel.name", kernelName)}- attrs := em.activityAttrs(e.PID, e.TID, extra...)+ attrList := em.activityAttrList(e.PID, e.TID, gpuebpf.DeviceIndex(e.DeviceIdx), "", extra...)+ attrs := metric.WithAttributes(attrList...)- em.kernelLaunchCalls.Add(ctx, 1, attrs)+ em.launchCounts.add(e.PID, 1, attrList)gridTotal := float64(e.GridX) * float64(e.GridY) * float64(e.GridZ)em.kernelGridSize.Record(ctx, gridTotal, attrs)@@ -293,8 +357,8 @@ func (em *EBPFMetrics) RecordLaunchActivity(ctx context.Context, e *gpuebpf.Kernem.kernelSharedMemory.Record(ctx, float64(e.SharedMemBytes), attrs)}-func (em *EBPFMetrics) RecordGraphLaunch(ctx context.Context, e *gpuebpf.GraphLaunchEvent) {- em.graphLaunchCalls.Add(ctx, 1, em.activityAttrs(e.PID, e.TID))+func (em *EBPFMetrics) RecordGraphLaunch(_ context.Context, e *gpuebpf.GraphLaunchEvent) {+ em.graphCounts.add(e.PID, 1, em.activityAttrList(e.PID, e.TID, gpuebpf.DeviceIndex(e.DeviceIdx), ""))}// HandleEvent processes activity-only events when used without SpanFanout.@@ -315,10 +379,10 @@ func (em *EBPFMetrics) HandleEvent(ev gpuebpf.CUDAEvent) {em.RecordGraphLaunch(ctx, e)case *gpuebpf.MallocEvent:- em.memoryAllocations.Add(ctx, int64(e.Size), em.activityAttrs(e.PID, e.TID))+ em.allocCounts.add(e.PID, int64(e.Size), em.activityAttrList(e.PID, e.TID, gpuebpf.DeviceIndex(e.DeviceIdx), ""))case *gpuebpf.MemcpyEvent:- em.memoryCopies.Record(ctx, float64(e.Size), em.activityAttrs(e.PID, e.TID,+ em.memoryCopies.Record(ctx, float64(e.Size), em.activityAttrs(e.PID, e.TID, gpuebpf.DeviceIndex(e.DeviceIdx), "",attribute.String("cuda.memcpy.kind", gpuebpf.MemcpyKindString(e.Kind)),))}diff --git a/opentelemetry-gpu-collector/internal/export/ebpfmetrics_duration_test.go b/opentelemetry-gpu-collector/internal/export/ebpfmetrics_duration_test.goindex c14aefdc..24d571ee 100644--- a/opentelemetry-gpu-collector/internal/export/ebpfmetrics_duration_test.go+++ b/opentelemetry-gpu-collector/internal/export/ebpfmetrics_duration_test.go@@ -3,6 +3,7 @@ package exportimport ("context""fmt"+ "os""testing""go.opentelemetry.io/otel/sdk/metric"@@ -11,6 +12,8 @@ import (gpuebpf "github.com/openlit/openlit/opentelemetry-gpu-collector/internal/ebpf")+func testPID() uint32 { return uint32(os.Getpid()) }+func newTestSpanFanout(t *testing.T) (*SpanFanout, *metric.ManualReader, func()) {t.Helper()reader := metric.NewManualReader()@@ -25,6 +28,7 @@ func newTestSpanFanout(t *testing.T) (*SpanFanout, *metric.ManualReader, func())}return NewSpanFanout(em, occ), reader, func() {occ.Close()+ em.Close()_ = provider.Shutdown(context.Background())}}@@ -34,14 +38,14 @@ func TestKernelDurationLaunchToSync(t *testing.T) {defer cleanup()launch := &gpuebpf.KernelLaunchEvent{KernelName: "matmul"}- launch.PID = 10+ launch.PID = testPID()launch.TID = 11launch.StreamID = 7launch.KtimeNs = 1_000_000_000fanout.HandleEvent(launch)sync := &gpuebpf.SyncEvent{}- sync.PID = 10+ sync.PID = testPID()sync.TID = 11sync.StreamID = 7sync.KtimeNs = 1_500_000_000@@ -82,14 +86,14 @@ func TestGraphDurationLaunchToSync(t *testing.T) {defer cleanup()launch := &gpuebpf.GraphLaunchEvent{}- launch.PID = 10+ launch.PID = testPID()launch.TID = 11launch.StreamID = 7launch.KtimeNs = 1_000_000_000fanout.HandleEvent(launch)sync := &gpuebpf.SyncEvent{}- sync.PID = 10+ sync.PID = testPID()sync.TID = 11sync.StreamID = 7sync.KtimeNs = 1_500_000_000@@ -160,14 +164,14 @@ func TestKernelDurationSkipsInvertedWindow(t *testing.T) {defer cleanup()launch := &gpuebpf.KernelLaunchEvent{KernelName: "late"}- launch.PID = 1+ launch.PID = testPID()launch.TID = 1launch.StreamID = 1launch.KtimeNs = 2_000_000_000fanout.HandleEvent(launch)sync := &gpuebpf.SyncEvent{}- sync.PID = 1+ sync.PID = testPID()sync.TID = 1sync.StreamID = 1sync.KtimeNs = 1_000_000_000diff --git a/opentelemetry-gpu-collector/internal/export/ebpfmetrics_test.go b/opentelemetry-gpu-collector/internal/export/ebpfmetrics_test.goindex 53bd26e8..b01558b2 100644--- a/opentelemetry-gpu-collector/internal/export/ebpfmetrics_test.go+++ b/opentelemetry-gpu-collector/internal/export/ebpfmetrics_test.go@@ -2,6 +2,7 @@ package exportimport ("context"+ "os""testing""go.opentelemetry.io/otel/sdk/metric"@@ -21,14 +22,15 @@ func TestGraphLaunchDoesNotIncrementKernelCalls(t *testing.T) {if err != nil {t.Fatal(err)}+ defer em.Close()kernel := &gpuebpf.KernelLaunchEvent{KernelName: "vector_add", GridX: 2, GridY: 1, GridZ: 1, BlockX: 32, BlockY: 1, BlockZ: 1}- kernel.PID = 42+ kernel.PID = uint32(os.Getpid())kernel.TID = 43em.HandleEvent(kernel)graph := &gpuebpf.GraphLaunchEvent{}- graph.PID = 42+ graph.PID = uint32(os.Getpid())graph.TID = 43em.HandleEvent(graph)@@ -85,7 +87,7 @@ func TestDeviceAttrsSoleGPU(t *testing.T) {&mockDevice{info: gpu.DeviceInfo{Vendor: gpu.VendorNVIDIA, UUID: "gpu-a", Index: 0, Name: "A100", PCIAddress: "0000:01:00.0"}},}em := &EBPFMetrics{devices: cudaspans.NewDeviceResolver(devs)}- attrs := em.deviceAttrs(1, 2)+ attrs := em.deviceAttrs(1, 2, -1, "")if len(attrs) < 2 {t.Fatalf("expected sole-GPU attrs, got %#v", attrs)}@@ -104,18 +106,23 @@ func TestDeviceAttrsSoleGPU(t *testing.T) {}}-func TestDeviceAttrsMultiGPUNeedsSetDevice(t *testing.T) {+func TestDeviceAttrsMultiGPUDefaultsToCUDAZero(t *testing.T) {devs := []gpu.Device{&mockDevice{info: gpu.DeviceInfo{Vendor: gpu.VendorNVIDIA, UUID: "gpu-a", Index: 0, Name: "A100"}},&mockDevice{info: gpu.DeviceInfo{Vendor: gpu.VendorNVIDIA, UUID: "gpu-b", Index: 1, Name: "A100"}},}em := &EBPFMetrics{devices: cudaspans.NewDeviceResolver(devs)}- if attrs := em.deviceAttrs(1, 2); len(attrs) != 0 {- t.Fatalf("expected no attrs before SetDevice, got %#v", attrs)+ attrs := em.deviceAttrs(1, 2, -1, "")+ m := map[string]string{}+ for _, a := range attrs {+ m[string(a.Key)] = a.Value.Emit()+ }+ if m["hw.id"] != "gpu-a" {+ t.Fatalf("hw.id = %q, want gpu-a (CUDA default device 0)", m["hw.id"])}em.devices.NoteSetDevice(1, 2, 1)- attrs := em.deviceAttrs(1, 2)- m := map[string]string{}+ attrs = em.deviceAttrs(1, 2, -1, "")+ m = map[string]string{}for _, a := range attrs {m[string(a.Key)] = a.Value.Emit()}@@ -127,14 +134,135 @@ func TestDeviceAttrsMultiGPUNeedsSetDevice(t *testing.T) {}}+func TestDeadPIDCountersStopExporting(t *testing.T) {+ reader := metric.NewManualReader()+ provider := metric.NewMeterProvider(metric.WithReader(reader))+ defer func() { _ = provider.Shutdown(context.Background()) }()++ em, err := NewEBPFMetrics(provider, nil, nil)+ if err != nil {+ t.Fatal(err)+ }+ defer em.Close()+ em.pidAlive = func(pid uint32) bool { return pid == 1 }++ live := &gpuebpf.KernelLaunchEvent{KernelName: "live"}+ live.PID = 1+ em.HandleEvent(live)+ dead := &gpuebpf.KernelLaunchEvent{KernelName: "dead"}+ dead.PID = 2+ em.HandleEvent(dead)++ var rm metricdata.ResourceMetrics+ if err := reader.Collect(context.Background(), &rm); err != nil {+ t.Fatal(err)+ }+ var pids []string+ for _, sm := range rm.ScopeMetrics {+ for _, m := range sm.Metrics {+ if m.Name != "gpu.kernel.launch.calls" {+ continue+ }+ sum := m.Data.(metricdata.Sum[int64])+ for _, dp := range sum.DataPoints {+ v, _ := dp.Attributes.Value("process.pid")+ pids = append(pids, v.AsString())+ }+ }+ }+ if len(pids) != 1 || pids[0] != "1" {+ t.Fatalf("exported pids = %v, want [1]", pids)+ }+ if n := sumInt64Counter(t, rm, "gpu.kernel.launch.calls"); n != 1 {+ t.Fatalf("calls = %d, want 1", n)+ }+}++func TestDeviceAttrsPrefersEventDeviceIndex(t *testing.T) {+ devs := []gpu.Device{+ &mockDevice{info: gpu.DeviceInfo{Vendor: gpu.VendorNVIDIA, UUID: "gpu-a", Index: 0, Name: "A100"}},+ &mockDevice{info: gpu.DeviceInfo{Vendor: gpu.VendorNVIDIA, UUID: "gpu-b", Index: 1, Name: "A100"}},+ }+ em := &EBPFMetrics{devices: cudaspans.NewDeviceResolver(devs)}+ attrs := em.deviceAttrs(1, 2, 1, "")+ m := map[string]string{}+ for _, a := range attrs {+ m[string(a.Key)] = a.Value.Emit()+ }+ if m["hw.id"] != "gpu-b" {+ t.Fatalf("hw.id = %q, want gpu-b from event DeviceIdx", m["hw.id"])+ }+}++func TestDeviceAttrsPrefersSpanUUID(t *testing.T) {+ devs := []gpu.Device{+ &mockDevice{info: gpu.DeviceInfo{Vendor: gpu.VendorNVIDIA, UUID: "gpu-a", Index: 0}},+ &mockDevice{info: gpu.DeviceInfo{Vendor: gpu.VendorNVIDIA, UUID: "gpu-b", Index: 1}},+ }+ em := &EBPFMetrics{devices: cudaspans.NewDeviceResolver(devs)}+ attrs := em.deviceAttrs(1, 2, 0, "gpu-b")+ m := map[string]string{}+ for _, a := range attrs {+ m[string(a.Key)] = a.Value.Emit()+ }+ if m["hw.id"] != "gpu-b" {+ t.Fatalf("hw.id = %q, want gpu-b from span UUID", m["hw.id"])+ }+}+func TestDeviceAttrsIgnoresUnknownIndex(t *testing.T) {devs := []gpu.Device{&mockDevice{info: gpu.DeviceInfo{Vendor: gpu.VendorNVIDIA, UUID: "gpu-a", Index: 0}},}em := &EBPFMetrics{devices: cudaspans.NewDeviceResolver(devs)}em.devices.NoteSetDevice(1, 2, 9) // unknown CUDA index — ignored- attrs := em.deviceAttrs(1, 2)+ attrs := em.deviceAttrs(1, 2, -1, "")if len(attrs) == 0 {t.Fatal("sole GPU should still attribute")}}++func TestLaunchCounterUsesEventDeviceIndex(t *testing.T) {+ reader := metric.NewManualReader()+ provider := metric.NewMeterProvider(metric.WithReader(reader))+ defer func() { _ = provider.Shutdown(context.Background()) }()++ devs := []gpu.Device{+ &mockDevice{info: gpu.DeviceInfo{Vendor: gpu.VendorNVIDIA, UUID: "gpu-a", Index: 0, Name: "A100"}},+ &mockDevice{info: gpu.DeviceInfo{Vendor: gpu.VendorNVIDIA, UUID: "gpu-b", Index: 1, Name: "A100"}},+ }+ em, err := NewEBPFMetrics(provider, cudaspans.NewDeviceResolver(devs), nil)+ if err != nil {+ t.Fatal(err)+ }+ defer em.Close()++ kernel := &gpuebpf.KernelLaunchEvent{KernelName: "matmul"}+ kernel.PID = uint32(os.Getpid())+ kernel.DeviceIdx = 1+ em.HandleEvent(kernel)++ var rm metricdata.ResourceMetrics+ if err := reader.Collect(context.Background(), &rm); err != nil {+ t.Fatal(err)+ }+ found := false+ for _, sm := range rm.ScopeMetrics {+ for _, m := range sm.Metrics {+ if m.Name != "gpu.kernel.launch.calls" {+ continue+ }+ sum := m.Data.(metricdata.Sum[int64])+ for _, dp := range sum.DataPoints {+ id, _ := dp.Attributes.Value("hw.id")+ if id.AsString() != "gpu-b" {+ t.Fatalf("hw.id = %q, want gpu-b", id.AsString())+ }+ found = true+ }+ }+ }+ if !found {+ t.Fatal("gpu.kernel.launch.calls missing hw.id")+ }+}diff --git a/opentelemetry-gpu-collector/internal/export/livecounter.go b/opentelemetry-gpu-collector/internal/export/livecounter.gonew file mode 100644index 00000000..c48c398c--- /dev/null+++ b/opentelemetry-gpu-collector/internal/export/livecounter.go@@ -0,0 +1,87 @@+package export++import (+ "os"+ "runtime"+ "strconv"+ "sync"++ "go.opentelemetry.io/otel/attribute"+)++const maxLiveCounterSeries = 16384++// liveCounter is a cumulative counter whose series can be dropped when the+// producing PID is gone. The OTel Go SDK keeps sync Int64Counter attribute+// sets forever; Prom remote-write then re-exports a frozen value and rate()+// stays 0. Observable counters plus this map omit dead PIDs so the series+// can go stale instead.+type liveCounter struct {+ mu sync.Mutex+ series map[string]*liveCounterSeries+}++type liveCounterSeries struct {+ pid uint32+ attrs []attribute.KeyValue+ value int64+}++func newLiveCounter() *liveCounter {+ return &liveCounter{series: make(map[string]*liveCounterSeries)}+}++func (c *liveCounter) add(pid uint32, n int64, attrs []attribute.KeyValue) {+ if c == nil || n == 0 {+ return+ }+ set := attribute.NewSet(attrs...)+ key := (&set).Encoded(attribute.DefaultEncoder())+ c.mu.Lock()+ defer c.mu.Unlock()+ s, ok := c.series[key]+ if !ok {+ if len(c.series) >= maxLiveCounterSeries {+ return+ }+ cloned := make([]attribute.KeyValue, len(attrs))+ copy(cloned, attrs)+ s = &liveCounterSeries{pid: pid, attrs: cloned}+ c.series[key] = s+ }+ s.value += n+}++func (c *liveCounter) collectLive(alive func(uint32) bool) (live []liveCounterSeries, deadPIDs []uint32) {+ if c == nil {+ return nil, nil+ }+ if alive == nil {+ alive = func(uint32) bool { return true }+ }+ c.mu.Lock()+ defer c.mu.Unlock()+ dead := map[uint32]struct{}{}+ live = make([]liveCounterSeries, 0, len(c.series))+ for k, s := range c.series {+ if alive(s.pid) {+ live = append(live, liveCounterSeries{pid: s.pid, attrs: s.attrs, value: s.value})+ continue+ }+ delete(c.series, k)+ dead[s.pid] = struct{}{}+ }+ for pid := range dead {+ deadPIDs = append(deadPIDs, pid)+ }+ return live, deadPIDs+}++func pidExists(pid uint32) bool {+ if runtime.GOOS != "linux" {+ // eBPF metrics are Linux-only; keep series alive in unit tests elsewhere.+ return true+ }+ _, err := os.Stat("/proc/" + strconv.FormatUint(uint64(pid), 10))+ return err == nil+}diff --git a/opentelemetry-gpu-collector/internal/export/occupancy.go b/opentelemetry-gpu-collector/internal/export/occupancy.goindex aef0206e..f078a0f6 100644--- a/opentelemetry-gpu-collector/internal/export/occupancy.go+++ b/opentelemetry-gpu-collector/internal/export/occupancy.go@@ -230,20 +230,23 @@ func (om *OccupancyMetrics) HandleEvent(ev gpuebpf.CUDAEvent) {om.engine.HandleLaunch(cudaoccupancy.KernelLaunch{PID: e.PID, TID: e.TID, StreamID: e.StreamID, KtimeNs: e.KtimeNs,Name: name, Kind: cudaspans.LaunchKindKernel,- GridX: e.GridX, GridY: e.GridY, GridZ: e.GridZ,+ DeviceIdx: gpuebpf.DeviceIndex(e.DeviceIdx),+ GridX: e.GridX, GridY: e.GridY, GridZ: e.GridZ,BlockX: e.BlockX, BlockY: e.BlockY, BlockZ: e.BlockZ,})case *gpuebpf.GraphLaunchEvent:om.engine.HandleLaunch(cudaoccupancy.KernelLaunch{PID: e.PID, TID: e.TID, StreamID: e.StreamID, KtimeNs: e.KtimeNs,Name: "graph", Kind: cudaspans.LaunchKindGraph,- GridX: 1, GridY: 1, GridZ: 1,+ DeviceIdx: gpuebpf.DeviceIndex(e.DeviceIdx),+ GridX: 1, GridY: 1, GridZ: 1,BlockX: 1, BlockY: 1, BlockZ: 1,})case *gpuebpf.SyncEvent:om.engine.HandleSync(cudaoccupancy.SyncEvent{PID: e.PID, TID: e.TID, StreamID: e.StreamID, KtimeNs: e.KtimeNs,DeviceWide: e.DeviceWide,+ DeviceIdx: gpuebpf.DeviceIndex(e.DeviceIdx),})case *gpuebpf.SetDeviceEvent:om.engine.HandleSetDevice(cudaoccupancy.SetDeviceEvent{diff --git a/opentelemetry-gpu-collector/internal/export/spanfanout.go b/opentelemetry-gpu-collector/internal/export/spanfanout.goindex c73949b5..ca90c060 100644--- a/opentelemetry-gpu-collector/internal/export/spanfanout.go+++ b/opentelemetry-gpu-collector/internal/export/spanfanout.go@@ -24,10 +24,11 @@ func (f *SpanFanout) HandleEvent(ev gpuebpf.CUDAEvent) {ctx := context.Background()switch e := ev.(type) {case *gpuebpf.SetDeviceEvent:+ if f.ebpf != nil && f.ebpf.devices != nil {+ f.ebpf.devices.NoteSetDevice(e.PID, e.TID, int(e.Device))+ }if f.occ != nil {f.occ.HandleEvent(ev)- } else if f.ebpf != nil && f.ebpf.devices != nil {- f.ebpf.devices.NoteSetDevice(e.PID, e.TID, int(e.Device))}case *gpuebpf.KernelLaunchEvent: