Skip to content

Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.

feat: use ebpf to improve GPU monitoring collector (#1098)

Aman Agarwalb4ebfefb2898
diff --git a/.github/workflows/gpu-collector-release.yml b/.github/workflows/gpu-collector-release.yml
deleted file mode 100644
index 2254f734..00000000
--- a/.github/workflows/gpu-collector-release.yml
+++ /dev/null
@@ -1,83 +0,0 @@
-name: Release OTel GPU Collector
-
-on:
- push:
- tags: [ 'otel-gpu-collector-*.*.*' ]
-
-env:
- REGISTRY: ghcr.io
-
-permissions:
- contents: write
-
-jobs:
- release:
- name: Docker
- runs-on: ubuntu-latest
- permissions:
- contents: write
- packages: write
- id-token: write
- env:
- IMAGE_NAME: openlit/otel-gpu-collector
-
- steps:
- - name: Checkout repository
- uses: actions/checkout@v6
-
- - name: Install cosign
- uses: sigstore/cosign-installer@ba7bc0a3fef59531c69a25acd34668d6d3fe6f22 #v4.1.0
- with:
- cosign-release: 'v2.2.2'
-
- - name: Extract version from tag
- id: extract_version
- run: echo "version=${GITHUB_REF#refs/tags/otel-gpu-collector-}" >> $GITHUB_ENV
- env:
- GITHUB_REF: ${{ github.ref }}
-
- - name: Set up Docker Buildx
- uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
-
- - name: Log into registry ${{ env.REGISTRY }}
- uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0
- with:
- registry: ${{ env.REGISTRY }}
- username: ${{ github.actor }}
- password: ${{ secrets.GITHUB_TOKEN }}
-
- - name: Extract Docker metadata
- id: meta
- uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v6.0.0
- with:
- images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
- tags: |
- type=semver,pattern={{version}},value=${{ env.version }}
- env:
- DOCKER_METADATA_ANNOTATIONS_LEVELS: manifest,index
-
- - name: Build and push Docker image
- id: build-and-push
- uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7.0.0
- with:
- context: ./otel-gpu-collector/
- provenance: false
- file: ./otel-gpu-collector/Dockerfile
- push: ${{ github.event_name != 'pull_request' }}
- tags: ${{ steps.meta.outputs.tags }}
- labels: ${{ steps.meta.outputs.labels }}
- platforms: linux/amd64,linux/arm64/v8
- annotations: |
- ${{ steps.meta.outputs.annotations }},
- io.artifacthub.package.logo-url=https://github.com/openlit/.github/blob/main/profile/assets/favicon.png?raw=true,
- io.artifacthub.package.readme-url=https://github.com/openlit/openlit/blob/main/otel-gpu-collector/README.md,
- io.artifacthub.package.license=Apache-2.0,
- org.opencontainers.image.vendor=OpenLIT,
- io.artifacthub.package.maintainers=[{'name':'OpenLIT','email':'[email protected]'}],
- io.artifacthub.package.keywords='Monitoring, Observability, GPUs, NVIDIA, OpenTelemetry, Metrics, LLMs, AI'
-
- - name: Sign the published Docker image
- env:
- TAGS: ${{ steps.meta.outputs.tags }}
- DIGEST: ${{ steps.build-and-push.outputs.digest }}
- run: echo "${TAGS}" | xargs -I {} cosign sign --yes {}@${DIGEST}
diff --git a/.github/workflows/otel-gpu-collector-release.yml b/.github/workflows/otel-gpu-collector-release.yml
new file mode 100644
index 00000000..00a828b6
--- /dev/null
+++ b/.github/workflows/otel-gpu-collector-release.yml
@@ -0,0 +1,225 @@
+name: Release OTel GPU Collector
+
+on:
+ push:
+ tags: [ 'otel-gpu-collector-*.*.*' ]
+
+env:
+ REGISTRY: ghcr.io
+ IMAGE_NAME: openlit/otel-gpu-collector
+ BINARY_NAME: opentelemetry-gpu-collector
+
+permissions:
+ contents: write
+
+jobs:
+ # ── 1. Unit tests ──────────────────────────────────────────────────────────
+ test:
+ name: Test
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+
+ - uses: actions/setup-go@v5
+ with:
+ go-version-file: opentelemetry-gpu-collector/go.mod
+ cache-dependency-path: opentelemetry-gpu-collector/go.sum
+
+ - name: Install eBPF build dependencies
+ run: |
+ sudo apt-get update -q
+ sudo apt-get install -y clang llvm libbpf-dev
+ # Download static bpftool binary from libbpf/bpftool releases — avoids
+ # linux-tools-$(uname -r) which fails on azure/custom kernels.
+ BPFTOOL_VERSION=$(curl -fsSL https://api.github.com/repos/libbpf/bpftool/releases/latest \
+ | jq -r .tag_name)
+ curl -fsSL "https://github.com/libbpf/bpftool/releases/download/${BPFTOOL_VERSION}/bpftool-${BPFTOOL_VERSION}-amd64.tar.gz" \
+ | sudo tar -xz -C /usr/local/bin/
+ sudo chmod +x /usr/local/bin/bpftool
+ bpftool version
+
+ - name: Generate eBPF objects
+ working-directory: opentelemetry-gpu-collector
+ run: make setup-bpf generate
+
+ - name: Run tests
+ working-directory: opentelemetry-gpu-collector
+ run: go test ./...
+
+ # ── 2. Cross-compile binaries ──────────────────────────────────────────────
+ build-binaries:
+ name: Build binaries (${{ matrix.goos }}/${{ matrix.goarch }})
+ needs: test
+ runs-on: ubuntu-latest
+ strategy:
+ matrix:
+ include:
+ - goos: linux
+ goarch: amd64
+ - goos: linux
+ goarch: arm64
+ - goos: linux
+ goarch: arm
+ goarm: "7"
+ - goos: darwin
+ goarch: amd64
+ - goos: darwin
+ goarch: arm64
+ - goos: windows
+ goarch: amd64
+ ext: .exe
+ - goos: windows
+ goarch: arm64
+ ext: .exe
+
+ steps:
+ - uses: actions/checkout@v4
+
+ - uses: actions/setup-go@v5
+ with:
+ go-version-file: opentelemetry-gpu-collector/go.mod
+ cache-dependency-path: opentelemetry-gpu-collector/go.sum
+
+ - name: Install eBPF build dependencies (linux targets only)
+ if: matrix.goos == 'linux'
+ run: |
+ sudo apt-get update -q
+ sudo apt-get install -y clang llvm libbpf-dev
+ BPFTOOL_VERSION=$(curl -fsSL https://api.github.com/repos/libbpf/bpftool/releases/latest \
+ | jq -r .tag_name)
+ curl -fsSL "https://github.com/libbpf/bpftool/releases/download/${BPFTOOL_VERSION}/bpftool-${BPFTOOL_VERSION}-amd64.tar.gz" \
+ | sudo tar -xz -C /usr/local/bin/
+ sudo chmod +x /usr/local/bin/bpftool
+
+ - name: Generate eBPF objects (linux targets only)
+ if: matrix.goos == 'linux'
+ working-directory: opentelemetry-gpu-collector
+ run: make setup-bpf generate
+
+ - name: Extract version
+ id: version
+ run: echo "version=${GITHUB_REF#refs/tags/otel-gpu-collector-}" >> $GITHUB_OUTPUT
+
+ - name: Build binary
+ working-directory: opentelemetry-gpu-collector
+ env:
+ GOOS: ${{ matrix.goos }}
+ GOARCH: ${{ matrix.goarch }}
+ GOARM: ${{ matrix.goarm }}
+ CGO_ENABLED: "0"
+ run: |
+ OUTPUT="${{ env.BINARY_NAME }}-${{ steps.version.outputs.version }}-${{ matrix.goos }}-${{ matrix.goarch }}${{ matrix.goarm && format('-armv{0}', matrix.goarm) || '' }}${{ matrix.ext }}"
+ go build -trimpath -ldflags "-s -w -X main.Version=${{ steps.version.outputs.version }}" \
+ -o "../dist/${OUTPUT}" ./cmd/collector
+
+ - name: Upload artifact
+ uses: actions/upload-artifact@v4
+ with:
+ name: binary-${{ matrix.goos }}-${{ matrix.goarch }}${{ matrix.goarm && format('-armv{0}', matrix.goarm) || '' }}
+ path: dist/
+
+ # ── 3. GitHub Release with binary assets ──────────────────────────────────
+ release:
+ name: GitHub Release
+ needs: build-binaries
+ runs-on: ubuntu-latest
+ permissions:
+ contents: write
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Download all artifacts
+ uses: actions/download-artifact@v4
+ with:
+ pattern: binary-*
+ path: dist/
+ merge-multiple: true
+
+ - name: Extract version
+ id: version
+ run: echo "version=${GITHUB_REF#refs/tags/otel-gpu-collector-}" >> $GITHUB_OUTPUT
+
+ - name: Create checksums
+ run: |
+ cd dist
+ sha256sum * > SHA256SUMS.txt
+
+ - name: Create GitHub Release
+ uses: softprops/action-gh-release@153bb8e04406b158c6c84fc1615b65b24149a1fe # v2
+ with:
+ name: "OTel GPU Collector ${{ steps.version.outputs.version }}"
+ files: |
+ dist/*
+ generate_release_notes: true
+
+ # ── 4. Docker multi-arch image → GHCR ─────────────────────────────────────
+ docker:
+ name: Docker
+ needs: test
+ runs-on: ubuntu-latest
+ permissions:
+ contents: read
+ packages: write
+ id-token: write
+
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Install cosign
+ uses: sigstore/cosign-installer@398d4b0eeef1380460a10c8013a76f728fb906ac # v3
+
+ - name: Set up QEMU
+ uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3
+
+ - name: Set up Docker Buildx
+ uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
+
+ - name: Log into GHCR
+ uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
+ with:
+ registry: ${{ env.REGISTRY }}
+ username: ${{ github.actor }}
+ password: ${{ secrets.GITHUB_TOKEN }}
+
+ - name: Extract version
+ id: version
+ run: echo "version=${GITHUB_REF#refs/tags/otel-gpu-collector-}" >> $GITHUB_OUTPUT
+
+ - name: Extract Docker metadata
+ id: meta
+ uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5
+ with:
+ images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+ tags: |
+ type=semver,pattern={{version}},value=${{ steps.version.outputs.version }}
+ type=semver,pattern={{major}}.{{minor}},value=${{ steps.version.outputs.version }}
+ type=raw,value=latest
+ env:
+ DOCKER_METADATA_ANNOTATIONS_LEVELS: manifest,index
+
+ - name: Build and push Docker image
+ id: build-and-push
+ uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
+ with:
+ context: ./opentelemetry-gpu-collector/
+ file: ./opentelemetry-gpu-collector/Dockerfile
+ push: true
+ tags: ${{ steps.meta.outputs.tags }}
+ labels: ${{ steps.meta.outputs.labels }}
+ platforms: linux/amd64,linux/arm64
+ cache-from: type=gha
+ cache-to: type=gha,mode=max
+ annotations: |
+ ${{ steps.meta.outputs.annotations }},
+ io.artifacthub.package.logo-url=https://github.com/openlit/.github/blob/main/profile/assets/favicon.png?raw=true,
+ io.artifacthub.package.readme-url=https://github.com/openlit/openlit/blob/main/opentelemetry-gpu-collector/README.md,
+ io.artifacthub.package.license=Apache-2.0,
+ org.opencontainers.image.vendor=OpenLIT,
+ io.artifacthub.package.maintainers=[{'name':'OpenLIT','email':'[email protected]'}],
+ io.artifacthub.package.keywords='Monitoring, Observability, GPUs, NVIDIA, OpenTelemetry, Metrics, LLMs, AI'
+
+ - name: Sign the published Docker image
+ env:
+ TAGS: ${{ steps.meta.outputs.tags }}
+ DIGEST: ${{ steps.build-and-push.outputs.digest }}
+ run: echo "${TAGS}" | xargs -I {} cosign sign --yes {}@${DIGEST}
diff --git a/.github/workflows/otel-gpu-collector-tests.yml b/.github/workflows/otel-gpu-collector-tests.yml
new file mode 100644
index 00000000..40d3222d
--- /dev/null
+++ b/.github/workflows/otel-gpu-collector-tests.yml
@@ -0,0 +1,47 @@
+name: OTel GPU Collector Tests
+
+on:
+ push:
+ branches: ["main"]
+ paths:
+ - 'opentelemetry-gpu-collector/**'
+ pull_request:
+ branches: ["main"]
+ paths:
+ - 'opentelemetry-gpu-collector/**'
+
+permissions:
+ contents: read
+
+jobs:
+ test:
+ name: Test
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+
+ - uses: actions/setup-go@v5
+ with:
+ go-version-file: opentelemetry-gpu-collector/go.mod
+ cache-dependency-path: opentelemetry-gpu-collector/go.sum
+
+ - name: Install eBPF build dependencies
+ run: |
+ sudo apt-get update -q
+ sudo apt-get install -y clang llvm libbpf-dev
+ # Download static bpftool binary from libbpf/bpftool releases — avoids
+ # linux-tools-$(uname -r) which fails on azure/custom kernels.
+ BPFTOOL_VERSION=$(curl -fsSL https://api.github.com/repos/libbpf/bpftool/releases/latest \
+ | jq -r .tag_name)
+ curl -fsSL "https://github.com/libbpf/bpftool/releases/download/${BPFTOOL_VERSION}/bpftool-${BPFTOOL_VERSION}-amd64.tar.gz" \
+ | sudo tar -xz -C /usr/local/bin/
+ sudo chmod +x /usr/local/bin/bpftool
+ bpftool version
+
+ - name: Generate eBPF objects
+ working-directory: opentelemetry-gpu-collector
+ run: make setup-bpf generate
+
+ - name: Run tests
+ working-directory: opentelemetry-gpu-collector
+ run: go test ./...
diff --git a/docs/docs.json b/docs/docs.json
index 0ed14191..594d1097 100644
--- a/docs/docs.json
+++ b/docs/docs.json
@@ -290,10 +290,11 @@
},
{
"group": "GPUs",
- "icon": "server",
+ "icon": "microchip",
"pages": [
- "latest/sdk/integrations/nvidia-gpu",
- "latest/sdk/integrations/amd-gpu"
+ "latest/gpu-collector/gpus/nvidia",
+ "latest/gpu-collector/gpus/amd",
+ "latest/gpu-collector/gpus/intel"
]
}
]
@@ -400,6 +401,35 @@
]
}
]
+ },
+ {
+ "tab": "GPU Collector",
+ "groups": [
+ {
+ "group": "Getting Started",
+ "pages": [
+ "latest/gpu-collector/overview",
+ "latest/gpu-collector/quickstart",
+ "latest/gpu-collector/installation"
+ ]
+ },
+ {
+ "group": "Reference",
+ "pages": [
+ "latest/gpu-collector/configuration",
+ "latest/gpu-collector/metrics"
+ ]
+ },
+ {
+ "group": "GPU Vendors",
+ "icon": "microchip",
+ "pages": [
+ "latest/gpu-collector/gpus/nvidia",
+ "latest/gpu-collector/gpus/amd",
+ "latest/gpu-collector/gpus/intel"
+ ]
+ }
+ ]
}
]
},
diff --git a/docs/latest/gpu-collector/configuration.mdx b/docs/latest/gpu-collector/configuration.mdx
new file mode 100644
index 00000000..8c3ba544
--- /dev/null
+++ b/docs/latest/gpu-collector/configuration.mdx
@@ -0,0 +1,63 @@
+---
+title: 'Configuration'
+description: 'Environment variables reference for the OpenTelemetry GPU Collector'
+icon: 'sliders'
+---
+
+The collector is configured entirely via environment variables. All variables follow the standard [OpenTelemetry SDK configuration](https://opentelemetry.io/docs/specs/otel/configuration/sdk-environment-variables/) spec where applicable.
+
+## Environment Variables
+
+| Variable | Default | Description |
+|---|---|---|
+| `OTEL_EXPORTER_OTLP_ENDPOINT` | *(required)* | OTLP endpoint URL, e.g. `http://localhost:4318` |
+| `OTEL_EXPORTER_OTLP_HEADERS` | | Auth headers in `key=val,key2=val2` format |
+| `OTEL_EXPORTER_OTLP_PROTOCOL` | `grpc` | `grpc` or `http/protobuf` |
+| `OTEL_SERVICE_NAME` | `default` | Service name attached to all metrics |
+| `OTEL_RESOURCE_ATTRIBUTES` | `deployment.environment=default` | Resource attributes in `key=val,key=val` format |
+| `OTEL_METRIC_EXPORT_INTERVAL` | `60000` | Metric polling interval in **milliseconds** |
+| `OTEL_GPU_EBPF_ENABLED` | `false` | Enable eBPF CUDA kernel tracing (Linux only) |
+
+## Common configurations
+
+### Minimal — send to a local OTel Collector
+
+```bash
+OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317 \
+./opentelemetry-gpu-collector
+```
+
+### Production — with service name, environment, and auth header
+
+```bash
+OTEL_SERVICE_NAME=gpu-worker \
+OTEL_RESOURCE_ATTRIBUTES=deployment.environment=production,team=ml \
+OTEL_EXPORTER_OTLP_ENDPOINT=https://ingest.example.com:4317 \
+OTEL_EXPORTER_OTLP_HEADERS=Authorization=Bearer\ my-token \
+OTEL_METRIC_EXPORT_INTERVAL=30000 \
+./opentelemetry-gpu-collector
+```
+
+### HTTP/protobuf instead of gRPC
+
+```bash
+OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf \
+OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318 \
+./opentelemetry-gpu-collector
+```
+
+### Enable eBPF CUDA tracing
+
+Requires `CAP_BPF` + `CAP_PERFMON` (or root) and NVIDIA CUDA runtime.
+
+```bash
+OTEL_GPU_EBPF_ENABLED=true \
+OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317 \
+./opentelemetry-gpu-collector
+```
+
+## Notes
+
+- `OTEL_METRIC_EXPORT_INTERVAL` is in **milliseconds** per the OTel spec. For a 30-second interval, set `30000`.
+- `deployment.environment` is extracted from `OTEL_RESOURCE_ATTRIBUTES` and attached as a resource attribute. Any key-value pairs in `OTEL_RESOURCE_ATTRIBUTES` are also forwarded to the OTel SDK resource via `resource.WithFromEnv()`.
+- If `OTEL_EXPORTER_OTLP_ENDPOINT` is not set, the collector starts but no metrics are exported. Check the logs for a warning.
diff --git a/docs/latest/gpu-collector/gpus/amd.mdx b/docs/latest/gpu-collector/gpus/amd.mdx
new file mode 100644
index 00000000..5247e9cf
--- /dev/null
+++ b/docs/latest/gpu-collector/gpus/amd.mdx
@@ -0,0 +1,105 @@
+---
+title: 'AMD GPUs'
+description: 'Monitor AMD GPU metrics via sysfs/hwmon using the OpenTelemetry GPU Collector'
+---
+
+The collector monitors AMD GPUs directly from the Linux kernel's sysfs and hwmon interfaces. No ROCm, no user-space libraries, and no additional drivers are needed beyond the standard AMDGPU kernel module.
+
+## Requirements
+
+- Linux with the `amdgpu` kernel driver
+- Kernel 5.x+ (sysfs/hwmon paths are stable from 5.x onwards)
+
+## Collected metrics
+
+| Metric | Description |
+|---|---|
+| `hw.gpu.utilization` | Compute utilization (0.0–1.0) |
+| `hw.gpu.memory.utilization` | Memory controller utilization (0.0–1.0) |
+| `hw.gpu.memory.limit` | Total VRAM (bytes) |
+| `hw.gpu.memory.usage` | Used VRAM (bytes) |
+| `hw.gpu.memory.free` | Free VRAM (bytes) |
+| `hw.gpu.temperature` | Die temperature (°C) |
+| `hw.gpu.fan_speed` | Fan speed (RPM) |
+| `hw.gpu.power.draw` | Current power draw (W) |
+| `hw.gpu.power.limit` | Power cap (W) |
+| `hw.gpu.energy.consumed` | Cumulative energy (J) |
+
+## Docker
+
+```bash
+docker run -d \
+ --name otel-gpu-collector \
+ --device /dev/kfd:/dev/kfd \
+ --device /dev/dri:/dev/dri \
+ -e OTEL_SERVICE_NAME=my-app \
+ -e OTEL_RESOURCE_ATTRIBUTES='deployment.environment=production' \
+ -e OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318 \
+ ghcr.io/openlit/otel-gpu-collector:latest
+```
+
+## Docker Compose
+
+```yaml
+services:
+ otel-gpu-collector:
+ image: ghcr.io/openlit/otel-gpu-collector:latest
+ environment:
+ OTEL_SERVICE_NAME: my-app
+ OTEL_RESOURCE_ATTRIBUTES: deployment.environment=production
+ OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318
+ devices:
+ - /dev/kfd:/dev/kfd
+ - /dev/dri:/dev/dri
+ restart: always
+```
+
+## Kubernetes (DaemonSet)
+
+```yaml
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+ name: otel-gpu-collector
+ namespace: monitoring
+spec:
+ selector:
+ matchLabels:
+ app: otel-gpu-collector
+ template:
+ metadata:
+ labels:
+ app: otel-gpu-collector
+ spec:
+ containers:
+ - name: collector
+ image: ghcr.io/openlit/otel-gpu-collector:latest
+ env:
+ - name: OTEL_SERVICE_NAME
+ value: gpu-collector
+ - name: OTEL_RESOURCE_ATTRIBUTES
+ value: deployment.environment=production
+ - name: OTEL_EXPORTER_OTLP_ENDPOINT
+ value: http://otel-collector.monitoring.svc.cluster.local:4318
+ securityContext:
+ privileged: false
+ volumeMounts:
+ - name: sys
+ mountPath: /sys
+ readOnly: true
+ volumes:
+ - name: sys
+ hostPath:
+ path: /sys
+```
+
+---
+
+<CardGroup cols={2}>
+ <Card title="Metrics reference" href="/latest/gpu-collector/metrics#gpu-hardware-telemetry" icon="table">
+ Full metrics list with types, units, and attributes
+ </Card>
+ <Card title="Configuration" href="/latest/gpu-collector/configuration" icon="sliders">
+ All environment variables and defaults
+ </Card>
+</CardGroup>
diff --git a/docs/latest/gpu-collector/gpus/intel.mdx b/docs/latest/gpu-collector/gpus/intel.mdx
new file mode 100644
index 00000000..45e2d34d
--- /dev/null
+++ b/docs/latest/gpu-collector/gpus/intel.mdx
@@ -0,0 +1,104 @@
+---
+title: 'Intel GPUs'
+description: 'Monitor Intel GPU metrics via sysfs/hwmon using the OpenTelemetry GPU Collector'
+---
+
+The collector monitors Intel GPUs via the Linux kernel's sysfs, hwmon, and DRM interfaces exposed by the i915 and Xe drivers. No Intel GPU tools, no OneAPI, and no user-space libraries are required.
+
+<Note>
+Intel GPU support provides thermal, power, energy, and clock metrics. Utilization and memory metrics are not available via the sysfs/hwmon interface — these would require the [Intel XPU Manager](https://github.com/intel/xpumanager) or similar tooling.
+</Note>
+
+## Requirements
+
+- Linux with the `i915` or `xe` kernel driver
+- Kernel 5.10+ for sysfs metric exposure
+- Kernel 6.16+ for fan speed (`fan1_input`)
+
+## Collected metrics
+
+| Metric | Source | Requirement |
+|---|---|---|
+| `hw.gpu.temperature` | hwmon `temp1_input` | kernel 5.10+ |
+| `hw.gpu.power.draw` | hwmon `power1_average` | kernel 5.10+ |
+| `hw.gpu.power.limit` | hwmon `power1_max` | kernel 5.10+ |
+| `hw.gpu.energy.consumed` | hwmon `energy1_input` | kernel 5.10+ |
+| `hw.gpu.clock.graphics` | DRM `gt_cur_freq_mhz` | Xe driver |
+| `hw.gpu.fan_speed` | hwmon `fan1_input` | kernel 6.16+ |
+
+## Docker
+
+```bash
+docker run -d \
+ --name otel-gpu-collector \
+ --device /dev/dri:/dev/dri \
+ -e OTEL_SERVICE_NAME=my-app \
+ -e OTEL_RESOURCE_ATTRIBUTES='deployment.environment=production' \
+ -e OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318 \
+ ghcr.io/openlit/otel-gpu-collector:latest
+```
+
+## Docker Compose
+
+```yaml
+services:
+ otel-gpu-collector:
+ image: ghcr.io/openlit/otel-gpu-collector:latest
+ environment:
+ OTEL_SERVICE_NAME: my-app
+ OTEL_RESOURCE_ATTRIBUTES: deployment.environment=production
+ OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318
+ devices:
+ - /dev/dri:/dev/dri
+ restart: always
+```
+
+## Kubernetes (DaemonSet)
+
+```yaml
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+ name: otel-gpu-collector
+ namespace: monitoring
+spec:
+ selector:
+ matchLabels:
+ app: otel-gpu-collector
+ template:
+ metadata:
+ labels:
+ app: otel-gpu-collector
+ spec:
+ containers:
+ - name: collector
+ image: ghcr.io/openlit/otel-gpu-collector:latest
+ env:
+ - name: OTEL_SERVICE_NAME
+ value: gpu-collector
+ - name: OTEL_RESOURCE_ATTRIBUTES
+ value: deployment.environment=production
+ - name: OTEL_EXPORTER_OTLP_ENDPOINT
+ value: http://otel-collector.monitoring.svc.cluster.local:4318
+ securityContext:
+ privileged: false
+ volumeMounts:
+ - name: sys
+ mountPath: /sys
+ readOnly: true
+ volumes:
+ - name: sys
+ hostPath:
+ path: /sys
+```
+
+---
+
+<CardGroup cols={2}>
+ <Card title="Metrics reference" href="/latest/gpu-collector/metrics#gpu-hardware-telemetry" icon="table">
+ Full metrics list with types, units, and attributes
+ </Card>
+ <Card title="Configuration" href="/latest/gpu-collector/configuration" icon="sliders">
+ All environment variables and defaults
+ </Card>
+</CardGroup>
diff --git a/docs/latest/gpu-collector/gpus/nvidia.mdx b/docs/latest/gpu-collector/gpus/nvidia.mdx
new file mode 100644
index 00000000..cdb966c2
--- /dev/null
+++ b/docs/latest/gpu-collector/gpus/nvidia.mdx
@@ -0,0 +1,117 @@
+---
+title: 'NVIDIA GPUs'
+description: 'Monitor NVIDIA GPU metrics via NVML using the OpenTelemetry GPU Collector'
+---
+
+The collector monitors NVIDIA GPUs via [NVML](https://developer.nvidia.com/nvidia-management-library-nvml) using the [go-nvml](https://github.com/NVIDIA/go-nvml) library, which loads `libnvidia-ml.so` at runtime. No CUDA toolkit or DCGM daemon is needed.
+
+## Requirements
+
+- Linux with NVIDIA GPU drivers installed
+- `libnvidia-ml.so` present on the host (installed with the NVIDIA driver)
+- For Docker: [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html)
+
+## Collected metrics
+
+| Metric | Description |
+|---|---|
+| `hw.gpu.utilization` | Compute, encoder, and decoder utilization (0.0–1.0) via `hw.gpu.task` |
+| `hw.gpu.memory.utilization` | Memory controller utilization (0.0–1.0) |
+| `hw.gpu.memory.limit` | Total VRAM (bytes) |
+| `hw.gpu.memory.usage` | Used VRAM (bytes) |
+| `hw.gpu.memory.free` | Free VRAM (bytes) |
+| `hw.gpu.temperature` | Die and memory temperature (°C) via `sensor` attribute |
+| `hw.gpu.fan_speed` | Fan speed (RPM) |
+| `hw.gpu.power.draw` | Current power draw (W) |
+| `hw.gpu.power.limit` | Power cap (W) |
+| `hw.gpu.energy.consumed` | Cumulative energy (J) |
+| `hw.gpu.clock.graphics` | Graphics/SM clock (MHz) |
+| `hw.gpu.clock.memory` | Memory clock (MHz) |
+| `hw.errors` | ECC correctable/uncorrectable errors and PCIe replay errors |
+
+## Docker
+
+```bash
+docker run -d \
+ --name otel-gpu-collector \
+ --gpus all \
+ -e OTEL_SERVICE_NAME=my-app \
+ -e OTEL_RESOURCE_ATTRIBUTES='deployment.environment=production' \
+ -e OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318 \
+ ghcr.io/openlit/otel-gpu-collector:latest
+```
+
+## Docker Compose
+
+```yaml
+services:
+ otel-gpu-collector:
+ image: ghcr.io/openlit/otel-gpu-collector:latest
+ environment:
+ OTEL_SERVICE_NAME: my-app
+ OTEL_RESOURCE_ATTRIBUTES: deployment.environment=production
+ OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318
+ deploy:
+ resources:
+ reservations:
+ devices:
+ - driver: nvidia
+ count: all
+ capabilities: [gpu]
+ restart: always
+```
+
+## Kubernetes (DaemonSet)
+
+To monitor GPUs on every node in a cluster, deploy the collector as a DaemonSet:
+
+```yaml
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+ name: otel-gpu-collector
+ namespace: monitoring
+spec:
+ selector:
+ matchLabels:
+ app: otel-gpu-collector
+ template:
+ metadata:
+ labels:
+ app: otel-gpu-collector
+ spec:
+ tolerations:
+ - key: nvidia.com/gpu
+ operator: Exists
+ effect: NoSchedule
+ containers:
+ - name: collector
+ image: ghcr.io/openlit/otel-gpu-collector:latest
+ env:
+ - name: OTEL_SERVICE_NAME
+ value: gpu-collector
+ - name: OTEL_RESOURCE_ATTRIBUTES
+ value: deployment.environment=production
+ - name: OTEL_EXPORTER_OTLP_ENDPOINT
+ value: http://otel-collector.monitoring.svc.cluster.local:4318
+ resources:
+ limits:
+ nvidia.com/gpu: 1
+ securityContext:
+ privileged: false
+```
+
+<Note>
+The collector does not need privileged mode for NVML — just access to `libnvidia-ml.so`. eBPF tracing requires `CAP_BPF` + `CAP_PERFMON`.
+</Note>
+
+---
+
+<CardGroup cols={2}>
+ <Card title="Metrics reference" href="/latest/gpu-collector/metrics#gpu-hardware-telemetry" icon="table">
+ Full metrics list with types, units, and attributes
+ </Card>
+ <Card title="Configuration" href="/latest/gpu-collector/configuration" icon="sliders">
+ All environment variables and defaults
+ </Card>
+</CardGroup>
diff --git a/docs/latest/gpu-collector/installation.mdx b/docs/latest/gpu-collector/installation.mdx
new file mode 100644
index 00000000..a7c39cb3
--- /dev/null
+++ b/docs/latest/gpu-collector/installation.mdx
@@ -0,0 +1,140 @@
+---
+title: 'Installation'
+description: 'Install the OpenTelemetry GPU Collector via Docker, binary, or from source'
+icon: 'circle-down'
+---
+
+## Docker (recommended)
+
+The easiest way to run the collector. The image is published to GitHub Container Registry and supports `linux/amd64` and `linux/arm64`.
+
+```bash
+docker pull ghcr.io/openlit/otel-gpu-collector:latest
+```
+
+### Tags
+
+| Tag | Description |
+|---|---|
+| `latest` | Most recent release |
+| `1.2.3` | Specific version |
+| `1.2` | Latest patch of a minor version |
+
+---
+
+## Pre-built binaries
+
+Download a binary for your platform from the [GitHub Releases](https://github.com/openlit/openlit/releases) page. Binaries are available for:
+
+| Platform | Architecture |
+|---|---|
+| Linux | amd64, arm64, armv7 |
+| macOS | amd64 (Intel), arm64 (Apple Silicon) |
+| Windows | amd64, arm64 |
+
+```bash
+# Example: Linux amd64
+curl -L https://github.com/openlit/openlit/releases/latest/download/opentelemetry-gpu-collector-<version>-linux-amd64 \
+ -o opentelemetry-gpu-collector
+chmod +x opentelemetry-gpu-collector
+
+OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317 \
+./opentelemetry-gpu-collector
+```
+
+<Note>
+GPU metrics (NVML, sysfs) and eBPF tracing only work on Linux. On macOS and Windows the binary runs with host and process metrics only.
+</Note>
+
+Verify the SHA256 checksum from the `SHA256SUMS.txt` file in the release:
+
+```bash
+sha256sum -c SHA256SUMS.txt --ignore-missing
+```
+
+---
+
+## Build from source
+
+Requirements: Go 1.21+, CGO enabled (required for NVML on Linux).
+
+```bash
+git clone https://github.com/openlit/openlit.git
+cd openlit/opentelemetry-gpu-collector
+make build
+./opentelemetry-gpu-collector
+```
+
+For eBPF CUDA tracing support, also run:
+
+```bash
+make setup-bpf # installs bpftool, generates vmlinux.h
+make generate # runs bpf2go code generation
+make build
+```
+
+---
+
+## Upgrade
+
+### Docker
+
+```bash
+docker pull ghcr.io/openlit/otel-gpu-collector:latest
+docker stop otel-gpu-collector
+docker rm otel-gpu-collector
+# re-run with same flags
+```
+
+### Binary
+
+Download the new binary from the [Releases](https://github.com/openlit/openlit/releases) page, replace the existing file, and restart the process.
+
+---
+
+## Uninstall
+
+### Docker
+
+```bash
+docker stop otel-gpu-collector
+docker rm otel-gpu-collector
+docker rmi ghcr.io/openlit/otel-gpu-collector:latest
+```
+
+### Binary
+
+```bash
+rm /usr/local/bin/opentelemetry-gpu-collector
+```
+
+---
+
+## Troubleshooting
+
+<AccordionGroup>
+ <Accordion title="No GPU metrics — collector starts but reports no hw.gpu.* metrics">
+ - Confirm the host has a supported GPU: `lspci | grep -E 'VGA|3D|Display'`
+ - For NVIDIA: verify `libnvidia-ml.so` is present: `ldconfig -p | grep nvidia-ml`
+ - For Docker: ensure `--gpus all` (NVIDIA) or `--device /dev/dri` (AMD/Intel) is passed
+ - Check logs: `docker logs otel-gpu-collector` for `"discovered GPU"` entries
+ </Accordion>
+ <Accordion title="eBPF tracing not working">
+ - Confirm `OTEL_GPU_EBPF_ENABLED=true` is set
+ - Check kernel version: `uname -r` (requires 5.8+)
+ - The process needs `CAP_BPF` and `CAP_PERFMON`, or run as root
+ - For Docker: add `--privileged` or `--cap-add CAP_BPF --cap-add CAP_PERFMON`
+ - Verify `libcudart.so` is present on the host: `ldconfig -p | grep cudart`
+ </Accordion>
+ <Accordion title="Connection refused / no data reaching backend">
+ - Verify `OTEL_EXPORTER_OTLP_ENDPOINT` is reachable from the container: `curl http://<endpoint>/health`
+ - For Docker networking: use the host IP or service name, not `localhost`
+ - Check if gRPC vs HTTP/protobuf matches the backend: set `OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf` for HTTP backends (port 4318)
+ </Accordion>
+ <Accordion title="Intel GPU not detected">
+ - Verify the i915 or Xe driver is loaded: `lsmod | grep -E 'i915|xe'`
+ - Check DRM entries exist: `ls /sys/class/drm/`
+ - Requires Linux kernel 5.10+ for sysfs metric exposure
+ - Fan speed requires kernel 6.16+
+ </Accordion>
+</AccordionGroup>
diff --git a/docs/latest/gpu-collector/metrics.mdx b/docs/latest/gpu-collector/metrics.mdx
new file mode 100644
index 00000000..81df8d8d
--- /dev/null
+++ b/docs/latest/gpu-collector/metrics.mdx
@@ -0,0 +1,106 @@
+---
+title: 'Metrics Reference'
+description: 'Complete list of all metrics exported by the OpenTelemetry GPU Collector'
+icon: 'table'
+---
+
+All metric names and attributes follow the [OpenTelemetry semantic conventions for hardware](https://opentelemetry.io/docs/specs/semconv/hardware/gpu/) and [system metrics](https://opentelemetry.io/docs/specs/semconv/system/).
+
+## GPU Hardware Telemetry
+
+Collected for each detected GPU on Linux. Availability depends on vendor and GPU model.
+
+### Metrics
+
+| Metric | Type | Unit | Description | NVIDIA | AMD | Intel |
+|---|---|---|---|:---:|:---:|:---:|
+| `hw.gpu.utilization` | Gauge | `1` | GPU compute/encoder/decoder utilization (0.0–1.0) | Yes | Yes | — |
+| `hw.gpu.memory.utilization` | Gauge | `1` | Memory controller utilization (0.0–1.0) | Yes | Yes | — |
+| `hw.gpu.memory.limit` | UpDownCounter | `By` | Total GPU memory | Yes | Yes | — |
+| `hw.gpu.memory.usage` | UpDownCounter | `By` | Used GPU memory | Yes | Yes | — |
+| `hw.gpu.memory.free` | UpDownCounter | `By` | Free GPU memory | Yes | Yes | — |
+| `hw.gpu.temperature` | Gauge | `Cel` | Die or memory temperature | Yes | Yes | Yes |
+| `hw.gpu.fan_speed` | Gauge | `{rpm}` | Fan speed | Yes | Yes | Yes* |
+| `hw.gpu.power.draw` | Gauge | `W` | Current power draw | Yes | Yes | Yes |
+| `hw.gpu.power.limit` | Gauge | `W` | Power limit/cap | Yes | Yes | Yes |
+| `hw.gpu.energy.consumed` | Counter | `J` | Cumulative energy consumed | Yes | Yes | Yes |
+| `hw.gpu.clock.graphics` | Gauge | `MHz` | Graphics/SM clock frequency | Yes | Yes | Yes* |
+| `hw.gpu.clock.memory` | Gauge | `MHz` | Memory clock frequency | Yes | Yes | — |
+| `hw.errors` | Counter | `{error}` | ECC and PCIe error count | Yes | — | — |
+
+\* Intel fan speed requires Linux kernel 6.16+. Intel graphics clock requires the Xe driver.
+
+### Attributes
+
+All GPU metrics carry these base attributes:
+
+| Attribute | Description | Example |
+|---|---|---|
+| `hw.id` | Unique device identifier (required by spec) | `GPU-a1b2c3d4-5678-...` |
+| `hw.name` | Product name | `NVIDIA A100-SXM4-80GB` |
+| `hw.vendor` | Vendor name | `nvidia`, `amd`, `intel` |
+| `gpu.index` | Zero-based device index | `0`, `1` |
+| `gpu.pci_address` | PCI bus address | `0000:01:00.0` |
+
+Additional per-metric attributes:
+
+| Metric | Attribute | Values |
+|---|---|---|
+| `hw.gpu.utilization` | `hw.gpu.task` | `general`, `encoder`, `decoder` |
+| `hw.gpu.temperature` | `sensor` | `die`, `memory` |
+| `hw.errors` | `error.type` | `corrected`, `uncorrected`, `pcie_replay` |
+| `hw.errors` | `hw.type` | `gpu` |
+
+---
+
+## System Metrics
+
+Collected on all platforms (Linux, macOS, Windows) via [gopsutil](https://github.com/shirou/gopsutil). Follows the [OTel semantic conventions for system metrics](https://opentelemetry.io/docs/specs/semconv/system/system-metrics/).
+
+| Metric | Type | Unit | Description | Attributes |
+|---|---|---|---|---|
+| `system.cpu.utilization` | Gauge | `1` | CPU utilization per logical core (0.0–1.0) | `cpu.logical_number` |
+| `system.cpu.logical.count` | UpDownCounter | `{cpu}` | Number of logical CPU cores | |
+| `system.memory.usage` | UpDownCounter | `By` | Memory bytes by state | `system.memory.state`={used,free,cached,buffers} |
+| `system.memory.utilization` | Gauge | `1` | Memory utilization (0.0–1.0) | |
+| `system.disk.io` | Counter | `By` | Disk I/O bytes | `system.device`, `disk.io.direction`={read,write} |
+| `system.disk.operations` | Counter | `{operation}` | Disk I/O operations | `system.device`, `disk.io.direction`={read,write} |
+| `system.filesystem.usage` | UpDownCounter | `By` | Filesystem space by state | `system.device`, `system.filesystem.mountpoint`, `system.filesystem.type`, `system.filesystem.state`={used,free} |
+| `system.filesystem.utilization` | Gauge | `1` | Filesystem utilization (0.0–1.0) | `system.device`, `system.filesystem.mountpoint`, `system.filesystem.type` |
+| `system.network.io` | Counter | `By` | Network I/O bytes | `network.interface.name`, `network.io.direction`={receive,transmit} |
+| `system.network.errors` | Counter | `{error}` | Network errors | `network.interface.name`, `network.io.direction`={receive,transmit} |
+
+<Note>
+`system.memory.state` values `cached` and `buffers` are only reported on Linux. Loopback interfaces (`lo`, `lo0`) are excluded from network metrics.
+</Note>
+
+---
+
+## Process Metrics
+
+Self-monitoring of the collector process. Follows the [OTel semantic conventions for process metrics](https://opentelemetry.io/docs/specs/semconv/system/process-metrics/).
+
+| Metric | Type | Unit | Description | Attributes |
+|---|---|---|---|---|
+| `process.cpu.time` | Counter | `s` | Cumulative CPU time | `cpu.mode`={user,system} |
+| `process.cpu.utilization` | Gauge | `1` | CPU utilization (0.0–1.0) | |
+| `process.memory.usage` | UpDownCounter | `By` | Resident memory (RSS) | |
+| `process.memory.virtual` | UpDownCounter | `By` | Virtual memory size | |
+| `process.thread.count` | UpDownCounter | `{thread}` | OS thread count | |
+| `process.unix.file_descriptor.count` | UpDownCounter | `{file_descriptor}` | Open file descriptors (Linux/macOS) | |
+| `process.runtime.go.goroutines` | Gauge | `{goroutine}` | Go goroutine count | |
+| `process.runtime.go.mem.heap_alloc` | Gauge | `By` | Go heap memory allocated | |
+
+---
+
+## eBPF CUDA Metrics (opt-in)
+
+Enable with `OTEL_GPU_EBPF_ENABLED=true`. Requires Linux, `CAP_BPF` + `CAP_PERFMON` (or root), and an NVIDIA CUDA runtime (`libcudart.so`). Attaches uprobes to `cudaLaunchKernel`, `cudaMalloc`, and `cudaMemcpy`.
+
+| Metric | Type | Unit | Description | Attributes |
+|---|---|---|---|---|
+| `gpu.kernel.launch.calls` | Counter | `{call}` | CUDA kernel launch count | `cuda.kernel.name` |
+| `gpu.kernel.grid.size` | Histogram | `{thread}` | Total threads in grid per launch | `cuda.kernel.name` |
+| `gpu.kernel.block.size` | Histogram | `{thread}` | Threads per block per launch | `cuda.kernel.name` |
+| `gpu.memory.allocations` | Counter | `By` | Bytes allocated via cudaMalloc | |
+| `gpu.memory.copies` | Histogram | `By` | Bytes per cudaMemcpy call | `cuda.memcpy.kind`={HostToHost,HostToDevice,DeviceToHost,DeviceToDevice} |
diff --git a/docs/latest/gpu-collector/overview.mdx b/docs/latest/gpu-collector/overview.mdx
new file mode 100644
index 00000000..d381ec1d
--- /dev/null
+++ b/docs/latest/gpu-collector/overview.mdx
@@ -0,0 +1,92 @@
+---
+title: 'Overview'
+description: 'OpenTelemetry-native GPU and host metrics collector for NVIDIA, AMD, and Intel GPUs'
+---
+
+The **OpenTelemetry GPU Collector** is a lightweight, single-binary metrics collector written in Go. It exports GPU hardware telemetry, host system metrics, and process metrics via OpenTelemetry (OTLP) — with no Python dependencies, no DCGM daemon, and no vendor-specific agents.
+
+It is fully configured via standard OpenTelemetry environment variables and follows the [OTel semantic conventions for hardware metrics](https://opentelemetry.io/docs/specs/semconv/hardware/gpu/).
+
+## Goals
+
+* **OpenTelemetry-native** — uses standard `OTEL_*` env vars, exports via OTLP gRPC or HTTP to any OTel-compatible backend
+* **Cross-vendor GPU support** — NVIDIA (NVML), AMD (sysfs/hwmon), and Intel (i915/Xe sysfs) from a single binary
+* **OTel semantic conventions** — `hw.gpu.*` metric names, `hw.id` / `hw.name` / `hw.vendor` attributes per spec
+* **Zero dependencies** — no DCGM, no Python, no CUDA toolkit needed at runtime for hardware metrics
+* **Resilient** — continues exporting host metrics even when no GPUs are present; retries GPU discovery every 30s
+
+## What it collects
+
+<CardGroup cols={3}>
+ <Card title="GPU Hardware Telemetry" icon="microchip">
+ Utilization, memory, temperature, power draw, energy, clock speeds, ECC errors, PCIe errors — for NVIDIA, AMD, and Intel GPUs
+ </Card>
+ <Card title="Host System Metrics" icon="server">
+ CPU utilization, memory usage, disk I/O, filesystem usage, and network I/O — on Linux, macOS, and Windows
+ </Card>
+ <Card title="eBPF CUDA Tracing" icon="code">
+ Kernel launch counts, grid/block sizes, memory allocations, and memory copies — via uprobes on libcudart.so (opt-in, Linux only)
+ </Card>
+</CardGroup>
+
+## GPU vendor support
+
+| Vendor | Backend | Metrics available |
+|---|---|---|
+| **NVIDIA** | NVML via [go-nvml](https://github.com/NVIDIA/go-nvml) — loads `libnvidia-ml.so` at runtime | Utilization, memory, temperature, power, energy, clocks, ECC errors, PCIe errors, fan speed |
+| **AMD** | sysfs + hwmon — zero external dependencies | Utilization, memory, temperature, power, energy, fan speed |
+| **Intel** | sysfs + hwmon + DRM (i915/Xe driver) | Temperature, power draw/limit, cumulative energy, graphics clock, fan speed (kernel 6.16+) |
+
+## Platform support
+
+| Feature | Linux | macOS | Windows |
+|---|:---:|:---:|:---:|
+| System metrics (CPU, memory, disk, network) | Yes | Yes | Yes |
+| Process metrics (CPU, memory, threads, FDs) | Yes | Yes | Yes |
+| GPU metrics (NVIDIA, AMD, Intel) | Yes | — | — |
+| eBPF CUDA tracing | Yes | — | — |
+
+## How it works
+
+```
+Host Metrics (all platforms via gopsutil)
+ +-- CPU utilization, memory, disk I/O, filesystem, network
+ +-- Process: self CPU, memory, threads, FDs, Go runtime
+
+GPU Metrics (Linux only)
+ +-- PCI Bus Scan (/sys/bus/pci/devices/)
+ | +-- NVIDIA (0x10de) --> NVML backend
+ | +-- AMD (0x1002) --> sysfs/hwmon backend
+ | +-- Intel (0x8086) --> sysfs/hwmon + DRM backend
+ |
+ +-- [Optional: eBPF CUDA tracing via uprobes on libcudart.so]
+
+Export
+ +-- OTel SDK --> OTLP gRPC/HTTP --> your OTel collector / backend
+```
+
+<Steps>
+ <Step title="Scans PCI bus">
+ On startup, the collector scans `/sys/bus/pci/devices/` for PCI class codes `0x0300` (VGA), `0x0302` (3D controller), and `0x0380` (display controller), identifying GPU vendor from the PCI vendor ID.
+ </Step>
+ <Step title="Initialises vendor backend">
+ Each detected GPU is handed to its vendor-specific backend. NVIDIA loads `libnvidia-ml.so` via NVML. AMD and Intel read directly from kernel sysfs/hwmon — no additional libraries needed.
+ </Step>
+ <Step title="Registers OTel instruments">
+ Observable gauge and counter instruments are registered with the OTel SDK meter. On each collection tick, the SDK calls back into the collector to read fresh values from each GPU.
+ </Step>
+ <Step title="Exports via OTLP">
+ Metrics are exported via OTLP to any compatible backend — OpenLIT, Grafana, Datadog, New Relic, or a standard OTel Collector.
+ </Step>
+</Steps>
+
+---
+
+<CardGroup cols={2}>
+ <Card title="Quickstart" href="/latest/gpu-collector/quickstart" icon="bolt">
+ Get the collector running in under 5 minutes with Docker
+ </Card>
+ <Card title="Configuration" href="/latest/gpu-collector/configuration" icon="sliders">
+ Full reference for all environment variables
+ </Card>
+</CardGroup>
diff --git a/docs/latest/gpu-collector/quickstart.mdx b/docs/latest/gpu-collector/quickstart.mdx
new file mode 100644
index 00000000..635a95d3
--- /dev/null
+++ b/docs/latest/gpu-collector/quickstart.mdx
@@ -0,0 +1,143 @@
+---
+title: 'Quickstart'
+description: 'Get the OpenTelemetry GPU Collector running in under 5 minutes'
+icon: 'bolt'
+---
+
+In this guide you'll pull the collector Docker image, point it at your OTel backend, and start seeing GPU and host metrics within minutes.
+
+## Prerequisites
+
+- **Linux host** with NVIDIA, AMD, or Intel GPU (for GPU metrics)
+- **Docker** installed
+- An **OpenTelemetry-compatible backend** (OpenLIT, Grafana, Datadog, or any OTLP endpoint)
+
+<AccordionGroup>
+ <Accordion title="Don't have an OTel backend? Start OpenLIT locally">
+ ```bash
+ docker run -d \
+ --name openlit \
+ -p 3000:3000 \
+ -p 4318:4318 \
+ ghcr.io/openlit/openlit:latest
+ ```
+ Then use `http://localhost:4318` as your `OTEL_EXPORTER_OTLP_ENDPOINT`.
+ </Accordion>
+</AccordionGroup>
+
+<Steps titleSize="h3">
+ <Step title="Pull the collector image">
+ ```bash
+ docker pull ghcr.io/openlit/otel-gpu-collector:latest
+ ```
+ </Step>
+
+ <Step title="Run the collector">
+
+ <Tabs>
+ <Tab title="NVIDIA GPU">
+ ```bash
+ docker run -d \
+ --name otel-gpu-collector \
+ --gpus all \
+ -e OTEL_SERVICE_NAME=my-app \
+ -e OTEL_RESOURCE_ATTRIBUTES='deployment.environment=production' \
+ -e OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318 \
+ ghcr.io/openlit/otel-gpu-collector:latest
+ ```
+ Requires the [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html) on the host.
+ </Tab>
+ <Tab title="AMD GPU">
+ ```bash
+ docker run -d \
+ --name otel-gpu-collector \
+ --device /dev/kfd:/dev/kfd \
+ --device /dev/dri:/dev/dri \
+ -e OTEL_SERVICE_NAME=my-app \
+ -e OTEL_RESOURCE_ATTRIBUTES='deployment.environment=production' \
+ -e OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318 \
+ ghcr.io/openlit/otel-gpu-collector:latest
+ ```
+ </Tab>
+ <Tab title="Intel GPU">
+ ```bash
+ docker run -d \
+ --name otel-gpu-collector \
+ --device /dev/dri:/dev/dri \
+ -e OTEL_SERVICE_NAME=my-app \
+ -e OTEL_RESOURCE_ATTRIBUTES='deployment.environment=production' \
+ -e OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318 \
+ ghcr.io/openlit/otel-gpu-collector:latest
+ ```
+ Requires Linux kernel 5.10+ with the i915 or Xe driver.
+ </Tab>
+ <Tab title="Host metrics only">
+ ```bash
+ docker run -d \
+ --name otel-gpu-collector \
+ -e OTEL_SERVICE_NAME=my-app \
+ -e OTEL_RESOURCE_ATTRIBUTES='deployment.environment=production' \
+ -e OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318 \
+ ghcr.io/openlit/otel-gpu-collector:latest
+ ```
+ The collector will export host and process metrics even without GPU access.
+ </Tab>
+ </Tabs>
+ </Step>
+
+ <Step title="Verify it's running">
+ ```bash
+ docker logs otel-gpu-collector
+ ```
+
+ You should see output like:
+ ```
+ time=2024-01-01T00:00:00Z level=INFO msg="starting opentelemetry-gpu-collector"
+ time=2024-01-01T00:00:00Z level=INFO msg="discovered GPU" address=0000:01:00.0 vendor=nvidia
+ time=2024-01-01T00:00:00Z level=INFO msg="system metrics collector initialized"
+ time=2024-01-01T00:00:00Z level=INFO msg="process metrics collector initialized"
+ time=2024-01-01T00:00:00Z level=INFO msg="collector running"
+ ```
+ </Step>
+
+ <Step title="View metrics in your backend">
+ Open your OTel backend and look for metrics in the `hw.gpu.*`, `system.*`, and `process.*` namespaces.
+
+ If using OpenLIT, navigate to `http://localhost:3000` and go to the **Metrics** section.
+ </Step>
+</Steps>
+
+## Docker Compose
+
+Add the collector as a service alongside your existing stack:
+
+```yaml
+services:
+ otel-gpu-collector:
+ image: ghcr.io/openlit/otel-gpu-collector:latest
+ environment:
+ OTEL_SERVICE_NAME: my-app
+ OTEL_RESOURCE_ATTRIBUTES: deployment.environment=production
+ OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318
+ deploy:
+ resources:
+ reservations:
+ devices:
+ - driver: nvidia
+ count: all
+ capabilities: [gpu]
+ depends_on:
+ - otel-collector
+ restart: always
+```
+
+---
+
+<CardGroup cols={2}>
+ <Card title="Configuration" href="/latest/gpu-collector/configuration" icon="sliders">
+ Full reference for all environment variables and defaults
+ </Card>
+ <Card title="Metrics reference" href="/latest/gpu-collector/metrics" icon="table">
+ Complete list of all metrics, types, units, and attributes
+ </Card>
+</CardGroup>
diff --git a/docs/latest/sdk/integrations/amd-gpu.mdx b/docs/latest/sdk/integrations/amd-gpu.mdx
index 0e4d9995..88ca73fa 100644
--- a/docs/latest/sdk/integrations/amd-gpu.mdx
+++ b/docs/latest/sdk/integrations/amd-gpu.mdx
@@ -49,14 +49,14 @@ OpenLIT uses OpenTelemetry to help you monitor AMD GPUs. This includes tracking
```sh
docker run --gpus all \
- -e GPU_APPLICATION_NAME='chatbot' \
- -e GPU_ENVIRONMENT='staging' \
+ -e OTEL_SERVICE_NAME='chatbot' \
+ -e OTEL_RESOURCE_ATTRIBUTES='deployment.environment=staging' \
-e OTEL_EXPORTER_OTLP_ENDPOINT="YOUR_OTEL_ENDPOINT" \
-e OTEL_EXPORTER_OTLP_HEADERS="YOUR_OTEL_HEADERS" \
ghcr.io/openlit/otel-gpu-collector:latest
```
- For more advanced configurations of the collector, visit the [OTel GPU Collector repository](https://github.com/openlit/openlit/tree/main/otel-gpu-collector/).
+ For more advanced configurations of the collector, visit the [OTel GPU Collector repository](https://github.com/openlit/openlit/tree/main/opentelemetry-gpu-collector/).
**Note:** If you've deployed **OpenLIT** using [Docker Compose](https://github.com/openlit/openlit/blob/main/docker-compose.yml), make sure to use the host's IP address or add OTel GPU Collector to the [Docker Compose](https://github.com/openlit/openlit/blob/main/docker-compose.yml):
@@ -67,8 +67,8 @@ OpenLIT uses OpenTelemetry to help you monitor AMD GPUs. This includes tracking
otel-gpu-collector:
image: ghcr.io/openlit/otel-gpu-collector:latest
environment:
- GPU_APPLICATION_NAME: 'chatbot'
- GPU_ENVIRONMENT: 'staging'
+ OTEL_SERVICE_NAME: 'chatbot'
+ OTEL_RESOURCE_ATTRIBUTES: 'deployment.environment=staging'
OTEL_EXPORTER_OTLP_ENDPOINT: "http://otel-collector:4318"
device_requests:
- driver: nvidia
@@ -90,14 +90,15 @@ OpenLIT uses OpenTelemetry to help you monitor AMD GPUs. This includes tracking
</AccordionGroup>
### Environment Variables
- OTel GPU Collector supports several environment variables for configuration. Below is a table that describes each variable:
+ OTel GPU Collector uses standard OpenTelemetry environment variables for configuration:
- | Environment Variable | Description | Default Value |
- |---------------------------------|---------------------------------------------------------------|-------------------------|
- | `GPU_APPLICATION_NAME` | Name of the application running on the GPU | `default_app` |
- | `GPU_ENVIRONMENT` | Environment name (e.g., staging, production) | `production` |
- | `OTEL_EXPORTER_OTLP_ENDPOINT` | OpenTelemetry OTLP endpoint URL | (required) |
- | `OTEL_EXPORTER_OTLP_HEADERS` | Headers for authenticating with the OTLP endpoint | Ignore if using OpenLIT |
+ | Environment Variable | Description | Default Value |
+ |-----------------------------------|---------------------------------------------------------------|-------------------------|
+ | `OTEL_SERVICE_NAME` | Service/application name attached to all metrics | `default` |
+ | `OTEL_RESOURCE_ATTRIBUTES` | Resource attributes, e.g. `deployment.environment=production` | `deployment.environment=default` |
+ | `OTEL_EXPORTER_OTLP_ENDPOINT` | OpenTelemetry OTLP endpoint URL | (required) |
+ | `OTEL_EXPORTER_OTLP_HEADERS` | Headers for authenticating with the OTLP endpoint | Ignore if using OpenLIT |
+ | `OTEL_METRIC_EXPORT_INTERVAL` | Metric polling interval in milliseconds | `10000` |
</Step>
</Steps>
</Accordion>
diff --git a/docs/latest/sdk/integrations/intel-gpu.mdx b/docs/latest/sdk/integrations/intel-gpu.mdx
new file mode 100644
index 00000000..cafa7a7d
--- /dev/null
+++ b/docs/latest/sdk/integrations/intel-gpu.mdx
@@ -0,0 +1,137 @@
+---
+title: 'Monitor Intel GPUs using OpenTelemetry'
+sidebarTitle: 'Intel GPUs'
+---
+
+import IntegrationMethods from '/snippets/integration-methods.mdx';
+
+OpenLIT uses OpenTelemetry to help you monitor Intel GPUs. This includes tracking GPU metrics like temperature, power consumption, energy usage, and clock frequency via the Linux i915/Xe kernel driver.
+
+<Note>
+Intel GPU support requires Linux with the **i915** or **Xe** kernel driver (kernel 5.10+). Metrics are read directly from the kernel's sysfs/hwmon interface — no additional software or libraries are required. Utilization and memory metrics are not available via this interface; use [Intel GPU Top](https://github.com/intel/igt-gpu-tools) or [XPUManager](https://github.com/intel/xpumanager) for those.
+</Note>
+
+## Collected Metrics
+
+| Metric | Description | Requirement |
+|---|---|---|
+| `hw.gpu.temperature` | Die temperature | hwmon `temp1_input` |
+| `hw.gpu.power.draw` | Current power draw (W) | hwmon `power1_average` |
+| `hw.gpu.power.limit` | Power cap (W) | hwmon `power1_max` |
+| `hw.gpu.energy.consumed` | Cumulative energy (J) | hwmon `energy1_input` |
+| `hw.gpu.clock.graphics` | Current graphics clock (MHz) | DRM `gt_cur_freq_mhz` |
+| `hw.gpu.fan_speed` | Fan speed (RPM) | hwmon `fan1_input`, kernel 6.16+ |
+
+## Get started
+
+<CardGroup cols={2}>
+ <Card title="Using the SDK" href="#using-the-sdk" icon="file-code">
+ Collect and send GPU performance metrics directly from your application to an OpenTelemetry endpoint.
+ </Card>
+ <Card title="Using the Collector" href="#using-the-collector" icon="robot">
+ Install the OpenTelemetry GPU Collector as a Docker container to collect and send GPU performance metrics to an OpenTelemetry endpoint.
+ </Card>
+</CardGroup>
+
+
+<AccordionGroup>
+ <Accordion title="Using the SDK">
+ <Steps>
+ <Step title="Install OpenLIT">
+ Open your command line or terminal and run:
+ ```shell
+ pip install openlit
+ ```
+ </Step>
+ <IntegrationMethods />
+ </Steps>
+ </Accordion>
+ <Accordion title="Using the Collector">
+ <Steps>
+ <Step title="Pull `otel-gpu-collector` Docker Image">
+
+ You can quickly start using the OTel GPU Collector by pulling the Docker image:
+
+ ```sh
+ docker pull ghcr.io/openlit/otel-gpu-collector:latest
+ ```
+ </Step>
+ <Step title="Run `otel-gpu-collector` Docker container">
+
+ Here's a quick example showing how to run the container with the required environment variables.
+
+ For Intel GPUs, pass the DRM device into the container using `--device`:
+
+ ```sh
+ docker run \
+ --device /dev/dri:/dev/dri \
+ -e OTEL_SERVICE_NAME='my-app' \
+ -e OTEL_RESOURCE_ATTRIBUTES='deployment.environment=staging' \
+ -e OTEL_EXPORTER_OTLP_ENDPOINT="YOUR_OTEL_ENDPOINT" \
+ -e OTEL_EXPORTER_OTLP_HEADERS="YOUR_OTEL_HEADERS" \
+ ghcr.io/openlit/otel-gpu-collector:latest
+ ```
+
+ For more advanced configurations of the collector, visit the [OTel GPU Collector repository](https://github.com/openlit/openlit/tree/main/opentelemetry-gpu-collector/).
+
+ **Note:** If you've deployed **OpenLIT** using [Docker Compose](https://github.com/openlit/openlit/blob/main/docker-compose.yml), make sure to use the host's IP address or add OTel GPU Collector to the [Docker Compose](https://github.com/openlit/openlit/blob/main/docker-compose.yml):
+
+ <AccordionGroup>
+ <Accordion title="Docker Compose: Add the following config under `services`">
+
+ ```yaml
+ otel-gpu-collector:
+ image: ghcr.io/openlit/otel-gpu-collector:latest
+ environment:
+ OTEL_SERVICE_NAME: 'my-app'
+ OTEL_RESOURCE_ATTRIBUTES: 'deployment.environment=staging'
+ OTEL_EXPORTER_OTLP_ENDPOINT: "http://otel-collector:4318"
+ devices:
+ - /dev/dri:/dev/dri
+ depends_on:
+ - otel-collector
+ restart: always
+ ```
+
+ </Accordion>
+ <Accordion title="Host IP: Use the Host IP to connect to OTel Collector">
+
+ ```sh
+ OTEL_EXPORTER_OTLP_ENDPOINT="http://192.168.10.15:4318"
+ ```
+
+ </Accordion>
+ </AccordionGroup>
+ ### Environment Variables
+
+ OTel GPU Collector uses standard OpenTelemetry environment variables for configuration:
+
+ | Environment Variable | Description | Default Value |
+ |-----------------------------------|---------------------------------------------------------------|-------------------------|
+ | `OTEL_SERVICE_NAME` | Service/application name attached to all metrics | `default` |
+ | `OTEL_RESOURCE_ATTRIBUTES` | Resource attributes, e.g. `deployment.environment=production` | `deployment.environment=default` |
+ | `OTEL_EXPORTER_OTLP_ENDPOINT` | OpenTelemetry OTLP endpoint URL | (required) |
+ | `OTEL_EXPORTER_OTLP_HEADERS` | Headers for authenticating with the OTLP endpoint | Ignore if using OpenLIT |
+ | `OTEL_METRIC_EXPORT_INTERVAL` | Metric polling interval in milliseconds | `10000` |
+ </Step>
+ </Steps>
+ </Accordion>
+</AccordionGroup>
+
+<Card title="Collected Metrics" href="/latest/openlit/observability/metrics#gpu-metrics" icon='table'>
+Details on the types of metrics collected and their descriptions.
+</Card>
+
+---
+
+<CardGroup cols={3}>
+ <Card title="Quickstart: LLM Observability" href="/latest/sdk/quickstart-ai-observability" icon='bolt'>
+ Production-ready AI monitoring setup in 2 simple steps with zero code changes
+ </Card>
+ <Card title="Configuration" href="/latest/sdk/configuration" icon='bolt'>
+ Configure the OpenLIT SDK according to your requirements.
+ </Card>
+ <Card title="Destinations" href="/latest/sdk/destinations/overview" icon='link'>
+ Send telemetry to Datadog, Grafana, New Relic, and other observability stacks
+ </Card>
+</CardGroup>
diff --git a/docs/latest/sdk/integrations/nvidia-gpu.mdx b/docs/latest/sdk/integrations/nvidia-gpu.mdx
index 088a96e0..066c13fa 100644
--- a/docs/latest/sdk/integrations/nvidia-gpu.mdx
+++ b/docs/latest/sdk/integrations/nvidia-gpu.mdx
@@ -49,14 +49,14 @@ OpenLIT uses OpenTelemetry to help you monitor NVIDIA GPUs. This includes tracki
```sh
docker run --gpus all \
- -e GPU_APPLICATION_NAME='chatbot' \
- -e GPU_ENVIRONMENT='staging' \
+ -e OTEL_SERVICE_NAME='chatbot' \
+ -e OTEL_RESOURCE_ATTRIBUTES='deployment.environment=staging' \
-e OTEL_EXPORTER_OTLP_ENDPOINT="YOUR_OTEL_ENDPOINT" \
-e OTEL_EXPORTER_OTLP_HEADERS="YOUR_OTEL_HEADERS" \
ghcr.io/openlit/otel-gpu-collector:latest
```
- For more advanced configurations of the collector, visit the [OTel GPU Collector repository](https://github.com/openlit/openlit/tree/main/otel-gpu-collector/).
+ For more advanced configurations of the collector, visit the [OTel GPU Collector repository](https://github.com/openlit/openlit/tree/main/opentelemetry-gpu-collector/).
**Note:** If you've deployed **OpenLIT** using [Docker Compose](https://github.com/openlit/openlit/blob/main/docker-compose.yml), make sure to use the host's IP address or add OTel GPU Collector to the [Docker Compose](https://github.com/openlit/openlit/blob/main/docker-compose.yml):
@@ -67,8 +67,8 @@ OpenLIT uses OpenTelemetry to help you monitor NVIDIA GPUs. This includes tracki
otel-gpu-collector:
image: ghcr.io/openlit/otel-gpu-collector:latest
environment:
- GPU_APPLICATION_NAME: 'chatbot'
- GPU_ENVIRONMENT: 'staging'
+ OTEL_SERVICE_NAME: 'chatbot'
+ OTEL_RESOURCE_ATTRIBUTES: 'deployment.environment=staging'
OTEL_EXPORTER_OTLP_ENDPOINT: "http://otel-collector:4318"
device_requests:
- driver: nvidia
@@ -90,14 +90,15 @@ OpenLIT uses OpenTelemetry to help you monitor NVIDIA GPUs. This includes tracki
</AccordionGroup>
### Environment Variables
- OTel GPU Collector supports several environment variables for configuration. Below is a table that describes each variable:
+ OTel GPU Collector uses standard OpenTelemetry environment variables for configuration:
- | Environment Variable | Description | Default Value |
- |---------------------------------|---------------------------------------------------------------|-------------------------|
- | `GPU_APPLICATION_NAME` | Name of the application running on the GPU | `default_app` |
- | `GPU_ENVIRONMENT` | Environment name (e.g., staging, production) | `production` |
- | `OTEL_EXPORTER_OTLP_ENDPOINT` | OpenTelemetry OTLP endpoint URL | (required) |
- | `OTEL_EXPORTER_OTLP_HEADERS` | Headers for authenticating with the OTLP endpoint | Ignore if using OpenLIT |
+ | Environment Variable | Description | Default Value |
+ |-----------------------------------|---------------------------------------------------------------|-------------------------|
+ | `OTEL_SERVICE_NAME` | Service/application name attached to all metrics | `default` |
+ | `OTEL_RESOURCE_ATTRIBUTES` | Resource attributes, e.g. `deployment.environment=production` | `deployment.environment=default` |
+ | `OTEL_EXPORTER_OTLP_ENDPOINT` | OpenTelemetry OTLP endpoint URL | (required) |
+ | `OTEL_EXPORTER_OTLP_HEADERS` | Headers for authenticating with the OTLP endpoint | Ignore if using OpenLIT |
+ | `OTEL_METRIC_EXPORT_INTERVAL` | Metric polling interval in milliseconds | `10000` |
</Step>
</Steps>
</Accordion>
diff --git a/docs/latest/sdk/integrations/overview.mdx b/docs/latest/sdk/integrations/overview.mdx
index b2b9433d..c270ec8e 100644
--- a/docs/latest/sdk/integrations/overview.mdx
+++ b/docs/latest/sdk/integrations/overview.mdx
@@ -3043,6 +3043,19 @@ Start integrating and monitoring your applications with OpenLIT in just one line
</div>
</Card>
+ <Card
+ title="Intel"
+ href="/latest/sdk/integrations/intel-gpu"
+ iconType="duotone"
+ icon={
+ <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 48 48" width="48" height="48"><path d="M24 4C13 4 4 13 4 24s9 20 20 20 20-9 20-20S35 4 24 4zm-2 28h-4V20h4v12zm-2-14a2 2 0 1 1 0-4 2 2 0 0 1 0 4zm10 14h-4V20h4v12z" fill="#0068b5"/></svg>
+ }
+ >
+ <div style={{position: "absolute", top: "12px", right: "12px", display: "flex", gap: "4px"}}>
+ <img src="https://cdn.simpleicons.org/python/ffa500" width="16" height="16" alt="Python SDK" title="Python SDK" />
+ </div>
+
+ </Card>
</CardGroup>
---
diff --git a/docs/snippets/quickstart-gpu.mdx b/docs/snippets/quickstart-gpu.mdx
index 703bd7a2..9c12f2c3 100644
--- a/docs/snippets/quickstart-gpu.mdx
+++ b/docs/snippets/quickstart-gpu.mdx
@@ -236,13 +236,13 @@
```sh
docker run --gpus all \
- -e GPU_APPLICATION_NAME='chatbot' \
- -e GPU_ENVIRONMENT='staging' \
+ -e OTEL_SERVICE_NAME='chatbot' \
+ -e OTEL_RESOURCE_ATTRIBUTES='deployment.environment=staging' \
-e OTEL_EXPORTER_OTLP_ENDPOINT="http://127.0.0.1:4318" \
ghcr.io/openlit/otel-gpu-collector:latest
```
- For more advanced configurations of the collector, visit the [OTel GPU Collector repository](https://github.com/openlit/openlit/tree/main/otel-gpu-collector/).
+ For more advanced configurations of the collector, visit the [OTel GPU Collector repository](https://github.com/openlit/openlit/tree/main/opentelemetry-gpu-collector/).
**Note:** If you've deployed **OpenLIT** using [Docker Compose](https://github.com/openlit/openlit/blob/main/docker-compose.yml), make sure to use the host's IP address or add OTel GPU Collector to the [Docker Compose](https://github.com/openlit/openlit/blob/main/docker-compose.yml):
@@ -253,8 +253,8 @@
otel-gpu-collector:
image: ghcr.io/openlit/otel-gpu-collector:latest
environment:
- GPU_APPLICATION_NAME: 'chatbot'
- GPU_ENVIRONMENT: 'staging'
+ OTEL_SERVICE_NAME: 'chatbot'
+ OTEL_RESOURCE_ATTRIBUTES: 'deployment.environment=staging'
OTEL_EXPORTER_OTLP_ENDPOINT: "http://otel-collector:4318"
device_requests:
- driver: nvidia
diff --git a/opentelemetry-gpu-collector/.gitignore b/opentelemetry-gpu-collector/.gitignore
new file mode 100644
index 00000000..6a4503bb
--- /dev/null
+++ b/opentelemetry-gpu-collector/.gitignore
@@ -0,0 +1,14 @@
+# Binary
+openlit-collector
+
+# Generated eBPF Go bindings (output of go generate / bpf2go)
+internal/ebpf/gpuevent_bpfel*.go
+internal/ebpf/gpuevent_bpfel*.o
+
+# Copied/generated eBPF build headers (from setup-bpf)
+internal/ebpf/bpf/vmlinux.h
+internal/ebpf/bpf/bpf_helpers.h
+internal/ebpf/bpf/bpf_helper_defs.h
+internal/ebpf/bpf/bpf_tracing.h
+internal/ebpf/bpf/bpf_core_read.h
+internal/ebpf/bpf/bpf_endian.h
diff --git a/opentelemetry-gpu-collector/Dockerfile b/opentelemetry-gpu-collector/Dockerfile
new file mode 100644
index 00000000..d9c107f2
--- /dev/null
+++ b/opentelemetry-gpu-collector/Dockerfile
@@ -0,0 +1,41 @@
+# Stage 1: Build the eBPF programs and Go binary
+FROM golang:1.25-bookworm AS builder
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+ clang \
+ llvm \
+ libbpf-dev \
+ bpftool \
+ make \
+ && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /src
+COPY go.mod go.sum ./
+RUN go mod download
+
+COPY . .
+
+# Generate eBPF Go bindings.
+# vmlinux.h is generated from the build host's kernel BTF; at runtime the
+# eBPF verifier uses the actual host kernel's BTF for CO-RE relocations.
+RUN if [ -f /sys/kernel/btf/vmlinux ]; then \
+ make setup-bpf && make generate; \
+ else \
+ echo "WARNING: /sys/kernel/btf/vmlinux not available — skipping eBPF generation. Pre-generated bindings must exist."; \
+ fi
+
+RUN GOOS=linux go build -o /opentelemetry-gpu-collector ./cmd/collector/
+
+# Stage 2: Minimal runtime image
+FROM debian:bookworm-slim
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+ ca-certificates \
+ && rm -rf /var/lib/apt/lists/*
+
+COPY --from=builder /opentelemetry-gpu-collector /usr/local/bin/opentelemetry-gpu-collector
+
+ENV OTEL_SERVICE_NAME=default
+ENV OTEL_RESOURCE_ATTRIBUTES=deployment.environment=default
+
+ENTRYPOINT ["/usr/local/bin/opentelemetry-gpu-collector"]
diff --git a/opentelemetry-gpu-collector/Makefile b/opentelemetry-gpu-collector/Makefile
new file mode 100644
index 00000000..56f12722
--- /dev/null
+++ b/opentelemetry-gpu-collector/Makefile
@@ -0,0 +1,46 @@
+.PHONY: all build generate setup-bpf test lint clean docker
+
+BINARY := opentelemetry-gpu-collector
+CMD_DIR := ./cmd/collector
+BPF_DIR := ./internal/ebpf
+
+GO := go
+
+build:
+ $(GO) build -o $(BINARY) $(CMD_DIR)
+
+all: setup-bpf generate build
+
+setup-bpf:
+ @echo "==> Checking eBPF build dependencies..."
+ @command -v clang >/dev/null 2>&1 || (echo "Installing clang/llvm..." && sudo apt-get update && sudo apt-get install -y clang llvm)
+ @command -v bpftool >/dev/null 2>&1 || { echo "bpftool not found. Download from https://github.com/libbpf/bpftool/releases or install linux-tools-generic" && exit 1; }
+ @dpkg -s libbpf-dev >/dev/null 2>&1 || (echo "Installing libbpf-dev..." && sudo apt-get update && sudo apt-get install -y libbpf-dev)
+ @echo "==> Generating vmlinux.h from kernel BTF..."
+ bpftool btf dump file /sys/kernel/btf/vmlinux format c > $(BPF_DIR)/bpf/vmlinux.h
+ @echo "==> Copying libbpf headers..."
+ cp /usr/include/bpf/bpf_helpers.h $(BPF_DIR)/bpf/
+ cp /usr/include/bpf/bpf_helper_defs.h $(BPF_DIR)/bpf/
+ cp /usr/include/bpf/bpf_tracing.h $(BPF_DIR)/bpf/
+ -cp /usr/include/bpf/bpf_core_read.h $(BPF_DIR)/bpf/ 2>/dev/null
+ -cp /usr/include/bpf/bpf_endian.h $(BPF_DIR)/bpf/ 2>/dev/null
+ @echo "==> eBPF headers ready."
+
+generate:
+ cd $(BPF_DIR) && $(GO) generate ./...
+
+test:
+ $(GO) test ./...
+
+lint:
+ golangci-lint run ./...
+
+clean:
+ rm -f $(BINARY)
+ rm -f $(BPF_DIR)/gpuevent_bpfel*.go $(BPF_DIR)/gpuevent_bpfel*.o
+ rm -f $(BPF_DIR)/bpf/vmlinux.h
+ rm -f $(BPF_DIR)/bpf/bpf_helpers.h $(BPF_DIR)/bpf/bpf_helper_defs.h
+ rm -f $(BPF_DIR)/bpf/bpf_tracing.h $(BPF_DIR)/bpf/bpf_core_read.h $(BPF_DIR)/bpf/bpf_endian.h
+
+docker:
+ docker build -t opentelemetry-gpu-collector:latest .
diff --git a/opentelemetry-gpu-collector/README.md b/opentelemetry-gpu-collector/README.md
new file mode 100644
index 00000000..928deeac
--- /dev/null
+++ b/opentelemetry-gpu-collector/README.md
@@ -0,0 +1,275 @@
+<div align="center">
+<img src="https://github.com/openlit/.github/blob/main/profile/assets/wide-logo-no-bg.png?raw=true" alt="OpenLIT Logo" width="30%"><h1>
+OpenTelemetry GPU Collector</h1>
+
+**[Documentation](https://docs.openlit.io/latest/features/gpu) | [Quickstart](#-getting-started) | [Metrics](#-metrics) | [Configuration](#-configuration)**
+
+**[Roadmap](#%EF%B8%8F-roadmap) | [Feature Request](https://github.com/openlit/openlit/issues/new?assignees=&labels=%3Araised_hand%3A+Up+for+Grabs%2C+%3Arocket%3A+Feature&projects=&template=feature-request.md&title=%5BFeat%5D%3A) | [Report a Bug](https://github.com/openlit/openlit/issues/new?assignees=&labels=%3Abug%3A+Bug%2C+%3Araised_hand%3A+Up+for+Grabs&projects=&template=bug.md&title=%5BBug%5D%3A)**
+
+[![OpenLIT](https://img.shields.io/badge/OpenLIT-orange)](https://openlit.io/)
+[![License](https://img.shields.io/github/license/openlit/openlit?label=License&logo=github&color=f80&logoColor=white)](https://github.com/openlit/openlit/blob/main/LICENSE)
+[![GitHub Last Commit](https://img.shields.io/github/last-commit/openlit/openlit)](https://github.com/openlit/openlit/pulse)
+[![GitHub Contributors](https://img.shields.io/github/contributors/openlit/openlit)](https://github.com/openlit/openlit/graphs/contributors)
+
+[![Slack](https://img.shields.io/badge/Slack-4A154B?logo=slack&logoColor=white)](https://join.slack.com/t/openlit/shared_invite/zt-2etnfttwg-TjP_7BZXfYg84oAukY8QRQ)
+[![X](https://img.shields.io/badge/follow-%40openlit__io-1DA1F2?logo=x&style=social)](https://twitter.com/openlit_io)
+
+</div>
+
+A high-performance host and GPU metrics collector written in Go. Exports host-level system metrics (CPU, memory, disk, network), process metrics, DCGM-style GPU hardware telemetry, and optional eBPF-based CUDA kernel tracing — all via OpenTelemetry (OTLP).
+
+Metric names and attributes follow the [OpenTelemetry semantic conventions for hardware](https://opentelemetry.io/docs/specs/semconv/hardware/gpu/) and [system metrics](https://opentelemetry.io/docs/specs/semconv/system/).
+
+## Features
+
+- **OpenTelemetry-native** — standard env vars (`OTEL_EXPORTER_OTLP_ENDPOINT`, `OTEL_METRIC_EXPORT_INTERVAL`, `OTEL_RESOURCE_ATTRIBUTES`), exports via OTLP gRPC or HTTP
+- **OTel semantic conventions** — `hw.gpu.*` metric names, `hw.id` / `hw.name` / `hw.vendor` attributes
+- **Host metrics** — CPU utilization, memory, disk I/O, filesystem, network I/O (Linux, macOS, Windows)
+- **Process metrics** — self-process CPU, memory, threads, file descriptors, Go runtime stats
+- **Cross-vendor GPU support** — NVIDIA (via NVML), AMD (via sysfs/hwmon), Intel (via sysfs/hwmon + DRM) on Linux
+- **eBPF CUDA tracing** (opt-in) — kernel launch counts, grid/block sizes, memory allocations, memory copies
+- **Lightweight** — single static binary, no Python dependencies
+- **Resilient** — stays alive on systems without GPUs, retries discovery every 30s
+
+## Platform Support
+
+| Feature | Linux | macOS | Windows |
+|---|:---:|:---:|:---:|
+| System metrics (CPU, memory, disk, network) | Yes | Yes | Yes |
+| Process metrics (CPU, memory, threads, FDs) | Yes | Yes | Yes |
+| GPU metrics — NVIDIA (NVML) | Yes | — | — |
+| GPU metrics — AMD (sysfs/hwmon) | Yes | — | — |
+| GPU metrics — Intel (sysfs/hwmon) | Yes | — | — |
+| eBPF CUDA tracing | Yes | — | — |
+
+On macOS/Windows the collector runs with host and process metrics only. GPU discovery is skipped gracefully. On Linux without GPUs, the collector retries discovery every 30 seconds while still exporting host metrics.
+
+## Architecture
+
+```
+Host Metrics (all platforms via gopsutil)
+ +-- CPU utilization, memory, disk I/O, filesystem, network
+ +-- Process: self CPU, memory, threads, FDs, Go runtime
+
+GPU Metrics (Linux only)
+ +-- PCI Bus Scan (/sys/bus/pci/devices/)
+ | +-- NVIDIA (0x10de) --> NVML backend (go-nvml / libnvidia-ml.so)
+ | +-- AMD (0x1002) --> sysfs/hwmon backend (zero dependencies)
+ | +-- Intel (0x8086) --> sysfs/hwmon + DRM backend
+ |
+ +-- [Optional: eBPF CUDA tracing via uprobes on libcudart.so]
+
+Export
+ +-- OTel SDK --> OTLP gRPC/HTTP --> your OTel collector / backend
+```
+
+## Getting Started
+
+### Prerequisites
+
+- An OpenTelemetry-compatible backend (e.g., OpenLIT, Grafana, Datadog)
+- For GPU metrics: Linux with NVIDIA/AMD/Intel GPU drivers installed
+- For eBPF tracing: Linux kernel 5.8+ with `CAP_BPF` + `CAP_PERFMON` (or root)
+
+### Docker
+
+```sh
+docker pull ghcr.io/openlit/otel-gpu-collector:latest
+
+docker run --gpus all \
+ -e OTEL_SERVICE_NAME=my-app \
+ -e OTEL_RESOURCE_ATTRIBUTES="deployment.environment=production" \
+ -e OTEL_EXPORTER_OTLP_ENDPOINT="http://otel-collector:4317" \
+ ghcr.io/openlit/otel-gpu-collector:latest
+```
+
+### Docker Compose
+
+```yaml
+services:
+ otel-gpu-collector:
+ image: ghcr.io/openlit/otel-gpu-collector:latest
+ environment:
+ OTEL_SERVICE_NAME: my-app
+ OTEL_RESOURCE_ATTRIBUTES: "deployment.environment=production"
+ OTEL_EXPORTER_OTLP_ENDPOINT: "http://otel-collector:4317"
+ deploy:
+ resources:
+ reservations:
+ devices:
+ - driver: nvidia
+ count: all
+ capabilities: [gpu]
+ depends_on:
+ - otel-collector
+ restart: always
+```
+
+### Binary
+
+Download a pre-built binary from the [Releases](https://github.com/openlit/openlit/releases) page:
+
+```sh
+# Linux amd64
+curl -L https://github.com/openlit/openlit/releases/latest/download/opentelemetry-gpu-collector-<version>-linux-amd64 \
+ -o opentelemetry-gpu-collector
+chmod +x opentelemetry-gpu-collector
+
+OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317 ./opentelemetry-gpu-collector
+```
+
+### Build from source
+
+```sh
+git clone https://github.com/openlit/openlit.git
+cd openlit/opentelemetry-gpu-collector
+make build
+./opentelemetry-gpu-collector
+```
+
+## Configuration
+
+All configuration uses standard OpenTelemetry environment variables.
+
+| Variable | Default | Description |
+|---|---|---|
+| `OTEL_EXPORTER_OTLP_ENDPOINT` | *(required)* | OTLP endpoint URL (e.g. `http://localhost:4317`) |
+| `OTEL_EXPORTER_OTLP_HEADERS` | | Auth headers (`key=val,key2=val2`) |
+| `OTEL_EXPORTER_OTLP_PROTOCOL` | `grpc` | `grpc` or `http/protobuf` |
+| `OTEL_SERVICE_NAME` | `default` | Service name attached to all metrics |
+| `OTEL_RESOURCE_ATTRIBUTES` | `deployment.environment=default` | Resource attributes (`deployment.environment=prod,team=ml`) |
+| `OTEL_METRIC_EXPORT_INTERVAL` | `60000` | Metric polling interval in **milliseconds** |
+| `OTEL_GPU_EBPF_ENABLED` | `false` | Enable eBPF CUDA kernel tracing (Linux only) |
+
+`deployment.environment` is read from `OTEL_RESOURCE_ATTRIBUTES` and defaults to `default` if not set.
+
+## Metrics
+
+### GPU Hardware Telemetry (Linux only)
+
+Follows the [OTel semantic conventions for hardware metrics](https://opentelemetry.io/docs/specs/semconv/hardware/gpu/).
+
+| Metric | Type | Unit | Description | NVIDIA | AMD | Intel |
+|---|---|---|---|:---:|:---:|:---:|
+| `hw.gpu.utilization` | Gauge | 1 | GPU compute/encoder/decoder utilization (0.0–1.0) via `hw.gpu.task` | Yes | Yes | — |
+| `hw.gpu.memory.utilization` | Gauge | 1 | Memory controller utilization (0.0–1.0) | Yes | Yes | — |
+| `hw.gpu.memory.limit` | UpDownCounter | By | Total GPU memory | Yes | Yes | — |
+| `hw.gpu.memory.usage` | UpDownCounter | By | Used GPU memory | Yes | Yes | — |
+| `hw.gpu.memory.free` | UpDownCounter | By | Free GPU memory | Yes | Yes | — |
+| `hw.gpu.temperature` | Gauge | Cel | Temperature via `sensor=die\|memory` | Yes | Yes | Yes |
+| `hw.gpu.fan_speed` | Gauge | {rpm} | Fan speed | Yes | Yes | Yes* |
+| `hw.gpu.power.draw` | Gauge | W | Current power draw | Yes | Yes | Yes |
+| `hw.gpu.power.limit` | Gauge | W | Power limit/cap | Yes | Yes | Yes |
+| `hw.gpu.energy.consumed` | Counter | J | Cumulative energy consumed | Yes | Yes | Yes |
+| `hw.gpu.clock.graphics` | Gauge | MHz | Graphics/SM clock frequency | Yes | Yes | Yes* |
+| `hw.gpu.clock.memory` | Gauge | MHz | Memory clock frequency | Yes | Yes | — |
+| `hw.errors` | Counter | {error} | ECC and PCIe errors via `error.type` + `hw.type=gpu` | Yes | — | — |
+
+\* Intel support depends on driver (i915/Xe) and kernel version.
+
+**Attributes on all GPU metrics:**
+
+| Attribute | Description | Example |
+|---|---|---|
+| `hw.id` | Unique device identifier (required by spec) | `GPU-a1b2c3d4-...` |
+| `hw.name` | Product name | `NVIDIA A100-SXM4-80GB` |
+| `hw.vendor` | Vendor name | `nvidia`, `amd`, `intel` |
+| `gpu.index` | Device index | `0`, `1` |
+| `gpu.pci_address` | PCI bus address | `0000:01:00.0` |
+
+**Additional attributes per metric:**
+
+| Metric | Extra Attribute | Values |
+|---|---|---|
+| `hw.gpu.utilization` | `hw.gpu.task` | `general`, `encoder`, `decoder` |
+| `hw.gpu.temperature` | `sensor` | `die`, `memory` |
+| `hw.errors` | `error.type` | `corrected`, `uncorrected`, `pcie_replay` |
+| `hw.errors` | `hw.type` | `gpu` |
+
+### System Metrics (all platforms)
+
+Follows the [OTel semantic conventions for system metrics](https://opentelemetry.io/docs/specs/semconv/system/system-metrics/).
+
+| Metric | Type | Unit | Description | Attributes |
+|---|---|---|---|---|
+| `system.cpu.utilization` | Gauge | 1 | CPU utilization per core (0.0–1.0) | `cpu.logical_number` |
+| `system.cpu.logical.count` | UpDownCounter | {cpu} | Logical CPU core count | |
+| `system.memory.usage` | UpDownCounter | By | Memory by state | `system.memory.state`={used,free,cached,buffers} |
+| `system.memory.utilization` | Gauge | 1 | Memory utilization (0.0–1.0) | |
+| `system.disk.io` | Counter | By | Disk I/O bytes | `system.device`, `disk.io.direction`={read,write} |
+| `system.disk.operations` | Counter | {operation} | Disk I/O operations | `system.device`, `disk.io.direction`={read,write} |
+| `system.filesystem.usage` | UpDownCounter | By | Filesystem space | `system.device`, `system.filesystem.mountpoint`, `system.filesystem.type`, `system.filesystem.state`={used,free} |
+| `system.filesystem.utilization` | Gauge | 1 | Filesystem utilization (0.0–1.0) | `system.device`, `system.filesystem.mountpoint`, `system.filesystem.type` |
+| `system.network.io` | Counter | By | Network I/O bytes | `network.interface.name`, `network.io.direction`={receive,transmit} |
+| `system.network.errors` | Counter | {error} | Network errors | `network.interface.name`, `network.io.direction`={receive,transmit} |
+
+### Process Metrics (all platforms)
+
+Follows the [OTel semantic conventions for process metrics](https://opentelemetry.io/docs/specs/semconv/system/process-metrics/).
+
+| Metric | Type | Unit | Description | Attributes |
+|---|---|---|---|---|
+| `process.cpu.time` | Counter | s | Process CPU time | `cpu.mode`={user,system} |
+| `process.cpu.utilization` | Gauge | 1 | Process CPU utilization | |
+| `process.memory.usage` | UpDownCounter | By | Resident memory (RSS) | |
+| `process.memory.virtual` | UpDownCounter | By | Virtual memory size | |
+| `process.thread.count` | UpDownCounter | {thread} | OS thread count | |
+| `process.unix.file_descriptor.count` | UpDownCounter | {file_descriptor} | Open file descriptors (Linux/macOS) | |
+| `process.runtime.go.goroutines` | Gauge | {goroutine} | Go goroutine count | |
+| `process.runtime.go.mem.heap_alloc` | Gauge | By | Go heap memory allocated | |
+
+### eBPF CUDA Tracing (opt-in, Linux only)
+
+Enable with `OTEL_GPU_EBPF_ENABLED=true`. Requires `CAP_BPF` + `CAP_PERFMON` or root, and NVIDIA CUDA runtime (`libcudart.so`).
+
+| Metric | Type | Unit | Description | Attributes |
+|---|---|---|---|---|
+| `gpu.kernel.launch.calls` | Counter | {call} | CUDA kernel launch count | `cuda.kernel.name` |
+| `gpu.kernel.grid.size` | Histogram | {thread} | Total threads in grid per launch | `cuda.kernel.name` |
+| `gpu.kernel.block.size` | Histogram | {thread} | Threads per block per launch | `cuda.kernel.name` |
+| `gpu.memory.allocations` | Counter | By | Bytes allocated via cudaMalloc | |
+| `gpu.memory.copies` | Histogram | By | Bytes per cudaMemcpy | `cuda.memcpy.kind`={HostToHost,HostToDevice,DeviceToHost,DeviceToDevice} |
+
+## How It Works
+
+### Device Discovery
+
+The collector scans `/sys/bus/pci/devices/` for PCI class codes `0x0300` (VGA), `0x0302` (3D controller), and `0x0380` (display controller), then maps the vendor ID:
+
+| Vendor ID | Backend | Collected metrics |
+|---|---|---|
+| `0x10de` (NVIDIA) | NVML via [go-nvml](https://github.com/NVIDIA/go-nvml) — loads `libnvidia-ml.so` at runtime | Utilization, memory, temperature, power, energy, clocks, ECC errors, PCIe errors, fan speed |
+| `0x1002` (AMD) | sysfs + hwmon — zero external dependencies | Utilization, memory, temperature, power, energy, fan speed |
+| `0x8086` (Intel) | sysfs + hwmon + DRM (i915/Xe driver) — requires Linux kernel 5.10+ | Temperature, power draw/limit, cumulative energy, graphics clock, fan speed (kernel 6.16+) |
+
+### eBPF CUDA Tracing
+
+Attaches uprobes to `libcudart.so` to intercept:
+- `cudaLaunchKernel` — kernel name, grid/block dimensions
+- `cudaMalloc` — allocation size
+- `cudaMemcpy` / `cudaMemcpyAsync` — copy size and direction
+
+Events flow through a BPF ring buffer to Go userspace, where kernel addresses are resolved to function names via ELF symbol tables.
+
+## Contributing
+
+Whether it's big or small, we love contributions. Check out our [Contribution guide](../../CONTRIBUTING.md) to get started.
+
+Join our [Slack](https://join.slack.com/t/openlit/shared_invite/zt-2etnfttwg-TjP_7BZXfYg84oAukY8QRQ) or [Discord](https://discord.gg/rjvTm6zd) community.
+
+## Roadmap
+
+| Feature | Status |
+|---|---|
+| NVIDIA GPU hardware telemetry (NVML) | Done |
+| AMD GPU hardware telemetry (sysfs/hwmon) | Done |
+| Intel GPU hardware telemetry (sysfs/hwmon) | Done |
+| eBPF CUDA kernel tracing | Done |
+| OTel semantic convention compliance (`hw.gpu.*`) | Done |
+| Prometheus `/metrics` endpoint | Planned |
+| ROCm HIP tracing (AMD eBPF) | Planned |
+| Per-process GPU utilization (DRM fdinfo) | Planned |
+
+## License
+
+OpenTelemetry GPU Collector is built and maintained by OpenLIT under the [Apache-2.0 license](../../LICENSE).
diff --git a/opentelemetry-gpu-collector/cmd/collector/main.go b/opentelemetry-gpu-collector/cmd/collector/main.go
new file mode 100644
index 00000000..e6ba24a7
--- /dev/null
+++ b/opentelemetry-gpu-collector/cmd/collector/main.go
@@ -0,0 +1,246 @@
+package main
+
+import (
+ "context"
+ "fmt"
+ "log/slog"
+ "os"
+ "os/signal"
+ "runtime"
+ "syscall"
+ "time"
+
+ sdkmetric "go.opentelemetry.io/otel/sdk/metric"
+
+ "github.com/openlit/openlit/opentelemetry-gpu-collector/internal/config"
+ "github.com/openlit/openlit/opentelemetry-gpu-collector/internal/discovery"
+ gpuebpf "github.com/openlit/openlit/opentelemetry-gpu-collector/internal/ebpf"
+ "github.com/openlit/openlit/opentelemetry-gpu-collector/internal/export"
+ "github.com/openlit/openlit/opentelemetry-gpu-collector/internal/gpu"
+ "github.com/openlit/openlit/opentelemetry-gpu-collector/internal/gpu/amd"
+ "github.com/openlit/openlit/opentelemetry-gpu-collector/internal/gpu/intel"
+ "github.com/openlit/openlit/opentelemetry-gpu-collector/internal/gpu/nvidia"
+ "github.com/openlit/openlit/opentelemetry-gpu-collector/internal/hostmetrics"
+)
+
+func main() {
+ logger := slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{
+ Level: slog.LevelInfo,
+ }))
+
+ if err := run(logger); err != nil {
+ logger.Error("fatal error", "error", err)
+ os.Exit(1)
+ }
+}
+
+func run(logger *slog.Logger) error {
+ cfg := config.Load()
+ logger.Info("starting opentelemetry-gpu-collector",
+ "service", cfg.ServiceName,
+ "environment", cfg.Environment,
+ "interval", cfg.CollectionInterval,
+ "ebpf_enabled", cfg.EBPFEnabled,
+ )
+
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+
+ // --- OTel meter provider (always created, even with zero GPUs) ---
+ provider, shutdownProvider, err := export.NewMeterProvider(ctx, cfg, logger)
+ if err != nil {
+ return fmt.Errorf("creating meter provider: %w", err)
+ }
+ defer shutdownProvider()
+
+ // --- Host metrics (always-on, works on all platforms) ---
+ sysColl, err := hostmetrics.NewSystemCollector(provider, logger)
+ if err != nil {
+ logger.Warn("system metrics unavailable", "error", err)
+ }
+
+ procColl, err := hostmetrics.NewProcessCollector(provider, logger)
+ if err != nil {
+ logger.Warn("process metrics unavailable", "error", err)
+ }
+
+ // --- GPU device discovery with retry ---
+ var devices []gpu.Device
+ var mc *export.MetricsCollector
+ var ebpfTracer *gpuebpf.Tracer
+
+ // GPU discovery only makes sense on Linux (PCI sysfs, NVML, etc.)
+ gpuCapable := runtime.GOOS == "linux"
+
+ if gpuCapable {
+ devices = tryDiscoverGPUs(logger)
+ if len(devices) > 0 {
+ mc, ebpfTracer = setupCollectors(ctx, cfg, provider, devices, logger)
+ } else {
+ logger.Warn("no GPUs discovered at startup; will retry periodically")
+ }
+ } else {
+ logger.Info("GPU monitoring not available on this platform", "os", runtime.GOOS)
+ }
+
+ // --- Signal handling ---
+ sigCh := make(chan os.Signal, 1)
+ signal.Notify(sigCh, syscall.SIGTERM, syscall.SIGINT)
+
+ retryTicker := time.NewTicker(30 * time.Second)
+ defer retryTicker.Stop()
+
+ for {
+ select {
+ case sig := <-sigCh:
+ logger.Info("received signal, shutting down", "signal", sig.String())
+ cancel()
+ if ebpfTracer != nil {
+ ebpfTracer.Close()
+ }
+ if mc != nil {
+ mc.Close()
+ }
+ for _, d := range devices {
+ d.Close()
+ }
+ if sysColl != nil {
+ sysColl.Close()
+ }
+ if procColl != nil {
+ procColl.Close()
+ }
+ logger.Info("shutdown complete")
+ return nil
+
+ case <-retryTicker.C:
+ if !gpuCapable || len(devices) > 0 {
+ continue
+ }
+ logger.Info("retrying GPU discovery...")
+ devices = tryDiscoverGPUs(logger)
+ if len(devices) > 0 {
+ mc, ebpfTracer = setupCollectors(ctx, cfg, provider, devices, logger)
+ logger.Info("GPUs discovered on retry", "count", len(devices))
+ }
+ }
+ }
+}
+
+// tryDiscoverGPUs attempts GPU discovery, returning nil on failure instead of crashing.
+func tryDiscoverGPUs(logger *slog.Logger) []gpu.Device {
+ devices, err := discoverAllGPUs(logger)
+ if err != nil {
+ logger.Warn("GPU discovery error", "error", err)
+ return nil
+ }
+ if len(devices) == 0 {
+ logger.Warn("no GPUs found on this system")
+ return nil
+ }
+ logger.Info("GPU discovery complete", "count", len(devices))
+ return devices
+}
+
+// setupCollectors initializes hardware metrics and optional eBPF tracing
+// after GPUs have been successfully discovered.
+func setupCollectors(
+ ctx context.Context,
+ cfg *config.Config,
+ provider *sdkmetric.MeterProvider,
+ devices []gpu.Device,
+ logger *slog.Logger,
+) (*export.MetricsCollector, *gpuebpf.Tracer) {
+ mc, err := export.NewMetricsCollector(provider, devices, logger)
+ if err != nil {
+ logger.Error("failed to create metrics collector", "error", err)
+ return nil, nil
+ }
+
+ var ebpfTracer *gpuebpf.Tracer
+ if cfg.EBPFEnabled {
+ ebpfMetrics, err := export.NewEBPFMetrics(provider, logger)
+ if err != nil {
+ logger.Warn("failed to create eBPF metrics instruments", "error", err)
+ } else {
+ tracer, err := gpuebpf.NewTracer(logger, ebpfMetrics.HandleEvent)
+ if err != nil {
+ logger.Warn("eBPF CUDA tracing unavailable", "error", err)
+ } else {
+ ebpfTracer = tracer
+ go ebpfTracer.Run(ctx)
+ logger.Info("eBPF CUDA tracing started")
+ }
+ }
+ }
+
+ return mc, ebpfTracer
+}
+
+// discoverAllGPUs finds GPUs on the PCI bus and instantiates appropriate backends.
+func discoverAllGPUs(logger *slog.Logger) ([]gpu.Device, error) {
+ pciDevices, err := discovery.Discover(logger)
+ if err != nil {
+ return nil, err
+ }
+
+ var (
+ allDevices []gpu.Device
+ nvidiaAddrs []string
+ amdAddrs []string
+ intelAddrs []string
+ )
+
+ for _, d := range pciDevices {
+ switch d.Vendor {
+ case gpu.VendorNVIDIA:
+ nvidiaAddrs = append(nvidiaAddrs, d.Address)
+ case gpu.VendorAMD:
+ amdAddrs = append(amdAddrs, d.Address)
+ case gpu.VendorIntel:
+ intelAddrs = append(intelAddrs, d.Address)
+ }
+ }
+
+ idx := 0
+
+ // NVIDIA: use NVML for all NVIDIA GPUs at once
+ if len(nvidiaAddrs) > 0 {
+ nvDevices, err := nvidia.DiscoverDevices(logger)
+ if err != nil {
+ logger.Warn("NVIDIA discovery failed", "error", err)
+ } else {
+ for _, d := range nvDevices {
+ allDevices = append(allDevices, d)
+ idx++
+ }
+ }
+ }
+
+ // AMD: use sysfs for each AMD GPU
+ if len(amdAddrs) > 0 {
+ amdDevices, err := amd.DiscoverDevices(amdAddrs, idx, logger)
+ if err != nil {
+ logger.Warn("AMD discovery failed", "error", err)
+ } else {
+ for _, d := range amdDevices {
+ allDevices = append(allDevices, d)
+ idx++
+ }
+ }
+ }
+
+ // Intel: use sysfs for each Intel GPU
+ if len(intelAddrs) > 0 {
+ intelDevices, err := intel.DiscoverDevices(intelAddrs, idx, logger)
+ if err != nil {
+ logger.Warn("Intel discovery failed", "error", err)
+ } else {
+ for _, d := range intelDevices {
+ allDevices = append(allDevices, d)
+ }
+ }
+ }
+
+ return allDevices, nil
+}
diff --git a/opentelemetry-gpu-collector/dev/otel-collector-config.yaml b/opentelemetry-gpu-collector/dev/otel-collector-config.yaml
new file mode 100644
index 00000000..a3ac3900
--- /dev/null
+++ b/opentelemetry-gpu-collector/dev/otel-collector-config.yaml
@@ -0,0 +1,17 @@
+receivers:
+ otlp:
+ protocols:
+ grpc:
+ endpoint: 0.0.0.0:4317
+ http:
+ endpoint: 0.0.0.0:4318
+
+exporters:
+ debug:
+ verbosity: detailed
+
+service:
+ pipelines:
+ metrics:
+ receivers: [otlp]
+ exporters: [debug]
diff --git a/opentelemetry-gpu-collector/go.mod b/opentelemetry-gpu-collector/go.mod
new file mode 100644
index 00000000..9b774deb
--- /dev/null
+++ b/opentelemetry-gpu-collector/go.mod
@@ -0,0 +1,42 @@
+module github.com/openlit/openlit/opentelemetry-gpu-collector
+
+go 1.25.0
+
+require (
+ github.com/NVIDIA/go-nvml v0.13.0-1
+ github.com/cilium/ebpf v0.21.0
+ github.com/ianlancetaylor/demangle v0.0.0-20251118225945-96ee0021ea0f
+ github.com/shirou/gopsutil/v4 v4.26.3
+ go.opentelemetry.io/otel v1.42.0
+ go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.42.0
+ go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.42.0
+ go.opentelemetry.io/otel/metric v1.42.0
+ go.opentelemetry.io/otel/sdk v1.42.0
+ go.opentelemetry.io/otel/sdk/metric v1.42.0
+)
+
+require (
+ github.com/cenkalti/backoff/v5 v5.0.3 // indirect
+ github.com/cespare/xxhash/v2 v2.3.0 // indirect
+ github.com/ebitengine/purego v0.10.0 // indirect
+ github.com/go-logr/logr v1.4.3 // indirect
+ github.com/go-logr/stdr v1.2.2 // indirect
+ github.com/go-ole/go-ole v1.2.6 // indirect
+ github.com/google/uuid v1.6.0 // indirect
+ github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 // indirect
+ github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
+ github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
+ github.com/tklauser/go-sysconf v0.3.16 // indirect
+ github.com/tklauser/numcpus v0.11.0 // indirect
+ github.com/yusufpapurcu/wmi v1.2.4 // indirect
+ go.opentelemetry.io/auto/sdk v1.2.1 // indirect
+ go.opentelemetry.io/otel/trace v1.42.0 // indirect
+ go.opentelemetry.io/proto/otlp v1.9.0 // indirect
+ golang.org/x/net v0.51.0 // indirect
+ golang.org/x/sys v0.41.0 // indirect
+ golang.org/x/text v0.34.0 // indirect
+ google.golang.org/genproto/googleapis/api v0.0.0-20260209200024-4cfbd4190f57 // indirect
+ google.golang.org/genproto/googleapis/rpc v0.0.0-20260209200024-4cfbd4190f57 // indirect
+ google.golang.org/grpc v1.79.2 // indirect
+ google.golang.org/protobuf v1.36.11 // indirect
+)
diff --git a/opentelemetry-gpu-collector/go.sum b/opentelemetry-gpu-collector/go.sum
new file mode 100644
index 00000000..e424e36e
--- /dev/null
+++ b/opentelemetry-gpu-collector/go.sum
@@ -0,0 +1,103 @@
+github.com/NVIDIA/go-nvml v0.13.0-1 h1:OLX8Jq3dONuPOQPC7rndB6+iDmDakw0XTYgzMxObkEw=
+github.com/NVIDIA/go-nvml v0.13.0-1/go.mod h1:+KNA7c7gIBH7SKSJ1ntlwkfN80zdx8ovl4hrK3LmPt4=
+github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM=
+github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
+github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
+github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/cilium/ebpf v0.21.0 h1:4dpx1J/B/1apeTmWBH5BkVLayHTkFrMovVPnHEk+l3k=
+github.com/cilium/ebpf v0.21.0/go.mod h1:1kHKv6Kvh5a6TePP5vvvoMa1bclRyzUXELSs272fmIQ=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/ebitengine/purego v0.10.0 h1:QIw4xfpWT6GWTzaW5XEKy3HXoqrJGx1ijYHzTF0/ISU=
+github.com/ebitengine/purego v0.10.0/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
+github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
+github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
+github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
+github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
+github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
+github.com/go-quicktest/qt v1.101.1-0.20240301121107-c6c8733fa1e6 h1:teYtXy9B7y5lHTp8V9KPxpYRAVA7dozigQcMiBust1s=
+github.com/go-quicktest/qt v1.101.1-0.20240301121107-c6c8733fa1e6/go.mod h1:p4lGIVX+8Wa6ZPNDvqcxq36XpUDLh42FLetFU7odllI=
+github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
+github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
+github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
+github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 h1:HWRh5R2+9EifMyIHV7ZV+MIZqgz+PMpZ14Jynv3O2Zs=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0/go.mod h1:JfhWUomR1baixubs02l85lZYYOm7LV6om4ceouMv45c=
+github.com/ianlancetaylor/demangle v0.0.0-20251118225945-96ee0021ea0f h1:Fnl4pzx8SR7k7JuzyW8lEtSFH6EQ8xgcypgIn8pcGIE=
+github.com/ianlancetaylor/demangle v0.0.0-20251118225945-96ee0021ea0f/go.mod h1:gx7rwoVhcfuVKG5uya9Hs3Sxj7EIvldVofAWIUtGouw=
+github.com/josharian/native v1.1.0 h1:uuaP0hAbW7Y4l0ZRQ6C9zfb7Mg1mbFKry/xzDAfmtLA=
+github.com/josharian/native v1.1.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w=
+github.com/jsimonetti/rtnetlink/v2 v2.0.1 h1:xda7qaHDSVOsADNouv7ukSuicKZO7GgVUCXxpaIEIlM=
+github.com/jsimonetti/rtnetlink/v2 v2.0.1/go.mod h1:7MoNYNbb3UaDHtF8udiJo/RH6VsTKP1pqKLUTVCvToE=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4=
+github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
+github.com/mdlayher/netlink v1.7.2 h1:/UtM3ofJap7Vl4QWCPDGXY8d3GIY2UGSDbK+QWmY8/g=
+github.com/mdlayher/netlink v1.7.2/go.mod h1:xraEF7uJbxLhc5fpHL4cPe221LI2bdttWlU+ZGLfQSw=
+github.com/mdlayher/socket v0.5.1 h1:VZaqt6RkGkt2OE9l3GcC6nZkqD3xKeQLyfleW/uBcos=
+github.com/mdlayher/socket v0.5.1/go.mod h1:TjPLHI1UgwEv5J1B5q0zTZq12A/6H7nKmtTanQE37IQ=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU=
+github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
+github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
+github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
+github.com/shirou/gopsutil/v4 v4.26.3 h1:2ESdQt90yU3oXF/CdOlRCJxrP+Am1aBYubTMTfxJ1qc=
+github.com/shirou/gopsutil/v4 v4.26.3/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ=
+github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
+github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
+github.com/tklauser/go-sysconf v0.3.16 h1:frioLaCQSsF5Cy1jgRBrzr6t502KIIwQ0MArYICU0nA=
+github.com/tklauser/go-sysconf v0.3.16/go.mod h1:/qNL9xxDhc7tx3HSRsLWNnuzbVfh3e7gh/BmM179nYI=
+github.com/tklauser/numcpus v0.11.0 h1:nSTwhKH5e1dMNsCdVBukSZrURJRoHbSEQjdEbY+9RXw=
+github.com/tklauser/numcpus v0.11.0/go.mod h1:z+LwcLq54uWZTX0u/bGobaV34u6V7KNlTZejzM6/3MQ=
+github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
+github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
+go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
+go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
+go.opentelemetry.io/otel v1.42.0 h1:lSQGzTgVR3+sgJDAU/7/ZMjN9Z+vUip7leaqBKy4sho=
+go.opentelemetry.io/otel v1.42.0/go.mod h1:lJNsdRMxCUIWuMlVJWzecSMuNjE7dOYyWlqOXWkdqCc=
+go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.42.0 h1:MdKucPl/HbzckWWEisiNqMPhRrAOQX8r4jTuGr636gk=
+go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.42.0/go.mod h1:RolT8tWtfHcjajEH5wFIZ4Dgh5jpPdFXYV9pTAk/qjc=
+go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.42.0 h1:H7O6RlGOMTizyl3R08Kn5pdM06bnH8oscSj7o11tmLA=
+go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.42.0/go.mod h1:mBFWu/WOVDkWWsR7Tx7h6EpQB8wsv7P0Yrh0Pb7othc=
+go.opentelemetry.io/otel/metric v1.42.0 h1:2jXG+3oZLNXEPfNmnpxKDeZsFI5o4J+nz6xUlaFdF/4=
+go.opentelemetry.io/otel/metric v1.42.0/go.mod h1:RlUN/7vTU7Ao/diDkEpQpnz3/92J9ko05BIwxYa2SSI=
+go.opentelemetry.io/otel/sdk v1.42.0 h1:LyC8+jqk6UJwdrI/8VydAq/hvkFKNHZVIWuslJXYsDo=
+go.opentelemetry.io/otel/sdk v1.42.0/go.mod h1:rGHCAxd9DAph0joO4W6OPwxjNTYWghRWmkHuGbayMts=
+go.opentelemetry.io/otel/sdk/metric v1.42.0 h1:D/1QR46Clz6ajyZ3G8SgNlTJKBdGp84q9RKCAZ3YGuA=
+go.opentelemetry.io/otel/sdk/metric v1.42.0/go.mod h1:Ua6AAlDKdZ7tdvaQKfSmnFTdHx37+J4ba8MwVCYM5hc=
+go.opentelemetry.io/otel/trace v1.42.0 h1:OUCgIPt+mzOnaUTpOQcBiM/PLQ/Op7oq6g4LenLmOYY=
+go.opentelemetry.io/otel/trace v1.42.0/go.mod h1:f3K9S+IFqnumBkKhRJMeaZeNk9epyhnCmQh/EysQCdc=
+go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A=
+go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4=
+golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo=
+golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y=
+golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
+golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
+golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k=
+golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
+golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk=
+golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
+gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
+google.golang.org/genproto/googleapis/api v0.0.0-20260209200024-4cfbd4190f57 h1:JLQynH/LBHfCTSbDWl+py8C+Rg/k1OVH3xfcaiANuF0=
+google.golang.org/genproto/googleapis/api v0.0.0-20260209200024-4cfbd4190f57/go.mod h1:kSJwQxqmFXeo79zOmbrALdflXQeAYcUbgS7PbpMknCY=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20260209200024-4cfbd4190f57 h1:mWPCjDEyshlQYzBpMNHaEof6UX1PmHcaUODUywQ0uac=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20260209200024-4cfbd4190f57/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ=
+google.golang.org/grpc v1.79.2 h1:fRMD94s2tITpyJGtBBn7MkMseNpOZU8ZxgC3MMBaXRU=
+google.golang.org/grpc v1.79.2/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ=
+google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
+google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
diff --git a/opentelemetry-gpu-collector/internal/config/config.go b/opentelemetry-gpu-collector/internal/config/config.go
new file mode 100644
index 00000000..6eb79150
--- /dev/null
+++ b/opentelemetry-gpu-collector/internal/config/config.go
@@ -0,0 +1,70 @@
+package config
+
+import (
+ "os"
+ "strconv"
+ "strings"
+ "time"
+)
+
+type Config struct {
+ Environment string
+ ServiceName string
+ CollectionInterval time.Duration
+ EBPFEnabled bool
+}
+
+func Load() *Config {
+ cfg := &Config{
+ ServiceName: envOrDefault("OTEL_SERVICE_NAME", "default"),
+ // OTEL_METRIC_EXPORT_INTERVAL is in milliseconds per the OTel spec.
+ // OTEL_EXPORTER_OTLP_ENDPOINT, OTEL_EXPORTER_OTLP_HEADERS, and
+ // OTEL_EXPORTER_OTLP_PROTOCOL are read directly by the OTel SDK exporters.
+ CollectionInterval: parseIntervalMS(os.Getenv("OTEL_METRIC_EXPORT_INTERVAL"), 60*time.Second),
+ EBPFEnabled: parseBool(envOrDefault("OTEL_GPU_EBPF_ENABLED", "false")),
+ }
+
+ // deployment.environment comes from OTEL_RESOURCE_ATTRIBUTES per the OTel spec.
+ cfg.Environment = parseResourceAttr(os.Getenv("OTEL_RESOURCE_ATTRIBUTES"), "deployment.environment", "default")
+
+ return cfg
+}
+
+func envOrDefault(key, fallback string) string {
+ if v := os.Getenv(key); v != "" {
+ return v
+ }
+ return fallback
+}
+
+// parseIntervalMS parses OTEL_METRIC_EXPORT_INTERVAL which is specified in milliseconds.
+func parseIntervalMS(s string, fallback time.Duration) time.Duration {
+ if s == "" {
+ return fallback
+ }
+ ms, err := strconv.ParseInt(s, 10, 64)
+ if err != nil || ms <= 0 {
+ return fallback
+ }
+ return time.Duration(ms) * time.Millisecond
+}
+
+func parseBool(s string) bool {
+ b, _ := strconv.ParseBool(s)
+ return b
+}
+
+// parseResourceAttr extracts a specific key from OTEL_RESOURCE_ATTRIBUTES format
+// "key1=val1,key2=val2".
+func parseResourceAttr(raw, key, fallback string) string {
+ if raw == "" {
+ return fallback
+ }
+ for _, pair := range strings.Split(raw, ",") {
+ kv := strings.SplitN(strings.TrimSpace(pair), "=", 2)
+ if len(kv) == 2 && strings.TrimSpace(kv[0]) == key {
+ return strings.TrimSpace(kv[1])
+ }
+ }
+ return fallback
+}
diff --git a/opentelemetry-gpu-collector/internal/config/config_test.go b/opentelemetry-gpu-collector/internal/config/config_test.go
new file mode 100644
index 00000000..46947421
--- /dev/null
+++ b/opentelemetry-gpu-collector/internal/config/config_test.go
@@ -0,0 +1,97 @@
+package config
+
+import (
+ "testing"
+ "time"
+)
+
+func TestParseIntervalMS(t *testing.T) {
+ tests := []struct {
+ input string
+ fallback time.Duration
+ want time.Duration
+ }{
+ {"5000", 10 * time.Second, 5 * time.Second},
+ {"1000", 10 * time.Second, 1 * time.Second},
+ {"", 10 * time.Second, 10 * time.Second},
+ {"0", 10 * time.Second, 10 * time.Second},
+ {"-100", 10 * time.Second, 10 * time.Second},
+ {"notanumber", 10 * time.Second, 10 * time.Second},
+ }
+
+ for _, tt := range tests {
+ got := parseIntervalMS(tt.input, tt.fallback)
+ if got != tt.want {
+ t.Errorf("parseIntervalMS(%q) = %v, want %v", tt.input, got, tt.want)
+ }
+ }
+}
+
+
+func TestParseResourceAttr(t *testing.T) {
+ tests := []struct {
+ raw string
+ key string
+ fallback string
+ want string
+ }{
+ {"", "deployment.environment", "default", "default"},
+ {"deployment.environment=production", "deployment.environment", "default", "production"},
+ {"team=ml,deployment.environment=staging", "deployment.environment", "default", "staging"},
+ {"deployment.environment=prod,team=ml", "deployment.environment", "default", "prod"},
+ {"other=value", "deployment.environment", "default", "default"},
+ {" deployment.environment = canary ", "deployment.environment", "default", "canary"},
+ }
+
+ for _, tt := range tests {
+ got := parseResourceAttr(tt.raw, tt.key, tt.fallback)
+ if got != tt.want {
+ t.Errorf("parseResourceAttr(%q, %q) = %q, want %q", tt.raw, tt.key, got, tt.want)
+ }
+ }
+}
+
+func TestLoadDefaults(t *testing.T) {
+ // Ensure no env vars interfere.
+ t.Setenv("OTEL_METRIC_EXPORT_INTERVAL", "")
+ t.Setenv("OTEL_GPU_EBPF_ENABLED", "")
+ t.Setenv("OTEL_SERVICE_NAME", "")
+ t.Setenv("OTEL_RESOURCE_ATTRIBUTES", "")
+
+ cfg := Load()
+
+ if cfg.ServiceName != "default" {
+ t.Errorf("ServiceName = %q, want %q", cfg.ServiceName, "default")
+ }
+ if cfg.CollectionInterval != 60*time.Second {
+ t.Errorf("CollectionInterval = %v, want %v", cfg.CollectionInterval, 60*time.Second)
+ }
+ if cfg.EBPFEnabled {
+ t.Error("EBPFEnabled should be false by default")
+ }
+ if cfg.Environment != "default" {
+ t.Errorf("Environment = %q, want %q", cfg.Environment, "default")
+ }
+}
+
+func TestLoadFromEnv(t *testing.T) {
+ t.Setenv("OTEL_METRIC_EXPORT_INTERVAL", "5000")
+ t.Setenv("OTEL_GPU_EBPF_ENABLED", "true")
+ t.Setenv("OTEL_SERVICE_NAME", "gpu-collector")
+ t.Setenv("OTEL_RESOURCE_ATTRIBUTES", "deployment.environment=production,team=ml")
+
+ cfg := Load()
+
+ if cfg.CollectionInterval != 5*time.Second {
+ t.Errorf("CollectionInterval = %v, want %v", cfg.CollectionInterval, 5*time.Second)
+ }
+ if !cfg.EBPFEnabled {
+ t.Error("EBPFEnabled should be true")
+ }
+ if cfg.ServiceName != "gpu-collector" {
+ t.Errorf("ServiceName = %q, want %q", cfg.ServiceName, "gpu-collector")
+ }
+ if cfg.Environment != "production" {
+ t.Errorf("Environment = %q, want %q", cfg.Environment, "production")
+ }
+}
diff --git a/opentelemetry-gpu-collector/internal/discovery/discovery.go b/opentelemetry-gpu-collector/internal/discovery/discovery.go
new file mode 100644
index 00000000..3c157299
--- /dev/null
+++ b/opentelemetry-gpu-collector/internal/discovery/discovery.go
@@ -0,0 +1,103 @@
+package discovery
+
+import (
+ "fmt"
+ "log/slog"
+ "os"
+ "path/filepath"
+ "strings"
+
+ "github.com/openlit/openlit/opentelemetry-gpu-collector/internal/gpu"
+)
+
+const pciDevicesPath = "/sys/bus/pci/devices"
+
+// PCI vendor IDs for GPU manufacturers.
+const (
+ pciVendorNVIDIA = "0x10de"
+ pciVendorAMD = "0x1002"
+ pciVendorIntel = "0x8086"
+)
+
+// PCI class codes for display/VGA/3D controllers.
+var gpuClassPrefixes = []string{
+ "0x0300", // VGA compatible controller
+ "0x0302", // 3D controller (e.g. NVIDIA Tesla/datacenter)
+ "0x0380", // Display controller
+}
+
+// PCIDevice holds the PCI-level information needed to instantiate a GPU backend.
+type PCIDevice struct {
+ Address string
+ Vendor gpu.Vendor
+}
+
+// Discover scans the PCI bus and returns all detected GPU devices.
+func Discover(logger *slog.Logger) ([]PCIDevice, error) {
+ entries, err := os.ReadDir(pciDevicesPath)
+ if err != nil {
+ return nil, fmt.Errorf("reading PCI devices: %w", err)
+ }
+
+ var devices []PCIDevice
+ for _, entry := range entries {
+ addr := entry.Name()
+ devPath := filepath.Join(pciDevicesPath, addr)
+
+ classCode := readSysfsString(filepath.Join(devPath, "class"))
+ if !isGPUClass(classCode) {
+ continue
+ }
+
+ vendorID := readSysfsString(filepath.Join(devPath, "vendor"))
+ vendor := mapVendor(vendorID)
+ if vendor == gpu.VendorUnknown {
+ logger.Debug("skipping unknown GPU vendor", "address", addr, "vendor_id", vendorID)
+ continue
+ }
+
+ logger.Info("discovered GPU", "address", addr, "vendor", vendor)
+ devices = append(devices, PCIDevice{
+ Address: addr,
+ Vendor: vendor,
+ })
+ }
+
+ return devices, nil
+}
+
+func readSysfsString(path string) string {
+ data, err := os.ReadFile(path)
+ if err != nil {
+ return ""
+ }
+ return strings.TrimSpace(string(data))
+}
+
+func isGPUClass(classCode string) bool {
+ // PCI class is 24 bits; the file contains e.g. "0x030000".
+ // We match on the first 6 hex chars (base class + sub class).
+ if len(classCode) < 6 {
+ return false
+ }
+ prefix := classCode[:6]
+ for _, gp := range gpuClassPrefixes {
+ if strings.EqualFold(prefix, gp) {
+ return true
+ }
+ }
+ return false
+}
+
+func mapVendor(vendorID string) gpu.Vendor {
+ switch strings.ToLower(vendorID) {
+ case pciVendorNVIDIA:
+ return gpu.VendorNVIDIA
+ case pciVendorAMD:
+ return gpu.VendorAMD
+ case pciVendorIntel:
+ return gpu.VendorIntel
+ default:
+ return gpu.VendorUnknown
+ }
+}
diff --git a/opentelemetry-gpu-collector/internal/discovery/discovery_test.go b/opentelemetry-gpu-collector/internal/discovery/discovery_test.go
new file mode 100644
index 00000000..615ce0fd
--- /dev/null
+++ b/opentelemetry-gpu-collector/internal/discovery/discovery_test.go
@@ -0,0 +1,66 @@
+package discovery
+
+import (
+ "testing"
+
+ "github.com/openlit/openlit/opentelemetry-gpu-collector/internal/gpu"
+)
+
+func TestIsGPUClass(t *testing.T) {
+ tests := []struct {
+ classCode string
+ want bool
+ }{
+ // VGA compatible controller
+ {"0x030000", true},
+ {"0x030001", true}, // sub-class variation
+ // 3D controller (NVIDIA Tesla / datacenter)
+ {"0x030200", true},
+ {"0x030201", true},
+ // Display controller
+ {"0x038000", true},
+ {"0x038001", true},
+ // Non-GPU classes
+ {"0x020000", false}, // Ethernet
+ {"0x010000", false}, // SCSI controller
+ {"0x060000", false}, // Host bridge
+ // Too short
+ {"0x030", false},
+ {"", false},
+ // Case insensitive
+ {"0X030000", true},
+ {"0X030200", true},
+ {"0X038000", true},
+ }
+
+ for _, tt := range tests {
+ got := isGPUClass(tt.classCode)
+ if got != tt.want {
+ t.Errorf("isGPUClass(%q) = %v, want %v", tt.classCode, got, tt.want)
+ }
+ }
+}
+
+func TestMapVendor(t *testing.T) {
+ tests := []struct {
+ vendorID string
+ want gpu.Vendor
+ }{
+ {"0x10de", gpu.VendorNVIDIA},
+ {"0x10DE", gpu.VendorNVIDIA}, // uppercase
+ {"0x1002", gpu.VendorAMD},
+ {"0x1002", gpu.VendorAMD},
+ {"0x8086", gpu.VendorIntel},
+ {"0x8086", gpu.VendorIntel},
+ {"0x1234", gpu.VendorUnknown},
+ {"", gpu.VendorUnknown},
+ {"unknown", gpu.VendorUnknown},
+ }
+
+ for _, tt := range tests {
+ got := mapVendor(tt.vendorID)
+ if got != tt.want {
+ t.Errorf("mapVendor(%q) = %q, want %q", tt.vendorID, got, tt.want)
+ }
+ }
+}
diff --git a/opentelemetry-gpu-collector/internal/ebpf/bpf/gpuevent.c b/opentelemetry-gpu-collector/internal/ebpf/bpf/gpuevent.c
new file mode 100644
index 00000000..6a2cc01c
--- /dev/null
+++ b/opentelemetry-gpu-collector/internal/ebpf/bpf/gpuevent.c
@@ -0,0 +1,133 @@
+// GPU event eBPF probes for CUDA runtime interception.
+// License: Apache-2.0
+
+//go:build ignore
+
+#include "vmlinux.h"
+#include "bpf_helpers.h"
+#include "bpf_tracing.h"
+#include "gpuevent.h"
+
+char LICENSE[] SEC("license") = "Dual MIT/GPL";
+
+struct {
+ __uint(type, BPF_MAP_TYPE_RINGBUF);
+ __uint(max_entries, 1 << 24); // 16 MB
+} gpu_events SEC(".maps");
+
+// Capture settings (toggled via map).
+struct {
+ __uint(type, BPF_MAP_TYPE_ARRAY);
+ __uint(max_entries, 1);
+ __type(key, __u32);
+ __type(value, __u32);
+} capture_config SEC(".maps");
+
+static __always_inline struct pid_info_t get_pid_info() {
+ struct pid_info_t pi = {};
+ __u64 pid_tgid = bpf_get_current_pid_tgid();
+ pi.host_pid = pid_tgid >> 32;
+ pi.user_pid = pid_tgid >> 32;
+
+ struct task_struct *task = (struct task_struct *)bpf_get_current_task();
+ if (task) {
+ struct nsproxy *ns;
+ bpf_probe_read_kernel(&ns, sizeof(ns), &task->nsproxy);
+ if (ns) {
+ struct pid_namespace *pidns;
+ bpf_probe_read_kernel(&pidns, sizeof(pidns), &ns->pid_ns_for_children);
+ if (pidns) {
+ unsigned int level;
+ bpf_probe_read_kernel(&level, sizeof(level), &pidns->level);
+ pi.ns = level;
+ }
+ }
+ }
+ return pi;
+}
+
+// uprobe/cudaLaunchKernel intercepts CUDA kernel launches.
+// Signature: cudaError_t cudaLaunchKernel(const void *func, dim3 gridDim,
+// dim3 blockDim, void **args, size_t sharedMem, cudaStream_t stream)
+//
+// ABI NOTE (x86_64 System V only):
+// dim3 is a struct of three uint32 values (12 bytes). Structs <= 16 bytes that
+// are "integer class" are split across consecutive argument registers. With func
+// in rdi (PARM1), the two dim3 structs consume rsi/rdx/rcx/r8 as follows:
+// gridDim.x, gridDim.y -> rsi (PARM2) low/high 32 bits
+// gridDim.z, blockDim.x -> rdx (PARM3) low/high 32 bits
+// blockDim.y, blockDim.z -> rcx (PARM4) low/high 32 bits
+// This packing was validated against CUDA 11.x–12.x on x86_64 gcc/clang.
+// It will NOT be correct on aarch64 (ARM64) or for future CUDA ABI changes.
+// TODO: replace with BTF CO-RE struct access once CUDA BTF info is available.
+SEC("uprobe/cudaLaunchKernel")
+int handle_cuda_launch(struct pt_regs *ctx) {
+ struct gpu_kernel_launch_t *ev;
+ ev = bpf_ringbuf_reserve(&gpu_events, sizeof(*ev), 0);
+ if (!ev)
+ return 0;
+
+ ev->flags = EVENT_GPU_KERNEL_LAUNCH;
+ ev->pid_info = get_pid_info();
+
+ // arg0: const void *func (kernel function pointer)
+ ev->kern_func_off = PT_REGS_PARM1(ctx);
+
+ // gridDim.x, gridDim.y packed into rsi; gridDim.z in low 32 bits of rdx
+ __u64 grid_xy = PT_REGS_PARM2(ctx);
+ ev->grid_x = grid_xy & 0xFFFFFFFF;
+ ev->grid_y = grid_xy >> 32;
+ ev->grid_z = PT_REGS_PARM3(ctx) & 0xFFFFFFFF;
+
+ // blockDim.x in high 32 bits of rdx; blockDim.y, blockDim.z in rcx
+ __u64 block_xy = PT_REGS_PARM3(ctx) >> 32;
+ ev->block_x = block_xy & 0xFFFFFFFF;
+ __u64 parm4 = PT_REGS_PARM4(ctx);
+ ev->block_y = parm4 & 0xFFFFFFFF;
+ ev->block_z = parm4 >> 32;
+
+ // arg4: cudaStream_t stream (6th parameter)
+ ev->stream = PT_REGS_PARM6(ctx);
+
+ // Capture user-space stack trace
+ bpf_get_stack(ctx, ev->ustack, sizeof(ev->ustack), BPF_F_USER_STACK);
+
+ bpf_ringbuf_submit(ev, 0);
+ return 0;
+}
+
+// uprobe/cudaMalloc intercepts GPU memory allocations.
+// Signature: cudaError_t cudaMalloc(void **devPtr, size_t size)
+SEC("uprobe/cudaMalloc")
+int handle_cuda_malloc(struct pt_regs *ctx) {
+ struct gpu_malloc_t *ev;
+ ev = bpf_ringbuf_reserve(&gpu_events, sizeof(*ev), 0);
+ if (!ev)
+ return 0;
+
+ ev->flags = EVENT_GPU_MALLOC;
+ ev->pid_info = get_pid_info();
+ ev->size = PT_REGS_PARM2(ctx);
+
+ bpf_ringbuf_submit(ev, 0);
+ return 0;
+}
+
+// uprobe/cudaMemcpy intercepts GPU memory copy operations.
+// Signature: cudaError_t cudaMemcpyAsync(void *dst, const void *src,
+// size_t count, enum cudaMemcpyKind kind, cudaStream_t stream)
+SEC("uprobe/cudaMemcpy")
+int handle_cuda_memcpy(struct pt_regs *ctx) {
+ struct gpu_memcpy_t *ev;
+ ev = bpf_ringbuf_reserve(&gpu_events, sizeof(*ev), 0);
+ if (!ev)
+ return 0;
+
+ ev->flags = EVENT_GPU_MEMCPY;
+ ev->pid_info = get_pid_info();
+ ev->size = PT_REGS_PARM3(ctx);
+ ev->kind = (__u8)PT_REGS_PARM4(ctx);
+
+ bpf_ringbuf_submit(ev, 0);
+ return 0;
+}
diff --git a/opentelemetry-gpu-collector/internal/ebpf/bpf/gpuevent.h b/opentelemetry-gpu-collector/internal/ebpf/bpf/gpuevent.h
new file mode 100644
index 00000000..786885fe
--- /dev/null
+++ b/opentelemetry-gpu-collector/internal/ebpf/bpf/gpuevent.h
@@ -0,0 +1,66 @@
+// GPU event eBPF data structures.
+// License: Apache-2.0
+
+#ifndef __GPUEVENT_H__
+#define __GPUEVENT_H__
+
+#define MAX_STACK_DEPTH 128
+#define MAX_KERNEL_ARGS 16
+
+// Event type flags (stored in first byte).
+#define EVENT_GPU_KERNEL_LAUNCH 1
+#define EVENT_GPU_MALLOC 2
+#define EVENT_GPU_MEMCPY 3
+
+// cudaMemcpyKind enum values.
+#define CUDA_MEMCPY_HOST_TO_HOST 0
+#define CUDA_MEMCPY_HOST_TO_DEVICE 1
+#define CUDA_MEMCPY_DEVICE_TO_HOST 2
+#define CUDA_MEMCPY_DEVICE_TO_DEVICE 3
+
+struct pid_info_t {
+ __u32 host_pid;
+ __u32 user_pid;
+ __u32 ns;
+};
+
+struct gpu_kernel_launch_t {
+ __u8 flags;
+ struct pid_info_t pid_info;
+ __u64 kern_func_off;
+ __u32 grid_x;
+ __u32 grid_y;
+ __u32 grid_z;
+ __u32 block_x;
+ __u32 block_y;
+ __u32 block_z;
+ __u64 stream;
+ __u64 args[MAX_KERNEL_ARGS];
+ __u64 ustack[MAX_STACK_DEPTH];
+};
+
+struct gpu_malloc_t {
+ __u8 flags;
+ struct pid_info_t pid_info;
+ __u64 size;
+};
+
+struct gpu_memcpy_t {
+ __u8 flags;
+ __u8 kind;
+ struct pid_info_t pid_info;
+ __u64 size;
+};
+
+// libbpf 0.3 (Debian Bullseye) only defines PT_REGS_PARM1–5.
+#ifndef PT_REGS_PARM6
+#if defined(__TARGET_ARCH_x86)
+#define PT_REGS_PARM6(x) ((__u64)(x)->r9)
+#elif defined(__TARGET_ARCH_arm64)
+#define PT_REGS_PARM6(x) ((__u64)(x)->regs[5])
+#else
+#error "PT_REGS_PARM6 not defined for this architecture"
+#endif
+#endif
+
+#endif /* __GPUEVENT_H__ */
diff --git a/opentelemetry-gpu-collector/internal/ebpf/demangle.go b/opentelemetry-gpu-collector/internal/ebpf/demangle.go
new file mode 100644
index 00000000..e095ce9e
--- /dev/null
+++ b/opentelemetry-gpu-collector/internal/ebpf/demangle.go
@@ -0,0 +1,15 @@
+//go:build linux
+
+package ebpf
+
+import (
+ "github.com/ianlancetaylor/demangle"
+)
+
+func tryDemangle(name string) string {
+ result, err := demangle.ToString(name)
+ if err != nil {
+ return name
+ }
+ return result
+}
diff --git a/opentelemetry-gpu-collector/internal/ebpf/symbols.go b/opentelemetry-gpu-collector/internal/ebpf/symbols.go
new file mode 100644
index 00000000..04c54081
--- /dev/null
+++ b/opentelemetry-gpu-collector/internal/ebpf/symbols.go
@@ -0,0 +1,199 @@
+//go:build linux
+
+package ebpf
+
+import (
+ "bufio"
+ "debug/elf"
+ "fmt"
+ "log/slog"
+ "os"
+ "sort"
+ "strconv"
+ "strings"
+ "sync"
+)
+
+// SymbolResolver maps kernel function addresses to human-readable names
+// by parsing ELF symbols from process memory maps.
+type SymbolResolver struct {
+ logger *slog.Logger
+ mu sync.RWMutex
+ // pid -> sorted symbol table
+ cache map[uint32]*symbolTable
+}
+
+type symbolEntry struct {
+ addr uint64
+ size uint64
+ name string
+}
+
+type symbolTable struct {
+ entries []symbolEntry
+ base uint64
+}
+
+func NewSymbolResolver(logger *slog.Logger) *SymbolResolver {
+ return &SymbolResolver{
+ logger: logger,
+ cache: make(map[uint32]*symbolTable),
+ }
+}
+
+// Resolve looks up the symbol name for a given PID and virtual address.
+func (sr *SymbolResolver) Resolve(pid uint32, addr uint64) string {
+ sr.mu.RLock()
+ st, ok := sr.cache[pid]
+ sr.mu.RUnlock()
+
+ if !ok {
+ st = sr.loadSymbols(pid)
+ sr.mu.Lock()
+ sr.cache[pid] = st
+ sr.mu.Unlock()
+ }
+
+ if st == nil {
+ return fmt.Sprintf("0x%x", addr)
+ }
+
+ name := st.lookup(addr)
+ if name == "" {
+ return fmt.Sprintf("0x%x", addr)
+ }
+
+ return demangleName(name)
+}
+
+func (st *symbolTable) lookup(addr uint64) string {
+ if len(st.entries) == 0 {
+ return ""
+ }
+
+ // Binary search for the largest entry.addr <= addr
+ idx := sort.Search(len(st.entries), func(i int) bool {
+ return st.entries[i].addr > addr
+ }) - 1
+
+ if idx < 0 {
+ return ""
+ }
+
+ entry := st.entries[idx]
+ if entry.size > 0 && addr >= entry.addr+entry.size {
+ return ""
+ }
+
+ return entry.name
+}
+
+// loadSymbols parses /proc/<pid>/maps to find CUDA-relevant libraries
+// and loads their ELF symbol tables.
+func (sr *SymbolResolver) loadSymbols(pid uint32) *symbolTable {
+ mapsPath := fmt.Sprintf("/proc/%d/maps", pid)
+ f, err := os.Open(mapsPath)
+ if err != nil {
+ sr.logger.Debug("cannot read process maps", "pid", pid, "error", err)
+ return nil
+ }
+ defer f.Close()
+
+ st := &symbolTable{}
+ seen := make(map[string]bool)
+
+ scanner := bufio.NewScanner(f)
+ for scanner.Scan() {
+ line := scanner.Text()
+ fields := strings.Fields(line)
+ if len(fields) < 6 {
+ continue
+ }
+
+ path := fields[5]
+ if !isCUDARelevant(path) || seen[path] {
+ continue
+ }
+ seen[path] = true
+
+ baseAddr := parseMapBase(fields[0])
+ symbols := loadELFSymbols(path)
+ for _, sym := range symbols {
+ st.entries = append(st.entries, symbolEntry{
+ addr: baseAddr + sym.addr,
+ size: sym.size,
+ name: sym.name,
+ })
+ }
+ }
+
+ sort.Slice(st.entries, func(i, j int) bool {
+ return st.entries[i].addr < st.entries[j].addr
+ })
+
+ return st
+}
+
+func isCUDARelevant(path string) bool {
+ if path == "" || path[0] != '/' {
+ return false
+ }
+ base := strings.ToLower(path)
+ return strings.Contains(base, "libcudart") ||
+ strings.Contains(base, "libtorch_cuda") ||
+ strings.Contains(base, "vllm") ||
+ strings.Contains(base, "ggml")
+}
+
+func parseMapBase(addrRange string) uint64 {
+ parts := strings.Split(addrRange, "-")
+ if len(parts) < 1 {
+ return 0
+ }
+ v, _ := strconv.ParseUint(parts[0], 16, 64)
+ return v
+}
+
+func loadELFSymbols(path string) []symbolEntry {
+ f, err := elf.Open(path)
+ if err != nil {
+ return nil
+ }
+ defer f.Close()
+
+ var entries []symbolEntry
+
+ // Try .symtab first, then .dynsym
+ for _, loader := range []func() ([]elf.Symbol, error){f.Symbols, f.DynamicSymbols} {
+ syms, err := loader()
+ if err != nil {
+ continue
+ }
+ for _, s := range syms {
+ if s.Value == 0 || s.Name == "" {
+ continue
+ }
+ if elf.ST_TYPE(s.Info) != elf.STT_FUNC {
+ continue
+ }
+ entries = append(entries, symbolEntry{
+ addr: s.Value,
+ size: s.Size,
+ name: s.Name,
+ })
+ }
+ }
+
+ return entries
+}
+
+// demangleName attempts to demangle a C++ mangled name.
+// Falls back to the original name if demangling fails.
+func demangleName(name string) string {
+ if !strings.HasPrefix(name, "_Z") {
+ return name
+ }
+ // Simple demangling: use the ianlancetaylor/demangle package if available,
+ // otherwise return as-is. The import is in a separate file to keep build flexibility.
+ return tryDemangle(name)
+}
diff --git a/opentelemetry-gpu-collector/internal/ebpf/tracer.go b/opentelemetry-gpu-collector/internal/ebpf/tracer.go
new file mode 100644
index 00000000..4c19d238
--- /dev/null
+++ b/opentelemetry-gpu-collector/internal/ebpf/tracer.go
@@ -0,0 +1,242 @@
+//go:build linux
+
+package ebpf
+
+import (
+ "context"
+ "encoding/binary"
+ "errors"
+ "fmt"
+ "log/slog"
+ "os"
+ "path/filepath"
+ "strings"
+ "sync"
+ "sync/atomic"
+
+ "github.com/cilium/ebpf"
+ "github.com/cilium/ebpf/link"
+ "github.com/cilium/ebpf/ringbuf"
+)
+
+//go:generate go run github.com/cilium/ebpf/cmd/bpf2go -cc clang -target bpfel gpuevent ./bpf/gpuevent.c -- -I./bpf -D__TARGET_ARCH_x86
+
+// Tracer manages eBPF programs for CUDA runtime interception.
+type Tracer struct {
+ logger *slog.Logger
+ objs *gpueventObjects
+ reader *ringbuf.Reader
+ links []link.Link
+ handler EventHandler
+ symbols *SymbolResolver
+ running atomic.Bool
+ wg sync.WaitGroup
+}
+
+// NewTracer loads eBPF programs and attaches uprobes to libcudart.so.
+func NewTracer(logger *slog.Logger, handler EventHandler) (*Tracer, error) {
+ cudaLib := findCudaLib()
+ if cudaLib == "" {
+ return nil, fmt.Errorf("libcudart.so not found; CUDA runtime is not installed")
+ }
+
+ logger.Info("found CUDA runtime library", "path", cudaLib)
+
+ spec, err := loadGpuevent()
+ if err != nil {
+ return nil, fmt.Errorf("loading eBPF spec: %w", err)
+ }
+
+ objs := new(gpueventObjects)
+ if err := spec.LoadAndAssign(objs, &ebpf.CollectionOptions{}); err != nil {
+ return nil, fmt.Errorf("loading eBPF objects: %w", err)
+ }
+
+ t := &Tracer{
+ logger: logger,
+ objs: objs,
+ handler: handler,
+ symbols: NewSymbolResolver(logger),
+ }
+
+ ex, err := link.OpenExecutable(cudaLib)
+ if err != nil {
+ t.Close()
+ return nil, fmt.Errorf("opening %s: %w", cudaLib, err)
+ }
+
+ probes := map[string]*ebpf.Program{
+ "cudaLaunchKernel": objs.HandleCudaLaunch,
+ "cudaMalloc": objs.HandleCudaMalloc,
+ "cudaMemcpy": objs.HandleCudaMemcpy,
+ "cudaMemcpyAsync": objs.HandleCudaMemcpy,
+ }
+
+ for sym, prog := range probes {
+ l, err := ex.Uprobe(sym, prog, nil)
+ if err != nil {
+ logger.Warn("failed to attach uprobe", "symbol", sym, "error", err)
+ continue
+ }
+ t.links = append(t.links, l)
+ logger.Info("attached uprobe", "symbol", sym)
+ }
+
+ if len(t.links) == 0 {
+ t.Close()
+ return nil, fmt.Errorf("no uprobes attached; CUDA symbols not found in %s", cudaLib)
+ }
+
+ t.reader, err = ringbuf.NewReader(objs.GpuEvents)
+ if err != nil {
+ t.Close()
+ return nil, fmt.Errorf("creating ring buffer reader: %w", err)
+ }
+
+ return t, nil
+}
+
+// Run starts reading events from the ring buffer. Blocks until ctx is cancelled.
+func (t *Tracer) Run(ctx context.Context) {
+ t.running.Store(true)
+ t.wg.Add(1)
+ defer t.wg.Done()
+
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ default:
+ }
+
+ record, err := t.reader.Read()
+ if err != nil {
+ if errors.Is(err, ringbuf.ErrClosed) {
+ return
+ }
+ t.logger.Warn("ring buffer read error", "error", err)
+ continue
+ }
+
+ t.processRecord(record.RawSample)
+ }
+}
+
+func (t *Tracer) processRecord(data []byte) {
+ if len(data) < 1 {
+ return
+ }
+
+ eventType := data[0]
+ switch eventType {
+ case EventTypeKernelLaunch:
+ t.processKernelLaunch(data)
+ case EventTypeMalloc:
+ t.processMalloc(data)
+ case EventTypeMemcpy:
+ t.processMemcpy(data)
+ default:
+ t.logger.Debug("unknown GPU event type", "type", eventType)
+ }
+}
+
+func (t *Tracer) processKernelLaunch(data []byte) {
+ // Parse the gpu_kernel_launch_t struct.
+ // Layout: flags(1) + pad(3) + pid_info(12) + kern_func_off(8) + grid(12) + block(12) + ...
+ if len(data) < 48 {
+ return
+ }
+
+ ev := &KernelLaunchEvent{}
+ ev.PID = binary.LittleEndian.Uint32(data[4:8])
+ ev.KernelAddr = binary.LittleEndian.Uint64(data[16:24])
+ ev.GridX = binary.LittleEndian.Uint32(data[24:28])
+ ev.GridY = binary.LittleEndian.Uint32(data[28:32])
+ ev.GridZ = binary.LittleEndian.Uint32(data[32:36])
+ ev.BlockX = binary.LittleEndian.Uint32(data[36:40])
+ ev.BlockY = binary.LittleEndian.Uint32(data[40:44])
+ ev.BlockZ = binary.LittleEndian.Uint32(data[44:48])
+
+ ev.KernelName = t.symbols.Resolve(ev.PID, ev.KernelAddr)
+
+ t.handler(ev)
+}
+
+func (t *Tracer) processMalloc(data []byte) {
+ // gpu_malloc_t: flags(1) + pad(3) + pid_info(12) + size(8) = 24 bytes
+ if len(data) < 24 {
+ return
+ }
+
+ ev := &MallocEvent{}
+ ev.PID = binary.LittleEndian.Uint32(data[4:8])
+ ev.Size = binary.LittleEndian.Uint64(data[16:24])
+
+ t.handler(ev)
+}
+
+func (t *Tracer) processMemcpy(data []byte) {
+ if len(data) < 24 {
+ return
+ }
+
+ ev := &MemcpyEvent{}
+ ev.Kind = data[1]
+ ev.PID = binary.LittleEndian.Uint32(data[4:8])
+ ev.Size = binary.LittleEndian.Uint64(data[16:24])
+
+ t.handler(ev)
+}
+
+// Close detaches all probes and frees resources.
+func (t *Tracer) Close() {
+ if t.reader != nil {
+ t.reader.Close()
+ }
+ for _, l := range t.links {
+ l.Close()
+ }
+ if t.objs != nil {
+ t.objs.Close()
+ }
+ t.wg.Wait()
+}
+
+// findCudaLib searches for libcudart.so in common locations.
+func findCudaLib() string {
+ candidates := []string{
+ "/usr/local/cuda/lib64/libcudart.so",
+ "/usr/lib/x86_64-linux-gnu/libcudart.so",
+ "/usr/lib64/libcudart.so",
+ "/usr/lib/libcudart.so",
+ }
+
+ // Also check LD_LIBRARY_PATH
+ if ldPath := os.Getenv("LD_LIBRARY_PATH"); ldPath != "" {
+ for _, dir := range strings.Split(ldPath, ":") {
+ candidates = append(candidates, filepath.Join(dir, "libcudart.so"))
+ }
+ }
+
+ // Check CUDA_HOME
+ if cudaHome := os.Getenv("CUDA_HOME"); cudaHome != "" {
+ candidates = append(candidates,
+ filepath.Join(cudaHome, "lib64", "libcudart.so"),
+ filepath.Join(cudaHome, "lib", "libcudart.so"),
+ )
+ }
+
+ for _, path := range candidates {
+ if _, err := os.Stat(path); err == nil {
+ return path
+ }
+ }
+
+ // Try to find via /etc/ld.so.cache by searching /usr/local/cuda*/lib64/
+ matches, _ := filepath.Glob("/usr/local/cuda*/lib64/libcudart.so")
+ if len(matches) > 0 {
+ return matches[len(matches)-1]
+ }
+
+ return ""
+}
diff --git a/opentelemetry-gpu-collector/internal/ebpf/tracer_stub.go b/opentelemetry-gpu-collector/internal/ebpf/tracer_stub.go
new file mode 100644
index 00000000..6a8f3e8e
--- /dev/null
+++ b/opentelemetry-gpu-collector/internal/ebpf/tracer_stub.go
@@ -0,0 +1,18 @@
+//go:build !linux
+
+package ebpf
+
+import (
+ "context"
+ "fmt"
+ "log/slog"
+)
+
+type Tracer struct{}
+
+func NewTracer(_ *slog.Logger, _ EventHandler) (*Tracer, error) {
+ return nil, fmt.Errorf("eBPF CUDA tracing is only supported on Linux")
+}
+
+func (t *Tracer) Run(_ context.Context) {}
+func (t *Tracer) Close() {}
diff --git a/opentelemetry-gpu-collector/internal/ebpf/types.go b/opentelemetry-gpu-collector/internal/ebpf/types.go
new file mode 100644
index 00000000..9e19c7d4
--- /dev/null
+++ b/opentelemetry-gpu-collector/internal/ebpf/types.go
@@ -0,0 +1,61 @@
+package ebpf
+
+// Event type constants shared across platforms.
+const (
+ EventTypeKernelLaunch = 1
+ EventTypeMalloc = 2
+ EventTypeMemcpy = 3
+)
+
+// CUDAEvent is the common interface for parsed ring buffer events.
+type CUDAEvent interface {
+ EventType() uint8
+}
+
+type KernelLaunchEvent struct {
+ PID uint32
+ KernelAddr uint64
+ GridX uint32
+ GridY uint32
+ GridZ uint32
+ BlockX uint32
+ BlockY uint32
+ BlockZ uint32
+ KernelName string
+}
+
+func (e *KernelLaunchEvent) EventType() uint8 { return EventTypeKernelLaunch }
+
+type MallocEvent struct {
+ PID uint32
+ Size uint64
+}
+
+func (e *MallocEvent) EventType() uint8 { return EventTypeMalloc }
+
+type MemcpyEvent struct {
+ PID uint32
+ Size uint64
+ Kind uint8
+}
+
+func (e *MemcpyEvent) EventType() uint8 { return EventTypeMemcpy }
+
+// MemcpyKindString returns a human-readable string for cudaMemcpyKind.
+func MemcpyKindString(kind uint8) string {
+ switch kind {
+ case 0:
+ return "HostToHost"
+ case 1:
+ return "HostToDevice"
+ case 2:
+ return "DeviceToHost"
+ case 3:
+ return "DeviceToDevice"
+ default:
+ return "Unknown"
+ }
+}
+
+// EventHandler is called for each parsed CUDA event.
+type EventHandler func(CUDAEvent)
diff --git a/opentelemetry-gpu-collector/internal/ebpf/types_test.go b/opentelemetry-gpu-collector/internal/ebpf/types_test.go
new file mode 100644
index 00000000..65b5f6cb
--- /dev/null
+++ b/opentelemetry-gpu-collector/internal/ebpf/types_test.go
@@ -0,0 +1,64 @@
+package ebpf
+
+import "testing"
+
+func TestEventTypeConstants(t *testing.T) {
+ if EventTypeKernelLaunch != 1 {
+ t.Errorf("EventTypeKernelLaunch = %d, want 1", EventTypeKernelLaunch)
+ }
+ if EventTypeMalloc != 2 {
+ t.Errorf("EventTypeMalloc = %d, want 2", EventTypeMalloc)
+ }
+ if EventTypeMemcpy != 3 {
+ t.Errorf("EventTypeMemcpy = %d, want 3", EventTypeMemcpy)
+ }
+}
+
+func TestKernelLaunchEventType(t *testing.T) {
+ e := &KernelLaunchEvent{}
+ if e.EventType() != EventTypeKernelLaunch {
+ t.Errorf("KernelLaunchEvent.EventType() = %d, want %d", e.EventType(), EventTypeKernelLaunch)
+ }
+}
+
+func TestMallocEventType(t *testing.T) {
+ e := &MallocEvent{}
+ if e.EventType() != EventTypeMalloc {
+ t.Errorf("MallocEvent.EventType() = %d, want %d", e.EventType(), EventTypeMalloc)
+ }
+}
+
+func TestMemcpyEventType(t *testing.T) {
+ e := &MemcpyEvent{}
+ if e.EventType() != EventTypeMemcpy {
+ t.Errorf("MemcpyEvent.EventType() = %d, want %d", e.EventType(), EventTypeMemcpy)
+ }
+}
+
+func TestMemcpyKindString(t *testing.T) {
+ tests := []struct {
+ kind uint8
+ want string
+ }{
+ {0, "HostToHost"},
+ {1, "HostToDevice"},
+ {2, "DeviceToHost"},
+ {3, "DeviceToDevice"},
+ {4, "Unknown"},
+ {255, "Unknown"},
+ }
+
+ for _, tt := range tests {
+ got := MemcpyKindString(tt.kind)
+ if got != tt.want {
+ t.Errorf("MemcpyKindString(%d) = %q, want %q", tt.kind, got, tt.want)
+ }
+ }
+}
+
+func TestCUDAEventInterface(t *testing.T) {
+ // Verify all event types satisfy the CUDAEvent interface at compile time.
+ var _ CUDAEvent = (*KernelLaunchEvent)(nil)
+ var _ CUDAEvent = (*MallocEvent)(nil)
+ var _ CUDAEvent = (*MemcpyEvent)(nil)
+}
diff --git a/opentelemetry-gpu-collector/internal/export/ebpfmetrics.go b/opentelemetry-gpu-collector/internal/export/ebpfmetrics.go
new file mode 100644
index 00000000..7947b662
--- /dev/null
+++ b/opentelemetry-gpu-collector/internal/export/ebpfmetrics.go
@@ -0,0 +1,114 @@
+package export
+
+import (
+ "context"
+ "fmt"
+ "log/slog"
+ "sync"
+
+ "go.opentelemetry.io/otel/attribute"
+ "go.opentelemetry.io/otel/metric"
+ sdkmetric "go.opentelemetry.io/otel/sdk/metric"
+
+ gpuebpf "github.com/openlit/openlit/opentelemetry-gpu-collector/internal/ebpf"
+)
+
+// EBPFMetrics records eBPF CUDA tracing events as OTel metrics.
+type EBPFMetrics struct {
+ logger *slog.Logger
+ mu sync.Mutex
+
+ kernelLaunchCalls metric.Int64Counter
+ kernelGridSize metric.Float64Histogram
+ kernelBlockSize metric.Float64Histogram
+ memoryAllocations metric.Int64Counter
+ memoryCopies metric.Float64Histogram
+}
+
+// NewEBPFMetrics creates OTel instruments for eBPF CUDA event metrics.
+func NewEBPFMetrics(provider *sdkmetric.MeterProvider, logger *slog.Logger) (*EBPFMetrics, error) {
+ meter := provider.Meter("otelcol.gpu.ebpf",
+ metric.WithInstrumentationVersion("1.0.0"),
+ )
+
+ kernelCalls, err := meter.Int64Counter("gpu.kernel.launch.calls",
+ metric.WithDescription("Number of CUDA kernel launches"),
+ metric.WithUnit("{call}"),
+ )
+ if err != nil {
+ return nil, fmt.Errorf("creating gpu.kernel.launch.calls: %w", err)
+ }
+
+ gridSize, err := meter.Float64Histogram("gpu.kernel.grid.size",
+ metric.WithDescription("Total threads in CUDA grid per kernel launch"),
+ metric.WithUnit("{thread}"),
+ )
+ if err != nil {
+ return nil, fmt.Errorf("creating gpu.kernel.grid.size: %w", err)
+ }
+
+ blockSize, err := meter.Float64Histogram("gpu.kernel.block.size",
+ metric.WithDescription("Threads per CUDA block per kernel launch"),
+ metric.WithUnit("{thread}"),
+ )
+ if err != nil {
+ return nil, fmt.Errorf("creating gpu.kernel.block.size: %w", err)
+ }
+
+ memAlloc, err := meter.Int64Counter("gpu.memory.allocations",
+ metric.WithDescription("Total bytes allocated via cudaMalloc"),
+ metric.WithUnit("bytes"),
+ )
+ if err != nil {
+ return nil, fmt.Errorf("creating gpu.memory.allocations: %w", err)
+ }
+
+ memCopies, err := meter.Float64Histogram("gpu.memory.copies",
+ metric.WithDescription("Bytes copied per cudaMemcpy operation"),
+ metric.WithUnit("bytes"),
+ )
+ if err != nil {
+ return nil, fmt.Errorf("creating gpu.memory.copies: %w", err)
+ }
+
+ return &EBPFMetrics{
+ logger: logger,
+ kernelLaunchCalls: kernelCalls,
+ kernelGridSize: gridSize,
+ kernelBlockSize: blockSize,
+ memoryAllocations: memAlloc,
+ memoryCopies: memCopies,
+ }, nil
+}
+
+// HandleEvent processes a single CUDA event and records it as OTel metrics.
+// This is the EventHandler passed to the eBPF Tracer.
+func (em *EBPFMetrics) HandleEvent(ev gpuebpf.CUDAEvent) {
+ ctx := context.Background()
+
+ switch e := ev.(type) {
+ case *gpuebpf.KernelLaunchEvent:
+ kernelName := e.KernelName
+ if kernelName == "" {
+ kernelName = fmt.Sprintf("0x%x", e.KernelAddr)
+ }
+ attrs := metric.WithAttributes(attribute.String("cuda.kernel.name", kernelName))
+
+ em.kernelLaunchCalls.Add(ctx, 1, attrs)
+
+ gridTotal := float64(e.GridX) * float64(e.GridY) * float64(e.GridZ)
+ em.kernelGridSize.Record(ctx, gridTotal, attrs)
+
+ blockTotal := float64(e.BlockX) * float64(e.BlockY) * float64(e.BlockZ)
+ em.kernelBlockSize.Record(ctx, blockTotal, attrs)
+
+ case *gpuebpf.MallocEvent:
+ em.memoryAllocations.Add(ctx, int64(e.Size))
+
+ case *gpuebpf.MemcpyEvent:
+ attrs := metric.WithAttributes(
+ attribute.String("cuda.memcpy.kind", gpuebpf.MemcpyKindString(e.Kind)),
+ )
+ em.memoryCopies.Record(ctx, float64(e.Size), attrs)
+ }
+}
diff --git a/opentelemetry-gpu-collector/internal/export/metrics.go b/opentelemetry-gpu-collector/internal/export/metrics.go
new file mode 100644
index 00000000..c4fad0ce
--- /dev/null
+++ b/opentelemetry-gpu-collector/internal/export/metrics.go
@@ -0,0 +1,270 @@
+package export
+
+import (
+ "context"
+ "fmt"
+ "log/slog"
+
+ "go.opentelemetry.io/otel/attribute"
+ "go.opentelemetry.io/otel/metric"
+ sdkmetric "go.opentelemetry.io/otel/sdk/metric"
+
+ "github.com/openlit/openlit/opentelemetry-gpu-collector/internal/gpu"
+)
+
+// MetricsCollector registers OTel observable instruments that poll GPU devices on each collection cycle.
+type MetricsCollector struct {
+ devices []gpu.Device
+ logger *slog.Logger
+ reg []metric.Registration
+}
+
+// NewMetricsCollector creates all GPU metric instruments and registers callbacks.
+// Metric names and attributes follow the OpenTelemetry semantic conventions for hardware:
+// https://opentelemetry.io/docs/specs/semconv/hardware/gpu/
+func NewMetricsCollector(provider *sdkmetric.MeterProvider, devices []gpu.Device, logger *slog.Logger) (*MetricsCollector, error) {
+ mc := &MetricsCollector{
+ devices: devices,
+ logger: logger,
+ }
+
+ meter := provider.Meter("otelcol.gpu.collector",
+ metric.WithInstrumentationVersion("1.0.0"),
+ )
+
+ // hw.gpu.utilization — general + encoder + decoder via hw.gpu.task attribute
+ gpuUtilization, err := meter.Float64ObservableGauge("hw.gpu.utilization",
+ metric.WithDescription("GPU utilization"),
+ metric.WithUnit("1"),
+ )
+ if err != nil {
+ return nil, fmt.Errorf("creating hw.gpu.utilization: %w", err)
+ }
+
+ // hw.gpu.memory.utilization
+ memUtilization, err := meter.Float64ObservableGauge("hw.gpu.memory.utilization",
+ metric.WithDescription("GPU memory utilization"),
+ metric.WithUnit("1"),
+ )
+ if err != nil {
+ return nil, fmt.Errorf("creating hw.gpu.memory.utilization: %w", err)
+ }
+
+ // hw.gpu.memory.limit — total memory
+ memLimit, err := meter.Int64ObservableUpDownCounter("hw.gpu.memory.limit",
+ metric.WithDescription("Total GPU memory"),
+ metric.WithUnit("By"),
+ )
+ if err != nil {
+ return nil, fmt.Errorf("creating hw.gpu.memory.limit: %w", err)
+ }
+
+ // hw.gpu.memory.usage — used memory
+ memUsage, err := meter.Int64ObservableUpDownCounter("hw.gpu.memory.usage",
+ metric.WithDescription("Used GPU memory"),
+ metric.WithUnit("By"),
+ )
+ if err != nil {
+ return nil, fmt.Errorf("creating hw.gpu.memory.usage: %w", err)
+ }
+
+ // hw.errors — ECC and PCIe errors
+ hwErrors, err := meter.Int64ObservableCounter("hw.errors",
+ metric.WithDescription("GPU hardware error count"),
+ metric.WithUnit("{error}"),
+ )
+ if err != nil {
+ return nil, fmt.Errorf("creating hw.errors: %w", err)
+ }
+
+ // Custom metrics not covered by the spec
+ temperature, err := meter.Float64ObservableGauge("hw.gpu.temperature",
+ metric.WithDescription("GPU temperature"),
+ metric.WithUnit("Cel"),
+ )
+ if err != nil {
+ return nil, fmt.Errorf("creating hw.gpu.temperature: %w", err)
+ }
+
+ fanSpeed, err := meter.Float64ObservableGauge("hw.gpu.fan_speed",
+ metric.WithDescription("GPU fan speed"),
+ metric.WithUnit("{rpm}"),
+ )
+ if err != nil {
+ return nil, fmt.Errorf("creating hw.gpu.fan_speed: %w", err)
+ }
+
+ memFree, err := meter.Int64ObservableUpDownCounter("hw.gpu.memory.free",
+ metric.WithDescription("Free GPU memory"),
+ metric.WithUnit("By"),
+ )
+ if err != nil {
+ return nil, fmt.Errorf("creating hw.gpu.memory.free: %w", err)
+ }
+
+ powerDraw, err := meter.Float64ObservableGauge("hw.gpu.power.draw",
+ metric.WithDescription("GPU power draw"),
+ metric.WithUnit("W"),
+ )
+ if err != nil {
+ return nil, fmt.Errorf("creating hw.gpu.power.draw: %w", err)
+ }
+
+ powerLimit, err := meter.Float64ObservableGauge("hw.gpu.power.limit",
+ metric.WithDescription("GPU power limit"),
+ metric.WithUnit("W"),
+ )
+ if err != nil {
+ return nil, fmt.Errorf("creating hw.gpu.power.limit: %w", err)
+ }
+
+ energyConsumed, err := meter.Float64ObservableCounter("hw.gpu.energy.consumed",
+ metric.WithDescription("Cumulative GPU energy consumed"),
+ metric.WithUnit("J"),
+ )
+ if err != nil {
+ return nil, fmt.Errorf("creating hw.gpu.energy.consumed: %w", err)
+ }
+
+ clockGraphics, err := meter.Float64ObservableGauge("hw.gpu.clock.graphics",
+ metric.WithDescription("GPU graphics clock frequency"),
+ metric.WithUnit("MHz"),
+ )
+ if err != nil {
+ return nil, fmt.Errorf("creating hw.gpu.clock.graphics: %w", err)
+ }
+
+ clockMemory, err := meter.Float64ObservableGauge("hw.gpu.clock.memory",
+ metric.WithDescription("GPU memory clock frequency"),
+ metric.WithUnit("MHz"),
+ )
+ if err != nil {
+ return nil, fmt.Errorf("creating hw.gpu.clock.memory: %w", err)
+ }
+
+ reg, err := meter.RegisterCallback(
+ func(ctx context.Context, o metric.Observer) error {
+ for _, dev := range mc.devices {
+ snap, err := dev.Collect()
+ if err != nil {
+ mc.logger.Warn("collection failed", "gpu", dev.Info().Index, "error", err)
+ continue
+ }
+
+ info := dev.Info()
+ attrs := deviceAttrs(info)
+
+ if snap.Utilization != nil {
+ generalAttrs := attribute.NewSet(append(attrs.ToSlice(), attribute.String("hw.gpu.task", "general"))...)
+ o.ObserveFloat64(gpuUtilization, *snap.Utilization/100.0, metric.WithAttributeSet(generalAttrs))
+ }
+ if snap.EncoderUtilization != nil {
+ encAttrs := attribute.NewSet(append(attrs.ToSlice(), attribute.String("hw.gpu.task", "encoder"))...)
+ o.ObserveFloat64(gpuUtilization, *snap.EncoderUtilization/100.0, metric.WithAttributeSet(encAttrs))
+ }
+ if snap.DecoderUtilization != nil {
+ decAttrs := attribute.NewSet(append(attrs.ToSlice(), attribute.String("hw.gpu.task", "decoder"))...)
+ o.ObserveFloat64(gpuUtilization, *snap.DecoderUtilization/100.0, metric.WithAttributeSet(decAttrs))
+ }
+
+ if snap.MemoryUtilization != nil {
+ o.ObserveFloat64(memUtilization, *snap.MemoryUtilization/100.0, metric.WithAttributeSet(attrs))
+ }
+
+ if snap.TemperatureGPU != nil {
+ dieAttrs := attribute.NewSet(append(attrs.ToSlice(), attribute.String("sensor", "die"))...)
+ o.ObserveFloat64(temperature, *snap.TemperatureGPU, metric.WithAttributeSet(dieAttrs))
+ }
+ if snap.TemperatureMemory != nil {
+ memTempAttrs := attribute.NewSet(append(attrs.ToSlice(), attribute.String("sensor", "memory"))...)
+ o.ObserveFloat64(temperature, *snap.TemperatureMemory, metric.WithAttributeSet(memTempAttrs))
+ }
+
+ if snap.FanSpeedRPM != nil {
+ o.ObserveFloat64(fanSpeed, *snap.FanSpeedRPM, metric.WithAttributeSet(attrs))
+ }
+
+ if snap.MemoryTotalBytes != nil {
+ o.ObserveInt64(memLimit, *snap.MemoryTotalBytes, metric.WithAttributeSet(attrs))
+ }
+ if snap.MemoryUsedBytes != nil {
+ o.ObserveInt64(memUsage, *snap.MemoryUsedBytes, metric.WithAttributeSet(attrs))
+ }
+ if snap.MemoryFreeBytes != nil {
+ o.ObserveInt64(memFree, *snap.MemoryFreeBytes, metric.WithAttributeSet(attrs))
+ }
+
+ if snap.PowerDrawWatts != nil {
+ o.ObserveFloat64(powerDraw, *snap.PowerDrawWatts, metric.WithAttributeSet(attrs))
+ }
+ if snap.PowerLimitWatts != nil {
+ o.ObserveFloat64(powerLimit, *snap.PowerLimitWatts, metric.WithAttributeSet(attrs))
+ }
+ if snap.EnergyJoules != nil {
+ o.ObserveFloat64(energyConsumed, *snap.EnergyJoules, metric.WithAttributeSet(attrs))
+ }
+
+ if snap.ClockGraphicsMHz != nil {
+ o.ObserveFloat64(clockGraphics, *snap.ClockGraphicsMHz, metric.WithAttributeSet(attrs))
+ }
+ if snap.ClockMemoryMHz != nil {
+ o.ObserveFloat64(clockMemory, *snap.ClockMemoryMHz, metric.WithAttributeSet(attrs))
+ }
+
+ if snap.PCIeReplayErrors != nil {
+ pcieAttrs := attribute.NewSet(append(attrs.ToSlice(),
+ attribute.String("hw.type", "gpu"),
+ attribute.String("error.type", "pcie_replay"),
+ )...)
+ o.ObserveInt64(hwErrors, *snap.PCIeReplayErrors, metric.WithAttributeSet(pcieAttrs))
+ }
+ if snap.ECCSingleBit != nil {
+ corrAttrs := attribute.NewSet(append(attrs.ToSlice(),
+ attribute.String("hw.type", "gpu"),
+ attribute.String("error.type", "corrected"),
+ )...)
+ o.ObserveInt64(hwErrors, *snap.ECCSingleBit, metric.WithAttributeSet(corrAttrs))
+ }
+ if snap.ECCDoubleBit != nil {
+ uncorrAttrs := attribute.NewSet(append(attrs.ToSlice(),
+ attribute.String("hw.type", "gpu"),
+ attribute.String("error.type", "uncorrected"),
+ )...)
+ o.ObserveInt64(hwErrors, *snap.ECCDoubleBit, metric.WithAttributeSet(uncorrAttrs))
+ }
+ }
+ return nil
+ },
+ gpuUtilization, memUtilization,
+ memLimit, memUsage, memFree,
+ temperature, fanSpeed,
+ powerDraw, powerLimit, energyConsumed,
+ clockGraphics, clockMemory,
+ hwErrors,
+ )
+ if err != nil {
+ return nil, fmt.Errorf("registering callback: %w", err)
+ }
+
+ mc.reg = append(mc.reg, reg)
+ return mc, nil
+}
+
+// Close unregisters all metric callbacks.
+func (mc *MetricsCollector) Close() {
+ for _, r := range mc.reg {
+ _ = r.Unregister()
+ }
+}
+
+// deviceAttrs returns the standard hw.* attribute set for a GPU device.
+// hw.id is required; hw.name, hw.vendor are recommended by the spec.
+func deviceAttrs(info gpu.DeviceInfo) attribute.Set {
+ return attribute.NewSet(
+ attribute.String("hw.id", info.UUID),
+ attribute.String("hw.name", info.Name),
+ attribute.String("hw.vendor", string(info.Vendor)),
+ attribute.Int("gpu.index", info.Index),
+ attribute.String("gpu.pci_address", info.PCIAddress),
+ )
+}
diff --git a/opentelemetry-gpu-collector/internal/export/metrics_test.go b/opentelemetry-gpu-collector/internal/export/metrics_test.go
new file mode 100644
index 00000000..6e041fcf
--- /dev/null
+++ b/opentelemetry-gpu-collector/internal/export/metrics_test.go
@@ -0,0 +1,160 @@
+package export
+
+import (
+ "log/slog"
+ "testing"
+
+ "go.opentelemetry.io/otel/sdk/metric"
+ "go.opentelemetry.io/otel/sdk/metric/metricdata"
+
+ "github.com/openlit/openlit/opentelemetry-gpu-collector/internal/gpu"
+)
+
+// mockDevice implements gpu.Device for testing without real hardware.
+type mockDevice struct {
+ info gpu.DeviceInfo
+ snapshot *gpu.Snapshot
+ err error
+}
+
+func (m *mockDevice) Info() gpu.DeviceInfo { return m.info }
+func (m *mockDevice) Collect() (*gpu.Snapshot, error) { return m.snapshot, m.err }
+func (m *mockDevice) Close() {}
+
+func ptr[T any](v T) *T { return &v }
+
+func TestNewMetricsCollectorRegisters(t *testing.T) {
+ reader := metric.NewManualReader()
+ provider := metric.NewMeterProvider(metric.WithReader(reader))
+ defer provider.Shutdown(t.Context())
+
+ dev := &mockDevice{
+ info: gpu.DeviceInfo{
+ Vendor: gpu.VendorNVIDIA,
+ Index: 0,
+ Name: "Test GPU",
+ UUID: "GPU-test-uuid-0000",
+ PCIAddress: "0000:01:00.0",
+ },
+ snapshot: &gpu.Snapshot{
+ Utilization: ptr(80.0),
+ EncoderUtilization: ptr(10.0),
+ DecoderUtilization: ptr(5.0),
+ MemoryUtilization: ptr(60.0),
+ TemperatureGPU: ptr(70.0),
+ MemoryTotalBytes: ptr(int64(8 * 1024 * 1024 * 1024)),
+ MemoryUsedBytes: ptr(int64(4 * 1024 * 1024 * 1024)),
+ MemoryFreeBytes: ptr(int64(4 * 1024 * 1024 * 1024)),
+ PowerDrawWatts: ptr(150.0),
+ PowerLimitWatts: ptr(250.0),
+ EnergyJoules: ptr(12345.0),
+ ClockGraphicsMHz: ptr(1800.0),
+ ClockMemoryMHz: ptr(9000.0),
+ PCIeReplayErrors: ptr(int64(0)),
+ ECCSingleBit: ptr(int64(0)),
+ ECCDoubleBit: ptr(int64(0)),
+ },
+ }
+
+ logger := slog.Default()
+ mc, err := NewMetricsCollector(provider, []gpu.Device{dev}, logger)
+ if err != nil {
+ t.Fatalf("NewMetricsCollector() error = %v", err)
+ }
+ defer mc.Close()
+
+ // Trigger a collection by gathering metrics.
+ var rm metricdata.ResourceMetrics
+ if err := reader.Collect(t.Context(), &rm); err != nil {
+ t.Fatalf("reader.Collect() error = %v", err)
+ }
+
+ // Find the scope metrics for our collector.
+ var scopeMetrics *metricdata.ScopeMetrics
+ for i := range rm.ScopeMetrics {
+ if rm.ScopeMetrics[i].Scope.Name == "otelcol.gpu.collector" {
+ scopeMetrics = &rm.ScopeMetrics[i]
+ break
+ }
+ }
+ if scopeMetrics == nil {
+ t.Fatal("no ScopeMetrics found for otelcol.gpu.collector")
+ }
+
+ // Build a set of reported metric names.
+ reported := make(map[string]bool)
+ for _, m := range scopeMetrics.Metrics {
+ reported[m.Name] = true
+ }
+
+ wantMetrics := []string{
+ "hw.gpu.utilization",
+ "hw.gpu.memory.utilization",
+ "hw.gpu.memory.limit",
+ "hw.gpu.memory.usage",
+ "hw.gpu.memory.free",
+ "hw.gpu.temperature",
+ "hw.gpu.power.draw",
+ "hw.gpu.power.limit",
+ "hw.gpu.energy.consumed",
+ "hw.gpu.clock.graphics",
+ "hw.gpu.clock.memory",
+ "hw.errors",
+ }
+ for _, name := range wantMetrics {
+ if !reported[name] {
+ t.Errorf("expected metric %q to be reported, but it was not", name)
+ }
+ }
+}
+
+func TestNewMetricsCollectorNoDevices(t *testing.T) {
+ provider := metric.NewMeterProvider()
+ defer provider.Shutdown(t.Context())
+
+ mc, err := NewMetricsCollector(provider, nil, slog.Default())
+ if err != nil {
+ t.Fatalf("NewMetricsCollector() with no devices error = %v", err)
+ }
+ mc.Close()
+}
+
+func TestMetricsCollectorCloseIdempotent(t *testing.T) {
+ provider := metric.NewMeterProvider()
+ defer provider.Shutdown(t.Context())
+
+ mc, err := NewMetricsCollector(provider, nil, slog.Default())
+ if err != nil {
+ t.Fatalf("NewMetricsCollector() error = %v", err)
+ }
+ // Double-close should not panic.
+ mc.Close()
+ mc.Close()
+}
+
+func TestDeviceAttrs(t *testing.T) {
+ info := gpu.DeviceInfo{
+ Vendor: gpu.VendorAMD,
+ Index: 2,
+ Name: "Radeon RX 7900 XT",
+ UUID: "GPU-amd-uuid-0001",
+ PCIAddress: "0000:03:00.0",
+ }
+ attrs := deviceAttrs(info)
+
+ lookup := attrs.ToSlice()
+ kvMap := make(map[string]string)
+ for _, kv := range lookup {
+ kvMap[string(kv.Key)] = kv.Value.AsString()
+ }
+
+ if kvMap["hw.id"] != info.UUID {
+ t.Errorf("hw.id = %q, want %q", kvMap["hw.id"], info.UUID)
+ }
+ if kvMap["hw.name"] != info.Name {
+ t.Errorf("hw.name = %q, want %q", kvMap["hw.name"], info.Name)
+ }
+ if kvMap["hw.vendor"] != string(info.Vendor) {
+ t.Errorf("hw.vendor = %q, want %q", kvMap["hw.vendor"], string(info.Vendor))
+ }
+}
diff --git a/opentelemetry-gpu-collector/internal/export/otel.go b/opentelemetry-gpu-collector/internal/export/otel.go
new file mode 100644
index 00000000..4d1d32b4
--- /dev/null
+++ b/opentelemetry-gpu-collector/internal/export/otel.go
@@ -0,0 +1,80 @@
+package export
+
+import (
+ "context"
+ "fmt"
+ "log/slog"
+ "os"
+ "time"
+
+ "go.opentelemetry.io/otel"
+ "go.opentelemetry.io/otel/attribute"
+ "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc"
+ "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp"
+ "go.opentelemetry.io/otel/sdk/metric"
+ "go.opentelemetry.io/otel/sdk/resource"
+
+ "github.com/openlit/openlit/opentelemetry-gpu-collector/internal/config"
+)
+
+type slogErrorHandler struct{ logger *slog.Logger }
+
+func (h *slogErrorHandler) Handle(err error) {
+ h.logger.Warn("otel sdk error", "error", err)
+}
+
+// NewMeterProvider creates an OTel MeterProvider configured with
+// the appropriate OTLP exporter and periodic reader.
+func NewMeterProvider(ctx context.Context, cfg *config.Config, logger *slog.Logger) (*metric.MeterProvider, func(), error) {
+ otel.SetErrorHandler(&slogErrorHandler{logger: logger})
+
+ res, err := resource.New(ctx,
+ resource.WithHost(),
+ resource.WithOS(),
+ resource.WithFromEnv(), // picks up OTEL_RESOURCE_ATTRIBUTES and OTEL_SERVICE_NAME
+ resource.WithAttributes(
+ attribute.String("service.name", cfg.ServiceName),
+ attribute.String("deployment.environment", cfg.Environment),
+ ),
+ )
+ if err != nil {
+ return nil, nil, fmt.Errorf("creating resource: %w", err)
+ }
+
+ // OTEL_EXPORTER_OTLP_ENDPOINT, OTEL_EXPORTER_OTLP_HEADERS, and
+ // OTEL_EXPORTER_OTLP_PROTOCOL are read automatically by the SDK exporters
+ // via their built-in env var support — no manual wiring needed.
+ var exporter metric.Exporter
+
+ protocol := os.Getenv("OTEL_EXPORTER_OTLP_PROTOCOL")
+ switch protocol {
+ case "http/protobuf", "http":
+ exporter, err = otlpmetrichttp.New(ctx)
+ default: // "grpc" or unset
+ exporter, err = otlpmetricgrpc.New(ctx)
+ }
+
+ if err != nil {
+ return nil, nil, fmt.Errorf("creating OTLP exporter: %w", err)
+ }
+
+ reader := metric.NewPeriodicReader(exporter,
+ metric.WithInterval(cfg.CollectionInterval),
+ )
+
+ provider := metric.NewMeterProvider(
+ metric.WithResource(res),
+ metric.WithReader(reader),
+ )
+
+ shutdown := func() {
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+ if err := provider.Shutdown(ctx); err != nil {
+ logger.Warn("meter provider shutdown (pending metrics may be lost)", "error", err)
+ }
+ }
+
+ return provider, shutdown, nil
+}
+
diff --git a/opentelemetry-gpu-collector/internal/gpu/amd/amd.go b/opentelemetry-gpu-collector/internal/gpu/amd/amd.go
new file mode 100644
index 00000000..0ccc4185
--- /dev/null
+++ b/opentelemetry-gpu-collector/internal/gpu/amd/amd.go
@@ -0,0 +1,227 @@
+package amd
+
+import (
+ "fmt"
+ "log/slog"
+ "os"
+ "path/filepath"
+ "strconv"
+ "strings"
+
+ "github.com/openlit/openlit/opentelemetry-gpu-collector/internal/gpu"
+)
+
+const (
+ drmClassPath = "/sys/class/drm"
+ hwmonClassPath = "/sys/class/hwmon"
+)
+
+// Device implements gpu.Device for AMD GPUs via sysfs/hwmon.
+type Device struct {
+ info gpu.DeviceInfo
+ drmPath string // e.g. /sys/class/drm/card0/device
+ hwmonPath string // e.g. /sys/class/hwmon/hwmon3
+ logger *slog.Logger
+}
+
+// DiscoverDevices scans sysfs for AMD GPU DRM cards and resolves their hwmon paths.
+func DiscoverDevices(pciAddresses []string, startIndex int, logger *slog.Logger) ([]*Device, error) {
+ if len(pciAddresses) == 0 {
+ return nil, nil
+ }
+
+ addrSet := make(map[string]bool, len(pciAddresses))
+ for _, a := range pciAddresses {
+ addrSet[strings.ToLower(a)] = true
+ }
+
+ entries, err := os.ReadDir(drmClassPath)
+ if err != nil {
+ return nil, fmt.Errorf("reading DRM class: %w", err)
+ }
+
+ var devices []*Device
+ idx := startIndex
+
+ for _, entry := range entries {
+ name := entry.Name()
+ if !strings.HasPrefix(name, "card") || strings.Contains(name, "-") {
+ continue
+ }
+
+ devPath := filepath.Join(drmClassPath, name, "device")
+ uevent := readFileString(filepath.Join(devPath, "uevent"))
+ pciAddr := extractUeventValue(uevent, "PCI_SLOT_NAME")
+ if pciAddr == "" {
+ continue
+ }
+
+ if !addrSet[strings.ToLower(pciAddr)] {
+ continue
+ }
+
+ driverLink, err := os.Readlink(filepath.Join(devPath, "driver"))
+ if err != nil || filepath.Base(driverLink) != "amdgpu" {
+ continue
+ }
+
+ hwmon := findHwmonForDevice(devPath)
+
+ productName := readFileString(filepath.Join(devPath, "product_name"))
+ if productName == "" {
+ productName = fmt.Sprintf("AMD GPU %s", pciAddr)
+ }
+
+ uniqueID := readFileString(filepath.Join(devPath, "unique_id"))
+ if uniqueID == "" {
+ uniqueID = pciAddr
+ }
+
+ driverVersion := readFileString(filepath.Join(devPath, "driver", "module", "version"))
+
+ logger.Info("discovered AMD GPU", "card", name, "pci", pciAddr, "hwmon", hwmon)
+
+ devices = append(devices, &Device{
+ info: gpu.DeviceInfo{
+ Vendor: gpu.VendorAMD,
+ Index: idx,
+ Name: productName,
+ UUID: uniqueID,
+ PCIAddress: pciAddr,
+ DriverVersion: driverVersion,
+ },
+ drmPath: devPath,
+ hwmonPath: hwmon,
+ logger: logger.With("gpu", idx, "vendor", "amd"),
+ })
+ idx++
+ }
+
+ return devices, nil
+}
+
+func (d *Device) Info() gpu.DeviceInfo {
+ return d.info
+}
+
+func (d *Device) Collect() (*gpu.Snapshot, error) {
+ s := &gpu.Snapshot{}
+
+ if v, err := readSysfsInt(filepath.Join(d.drmPath, "gpu_busy_percent")); err == nil {
+ f := float64(v)
+ s.Utilization = &f
+ }
+
+ if v, err := readSysfsInt(filepath.Join(d.drmPath, "mem_busy_percent")); err == nil {
+ f := float64(v)
+ s.MemoryUtilization = &f
+ }
+
+ if v, err := readSysfsInt(filepath.Join(d.drmPath, "mem_info_vram_total")); err == nil {
+ s.MemoryTotalBytes = &v
+ }
+ if v, err := readSysfsInt(filepath.Join(d.drmPath, "mem_info_vram_used")); err == nil {
+ s.MemoryUsedBytes = &v
+ }
+ if s.MemoryTotalBytes != nil && s.MemoryUsedBytes != nil {
+ free := *s.MemoryTotalBytes - *s.MemoryUsedBytes
+ s.MemoryFreeBytes = &free
+ }
+
+ if d.hwmonPath != "" {
+ d.collectHwmon(s)
+ }
+
+ return s, nil
+}
+
+func (d *Device) collectHwmon(s *gpu.Snapshot) {
+ // Temperature: temp1_input is GPU edge temp in millidegrees C
+ if v, err := readSysfsInt(filepath.Join(d.hwmonPath, "temp1_input")); err == nil {
+ f := float64(v) / 1000.0
+ s.TemperatureGPU = &f
+ }
+ // temp2_input is junction/memory temp on SOC15+ dGPUs
+ if v, err := readSysfsInt(filepath.Join(d.hwmonPath, "temp2_input")); err == nil {
+ f := float64(v) / 1000.0
+ s.TemperatureMemory = &f
+ }
+
+ // Power: power1_average in microwatts
+ if v, err := readSysfsInt(filepath.Join(d.hwmonPath, "power1_average")); err == nil {
+ f := float64(v) / 1e6
+ s.PowerDrawWatts = &f
+ }
+
+ // Power cap: power1_cap in microwatts
+ if v, err := readSysfsInt(filepath.Join(d.hwmonPath, "power1_cap")); err == nil {
+ f := float64(v) / 1e6
+ s.PowerLimitWatts = &f
+ }
+
+ // Energy: energy1_input in microjoules
+ if v, err := readSysfsInt(filepath.Join(d.hwmonPath, "energy1_input")); err == nil {
+ f := float64(v) / 1e6
+ s.EnergyJoules = &f
+ }
+
+ // Graphics clock: freq1_input in Hz
+ if v, err := readSysfsInt(filepath.Join(d.hwmonPath, "freq1_input")); err == nil {
+ f := float64(v) / 1e6 // Hz to MHz
+ s.ClockGraphicsMHz = &f
+ }
+
+ // Memory clock: freq2_input in Hz (dGPU only)
+ if v, err := readSysfsInt(filepath.Join(d.hwmonPath, "freq2_input")); err == nil {
+ f := float64(v) / 1e6
+ s.ClockMemoryMHz = &f
+ }
+
+ // Fan speed: fan1_input in RPM
+ if v, err := readSysfsInt(filepath.Join(d.hwmonPath, "fan1_input")); err == nil {
+ f := float64(v)
+ s.FanSpeedRPM = &f
+ }
+}
+
+func (d *Device) Close() {}
+
+// findHwmonForDevice locates the hwmon directory associated with a DRM device path.
+func findHwmonForDevice(drmDevicePath string) string {
+ hwmonDir := filepath.Join(drmDevicePath, "hwmon")
+ entries, err := os.ReadDir(hwmonDir)
+ if err != nil {
+ return ""
+ }
+ for _, e := range entries {
+ if strings.HasPrefix(e.Name(), "hwmon") {
+ return filepath.Join(hwmonDir, e.Name())
+ }
+ }
+ return ""
+}
+
+func readFileString(path string) string {
+ data, err := os.ReadFile(path)
+ if err != nil {
+ return ""
+ }
+ return strings.TrimSpace(string(data))
+}
+
+func readSysfsInt(path string) (int64, error) {
+ s := readFileString(path)
+ if s == "" {
+ return 0, fmt.Errorf("empty or missing: %s", path)
+ }
+ return strconv.ParseInt(s, 10, 64)
+}
+
+func extractUeventValue(uevent, key string) string {
+ for _, line := range strings.Split(uevent, "\n") {
+ if strings.HasPrefix(line, key+"=") {
+ return strings.TrimPrefix(line, key+"=")
+ }
+ }
+ return ""
+}
diff --git a/opentelemetry-gpu-collector/internal/gpu/gpu.go b/opentelemetry-gpu-collector/internal/gpu/gpu.go
new file mode 100644
index 00000000..18138579
--- /dev/null
+++ b/opentelemetry-gpu-collector/internal/gpu/gpu.go
@@ -0,0 +1,56 @@
+package gpu
+
+// Vendor identifies the GPU manufacturer.
+type Vendor string
+
+const (
+ VendorNVIDIA Vendor = "nvidia"
+ VendorAMD Vendor = "amd"
+ VendorIntel Vendor = "intel"
+ VendorUnknown Vendor = "unknown"
+)
+
+// DeviceInfo holds static identification data for a GPU.
+type DeviceInfo struct {
+ Vendor Vendor
+ Index int
+ Name string
+ UUID string
+ PCIAddress string
+ DriverVersion string
+}
+
+// Snapshot holds a point-in-time reading of all GPU metrics.
+// Nil pointer fields indicate the metric is unavailable for this device.
+type Snapshot struct {
+ Utilization *float64 // GPU compute busy (%)
+ MemoryUtilization *float64 // memory controller busy (%)
+ EncoderUtilization *float64 // video encoder busy (%)
+ DecoderUtilization *float64 // video decoder busy (%)
+
+ TemperatureGPU *float64 // die temperature (celsius)
+ TemperatureMemory *float64 // memory temperature (celsius)
+ FanSpeedRPM *float64 // fan speed (RPM)
+
+ MemoryTotalBytes *int64 // total VRAM (bytes)
+ MemoryUsedBytes *int64 // used VRAM (bytes)
+ MemoryFreeBytes *int64 // free VRAM (bytes)
+
+ PowerDrawWatts *float64 // current power draw (watts)
+ PowerLimitWatts *float64 // power limit (watts)
+ EnergyJoules *float64 // cumulative energy consumed (joules)
+
+ ClockGraphicsMHz *float64 // current graphics/SM clock (MHz)
+ ClockMemoryMHz *float64 // current memory clock (MHz)
+
+ PCIeReplayErrors *int64 // cumulative PCIe replay counter
+ ECCSingleBit *int64 // correctable ECC errors
+ ECCDoubleBit *int64 // uncorrectable ECC errors
+}
+
+// Device is the interface that all vendor GPU backends implement.
+type Device interface {
+ Info() DeviceInfo
+ Collect() (*Snapshot, error)
+ Close()
+}
diff --git a/opentelemetry-gpu-collector/internal/gpu/intel/intel.go b/opentelemetry-gpu-collector/internal/gpu/intel/intel.go
new file mode 100644
index 00000000..a04ef1b9
--- /dev/null
+++ b/opentelemetry-gpu-collector/internal/gpu/intel/intel.go
@@ -0,0 +1,192 @@
+package intel
+
+import (
+ "fmt"
+ "log/slog"
+ "os"
+ "path/filepath"
+ "strconv"
+ "strings"
+
+ "github.com/openlit/openlit/opentelemetry-gpu-collector/internal/gpu"
+)
+
+const (
+ drmClassPath = "/sys/class/drm"
+ hwmonClassPath = "/sys/class/hwmon"
+)
+
+// Device implements gpu.Device for Intel GPUs via sysfs/hwmon/DRM.
+type Device struct {
+ info gpu.DeviceInfo
+ drmPath string // e.g. /sys/class/drm/card0/device
+ hwmonPath string
+ logger *slog.Logger
+}
+
+// DiscoverDevices scans sysfs for Intel GPU DRM cards driven by i915 or xe.
+func DiscoverDevices(pciAddresses []string, startIndex int, logger *slog.Logger) ([]*Device, error) {
+ if len(pciAddresses) == 0 {
+ return nil, nil
+ }
+
+ addrSet := make(map[string]bool, len(pciAddresses))
+ for _, a := range pciAddresses {
+ addrSet[strings.ToLower(a)] = true
+ }
+
+ entries, err := os.ReadDir(drmClassPath)
+ if err != nil {
+ return nil, fmt.Errorf("reading DRM class: %w", err)
+ }
+
+ var devices []*Device
+ idx := startIndex
+
+ for _, entry := range entries {
+ name := entry.Name()
+ if !strings.HasPrefix(name, "card") || strings.Contains(name, "-") {
+ continue
+ }
+
+ devPath := filepath.Join(drmClassPath, name, "device")
+ uevent := readFileString(filepath.Join(devPath, "uevent"))
+ pciAddr := extractUeventValue(uevent, "PCI_SLOT_NAME")
+ if pciAddr == "" {
+ continue
+ }
+
+ if !addrSet[strings.ToLower(pciAddr)] {
+ continue
+ }
+
+ driverLink, err := os.Readlink(filepath.Join(devPath, "driver"))
+ if err != nil {
+ continue
+ }
+ driverName := filepath.Base(driverLink)
+ if driverName != "i915" && driverName != "xe" {
+ continue
+ }
+
+ hwmon := findHwmonForDevice(devPath)
+
+ productName := readFileString(filepath.Join(devPath, "label"))
+ if productName == "" {
+ productName = fmt.Sprintf("Intel GPU %s", pciAddr)
+ }
+
+ logger.Info("discovered Intel GPU", "card", name, "pci", pciAddr, "driver", driverName, "hwmon", hwmon)
+
+ devices = append(devices, &Device{
+ info: gpu.DeviceInfo{
+ Vendor: gpu.VendorIntel,
+ Index: idx,
+ Name: productName,
+ UUID: pciAddr,
+ PCIAddress: pciAddr,
+ DriverVersion: driverName,
+ },
+ drmPath: devPath,
+ hwmonPath: hwmon,
+ logger: logger.With("gpu", idx, "vendor", "intel"),
+ })
+ idx++
+ }
+
+ return devices, nil
+}
+
+func (d *Device) Info() gpu.DeviceInfo {
+ return d.info
+}
+
+func (d *Device) Collect() (*gpu.Snapshot, error) {
+ s := &gpu.Snapshot{}
+
+ // Intel Xe driver exposes some metrics via DRM sysfs
+ if v, err := readSysfsInt(filepath.Join(d.drmPath, "gt_cur_freq_mhz")); err == nil {
+ f := float64(v)
+ s.ClockGraphicsMHz = &f
+ }
+
+ if d.hwmonPath != "" {
+ d.collectHwmon(s)
+ }
+
+ return s, nil
+}
+
+func (d *Device) collectHwmon(s *gpu.Snapshot) {
+ // Temperature
+ if v, err := readSysfsInt(filepath.Join(d.hwmonPath, "temp1_input")); err == nil {
+ f := float64(v) / 1000.0
+ s.TemperatureGPU = &f
+ }
+
+ // Power: Xe hwmon exposes energy1_input (microjoules).
+ // Compute average power from energy delta externally or report instantaneous if available.
+ if v, err := readSysfsInt(filepath.Join(d.hwmonPath, "power1_average")); err == nil {
+ f := float64(v) / 1e6
+ s.PowerDrawWatts = &f
+ }
+
+ // Power limit (Xe: power1_max in microwatts)
+ if v, err := readSysfsInt(filepath.Join(d.hwmonPath, "power1_max")); err == nil {
+ f := float64(v) / 1e6
+ s.PowerLimitWatts = &f
+ }
+
+ // Cumulative energy (microjoules)
+ if v, err := readSysfsInt(filepath.Join(d.hwmonPath, "energy1_input")); err == nil {
+ f := float64(v) / 1e6
+ s.EnergyJoules = &f
+ }
+
+ // Fan speed (Xe hwmon, Linux 6.16+)
+ if v, err := readSysfsInt(filepath.Join(d.hwmonPath, "fan1_input")); err == nil {
+ f := float64(v)
+ s.FanSpeedRPM = &f
+ }
+}
+
+func (d *Device) Close() {}
+
+func findHwmonForDevice(drmDevicePath string) string {
+ hwmonDir := filepath.Join(drmDevicePath, "hwmon")
+ entries, err := os.ReadDir(hwmonDir)
+ if err != nil {
+ return ""
+ }
+ for _, e := range entries {
+ if strings.HasPrefix(e.Name(), "hwmon") {
+ return filepath.Join(hwmonDir, e.Name())
+ }
+ }
+ return ""
+}
+
+func readFileString(path string) string {
+ data, err := os.ReadFile(path)
+ if err != nil {
+ return ""
+ }
+ return strings.TrimSpace(string(data))
+}
+
+func readSysfsInt(path string) (int64, error) {
+ s := readFileString(path)
+ if s == "" {
+ return 0, fmt.Errorf("empty or missing: %s", path)
+ }
+ return strconv.ParseInt(s, 10, 64)
+}
+
+func extractUeventValue(uevent, key string) string {
+ for _, line := range strings.Split(uevent, "\n") {
+ if strings.HasPrefix(line, key+"=") {
+ return strings.TrimPrefix(line, key+"=")
+ }
+ }
+ return ""
+}
diff --git a/opentelemetry-gpu-collector/internal/gpu/nvidia/nvidia.go b/opentelemetry-gpu-collector/internal/gpu/nvidia/nvidia.go
new file mode 100644
index 00000000..df813792
--- /dev/null
+++ b/opentelemetry-gpu-collector/internal/gpu/nvidia/nvidia.go
@@ -0,0 +1,190 @@
+//go:build linux
+
+package nvidia
+
+import (
+ "fmt"
+ "log/slog"
+ "sync"
+
+ "github.com/NVIDIA/go-nvml/pkg/nvml"
+ "github.com/openlit/openlit/opentelemetry-gpu-collector/internal/gpu"
+)
+
+var (
+ initOnce sync.Once
+ initErr error
+)
+
+// InitNVML initializes the NVML library. Safe to call multiple times.
+func InitNVML() error {
+ initOnce.Do(func() {
+ ret := nvml.Init()
+ if ret != nvml.SUCCESS {
+ initErr = fmt.Errorf("nvml.Init: %s", nvml.ErrorString(ret))
+ }
+ })
+ return initErr
+}
+
+// ShutdownNVML shuts down the NVML library.
+func ShutdownNVML() {
+ nvml.Shutdown()
+}
+
+// Device implements gpu.Device for NVIDIA GPUs via NVML.
+type Device struct {
+ handle nvml.Device
+ info gpu.DeviceInfo
+ logger *slog.Logger
+}
+
+// DiscoverDevices returns all NVIDIA GPUs detected by NVML.
+func DiscoverDevices(logger *slog.Logger) ([]*Device, error) {
+ if err := InitNVML(); err != nil {
+ return nil, err
+ }
+
+ count, ret := nvml.DeviceGetCount()
+ if ret != nvml.SUCCESS {
+ return nil, fmt.Errorf("DeviceGetCount: %s", nvml.ErrorString(ret))
+ }
+
+ driverVersion, ret := nvml.SystemGetDriverVersion()
+ if ret != nvml.SUCCESS {
+ driverVersion = "unknown"
+ }
+
+ devices := make([]*Device, 0, count)
+ for i := 0; i < count; i++ {
+ handle, ret := nvml.DeviceGetHandleByIndex(i)
+ if ret != nvml.SUCCESS {
+ logger.Warn("skipping NVIDIA GPU", "index", i, "error", nvml.ErrorString(ret))
+ continue
+ }
+
+ name, _ := handle.GetName()
+ uuid, _ := handle.GetUUID()
+ pciInfo, _ := handle.GetPciInfo()
+
+ devices = append(devices, &Device{
+ handle: handle,
+ info: gpu.DeviceInfo{
+ Vendor: gpu.VendorNVIDIA,
+ Index: i,
+ Name: name,
+ UUID: uuid,
+ PCIAddress: pciAddressString(pciInfo),
+ DriverVersion: driverVersion,
+ },
+ logger: logger.With("gpu", i, "vendor", "nvidia"),
+ })
+ }
+
+ return devices, nil
+}
+
+func (d *Device) Info() gpu.DeviceInfo {
+ return d.info
+}
+
+func (d *Device) Collect() (*gpu.Snapshot, error) {
+ s := &gpu.Snapshot{}
+
+ if util, ret := d.handle.GetUtilizationRates(); ret == nvml.SUCCESS {
+ gpuUtil := float64(util.Gpu)
+ memUtil := float64(util.Memory)
+ s.Utilization = &gpuUtil
+ s.MemoryUtilization = &memUtil
+ }
+
+ if encUtil, _, ret := d.handle.GetEncoderUtilization(); ret == nvml.SUCCESS {
+ v := float64(encUtil)
+ s.EncoderUtilization = &v
+ }
+
+ if decUtil, _, ret := d.handle.GetDecoderUtilization(); ret == nvml.SUCCESS {
+ v := float64(decUtil)
+ s.DecoderUtilization = &v
+ }
+
+ if temp, ret := d.handle.GetTemperature(nvml.TEMPERATURE_GPU); ret == nvml.SUCCESS {
+ v := float64(temp)
+ s.TemperatureGPU = &v
+ }
+
+ if fanSpeed, ret := d.handle.GetFanSpeed(); ret == nvml.SUCCESS {
+ v := float64(fanSpeed)
+ s.FanSpeedRPM = &v
+ }
+
+ if mem, ret := d.handle.GetMemoryInfo(); ret == nvml.SUCCESS {
+ total := int64(mem.Total)
+ used := int64(mem.Used)
+ free := int64(mem.Free)
+ s.MemoryTotalBytes = &total
+ s.MemoryUsedBytes = &used
+ s.MemoryFreeBytes = &free
+ }
+
+ // NVML returns power in milliwatts
+ if power, ret := d.handle.GetPowerUsage(); ret == nvml.SUCCESS {
+ v := float64(power) / 1000.0
+ s.PowerDrawWatts = &v
+ }
+
+ if limit, ret := d.handle.GetPowerManagementLimit(); ret == nvml.SUCCESS {
+ v := float64(limit) / 1000.0
+ s.PowerLimitWatts = &v
+ }
+
+ // NVML returns energy in millijoules
+ if energy, ret := d.handle.GetTotalEnergyConsumption(); ret == nvml.SUCCESS {
+ v := float64(energy) / 1000.0
+ s.EnergyJoules = &v
+ }
+
+ if clock, ret := d.handle.GetClockInfo(nvml.CLOCK_GRAPHICS); ret == nvml.SUCCESS {
+ v := float64(clock)
+ s.ClockGraphicsMHz = &v
+ }
+
+ if clock, ret := d.handle.GetClockInfo(nvml.CLOCK_MEM); ret == nvml.SUCCESS {
+ v := float64(clock)
+ s.ClockMemoryMHz = &v
+ }
+
+ if replay, ret := d.handle.GetPcieReplayCounter(); ret == nvml.SUCCESS {
+ v := int64(replay)
+ s.PCIeReplayErrors = &v
+ }
+
+ if ecc, ret := d.handle.GetTotalEccErrors(nvml.MEMORY_ERROR_TYPE_CORRECTED, nvml.VOLATILE_ECC); ret == nvml.SUCCESS {
+ v := int64(ecc)
+ s.ECCSingleBit = &v
+ }
+
+ if ecc, ret := d.handle.GetTotalEccErrors(nvml.MEMORY_ERROR_TYPE_UNCORRECTED, nvml.VOLATILE_ECC); ret == nvml.SUCCESS {
+ v := int64(ecc)
+ s.ECCDoubleBit = &v
+ }
+
+ return s, nil
+}
+
+func (d *Device) Close() {}
+
+func pciAddressString(pci nvml.PciInfo) string {
+ // PciInfo.BusId is a fixed-size byte array; convert to string.
+ var buf []byte
+ for _, b := range pci.BusId {
+ if b == 0 {
+ break
+ }
+ buf = append(buf, byte(b))
+ }
+ if len(buf) > 0 {
+ return string(buf)
+ }
+ return fmt.Sprintf("%04x:%02x:%02x.0", pci.Domain, pci.Bus, pci.Device)
+}
diff --git a/opentelemetry-gpu-collector/internal/gpu/nvidia/nvidia_stub.go b/opentelemetry-gpu-collector/internal/gpu/nvidia/nvidia_stub.go
new file mode 100644
index 00000000..c61500e6
--- /dev/null
+++ b/opentelemetry-gpu-collector/internal/gpu/nvidia/nvidia_stub.go
@@ -0,0 +1,25 @@
+//go:build !linux
+
+package nvidia
+
+import (
+ "fmt"
+ "log/slog"
+
+ "github.com/openlit/openlit/opentelemetry-gpu-collector/internal/gpu"
+)
+
+// Device is a stub for non-Linux platforms.
+type Device struct {
+ info gpu.DeviceInfo
+}
+
+func (d *Device) Info() gpu.DeviceInfo { return d.info }
+func (d *Device) Collect() (*gpu.Snapshot, error) { return nil, fmt.Errorf("not supported on this OS") }
+func (d *Device) Close() {}
+
+func DiscoverDevices(_ *slog.Logger) ([]*Device, error) {
+ return nil, fmt.Errorf("NVIDIA GPU support requires Linux")
+}
+
+func ShutdownNVML() {}
diff --git a/opentelemetry-gpu-collector/internal/hostmetrics/hostmetrics_test.go b/opentelemetry-gpu-collector/internal/hostmetrics/hostmetrics_test.go
new file mode 100644
index 00000000..78e8cc36
--- /dev/null
+++ b/opentelemetry-gpu-collector/internal/hostmetrics/hostmetrics_test.go
@@ -0,0 +1,174 @@
+package hostmetrics
+
+import (
+ "log/slog"
+ "testing"
+
+ "go.opentelemetry.io/otel/sdk/metric"
+ "go.opentelemetry.io/otel/sdk/metric/metricdata"
+)
+
+func TestNewSystemCollectorInitializes(t *testing.T) {
+ reader := metric.NewManualReader()
+ provider := metric.NewMeterProvider(metric.WithReader(reader))
+ defer provider.Shutdown(t.Context())
+
+ sc, err := NewSystemCollector(provider, slog.Default())
+ if err != nil {
+ t.Fatalf("NewSystemCollector() error = %v", err)
+ }
+ defer sc.Close()
+
+ var rm metricdata.ResourceMetrics
+ if err := reader.Collect(t.Context(), &rm); err != nil {
+ t.Fatalf("reader.Collect() error = %v", err)
+ }
+
+ var found bool
+ for _, sm := range rm.ScopeMetrics {
+ if sm.Scope.Name == "otelcol.system" {
+ found = true
+ break
+ }
+ }
+ if !found {
+ t.Error("expected ScopeMetrics for otelcol.system")
+ }
+}
+
+func TestSystemCollectorInstrumentNames(t *testing.T) {
+ reader := metric.NewManualReader()
+ provider := metric.NewMeterProvider(metric.WithReader(reader))
+ defer provider.Shutdown(t.Context())
+
+ sc, err := NewSystemCollector(provider, slog.Default())
+ if err != nil {
+ t.Fatalf("NewSystemCollector() error = %v", err)
+ }
+ defer sc.Close()
+
+ var rm metricdata.ResourceMetrics
+ if err := reader.Collect(t.Context(), &rm); err != nil {
+ t.Fatalf("reader.Collect() error = %v", err)
+ }
+
+ reported := make(map[string]bool)
+ for _, sm := range rm.ScopeMetrics {
+ if sm.Scope.Name == "otelcol.system" {
+ for _, m := range sm.Metrics {
+ reported[m.Name] = true
+ }
+ }
+ }
+
+ wantMetrics := []string{
+ "system.cpu.utilization",
+ "system.cpu.logical.count",
+ "system.memory.usage",
+ "system.memory.utilization",
+ "system.disk.io",
+ "system.disk.operations",
+ "system.filesystem.usage",
+ "system.filesystem.utilization",
+ "system.network.io",
+ "system.network.errors",
+ }
+ for _, name := range wantMetrics {
+ if !reported[name] {
+ t.Errorf("expected metric %q to be reported", name)
+ }
+ }
+}
+
+func TestSystemCollectorCloseIdempotent(t *testing.T) {
+ provider := metric.NewMeterProvider()
+ defer provider.Shutdown(t.Context())
+
+ sc, err := NewSystemCollector(provider, slog.Default())
+ if err != nil {
+ t.Fatalf("NewSystemCollector() error = %v", err)
+ }
+ sc.Close()
+ sc.Close() // should not panic
+}
+
+func TestNewProcessCollectorInitializes(t *testing.T) {
+ reader := metric.NewManualReader()
+ provider := metric.NewMeterProvider(metric.WithReader(reader))
+ defer provider.Shutdown(t.Context())
+
+ pc, err := NewProcessCollector(provider, slog.Default())
+ if err != nil {
+ t.Fatalf("NewProcessCollector() error = %v", err)
+ }
+ defer pc.Close()
+
+ var rm metricdata.ResourceMetrics
+ if err := reader.Collect(t.Context(), &rm); err != nil {
+ t.Fatalf("reader.Collect() error = %v", err)
+ }
+
+ var found bool
+ for _, sm := range rm.ScopeMetrics {
+ if sm.Scope.Name == "otelcol.process" {
+ found = true
+ break
+ }
+ }
+ if !found {
+ t.Error("expected ScopeMetrics for otelcol.process")
+ }
+}
+
+func TestProcessCollectorInstrumentNames(t *testing.T) {
+ reader := metric.NewManualReader()
+ provider := metric.NewMeterProvider(metric.WithReader(reader))
+ defer provider.Shutdown(t.Context())
+
+ pc, err := NewProcessCollector(provider, slog.Default())
+ if err != nil {
+ t.Fatalf("NewProcessCollector() error = %v", err)
+ }
+ defer pc.Close()
+
+ var rm metricdata.ResourceMetrics
+ if err := reader.Collect(t.Context(), &rm); err != nil {
+ t.Fatalf("reader.Collect() error = %v", err)
+ }
+
+ reported := make(map[string]bool)
+ for _, sm := range rm.ScopeMetrics {
+ if sm.Scope.Name == "otelcol.process" {
+ for _, m := range sm.Metrics {
+ reported[m.Name] = true
+ }
+ }
+ }
+
+ wantMetrics := []string{
+ "process.cpu.time",
+ "process.cpu.utilization",
+ "process.memory.usage",
+ "process.memory.virtual",
+ "process.thread.count",
+ "process.runtime.go.goroutines",
+ "process.runtime.go.mem.heap_alloc",
+ }
+ for _, name := range wantMetrics {
+ if !reported[name] {
+ t.Errorf("expected metric %q to be reported", name)
+ }
+ }
+}
+
+func TestProcessCollectorCloseIdempotent(t *testing.T) {
+ provider := metric.NewMeterProvider()
+ defer provider.Shutdown(t.Context())
+
+ pc, err := NewProcessCollector(provider, slog.Default())
+ if err != nil {
+ t.Fatalf("NewProcessCollector() error = %v", err)
+ }
+ pc.Close()
+ pc.Close() // should not panic
+}
diff --git a/opentelemetry-gpu-collector/internal/hostmetrics/process.go b/opentelemetry-gpu-collector/internal/hostmetrics/process.go
new file mode 100644
index 00000000..19dae10a
--- /dev/null
+++ b/opentelemetry-gpu-collector/internal/hostmetrics/process.go
@@ -0,0 +1,207 @@
+package hostmetrics
+
+import (
+ "context"
+ "fmt"
+ "log/slog"
+ "os"
+ "runtime"
+
+ "github.com/shirou/gopsutil/v4/process"
+
+ "go.opentelemetry.io/otel/attribute"
+ "go.opentelemetry.io/otel/metric"
+ sdkmetric "go.opentelemetry.io/otel/sdk/metric"
+)
+
+// ProcessCollector reports metrics about the collector process itself,
+// following the OpenTelemetry semantic conventions for process metrics.
+// https://opentelemetry.io/docs/specs/semconv/system/process-metrics/
+type ProcessCollector struct {
+ logger *slog.Logger
+ proc *process.Process
+ reg []metric.Registration
+}
+
+// NewProcessCollector creates process-level metric instruments for the current process.
+func NewProcessCollector(provider *sdkmetric.MeterProvider, logger *slog.Logger) (*ProcessCollector, error) {
+ pid := int32(os.Getpid())
+ proc, err := process.NewProcess(pid)
+ if err != nil {
+ return nil, fmt.Errorf("accessing process %d: %w", pid, err)
+ }
+
+ pc := &ProcessCollector{
+ logger: logger,
+ proc: proc,
+ }
+
+ meter := provider.Meter("otelcol.process",
+ metric.WithInstrumentationVersion("1.0.0"),
+ )
+
+ // spec: process.cpu.time (Counter), attribute: cpu.mode
+ cpuTime, err := meter.Float64ObservableCounter("process.cpu.time",
+ metric.WithDescription("Process CPU time"),
+ metric.WithUnit("s"),
+ )
+ if err != nil {
+ return nil, fmt.Errorf("creating process.cpu.time: %w", err)
+ }
+
+ // spec: process.cpu.utilization (Gauge), attribute: cpu.mode
+ cpuUtil, err := meter.Float64ObservableGauge("process.cpu.utilization",
+ metric.WithDescription("Process CPU utilization"),
+ metric.WithUnit("1"),
+ )
+ if err != nil {
+ return nil, fmt.Errorf("creating process.cpu.utilization: %w", err)
+ }
+
+ // spec: process.memory.usage (UpDownCounter)
+ memUsage, err := meter.Int64ObservableUpDownCounter("process.memory.usage",
+ metric.WithDescription("Process resident memory (RSS)"),
+ metric.WithUnit("By"),
+ )
+ if err != nil {
+ return nil, fmt.Errorf("creating process.memory.usage: %w", err)
+ }
+
+ // spec: process.memory.virtual (UpDownCounter)
+ memVirtual, err := meter.Int64ObservableUpDownCounter("process.memory.virtual",
+ metric.WithDescription("Process virtual memory size"),
+ metric.WithUnit("By"),
+ )
+ if err != nil {
+ return nil, fmt.Errorf("creating process.memory.virtual: %w", err)
+ }
+
+ // spec: process.thread.count (UpDownCounter)
+ threadCount, err := meter.Int64ObservableUpDownCounter("process.thread.count",
+ metric.WithDescription("Process thread count"),
+ metric.WithUnit("{thread}"),
+ )
+ if err != nil {
+ return nil, fmt.Errorf("creating process.thread.count: %w", err)
+ }
+
+ // spec: process.unix.file_descriptor.count (UpDownCounter), unit: {file_descriptor}
+ fdCount, err := meter.Int64ObservableUpDownCounter("process.unix.file_descriptor.count",
+ metric.WithDescription("Open file descriptor count"),
+ metric.WithUnit("{file_descriptor}"),
+ )
+ if err != nil {
+ return nil, fmt.Errorf("creating process.unix.file_descriptor.count: %w", err)
+ }
+
+ // Custom Go runtime metrics (no spec equivalent)
+ goRoutines, err := meter.Int64ObservableGauge("process.runtime.go.goroutines",
+ metric.WithDescription("Number of Go goroutines"),
+ metric.WithUnit("{goroutine}"),
+ )
+ if err != nil {
+ return nil, fmt.Errorf("creating process.runtime.go.goroutines: %w", err)
+ }
+
+ goMemAlloc, err := meter.Int64ObservableGauge("process.runtime.go.mem.heap_alloc",
+ metric.WithDescription("Go heap memory allocated"),
+ metric.WithUnit("By"),
+ )
+ if err != nil {
+ return nil, fmt.Errorf("creating process.runtime.go.mem.heap_alloc: %w", err)
+ }
+
+ reg, err := meter.RegisterCallback(
+ func(ctx context.Context, o metric.Observer) error {
+ pc.collectCPU(ctx, o, cpuTime, cpuUtil)
+ pc.collectMemory(ctx, o, memUsage, memVirtual)
+ pc.collectThreads(ctx, o, threadCount)
+ pc.collectFDs(ctx, o, fdCount)
+ pc.collectGoRuntime(ctx, o, goRoutines, goMemAlloc)
+ return nil
+ },
+ cpuTime, cpuUtil,
+ memUsage, memVirtual,
+ threadCount, fdCount,
+ goRoutines, goMemAlloc,
+ )
+ if err != nil {
+ return nil, fmt.Errorf("registering process callback: %w", err)
+ }
+
+ pc.reg = append(pc.reg, reg)
+ logger.Info("process metrics collector initialized", "pid", pid)
+ return pc, nil
+}
+
+func (pc *ProcessCollector) Close() {
+ for _, r := range pc.reg {
+ _ = r.Unregister()
+ }
+}
+
+func (pc *ProcessCollector) collectCPU(_ context.Context, o metric.Observer,
+ cpuTime metric.Float64ObservableCounter,
+ cpuUtil metric.Float64ObservableGauge,
+) {
+ times, err := pc.proc.Times()
+ if err == nil {
+ // spec: cpu.mode attribute values: user, system
+ o.ObserveFloat64(cpuTime, times.User,
+ metric.WithAttributes(attribute.String("cpu.mode", "user")),
+ )
+ o.ObserveFloat64(cpuTime, times.System,
+ metric.WithAttributes(attribute.String("cpu.mode", "system")),
+ )
+ }
+
+ pct, err := pc.proc.Percent(0)
+ if err == nil {
+ o.ObserveFloat64(cpuUtil, pct/100.0)
+ }
+}
+
+func (pc *ProcessCollector) collectMemory(_ context.Context, o metric.Observer,
+ usage metric.Int64ObservableUpDownCounter,
+ virtual metric.Int64ObservableUpDownCounter,
+) {
+ memInfo, err := pc.proc.MemoryInfo()
+ if err != nil {
+ return
+ }
+
+ o.ObserveInt64(usage, int64(memInfo.RSS))
+ o.ObserveInt64(virtual, int64(memInfo.VMS))
+}
+
+func (pc *ProcessCollector) collectThreads(_ context.Context, o metric.Observer,
+ threadCount metric.Int64ObservableUpDownCounter,
+) {
+ threads, err := pc.proc.NumThreads()
+ if err == nil {
+ o.ObserveInt64(threadCount, int64(threads))
+ }
+}
+
+func (pc *ProcessCollector) collectFDs(_ context.Context, o metric.Observer,
+ fdCount metric.Int64ObservableUpDownCounter,
+) {
+ if runtime.GOOS == "windows" {
+ return
+ }
+ fds, err := pc.proc.NumFDs()
+ if err == nil {
+ o.ObserveInt64(fdCount, int64(fds))
+ }
+}
+
+func (pc *ProcessCollector) collectGoRuntime(_ context.Context, o metric.Observer,
+ goRoutines metric.Int64ObservableGauge,
+ heapAlloc metric.Int64ObservableGauge,
+) {
+ o.ObserveInt64(goRoutines, int64(runtime.NumGoroutine()))
+
+ var m runtime.MemStats
+ runtime.ReadMemStats(&m)
+ o.ObserveInt64(heapAlloc, int64(m.HeapAlloc))
+}
diff --git a/opentelemetry-gpu-collector/internal/hostmetrics/system.go b/opentelemetry-gpu-collector/internal/hostmetrics/system.go
new file mode 100644
index 00000000..a80a5cb3
--- /dev/null
+++ b/opentelemetry-gpu-collector/internal/hostmetrics/system.go
@@ -0,0 +1,313 @@
+package hostmetrics
+
+import (
+ "context"
+ "fmt"
+ "log/slog"
+ "runtime"
+ "time"
+
+ "github.com/shirou/gopsutil/v4/cpu"
+ "github.com/shirou/gopsutil/v4/disk"
+ "github.com/shirou/gopsutil/v4/mem"
+ "github.com/shirou/gopsutil/v4/net"
+
+ "go.opentelemetry.io/otel/attribute"
+ "go.opentelemetry.io/otel/metric"
+ sdkmetric "go.opentelemetry.io/otel/sdk/metric"
+)
+
+// SystemCollector registers OTel instruments for host-level system metrics
+// following the OpenTelemetry semantic conventions for system metrics.
+// https://opentelemetry.io/docs/specs/semconv/system/system-metrics/
+type SystemCollector struct {
+ logger *slog.Logger
+ reg []metric.Registration
+}
+
+// NewSystemCollector creates system-level metric instruments and registers callbacks.
+func NewSystemCollector(provider *sdkmetric.MeterProvider, logger *slog.Logger) (*SystemCollector, error) {
+ sc := &SystemCollector{logger: logger}
+
+ meter := provider.Meter("otelcol.system",
+ metric.WithInstrumentationVersion("1.0.0"),
+ )
+
+ // --- CPU ---
+ cpuUtilization, err := meter.Float64ObservableGauge("system.cpu.utilization",
+ metric.WithDescription("CPU utilization (0.0-1.0)"),
+ metric.WithUnit("1"),
+ )
+ if err != nil {
+ return nil, fmt.Errorf("creating system.cpu.utilization: %w", err)
+ }
+
+ // spec: system.cpu.logical.count (UpDownCounter)
+ cpuCount, err := meter.Int64ObservableUpDownCounter("system.cpu.logical.count",
+ metric.WithDescription("Number of CPU logical cores"),
+ metric.WithUnit("{cpu}"),
+ )
+ if err != nil {
+ return nil, fmt.Errorf("creating system.cpu.logical.count: %w", err)
+ }
+
+ // --- Memory ---
+ // spec: system.memory.usage (UpDownCounter), attribute: system.memory.state
+ memUsage, err := meter.Int64ObservableUpDownCounter("system.memory.usage",
+ metric.WithDescription("Memory usage by state"),
+ metric.WithUnit("By"),
+ )
+ if err != nil {
+ return nil, fmt.Errorf("creating system.memory.usage: %w", err)
+ }
+
+ memUtilization, err := meter.Float64ObservableGauge("system.memory.utilization",
+ metric.WithDescription("Memory utilization (0.0-1.0)"),
+ metric.WithUnit("1"),
+ )
+ if err != nil {
+ return nil, fmt.Errorf("creating system.memory.utilization: %w", err)
+ }
+
+ // --- Disk ---
+ // spec: system.disk.io, attributes: system.device + disk.io.direction
+ diskIO, err := meter.Int64ObservableCounter("system.disk.io",
+ metric.WithDescription("Disk I/O bytes"),
+ metric.WithUnit("By"),
+ )
+ if err != nil {
+ return nil, fmt.Errorf("creating system.disk.io: %w", err)
+ }
+
+ diskOps, err := meter.Int64ObservableCounter("system.disk.operations",
+ metric.WithDescription("Disk I/O operations"),
+ metric.WithUnit("{operation}"),
+ )
+ if err != nil {
+ return nil, fmt.Errorf("creating system.disk.operations: %w", err)
+ }
+
+ // spec: system.filesystem.usage, attributes: system.device + system.filesystem.*
+ fsUsage, err := meter.Int64ObservableUpDownCounter("system.filesystem.usage",
+ metric.WithDescription("Filesystem space usage"),
+ metric.WithUnit("By"),
+ )
+ if err != nil {
+ return nil, fmt.Errorf("creating system.filesystem.usage: %w", err)
+ }
+
+ fsUtilization, err := meter.Float64ObservableGauge("system.filesystem.utilization",
+ metric.WithDescription("Filesystem utilization (0.0-1.0)"),
+ metric.WithUnit("1"),
+ )
+ if err != nil {
+ return nil, fmt.Errorf("creating system.filesystem.utilization: %w", err)
+ }
+
+ // --- Network ---
+ // spec: system.network.io, attributes: network.interface.name + network.io.direction
+ netIO, err := meter.Int64ObservableCounter("system.network.io",
+ metric.WithDescription("Network I/O bytes"),
+ metric.WithUnit("By"),
+ )
+ if err != nil {
+ return nil, fmt.Errorf("creating system.network.io: %w", err)
+ }
+
+ netErrors, err := meter.Int64ObservableCounter("system.network.errors",
+ metric.WithDescription("Network error count"),
+ metric.WithUnit("{error}"),
+ )
+ if err != nil {
+ return nil, fmt.Errorf("creating system.network.errors: %w", err)
+ }
+
+ reg, err := meter.RegisterCallback(
+ func(ctx context.Context, o metric.Observer) error {
+ sc.collectCPU(ctx, o, cpuUtilization, cpuCount)
+ sc.collectMemory(ctx, o, memUsage, memUtilization)
+ sc.collectDisk(ctx, o, diskIO, diskOps)
+ sc.collectFilesystem(ctx, o, fsUsage, fsUtilization)
+ sc.collectNetwork(ctx, o, netIO, netErrors)
+ return nil
+ },
+ cpuUtilization, cpuCount,
+ memUsage, memUtilization,
+ diskIO, diskOps,
+ fsUsage, fsUtilization,
+ netIO, netErrors,
+ )
+ if err != nil {
+ return nil, fmt.Errorf("registering system callback: %w", err)
+ }
+
+ sc.reg = append(sc.reg, reg)
+ logger.Info("system metrics collector initialized", "os", runtime.GOOS, "arch", runtime.GOARCH)
+ return sc, nil
+}
+
+func (sc *SystemCollector) Close() {
+ for _, r := range sc.reg {
+ _ = r.Unregister()
+ }
+}
+
+func (sc *SystemCollector) collectCPU(_ context.Context, o metric.Observer,
+ utilization metric.Float64ObservableGauge,
+ count metric.Int64ObservableUpDownCounter,
+) {
+ logicalCores, err := cpu.Counts(true)
+ if err == nil {
+ o.ObserveInt64(count, int64(logicalCores))
+ }
+
+ percents, err := cpu.Percent(0, true)
+ if err != nil {
+ sc.logger.Debug("cpu percent error", "error", err)
+ return
+ }
+
+ // spec: cpu.logical_number attribute for per-core, cpu.mode for mode
+ for i, pct := range percents {
+ attrs := metric.WithAttributes(attribute.Int("cpu.logical_number", i))
+ o.ObserveFloat64(utilization, pct/100.0, attrs)
+ }
+
+ // Aggregate across all cores
+ aggPercents, err := cpu.Percent(0, false)
+ if err == nil && len(aggPercents) > 0 {
+ o.ObserveFloat64(utilization, aggPercents[0]/100.0)
+ }
+}
+
+func (sc *SystemCollector) collectMemory(_ context.Context, o metric.Observer,
+ usage metric.Int64ObservableUpDownCounter,
+ utilization metric.Float64ObservableGauge,
+) {
+ v, err := mem.VirtualMemory()
+ if err != nil {
+ sc.logger.Debug("memory error", "error", err)
+ return
+ }
+
+ // spec: system.memory.state attribute values: used, free, cached, buffers
+ o.ObserveInt64(usage, int64(v.Used),
+ metric.WithAttributes(attribute.String("system.memory.state", "used")),
+ )
+ o.ObserveInt64(usage, int64(v.Free),
+ metric.WithAttributes(attribute.String("system.memory.state", "free")),
+ )
+
+ if runtime.GOOS == "linux" {
+ o.ObserveInt64(usage, int64(v.Cached),
+ metric.WithAttributes(attribute.String("system.memory.state", "cached")),
+ )
+ o.ObserveInt64(usage, int64(v.Buffers),
+ metric.WithAttributes(attribute.String("system.memory.state", "buffers")),
+ )
+ }
+
+ o.ObserveFloat64(utilization, v.UsedPercent/100.0)
+}
+
+func (sc *SystemCollector) collectDisk(_ context.Context, o metric.Observer,
+ ioBytes metric.Int64ObservableCounter,
+ ops metric.Int64ObservableCounter,
+) {
+ counters, err := disk.IOCounters()
+ if err != nil {
+ sc.logger.Debug("disk io error", "error", err)
+ return
+ }
+
+ for device, stat := range counters {
+ // spec: system.device + disk.io.direction
+ readAttrs := metric.WithAttributes(
+ attribute.String("system.device", device),
+ attribute.String("disk.io.direction", "read"),
+ )
+ writeAttrs := metric.WithAttributes(
+ attribute.String("system.device", device),
+ attribute.String("disk.io.direction", "write"),
+ )
+
+ o.ObserveInt64(ioBytes, int64(stat.ReadBytes), readAttrs)
+ o.ObserveInt64(ioBytes, int64(stat.WriteBytes), writeAttrs)
+ o.ObserveInt64(ops, int64(stat.ReadCount), readAttrs)
+ o.ObserveInt64(ops, int64(stat.WriteCount), writeAttrs)
+ }
+}
+
+func (sc *SystemCollector) collectFilesystem(_ context.Context, o metric.Observer,
+ usage metric.Int64ObservableUpDownCounter,
+ utilization metric.Float64ObservableGauge,
+) {
+ partitions, err := disk.Partitions(false)
+ if err != nil {
+ sc.logger.Debug("filesystem partitions error", "error", err)
+ return
+ }
+
+ for _, p := range partitions {
+ stat, err := disk.Usage(p.Mountpoint)
+ if err != nil {
+ continue
+ }
+
+ // spec: system.device, system.filesystem.mountpoint, system.filesystem.type, system.filesystem.state
+ baseAttrs := []attribute.KeyValue{
+ attribute.String("system.device", p.Device),
+ attribute.String("system.filesystem.mountpoint", p.Mountpoint),
+ attribute.String("system.filesystem.type", p.Fstype),
+ }
+
+ o.ObserveInt64(usage, int64(stat.Used),
+ metric.WithAttributes(append(baseAttrs, attribute.String("system.filesystem.state", "used"))...),
+ )
+ o.ObserveInt64(usage, int64(stat.Free),
+ metric.WithAttributes(append(baseAttrs, attribute.String("system.filesystem.state", "free"))...),
+ )
+ o.ObserveFloat64(utilization, stat.UsedPercent/100.0,
+ metric.WithAttributes(baseAttrs...),
+ )
+ }
+}
+
+func (sc *SystemCollector) collectNetwork(_ context.Context, o metric.Observer,
+ ioBytes metric.Int64ObservableCounter,
+ errors metric.Int64ObservableCounter,
+) {
+ counters, err := net.IOCounters(true)
+ if err != nil {
+ sc.logger.Debug("network io error", "error", err)
+ return
+ }
+
+ for _, stat := range counters {
+ if stat.Name == "lo" || stat.Name == "lo0" {
+ continue
+ }
+
+ // spec: network.interface.name + network.io.direction
+ rxAttrs := metric.WithAttributes(
+ attribute.String("network.interface.name", stat.Name),
+ attribute.String("network.io.direction", "receive"),
+ )
+ txAttrs := metric.WithAttributes(
+ attribute.String("network.interface.name", stat.Name),
+ attribute.String("network.io.direction", "transmit"),
+ )
+
+ o.ObserveInt64(ioBytes, int64(stat.BytesRecv), rxAttrs)
+ o.ObserveInt64(ioBytes, int64(stat.BytesSent), txAttrs)
+ o.ObserveInt64(errors, int64(stat.Errin), rxAttrs)
+ o.ObserveInt64(errors, int64(stat.Errout), txAttrs)
+ }
+}
+
+// cpuTimeSinceStart primes the CPU percent calculation so the first real call returns meaningful data.
+func init() {
+ go func() {
+ _, _ = cpu.Percent(200*time.Millisecond, false)
+ }()
+}
diff --git a/otel-gpu-collector/README.md b/otel-gpu-collector/README.md
deleted file mode 100644
index 1a33b3cc..00000000
--- a/otel-gpu-collector/README.md
+++ /dev/null
@@ -1,170 +0,0 @@
-<div align="center">
-<img src="https://github.com/openlit/.github/blob/main/profile/assets/wide-logo-no-bg.png?raw=true" alt="OpenLIT Logo" width="30%"><h1>
-OpenTelemetry GPU Collector</h1>
-
-**[Documentation](https://docs.openlit.io/latest/features/gpu) | [Quickstart](-getting-started-with-gpu-monitoring) | [Python SDK](https://github.com/openlit/openlit/tree/main/sdk/python) | [Metrics](#metrics)**
-
-**[Roadmap](#️-roadmap) | [Feature Request](https://github.com/openlit/openlit/issues/new?assignees=&labels=%3Araised_hand%3A+Up+for+Grabs%2C+%3Arocket%3A+Feature&projects=&template=feature-request.md&title=%5BFeat%5D%3A) | [Report a Bug](https://github.com/openlit/openlit/issues/new?assignees=&labels=%3Abug%3A+Bug%2C+%3Araised_hand%3A+Up+for+Grabs&projects=&template=bug.md&title=%5BBug%5D%3A)**
-
-[![OpenLIT](https://img.shields.io/badge/OpenLIT-orange)](https://openlit.io/)
-[![License](https://img.shields.io/github/license/openlit/openlit?label=License&logo=github&color=f80&logoColor=white)](https://github.com/openlit/openlit/blob/main/LICENSE)
-[![GitHub Last Commit](https://img.shields.io/github/last-commit/openlit/openlit)](https://github.com/openlit/openlit/pulse)
-[![GitHub Contributors](https://img.shields.io/github/contributors/openlit/openlit)](https://github.com/openlit/openlit/graphs/contributors)
-
-[![Slack](https://img.shields.io/badge/Slack-4A154B?logo=slack&logoColor=white)](https://join.slack.com/t/openlit/shared_invite/zt-2etnfttwg-TjP_7BZXfYg84oAukY8QRQ)
-[![X](https://img.shields.io/badge/follow-%40openlit__io-1DA1F2?logo=x&style=social)](https://twitter.com/openlit_io)
-
-</div>
-
-OpenTelemetry GPU Collector is a lightweight, efficient COLLECTOR designed to collect GPU performance metrics and send them to an OpenTelemetry-compatible endpoint for monitoring and observability. This tool is particularly useful for monitoring GPUs in high-performance computing environments, AI/ML tasks, and LLMs.
-
-## ⚡ Features
-
-- Collects detailed GPU performance metrics
-- OpenTelemetry-native
-- Lightweight and efficient
-- Supports NVIDIA and AMD GPUs
-
-## 🚀 Getting Started with GPU Monitoring
-
-### Prerequisites
-
-- Docker installed on your system
-
-### Step 1: Pull the Docker Image
-
-You can quickly start using the OTel GPU Collector by pulling the Docker image:
-
-```sh
-docker pull ghcr.io/openlit/otel-gpu-collector:latest
-```
-
-### Step 2: Run the Container
-
-Here's a quick example showing how to run the container with the required environment variables:
-
-```sh
-docker run --gpus all \
- -e GPU_APPLICATION_NAME='chatbot' \
- -e GPU_ENVIRONMENT='staging' \
- -e OTEL_EXPORTER_OTLP_ENDPOINT="YOUR_OTEL_ENDPOINT" \
- -e OTEL_EXPORTER_OTLP_HEADERS="YOUR_OTEL_HEADERS" \
- ghcr.io/openlit/otel-gpu-collector:latest
-```
-
-**Note:** If you've deployed **OpenLIT** using [Docker Compose](https://github.com/openlit/openlit/blob/main/docker-compose.yml), make sure to use the host's IP address or add OTel GPU Collector to the [Docker Compose](https://github.com/openlit/openlit/blob/main/docker-compose.yml):
-
-<details>
-<summary>Docker Compose: Add the following config under `services`</summary>
-
-```yaml
-otel-gpu-collector:
- image: ghcr.io/openlit/otel-gpu-collector:latest
- environment:
- GPU_APPLICATION_NAME: 'chatbot'
- GPU_ENVIRONMENT: 'staging'
- OTEL_EXPORTER_OTLP_ENDPOINT: "http://otel-collector:4318"
- device_requests:
- - driver: nvidia
- count: all
- capabilities: [gpu]
- depends_on:
- - otel-collector
- restart: always
-```
-
-</details>
-
-<details>
-<summary>Host IP: Use the Host IP to connect to OTel Collector</summary>
-
-```sh
-OTEL_EXPORTER_OTLP_ENDPOINT="http://192.168.10.15:4318"
-```
-</details>
-
-### Environment Variables
-
-OTel GPU Collector supports several environment variables for configuration. Below is a table that describes each variable:
-
-| Environment Variable | Description | Default Value |
-|---------------------------------|---------------------------------------------------------------|-------------------------|
-| `GPU_APPLICATION_NAME` | Name of the application running on the GPU | `default_app` |
-| `GPU_ENVIRONMENT` | Environment name (e.g., staging, production) | `production` |
-| `OTEL_EXPORTER_OTLP_ENDPOINT` | OpenTelemetry OTLP endpoint URL | (required) |
-| `OTEL_EXPORTER_OTLP_HEADERS` | Headers for authenticating with the OTLP endpoint | Ignore if using OpenLIT |
-
-## Alternative: Using OpenLIT SDK
-
-You can also collect GPU metrics directly using the OpenLIT SDK in your Python application. Here’s an example:
-
-```python
-import openlit
-
-openlit.init(collect_gpu_stats=True)
-```
-
-For more details, check out the [OpenLIT documentation](https://docs.openlit.io/) or the [SDK source code](https://github.com/openlit/openlit/tree/main/sdk/python).
-
-## Metrics
-
-| Metric Name | Description | Unit | Type | Attributes |
-|-------------------------------------|------------------------------------------|------------|-------|------------------------------------------------------------------------------------------------------------|
-| `gpu.utilization` | GPU Utilization in percentage | `percent` | Gauge | `telemetry.sdk.name`, `gen_ai.application_name`, `gen_ai.environment`, `gpu_index`, `gpu_name`, `gpu_uuid` |
-| `gpu.enc.utilization` | GPU encoder Utilization in percentage | `percent` | Gauge | `telemetry.sdk.name`, `gen_ai.application_name`, `gen_ai.environment`, `gpu_index`, `gpu_name`, `gpu_uuid` |
-| `gpu.dec.utilization` | GPU decoder Utilization in percentage | `percent` | Gauge | `telemetry.sdk.name`, `gen_ai.application_name`, `gen_ai.environment`, `gpu_index`, `gpu_name`, `gpu_uuid` |
-| `gpu.temperature` | GPU Temperature in Celsius | `Celsius` | Gauge | `telemetry.sdk.name`, `gen_ai.application_name`, `gen_ai.environment`, `gpu_index`, `gpu_name`, `gpu_uuid` |
-| `gpu.fan_speed` | GPU Fan Speed (0-100) as an integer | `Integer` | Gauge | `telemetry.sdk.name`, `gen_ai.application_name`, `gen_ai.environment`, `gpu_index`, `gpu_name`, `gpu_uuid` |
-| `gpu.memory.available` | Available GPU Memory in MB | `MB` | Gauge | `telemetry.sdk.name`, `gen_ai.application_name`, `gen_ai.environment`, `gpu_index`, `gpu_name`, `gpu_uuid` |
-| `gpu.memory.total` | Total GPU Memory in MB | `MB` | Gauge | `telemetry.sdk.name`, `gen_ai.application_name`, `gen_ai.environment`, `gpu_index`, `gpu_name`, `gpu_uuid` |
-| `gpu.memory.used` | Used GPU Memory in MB | `MB` | Gauge | `telemetry.sdk.name`, `gen_ai.application_name`, `gen_ai.environment`, `gpu_index`, `gpu_name`, `gpu_uuid` |
-| `gpu.memory.free` | Free GPU Memory in MB | `MB` | Gauge | `telemetry.sdk.name`, `gen_ai.application_name`, `gen_ai.environment`, `gpu_index`, `gpu_name`, `gpu_uuid` |
-| `gpu.power.draw` | GPU Power Draw in Watts | `Watt` | Gauge | `telemetry.sdk.name`, `gen_ai.application_name`, `gen_ai.environment`, `gpu_index`, `gpu_name`, `gpu_uuid` |
-| `gpu.power.limit` | GPU Power Limit in Watts | `Watt` | Gauge | `telemetry.sdk.name`, `gen_ai.application_name`, `gen_ai.environment`, `gpu_index`, `gpu_name`, `gpu_uuid` |
-
-## Building the Docker Image
-
-To build the Docker image yourself, you can clone the repository and execute the following commands:
-
-```sh
-# Clone the OpenLIT repository and set directory
-git clone https://github.com/openlit/openlit.git
-cd otel-gpu-collector
-
-# Build the Docker image
-docker build -t otel-gpu-collector .
-```
-
-## 🛣️ Roadmap
-
-We are dedicated to continuously improving OpenTelemetry GPU Collector. Here's a look at what's been accomplished and what's on the horizon:
-
-| Feature | Status |
-|----------------------------------------------------------------------------------------------|---------------|
-| [OpenTelmetry-native AMD GPU Monitoring]() | ✅ Completed |
-| [OpenTelmetry-native NVIDIA GPU Monitoring]() | ✅ Completed |
-
-
-
-## 🌱 Contributing
-
-Whether it's big or small, we love contributions 💚. Check out our [Contribution guide](../../CONTRIBUTING.md) to get started
-
-Unsure where to start? Here are a few ways to get involved:
-
-- Join our [Slack](https://join.slack.com/t/openlit/shared_invite/zt-2etnfttwg-TjP_7BZXfYg84oAukY8QRQ) or [Discord](https://discord.gg/rjvTm6zd) community to discuss ideas, share feedback, and connect with both our team and the wider OpenLIT community.
-
-Your input helps us grow and improve, and we're here to support you every step of the way.
-
-## 💚 Community & Support
-
-Connect with OpenLIT community and maintainers for support, discussions, and updates:
-
-- 🌟 If you like it, Leave a star on our [GitHub](https://github.com/openlit/openlit/)
-- 🌍 Join our [Slack](https://join.slack.com/t/openlit/shared_invite/zt-2etnfttwg-TjP_7BZXfYg84oAukY8QRQ) or [Discord](https://discord.gg/rjvTm6zd) community for live interactions and questions.
-- 🐞 Report bugs on our [GitHub Issues](https://github.com/openlit/openlit/issues) to help us improve OpenLIT.
-- 𝕏 Follow us on [X](https://twitter.com/openlit_io) for the latest updates and news.
-
-## License
-
-OpenTelemetry GPU Collector is built and maintained by OpenLIT under the [Apache-2.0 license](LICENSE).
diff --git a/otel-gpu-collector/collector.py b/otel-gpu-collector/collector.py
deleted file mode 100644
index 96fb1c81..00000000
--- a/otel-gpu-collector/collector.py
+++ /dev/null
@@ -1,111 +0,0 @@
-import openlit
-import os
-import logging
-import sys
-import json
-import signal
-import time
-from threading import Event
-
-
-# Create a JSON logger
-class JSONFormatter(logging.Formatter):
- def format(self, record):
- log_record = {
- "level": record.levelname,
- "message": record.getMessage(),
- "timestamp": time.strftime(
- "%Y-%m-%d %H:%M:%S", time.localtime(record.created)
- ),
- "name": record.name,
- "application_name": record.application_name,
- "environment": record.environment,
- }
- if record.exc_info:
- log_record["exception"] = self.formatException(record.exc_info)
- return json.dumps(log_record)
-
-
-# Custom filter to add fixed information to log records
-class ContextFilter(logging.Filter):
- def __init__(self, application_name, environment):
- super().__init__()
- self.application_name = application_name
- self.environment = environment
-
- def filter(self, record):
- record.application_name = self.application_name
- record.environment = self.environment
- return True
-
-
-def setup_logger(application_name, environment):
- logger = logging.getLogger("otel-gpu-collector")
- logger.setLevel(logging.INFO)
-
- # Log to stdout
- handler = logging.StreamHandler(sys.stdout)
- handler.setFormatter(JSONFormatter())
-
- logger.addHandler(handler)
-
- # Add context filter to logger
- logger.addFilter(ContextFilter(application_name, environment))
-
- return logger
-
-
-# Load environment variables
-application_name = os.getenv("GPU_APPLICATION_NAME", "default")
-environment = os.getenv("GPU_ENVIRONMENT", "default")
-
-# Initialize logger
-logger = setup_logger(application_name, environment)
-
-# Event to control the infinite loop
-keep_running = Event()
-
-
-def signal_handler(sig, frame):
- global keep_running
- logger.info("Received termination signal", extra={"signal": sig})
- keep_running.set()
-
-
-def main():
- logger.info("Starting otel-gpu-collector")
-
- try:
- # Log environment variables
- logger.info("Environment variables loaded")
-
- # Initialize OpenLit with provided parameters
- logger.info("Initializing GPU OpenTelemetry Instrumentation library OpenLIT")
- openlit.init(
- collect_gpu_stats=True,
- application_name=application_name,
- environment=environment,
- )
- logger.info("Initialization complete")
-
- # Register signal handlers
- signal.signal(signal.SIGTERM, signal_handler)
- signal.signal(signal.SIGINT, signal_handler)
-
- # Log that signal handlers are registered
- logger.info(
- "Signal handlers registered", extra={"signals": ["SIGTERM", "SIGINT"]}
- )
-
- # Keep the script running indefinitely
- global keep_running
- keep_running.wait()
-
- except Exception as e:
- logger.error("An unexpected error occurred", exc_info=True)
- finally:
- logger.info("Shutting down GPU Monitor Service")
-
-
-if __name__ == "__main__":
- main()
diff --git a/otel-gpu-collector/dockerfile b/otel-gpu-collector/dockerfile
deleted file mode 100644
index 08233999..00000000
--- a/otel-gpu-collector/dockerfile
+++ /dev/null
@@ -1,30 +0,0 @@
-# Use an official Python runtime as a parent image
-FROM python:3.9-slim
-
-# Set the working directory in the container
-WORKDIR /usr/src/app
-
-# Install build dependencies and necessary libraries
-RUN apt-get update && apt-get install -y --no-install-recommends \
- gcc \
- python3-dev \
- libffi-dev \
- libssl-dev \
- && apt-get clean \
- && rm -rf /var/lib/apt/lists/*
-
-# Copy the current directory contents into the container at /usr/src/app
-COPY . .
-
-# Install necessary Python packages
-RUN pip install --no-cache-dir --no-binary :psutil: psutil==6.1.0 nvidia-ml-py==12.560.30 amdsmi==6.2.4 openlit==1.29.0
-
-# Expose any required ports, if necessary
-EXPOSE 80
-
-# Define default values for the environment variables
-ENV GPU_APPLICATION_NAME=default
-ENV GPU_ENVIRONMENT=default
-
-# Run the script continuously
-CMD ["python", "collector.py"]