Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.
TypeScript agent-harnessagent-harness-engineeringai-engineeringai-observabilityamd-gpuclaude-codeclickhousecodexcoding-agentcoding-agentscursorgenaigpu-monitoringllm-observabilityllmopsnvidia-smiobservabilityopen-sourceopenaiopentelemetry
feat: replace bundled OTel collector with a first-party OTLP receiver (#1588)
Aman Agarwal8d38c3947703
diff --git a/README.md b/README.mdindex f72c05ad..d2498d4e 100644--- a/README.md+++ b/README.md@@ -356,17 +356,22 @@ Your telemetry can flow through the OpenTelemetry ecosystem:```mermaidflowchart TD- A["AI App / AI Agent"] -->|OpenTelemetry| C[OpenTelemetry Collector]- C --> B[OpenLIT Backend]+ A["AI App / AI Agent"] -->|OTLP| R[OpenLIT OTLP receiver]+ A -->|optional sidecar| C[Your OpenTelemetry Collector]+ C --> RC --> O["Other OTel backends<br/>(Datadog, Grafana, Honeycomb, ...)"]+ R --> B[ClickHouse]B --> D[OpenLIT Dashboard]style A fill:#111827,stroke:#F97316,stroke-width:2px,color:#fff+ style R fill:#F97316,stroke:#7C2D12,color:#fffstyle C fill:#111827,stroke:#F97316,stroke-width:2px,color:#fff- style B fill:#F97316,stroke:#7C2D12,color:#fff+ style B fill:#111827,stroke:#F97316,stroke-width:2px,color:#fffstyle D fill:#F97316,stroke:#7C2D12,color:#fff```+OpenLIT listens for OTLP on `:4317` (gRPC) and `:4318` (HTTP). A bundled Collector is not required. You can still put your own Collector in front for fan-out or processing.+This means you can integrate OpenLIT into an existing OpenTelemetry architecture instead of replacing it.---@@ -437,12 +442,12 @@ flowchart TDdirection LRAgent --> LLM --> Tools --> RAG --> DBend- App -->|OpenTelemetry| Collector[OpenTelemetry Collector]- Collector --> CH[(ClickHouse)]+ App -->|OTLP| Receiver[OpenLIT OTLP receiver]+ Receiver --> CH[(ClickHouse)]CH --> Dash[OpenLIT Dashboard]style App fill:#1F2937,stroke:#F97316,stroke-width:2px,color:#fff- style Collector fill:#111827,stroke:#F97316,stroke-width:2px,color:#fff+ style Receiver fill:#111827,stroke:#F97316,stroke-width:2px,color:#fffstyle CH fill:#111827,stroke:#F97316,stroke-width:2px,color:#fffstyle Dash fill:#F97316,stroke:#7C2D12,color:#fff```diff --git a/assets/otel-collector-config.yaml b/assets/otel-collector-config.yamlindex 67c8168f..09e45370 100644--- a/assets/otel-collector-config.yaml+++ b/assets/otel-collector-config.yaml@@ -1,3 +1,5 @@+# Optional sidecar: run your own OpenTelemetry Collector and forward OTLP to OpenLIT.+# OpenLIT no longer bundles otelcol. The image listens on :4317 (gRPC) and :4318 (HTTP).receivers:otlp:protocols:@@ -6,47 +8,20 @@ receivers:http:endpoint: 0.0.0.0:4318-processors:- batch:- memory_limiter:- # 80% of maximum memory up to 2G- limit_mib: 1500- # 25% of limit up to 2G- spike_limit_mib: 512- check_interval: 5s-exporters:- clickhouse:- endpoint: tcp://${env:INIT_DB_HOST}:9000?dial_timeout=10s- database: ${env:INIT_DB_DATABASE}- username: ${env:INIT_DB_USERNAME}- password: ${env:INIT_DB_PASSWORD}- ttl: 730h- logs_table_name: otel_logs- traces_table_name: otel_traces- # Metrics use separate tables by type: otel_metrics_gauge, otel_metrics_sum,- # otel_metrics_histogram, otel_metrics_summary, otel_metrics_exponential_histogram- timeout: 5s- retry_on_failure:- enabled: true- initial_interval: 5s- max_interval: 30s- max_elapsed_time: 300s+ otlphttp/openlit:+ endpoint: ${env:OPENLIT_OTLP_ENDPOINT}+ headers:+ x-openlit-api-key: ${env:OPENLIT_API_KEY}service:pipelines:- logs:- receivers: [otlp]- processors: [batch]- exporters: [clickhouse]traces:receivers: [otlp]- processors: [memory_limiter, batch]- exporters: [clickhouse]+ exporters: [otlphttp/openlit]metrics:receivers: [otlp]- processors: [memory_limiter, batch]- exporters: [clickhouse]- # telemetry:- # metrics:- # address: localhost:8888\ No newline at end of file+ exporters: [otlphttp/openlit]+ logs:+ receivers: [otlp]+ exporters: [otlphttp/openlit]diff --git a/docker-compose.yml b/docker-compose.ymlindex f5cb6b85..16b44561 100644--- a/docker-compose.yml+++ b/docker-compose.yml@@ -41,15 +41,6 @@ services:GOOGLE_CLIENT_SECRET: ${GOOGLE_CLIENT_SECRET:-}GITHUB_CLIENT_ID: ${GITHUB_CLIENT_ID:-}GITHUB_CLIENT_SECRET: ${GITHUB_CLIENT_SECRET:-}-- # TLS Configuration (user-configurable)- OPAMP_ENVIRONMENT: ${OPAMP_ENVIRONMENT:-production}- OPAMP_CERTS_DIR: "/app/opamp/certs"- OPAMP_TLS_INSECURE_SKIP_VERIFY: ${OPAMP_TLS_INSECURE_SKIP_VERIFY:-false}- OPAMP_TLS_REQUIRE_CLIENT_CERT: ${OPAMP_TLS_REQUIRE_CLIENT_CERT:-true}- OPAMP_TLS_MIN_VERSION: ${OPAMP_TLS_MIN_VERSION:-1.2}- OPAMP_TLS_MAX_VERSION: ${OPAMP_TLS_MAX_VERSION:-1.3}- OPAMP_LOG_LEVEL: ${OPAMP_LOG_LEVEL:-info}env_file:- path: .env@@ -63,7 +54,6 @@ services:condition: service_healthyvolumes:- openlit-data:/app/client/data- - ./assets/otel-collector-config.yaml:/etc/otel/otel-collector-config.yamlhealthcheck:test: ["CMD", "node", "-e", "fetch('http://localhost:3000').then(r=>{process.exit(r.ok?0:1)}).catch(()=>process.exit(1))"]interval: 5sdiff --git a/docs/docs.json b/docs/docs.jsonindex ad0e5dd7..12b595e6 100644--- a/docs/docs.json+++ b/docs/docs.json@@ -42,6 +42,7 @@"pages": ["latest/overview","latest/openlit/installation",+ "latest/openlit/otlp-receiver",{"group": "Quickstarts","icon": "bolt",diff --git a/docs/latest/cli/installation.mdx b/docs/latest/cli/installation.mdxindex 39a1d8e0..49ea7722 100644--- a/docs/latest/cli/installation.mdx+++ b/docs/latest/cli/installation.mdx@@ -96,7 +96,7 @@ iwr -useb https://raw.githubusercontent.com/openlit/openlit/main/cli/scripts/uni<CardGroup cols={2}><Card title="Configuration" href="/latest/cli/configuration" icon="sliders">- Point the CLI at your OpenLIT collector+ Point the CLI at your OpenLIT OTLP endpoint</Card><Card title="Wire your coding agents" href="/latest/openlit/coding-agents/setup-and-configure" icon="code">Claude Code, Cursor, and Codex in under a minutediff --git a/docs/latest/cli/overview.mdx b/docs/latest/cli/overview.mdxindex fbc4b696..33526fd5 100644--- a/docs/latest/cli/overview.mdx+++ b/docs/latest/cli/overview.mdx@@ -4,7 +4,7 @@ sidebarTitle: "Overview"description: "The command-line interface for OpenLIT - observe coding agents today, control the whole platform tomorrow"----The **`openlit` CLI** is a single Go binary that connects your local tooling to an OpenLIT instance. Today it ships first-class observability for AI **coding agents** - Claude Code, Cursor, and Codex - by wiring into each vendor's hook system, normalizing events onto OpenTelemetry's `gen_ai.*` and OpenLIT's `coding_agent.*` semantic conventions, and exporting them via OTLP to your collector.+The **`openlit` CLI** is a single Go binary that connects your local tooling to an OpenLIT instance. Today it ships first-class observability for AI **coding agents** - Claude Code, Cursor, and Codex - by wiring into each vendor's hook system, normalizing events onto OpenTelemetry's `gen_ai.*` and OpenLIT's `coding_agent.*` semantic conventions, and exporting them via OTLP to OpenLIT's [OTLP receiver](/latest/openlit/otlp-receiver).It talks **only** to your OpenLIT endpoint. There is no managed cloud - your telemetry stays on your infrastructure.@@ -51,7 +51,7 @@ openlit coding hook --vendor=<v> --event=<name> (invoked by the host plugin ma+-- 5s hard timeout, always exits 0 (never blocks the developer)|v-OTLP/HTTP --> your OpenLIT collector --> /agents page+OTLP/HTTP --> your OpenLIT OTLP receiver --> /agents page```The hook is the CLI's hot path - invoked once per agent event. It is crash-isolated: a 5-second hard timeout with ~3 seconds reserved for OTLP flush, and it **always exits 0** even when telemetry fails, so a broken pipeline never blocks your coding session.diff --git a/docs/latest/gpu-collector/configuration.mdx b/docs/latest/gpu-collector/configuration.mdxindex d36e4101..7bda7fe6 100644--- a/docs/latest/gpu-collector/configuration.mdx+++ b/docs/latest/gpu-collector/configuration.mdx@@ -103,7 +103,7 @@ env:- name: OTEL_RESOURCE_ATTRIBUTESvalue: "host.name=$(K8S_NODE_NAME),k8s.node.name=$(K8S_NODE_NAME)"- name: OTEL_EXPORTER_OTLP_ENDPOINT- value: "http://otel-collector:4317"+ value: "http://openlit:4318"# Optional when cloud auto-detect is unavailable (on-prem / self-managed):# - name: K8S_CLUSTER_NAME# value: my-cluster@@ -145,10 +145,10 @@ On EKS, pods without `hostNetwork` may fail IMDSv2 when the node `httpPutRespons## Common configurations-### Minimal - send to a local OTel Collector+### Minimal - send to OpenLIT or any OTLP endpoint```bash-OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317 \+OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318 \./opentelemetry-gpu-collector```diff --git a/docs/latest/gpu-collector/gpus/amd.mdx b/docs/latest/gpu-collector/gpus/amd.mdxindex df991afc..ebee3909 100644--- a/docs/latest/gpu-collector/gpus/amd.mdx+++ b/docs/latest/gpu-collector/gpus/amd.mdx@@ -36,7 +36,7 @@ docker run -d \--pid=host \-e OTEL_SERVICE_NAME=my-app \-e OTEL_RESOURCE_ATTRIBUTES='deployment.environment=production' \- -e OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318 \+ -e OTEL_EXPORTER_OTLP_ENDPOINT=http://openlit:4318 \ghcr.io/openlit/otel-gpu-collector:latest```@@ -52,7 +52,7 @@ services:environment:OTEL_SERVICE_NAME: my-appOTEL_RESOURCE_ATTRIBUTES: deployment.environment=production- OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318+ OTEL_EXPORTER_OTLP_ENDPOINT: http://openlit:4318devices:- /dev/kfd:/dev/kfd- /dev/dri:/dev/dri@@ -86,7 +86,7 @@ spec:- name: OTEL_RESOURCE_ATTRIBUTESvalue: deployment.environment=production- name: OTEL_EXPORTER_OTLP_ENDPOINT- value: http://otel-collector.monitoring.svc.cluster.local:4318+ value: http://openlit.openlit.svc.cluster.local:4318securityContext:privileged: falsevolumeMounts:diff --git a/docs/latest/gpu-collector/gpus/intel.mdx b/docs/latest/gpu-collector/gpus/intel.mdxindex a27266c5..43c3f09d 100644--- a/docs/latest/gpu-collector/gpus/intel.mdx+++ b/docs/latest/gpu-collector/gpus/intel.mdx@@ -35,7 +35,7 @@ docker run -d \--pid=host \-e OTEL_SERVICE_NAME=my-app \-e OTEL_RESOURCE_ATTRIBUTES='deployment.environment=production' \- -e OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318 \+ -e OTEL_EXPORTER_OTLP_ENDPOINT=http://openlit:4318 \ghcr.io/openlit/otel-gpu-collector:latest```@@ -51,7 +51,7 @@ services:environment:OTEL_SERVICE_NAME: my-appOTEL_RESOURCE_ATTRIBUTES: deployment.environment=production- OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318+ OTEL_EXPORTER_OTLP_ENDPOINT: http://openlit:4318devices:- /dev/dri:/dev/drirestart: always@@ -84,7 +84,7 @@ spec:- name: OTEL_RESOURCE_ATTRIBUTESvalue: deployment.environment=production- name: OTEL_EXPORTER_OTLP_ENDPOINT- value: http://otel-collector.monitoring.svc.cluster.local:4318+ value: http://openlit.openlit.svc.cluster.local:4318securityContext:privileged: falsevolumeMounts:diff --git a/docs/latest/gpu-collector/gpus/nvidia.mdx b/docs/latest/gpu-collector/gpus/nvidia.mdxindex f56189ef..f0ac9865 100644--- a/docs/latest/gpu-collector/gpus/nvidia.mdx+++ b/docs/latest/gpu-collector/gpus/nvidia.mdx@@ -39,7 +39,7 @@ docker run -d \--pid=host \-e OTEL_SERVICE_NAME=my-app \-e OTEL_RESOURCE_ATTRIBUTES='deployment.environment=production' \- -e OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318 \+ -e OTEL_EXPORTER_OTLP_ENDPOINT=http://openlit:4318 \ghcr.io/openlit/otel-gpu-collector:latest```@@ -55,7 +55,7 @@ services:environment:OTEL_SERVICE_NAME: my-appOTEL_RESOURCE_ATTRIBUTES: deployment.environment=production- OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318+ OTEL_EXPORTER_OTLP_ENDPOINT: http://openlit:4318deploy:resources:reservations:@@ -99,7 +99,7 @@ spec:- name: OTEL_RESOURCE_ATTRIBUTESvalue: deployment.environment=production- name: OTEL_EXPORTER_OTLP_ENDPOINT- value: http://otel-collector.monitoring.svc.cluster.local:4318+ value: http://openlit.openlit.svc.cluster.local:4318resources:limits:nvidia.com/gpu: 1diff --git a/docs/latest/gpu-collector/installation.mdx b/docs/latest/gpu-collector/installation.mdxindex f2a3e906..c2ef891f 100644--- a/docs/latest/gpu-collector/installation.mdx+++ b/docs/latest/gpu-collector/installation.mdx@@ -137,7 +137,7 @@ spec:- name: OTEL_RESOURCE_ATTRIBUTESvalue: "host.name=$(K8S_NODE_NAME),k8s.node.name=$(K8S_NODE_NAME)"- name: OTEL_EXPORTER_OTLP_ENDPOINT- value: "http://otel-collector:4317"+ value: "http://openlit:4318"# Optional for on-prem / self-managed clusters:# - name: K8S_CLUSTER_NAME# value: my-clusterdiff --git a/docs/latest/gpu-collector/quickstart.mdx b/docs/latest/gpu-collector/quickstart.mdxindex bd1cb0e0..92b0f4f7 100644--- a/docs/latest/gpu-collector/quickstart.mdx+++ b/docs/latest/gpu-collector/quickstart.mdx@@ -123,7 +123,7 @@ services:environment:OTEL_SERVICE_NAME: my-appOTEL_RESOURCE_ATTRIBUTES: deployment.environment=production- OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318+ OTEL_EXPORTER_OTLP_ENDPOINT: http://openlit:4318deploy:resources:reservations:@@ -132,7 +132,7 @@ services:count: allcapabilities: [gpu]depends_on:- - otel-collector+ - openlitrestart: always```diff --git a/docs/latest/openlit/coding-agents/analytics.mdx b/docs/latest/openlit/coding-agents/analytics.mdxindex 3c59f309..da296dfd 100644--- a/docs/latest/openlit/coding-agents/analytics.mdx+++ b/docs/latest/openlit/coding-agents/analytics.mdx@@ -34,7 +34,7 @@ WHERE SpanName IN ()```-The metrics pipeline emits the same rollups as counters, if you'd rather point a Prometheus/Mimir/Datadog backend at the OpenLIT collector directly: `coding_agent.session.count`, `coding_agent.session.duration`, `coding_agent.tool.call.count`, `coding_agent.lines_of_code.count`, `coding_agent.edit.decision.count`, `coding_agent.commit.count`, `coding_agent.pull_request.count`.+The metrics pipeline emits the same rollups as counters, if you'd rather point a Prometheus/Mimir/Datadog backend at OpenLIT's OTLP endpoint directly: `coding_agent.session.count`, `coding_agent.session.duration`, `coding_agent.tool.call.count`, `coding_agent.lines_of_code.count`, `coding_agent.edit.decision.count`, `coding_agent.commit.count`, `coding_agent.pull_request.count`.---diff --git a/docs/latest/openlit/coding-agents/overview.mdx b/docs/latest/openlit/coding-agents/overview.mdxindex a5e388d7..36eab3c0 100644--- a/docs/latest/openlit/coding-agents/overview.mdx+++ b/docs/latest/openlit/coding-agents/overview.mdx@@ -3,7 +3,7 @@ title: "Overview"description: "OpenLIT ships first-class observability for AI coding agents - Claude Code, Cursor, Codex, and Windsurf - with no SDK and no code changes"----OpenLIT ships first-class observability for AI coding agents. The `openlit` CLI ingests events from each vendor's hook system, normalizes them onto OpenTelemetry's `gen_ai.*` and OpenLIT's `coding_agent.*` semantic conventions, and ships them to your existing OpenLIT collector. The [Agents page](/latest/openlit/observability/agents/overview)'s Coding Agents tab detects them automatically - no controller, no SDK, no code change in your repos.+OpenLIT ships first-class observability for AI coding agents. The `openlit` CLI ingests events from each vendor's hook system, normalizes them onto OpenTelemetry's `gen_ai.*` and OpenLIT's `coding_agent.*` semantic conventions, and ships them to your OpenLIT OTLP endpoint. The [Agents page](/latest/openlit/observability/agents/overview)'s Coding Agents tab detects them automatically - no controller, no SDK, no code change in your repos.<Note>You only need an OpenLIT instance and the `openlit` CLI on each developer's machine. The CLI talks **only** to your OpenLIT endpoint. There is no managed cloud for coding-agent telemetry.@@ -39,4 +39,4 @@ You only need an OpenLIT instance and the `openlit` CLI on each developer's mach- A running OpenLIT instance - see [Installation](/latest/openlit/installation) if you don't have one.- An OpenLIT API key (Settings → API Keys).-- The OTLP endpoint of your OpenLIT collector (defaults to `http://localhost:4318` for local installs).+- The OTLP endpoint of your OpenLIT instance (defaults to `http://localhost:4318` for local installs).diff --git a/docs/latest/openlit/coding-agents/setup-and-configure.mdx b/docs/latest/openlit/coding-agents/setup-and-configure.mdxindex 3182efc5..9d23dbb3 100644--- a/docs/latest/openlit/coding-agents/setup-and-configure.mdx+++ b/docs/latest/openlit/coding-agents/setup-and-configure.mdx@@ -48,7 +48,7 @@ description: "Install the openlit CLI, wire each vendor's hooks, then configureManifest path: `~/.claude/plugins/openlit-cc/`. Hooks wired: `SessionStart`, `UserPromptSubmit`, `PreToolUse`, `PostToolUse`, `Stop`, `SubagentStop`, `SessionEnd`. Claude Code's transcript JSONL (`transcript_path` in the hook payload) is tailed on `SessionStart` for early model attribution and on `SessionEnd` for authoritative token usage and cost.- **Optional: Claude Code's native OpenTelemetry exporter.** If you'd rather not install the plugin, or you want a second signal source for cross-checking, you can point Claude Code's built-in OTel exporter at your OpenLIT collector instead. Both paths can run side by side - OpenLIT dedupes per `session.id` (Claude Code's authoritative cost/tokens from the native path win, while the hook path provides the repository/working-folder context that Claude Code itself can't see).+ **Optional: Claude Code's native OpenTelemetry exporter.** If you'd rather not install the plugin, or you want a second signal source for cross-checking, you can point Claude Code's built-in OTel exporter at your OpenLIT OTLP endpoint instead. Both paths can run side by side - OpenLIT dedupes per `session.id` (Claude Code's authoritative cost/tokens from the native path win, while the hook path provides the repository/working-folder context that Claude Code itself can't see).```bashexport CLAUDE_CODE_ENABLE_TELEMETRY=1@@ -129,7 +129,7 @@ Equivalent environment variables (precedence: flags > env > config file):| Variable | Purpose ||---|---|-| `OPENLIT_OTLP_ENDPOINT` | OTLP/HTTP endpoint of your collector |+| `OPENLIT_OTLP_ENDPOINT` | OTLP/HTTP endpoint of your OpenLIT receiver (`:4318`) || `OPENLIT_API_KEY` | Authenticates the OTLP exporter || `OPENLIT_USER` | Optional override for `gen_ai.user.name` (email or login) || `OPENLIT_CODING_CONTENT_CAPTURE` | `minimal` \| `metadata_only` \| `full` *(default: `full`)* - see [Privacy & Governance](/latest/openlit/coding-agents/privacy-and-governance) |diff --git a/docs/latest/openlit/developer-resources/api-reference/endpoint/manage-models/public-pricing.mdx b/docs/latest/openlit/developer-resources/api-reference/endpoint/manage-models/public-pricing.mdxindex fae8a1fc..6e0e4588 100644--- a/docs/latest/openlit/developer-resources/api-reference/endpoint/manage-models/public-pricing.mdx+++ b/docs/latest/openlit/developer-resources/api-reference/endpoint/manage-models/public-pricing.mdx@@ -21,7 +21,7 @@ Replace the host and database config id with values from your OpenLIT deploymentimport openlitopenlit.init(- otlp_endpoint="http://<otel-collector-host>:4318",+ otlp_endpoint="http://<openlit-host>:4318",pricing_json="https://<your-openlit-host>/api/pricing/export/<your-db-config-id>",)```@@ -31,7 +31,7 @@ Replace the host and database config id with values from your OpenLIT deploymentimport openlit from "openlit";openlit.init({- otlpEndpoint: "http://<otel-collector-host>:4318",+ otlpEndpoint: "http://<openlit-host>:4318",pricingJson: "https://<your-openlit-host>/api/pricing/export/<your-db-config-id>",});```@@ -44,7 +44,7 @@ Replace the host and database config id with values from your OpenLIT deploymentfunc main() {openlit.Init(openlit.Config{- OtlpEndpoint: "http://<otel-collector-host>:4318",+ OtlpEndpoint: "http://<openlit-host>:4318",PricingJson: "https://<your-openlit-host>/api/pricing/export/<your-db-config-id>",})}diff --git a/docs/latest/openlit/installation.mdx b/docs/latest/openlit/installation.mdxindex 4e03bb60..74304292 100644--- a/docs/latest/openlit/installation.mdx+++ b/docs/latest/openlit/installation.mdx@@ -1,22 +1,22 @@---title: "Self-Host OpenLIT"sidebarTitle: Self Host-description: "Self-host OpenLIT with Docker or Kubernetes. Deploy OpenLIT, ClickHouse, and an OpenTelemetry Collector on your own infrastructure."+description: "Self-host OpenLIT with Docker or Kubernetes. Deploy OpenLIT (dashboard plus OTLP receiver) and ClickHouse on your own infrastructure."icon: 'circle-down'----OpenLIT is open source and is very easy and lightweight to self-host as it needs only 3 components to run:-1. OpenLIT itself-2. ClickHouse for storage-3. OpenTelemetry Collector for telemetry collection.+OpenLIT is open source and is lightweight to self-host. You need two components:++1. **OpenLIT** — dashboard, APIs, and the first-party [OTLP receiver](/latest/openlit/otlp-receiver) (`:4317` gRPC, `:4318` HTTP)+2. **ClickHouse** — storage for traces, metrics, and logs (`otel_*` tables)<Info>-**Already have ClickHouse or OpenTelemetry Collector?**+**Already have ClickHouse?**-You can reuse your existing infrastructure and skip deploying these components. OpenLIT can be configured to connect to your existing ClickHouse database and OTel Collector setup. [Learn more about connecting to existing databases →](/latest/openlit/developer-resources/multiple-db)+You can reuse it and skip deploying a new database. OpenLIT connects through `INIT_DB_*` or a project [database config](/latest/openlit/developer-resources/multiple-db). You do **not** need a bundled OpenTelemetry Collector — OTLP ingest is built into the OpenLIT container. An optional sidecar Collector can still forward OTLP to OpenLIT.</Info>-This section contains guides for different deployment scenarios.+This section contains guides for different deployment scenarios.<CardGroup cols={2}>@@ -27,20 +27,21 @@ This section contains guides for different deployment scenarios.<path fill="#FFA500" d="M10.204 14.35l.007.01-.999 2.413a5.171 5.171 0 0 1-2.075-2.597l2.578-.437.004.005a.44.44 0 0 1 .484.606zm-.833-2.129a.44.44 0 0 0 .173-.756l.002-.011L7.585 9.7a5.143 5.143 0 0 0-.73 3.255l2.514-.725.002-.009zm1.145-1.98a.44.44 0 0 0 .699-.337l.01-.005.15-2.62a5.144 5.144 0 0 0-3.01 1.442l2.147 1.523.004-.002zm.76 2.75l.723.349.722-.347.18-.78-.5-.623h-.804l-.5.623.179.779zm1.5-3.095a.44.44 0 0 0 .7.336l.008.003 2.134-1.513a5.188 5.188 0 0 0-2.992-1.442l.148 2.615.002.001zm10.876 5.97l-5.773 7.181a1.6 1.6 0 0 1-1.248.594l-9.261.003a1.6 1.6 0 0 1-1.247-.596l-5.776-7.18a1.583 1.583 0 0 1-.307-1.34L2.1 5.573c.108-.47.425-.864.863-1.073L11.305.513a1.606 1.606 0 0 1 1.385 0l8.345 3.985c.438.209.755.604.863 1.073l2.062 8.955c.108.47-.005.963-.308 1.34zm-3.289-2.057c-.042-.01-.103-.026-.145-.034-.174-.033-.315-.025-.479-.038-.35-.037-.638-.067-.895-.148-.105-.04-.18-.165-.216-.216l-.201-.059a6.45 6.45 0 0 0-.105-2.332 6.465 6.465 0 0 0-.936-2.163c.052-.047.15-.133.177-.159.008-.09.001-.183.094-.282.197-.185.444-.338.743-.522.142-.084.273-.137.415-.242.032-.024.076-.062.11-.089.24-.191.295-.52.123-.736-.172-.216-.506-.236-.745-.045-.034.027-.08.062-.111.088-.134.116-.217.23-.33.35-.246.25-.45.458-.673.609-.097.056-.239.037-.303.033l-.19.135a6.545 6.545 0 0 0-4.146-2.003l-.012-.223c-.065-.062-.143-.115-.163-.25-.022-.268.015-.557.057-.905.023-.163.061-.298.068-.475.001-.04-.001-.099-.001-.142 0-.306-.224-.555-.5-.555-.275 0-.499.249-.499.555l.001.014c0 .041-.002.092 0 .128.006.177.044.312.067.475.042.348.078.637.056.906a.545.545 0 0 1-.162.258l-.012.211a6.424 6.424 0 0 0-4.166 2.003 8.373 8.373 0 0 1-.18-.128c-.09.012-.18.04-.297-.029-.223-.15-.427-.358-.673-.608-.113-.12-.195-.234-.329-.349-.03-.026-.077-.062-.111-.088a.594.594 0 0 0-.348-.132.481.481 0 0 0-.398.176c-.172.216-.117.546.123.737l.007.005.104.083c.142.105.272.159.414.242.299.185.546.338.743.522.076.082.09.226.1.288l.16.143a6.462 6.462 0 0 0-1.02 4.506l-.208.06c-.055.072-.133.184-.215.217-.257.081-.546.11-.895.147-.164.014-.305.006-.48.039-.037.007-.09.02-.133.03l-.004.002-.007.002c-.295.071-.484.342-.423.608.061.267.349.429.645.365l.007-.001.01-.003.129-.029c.17-.046.294-.113.448-.172.33-.118.604-.217.87-.256.112-.009.23.069.288.101l.217-.037a6.5 6.5 0 0 0 2.88 3.596l-.09.218c.033.084.069.199.044.282-.097.252-.263.517-.452.813-.091.136-.185.242-.268.399-.02.037-.045.095-.064.134-.128.275-.034.591.213.71.248.12.556-.007.69-.282v-.002c.02-.039.046-.09.062-.127.07-.162.094-.301.144-.458.132-.332.205-.68.387-.897.05-.06.13-.082.215-.105l.113-.205a6.453 6.453 0 0 0 4.609.012l.106.192c.086.028.18.042.256.155.136.232.229.507.342.84.05.156.074.295.145.457.016.037.043.09.062.129.133.276.442.402.69.282.247-.118.341-.435.213-.71-.02-.039-.045-.096-.065-.134-.083-.156-.177-.261-.268-.398-.19-.296-.346-.541-.443-.793-.04-.13.007-.21.038-.294-.018-.022-.059-.144-.083-.202a6.499 6.499 0 0 0 2.88-3.622c.064.01.176.03.213.038.075-.05.144-.114.28-.104.266.039.54.138.87.256.154.06.277.128.448.173.036.01.088.019.13.028l.009.003.007.001c.297.064.584-.098.645-.365.06-.266-.128-.537-.423-.608zM16.4 9.701l-1.95 1.746v.005a.44.44 0 0 0 .173.757l.003.01 2.526.728a5.199 5.199 0 0 0-.108-1.674A5.208 5.208 0 0 0 16.4 9.7zm-4.013 5.325a.437.437 0 0 0-.404-.232.44.44 0 0 0-.372.233h-.002l-1.268 2.292a5.164 5.164 0 0 0 3.326.003l-1.27-2.296h-.01zm1.888-1.293a.44.44 0 0 0-.27.036.44.44 0 0 0-.214.572l-.003.004 1.01 2.438a5.15 5.15 0 0 0 2.081-2.615l-2.6-.44-.004.005z"/></svg>}href="/latest/openlit/installation#kubernetes">- Deploy OpenLIT's lightweight 3-component architecture on Kubernetes for scalable AI Engineering capabilities.+ Deploy OpenLIT and ClickHouse on Kubernetes. OTLP ingest is built into the OpenLIT container.</Card><Cardtitle="Docker"icon="docker"href="/latest/openlit/installation#docker">- Deploy OpenLIT's lightweight 3-component stack using Docker for an easy-to-manage AI Engineering setup.+ Deploy OpenLIT and ClickHouse with Docker Compose. Ports 3000, 4317, and 4318 are on the OpenLIT service.</Card></CardGroup>## Kubernetes-Deploy OpenLIT on your Kubernetes cluster using the [OpenLIT Helm chart](https://github.com/openlit/helm/tree/main/charts/openlit). The Helm chart deploys all 3 components (OpenLIT platform, ClickHouse, and OpenTelemetry Collector) for a reliable and scalable solution.+Deploy OpenLIT on your Kubernetes cluster using the [OpenLIT Helm chart](https://github.com/openlit/helm/tree/main/charts/openlit). The chart deploys the OpenLIT platform (including the OTLP receiver) and ClickHouse.+<Steps><Step title="Helm Repo Setup">```shell shell@@ -55,8 +56,10 @@ Deploy OpenLIT on your Kubernetes cluster using the [OpenLIT Helm chart](https:/</Step></Steps>+Send OTLP to the OpenLIT service on `4317` (gRPC) or `4318` (HTTP). See [OTLP receiver](/latest/openlit/otlp-receiver).+## Docker-For a quick and straightforward setup, Docker Compose can be used to deploy OpenLIT's complete stack: the OpenLIT platform, ClickHouse database, and OpenTelemetry Collector.+For a quick setup, Docker Compose deploys OpenLIT and ClickHouse. The OpenLIT container listens on `3000` (UI), `4317` (OTLP/gRPC), and `4318` (OTLP/HTTP). There is no separate `otel-collector` service.<Steps><Step title="Git Clone OpenLIT Repository">@@ -72,14 +75,18 @@ For a quick and straightforward setup, Docker Compose can be used to deploy Open</Step></Steps>+<Warning>+If you previously ran the old stack, restart Compose with `--remove-orphans`. The standalone `otel-collector` container is no longer used and will conflict on ports 4317 and 4318 if it is left running.+</Warning>+## Frequently asked questions<AccordionGroup><Accordion title="What do I need to self-host OpenLIT?">- Just Docker and Docker Compose for the quickest path, or a Kubernetes cluster with Helm if you're deploying at scale. Both methods deploy the same 3 components: OpenLIT, ClickHouse, and an OpenTelemetry Collector.+ Docker and Docker Compose for the quickest path, or a Kubernetes cluster with Helm. Both deploy OpenLIT (dashboard + OTLP receiver) and ClickHouse.</Accordion><Accordion title="Can I reuse my existing ClickHouse or OpenTelemetry Collector?">- Yes. OpenLIT can connect to infrastructure you already run instead of deploying its own ClickHouse or OTel Collector. See [Connect Multiple Databases](/latest/openlit/developer-resources/multiple-db) for setup.+ Yes for ClickHouse — see [Connect Multiple Databases](/latest/openlit/developer-resources/multiple-db). You do not need OpenLIT to run `otelcol`. If you already operate a Collector, [forward OTLP to OpenLIT](/latest/openlit/otlp-receiver#optional-sidecar-collector).</Accordion><Accordion title="Is OpenLIT free to self-host?">Yes. OpenLIT is licensed under Apache-2.0 - every feature in this documentation is free to self-host, with no license key or usage limits.diff --git a/docs/latest/openlit/observability/telemetry/logs.mdx b/docs/latest/openlit/observability/telemetry/logs.mdxindex 597d564a..5dc4b149 100644--- a/docs/latest/openlit/observability/telemetry/logs.mdx+++ b/docs/latest/openlit/observability/telemetry/logs.mdx@@ -1,11 +1,11 @@---title: "Logs"-description: "A searchable, filterable explorer for OTel log records ingested through OpenLIT's built-in OpenTelemetry Collector"+description: "A searchable, filterable explorer for OTel log records ingested through OpenLIT's OTLP receiver"---Navigate to **Telemetry > Logs** for a dedicated logs explorer - a searchable, filterable list of ingested log records with a detail view per log entry.-OpenLIT ingests OTel logs the same way it ingests traces and metrics: through the built-in OpenTelemetry Collector's `otlp` receiver (gRPC `:4317`, HTTP `:4318`), which exports them into the `otel_logs` table in ClickHouse. Point your application's log exporter at the same OTLP endpoint you use for traces and metrics to have logs show up here automatically - no separate pipeline is required.+OpenLIT ingests OTel logs the same way it ingests traces and metrics: through the first-party OTLP receiver (gRPC `:4317`, HTTP `:4318`), which writes them into the `otel_logs` table in ClickHouse. Point your application's log exporter at the same OTLP endpoint you use for traces and metrics to have logs show up here automatically - no separate pipeline is required.The logs list shows, per entry: Time, Severity, Service, Body, Trace ID, and Span ID. Like traces and metrics, logs support attribute-based filtering - Log Attributes, Resource Attributes, Scope Attributes, or top-level Field - so you can narrow down to a specific service, severity, or custom field before inspecting individual entries.diff --git a/docs/latest/openlit/observability/telemetry/overview.mdx b/docs/latest/openlit/observability/telemetry/overview.mdxindex 2c481e05..7355e485 100644--- a/docs/latest/openlit/observability/telemetry/overview.mdx+++ b/docs/latest/openlit/observability/telemetry/overview.mdx@@ -14,7 +14,7 @@ description: "LLM observability for traces, metrics, logs, and exceptions - all></video>-The **Telemetry** page is the single home for every OpenTelemetry signal OpenLIT ingests from your AI applications. Navigate to it at `/telemetry`. It has four tabs, all backed by the same ClickHouse tables (`otel_traces`, `otel_metrics_*`, `otel_logs`) populated by the OpenTelemetry Collector built into OpenLIT:+The **Telemetry** page is the single home for every OpenTelemetry signal OpenLIT ingests from your AI applications. Navigate to it at `/telemetry`. It has four tabs, all backed by the same ClickHouse tables (`otel_traces`, `otel_metrics_*`, `otel_logs`) populated by OpenLIT's first-party OTLP receiver:<CardGroup cols={2}><Card title="Traces" href="/latest/openlit/observability/telemetry/traces" icon="diagram-project">@@ -35,10 +35,13 @@ The **Exceptions** tab is documented alongside Traces since it reuses the same t----<CardGroup cols={3}>+<CardGroup cols={2}><Card title="Quickstart: LLM Observability" href="/latest/openlit/quickstart-ai-observability" icon='bolt'>Production-ready AI monitoring setup in 2 simple steps with zero code changes</Card>+ <Card title="OTLP receiver" href="/latest/openlit/otlp-receiver" icon='satellite-dish'>+ Send traces, metrics, and logs to OpenLIT over OTLP HTTP and gRPC — no bundled Collector+ </Card><Card title="Create a dashboard" href="/latest/openlit/dashboards/overview" icon='grid'>Create custom visualizations with flexible widgets, queries, and real-time AI monitoring</Card>@@ -54,7 +57,7 @@ The **Exceptions** tab is documented alongside Traces since it reuses the same tAI observability is the practice of collecting traces, metrics, and logs from LLM applications and agents so you can understand behavior, cost, latency, and errors in production - OpenLIT does this via standard OpenTelemetry data.</Accordion><Accordion title="Does OpenLIT support log monitoring, not just traces?">- Yes. Logs have their own dedicated explorer under Telemetry, ingested the same way as traces and metrics through the built-in OpenTelemetry Collector's OTLP receiver.+ Yes. Logs have their own dedicated explorer under Telemetry, ingested the same way as traces and metrics through OpenLIT's OTLP receiver.</Accordion><Accordion title="Can I still use dashboards alongside the Metrics explorer?">Yes. The Metrics explorer and Dashboard Widgets read from the same underlying data - use the explorer to inspect a single metric, and dashboards for cross-signal, trend-style visualizations.diff --git a/docs/latest/openlit/organisation/signal-routing.mdx b/docs/latest/openlit/organisation/signal-routing.mdxindex cf60bdd7..d2cdc54d 100644--- a/docs/latest/openlit/organisation/signal-routing.mdx+++ b/docs/latest/openlit/organisation/signal-routing.mdx@@ -93,7 +93,7 @@ Regardless of signal routing:- Dashboard / board definitions and widget layout metadata- Rules, vault secrets materialization, controller-derived intelligence-Raw OTLP **write** paths (SDK exporters, collectors) are separate from these **read** connectors. To mirror only AI telemetry into ClickHouse, see the [OTel Collector AI filter recipe](https://github.com/openlit/openlit/blob/main/assets/otel-collector-ai-filter.yaml).+Raw OTLP **write** paths (SDK exporters, optional sidecar collectors) are separate from these **read** connectors. Send OTLP into OpenLIT on `:4317` / `:4318` — see the [OTLP receiver](/latest/openlit/otlp-receiver). To forward from a Collector you operate, use the [sidecar collector example](https://github.com/openlit/openlit/blob/main/assets/otel-collector-config.yaml).## Permissions and auditdiff --git a/docs/latest/openlit/otlp-receiver.mdx b/docs/latest/openlit/otlp-receiver.mdxnew file mode 100644index 00000000..c46fc777--- /dev/null+++ b/docs/latest/openlit/otlp-receiver.mdx@@ -0,0 +1,77 @@+---+title: "OTLP receiver"+sidebarTitle: OTLP receiver+description: "How OpenLIT ingests traces, metrics, and logs over OTLP HTTP and gRPC without a bundled OpenTelemetry Collector."+---++OpenLIT's image includes a first-party **OTLP receiver**. Applications, SDKs, and the `openlit` CLI send traces, metrics, and logs to it. The receiver writes ClickHouse `otel_*` tables. The OpenLIT image does **not** ship `otelcol`.++```mermaid+flowchart LR+ App[AI app / SDK / CLI] -->|OTLP| Rec[OpenLIT OTLP receiver]+ Rec --> CH[(ClickHouse otel_* tables)]+ CH --> UI[OpenLIT UI]+```++## Endpoints++| Protocol | Address | Paths |+| --- | --- | --- |+| OTLP/HTTP | `:4318` | `POST /v1/traces`, `POST /v1/metrics`, `POST /v1/logs` (protobuf or JSON; gzip optional) |+| OTLP/gRPC | `:4317` | Standard OTLP gRPC |+| Health | `:4318` | `GET /health` |++The OpenLIT UI also proxies `/v1/*` from the app origin to this process, so the same HTTP paths work through the dashboard host when you cannot reach `:4318` directly.++Docker Compose and Kubernetes examples publish `4317` and `4318` on the **OpenLIT** service (`openlit`), not a separate `otel-collector` container.++Point exporters at the receiver:++```bash+export OTEL_EXPORTER_OTLP_ENDPOINT="http://127.0.0.1:4318"+```++```python+import openlit++openlit.init(otlp_endpoint="http://127.0.0.1:4318")+```++Supported metric types: gauge, sum, histogram, summary, and exponential histogram (`otel_metrics_*` tables).++## Authentication++Send an OpenLIT API key so ingest is scoped to organisation → project → environment and writes to that project's ClickHouse:++```bash+export OTEL_EXPORTER_OTLP_HEADERS="Authorization=Bearer <openlit-api-key>"+```++`x-openlit-api-key` is also accepted. Client `x-database-config-id` headers are ignored.++A valid key stamps `organisation.environment.name`, `openlit.organisation.id`, and `openlit.project.id`. It does **not** overwrite client `deployment.environment`.++Without a key, data is written to `INIT_DB_*` unless `OTLP_REQUIRE_API_KEY=true`.++## Schema++On first ingest, and again if an insert fails because `otel_*` tables were dropped, the receiver runs `CREATE TABLE IF NOT EXISTS` for the OTLP schema and retries the write once. Compose and Kubernetes ClickHouse init scripts still create the same tables on an empty volume.++The ClickHouse user needs `CREATE TABLE` (or the tables must already exist). If create is denied and the tables are missing, ingest fails with an explicit error.++## Optional sidecar Collector++Keep using your own OpenTelemetry Collector for fan-out, sampling, or processors. Forward OTLP to OpenLIT instead of writing ClickHouse from `otelcol`. A starter config is in [`assets/otel-collector-config.yaml`](https://github.com/openlit/openlit/blob/main/assets/otel-collector-config.yaml).++To send SDK data to a Collector you operate (not into OpenLIT), see [OpenTelemetry Collector destination](/latest/sdk/destinations/otelcol).++## Related++<CardGroup cols={2}>+ <Card title="Self-host OpenLIT" href="/latest/openlit/installation" icon="circle-down">+ Docker Compose and Helm deploy OpenLIT plus ClickHouse. OTLP ingest is built in.+ </Card>+ <Card title="Telemetry" href="/latest/openlit/observability/telemetry/overview" icon="wave-pulse">+ Explore traces, metrics, and logs after they land in ClickHouse.+ </Card>+</CardGroup>diff --git a/docs/latest/sdk/integrations/amd-gpu.mdx b/docs/latest/sdk/integrations/amd-gpu.mdxindex 02f273bb..7182a871 100644--- a/docs/latest/sdk/integrations/amd-gpu.mdx+++ b/docs/latest/sdk/integrations/amd-gpu.mdx@@ -73,18 +73,18 @@ OpenLIT uses OpenTelemetry to help you monitor AMD GPUs. This includes trackingenvironment:OTEL_SERVICE_NAME: 'chatbot'OTEL_RESOURCE_ATTRIBUTES: 'deployment.environment=staging'- OTEL_EXPORTER_OTLP_ENDPOINT: "http://otel-collector:4318"+ OTEL_EXPORTER_OTLP_ENDPOINT: "http://openlit:4318"device_requests:- driver: nvidiacount: allcapabilities: [gpu]depends_on:- - otel-collector+ - openlitrestart: always```</Accordion>- <Accordion title="Host IP: Use the Host IP to connect to OTel Collector">+ <Accordion title="Host IP: Use the Host IP to connect to OpenLIT">```shOTEL_EXPORTER_OTLP_ENDPOINT="http://192.168.10.15:4318"diff --git a/docs/latest/sdk/integrations/intel-gpu.mdx b/docs/latest/sdk/integrations/intel-gpu.mdxindex 10d0635d..66a450c0 100644--- a/docs/latest/sdk/integrations/intel-gpu.mdx+++ b/docs/latest/sdk/integrations/intel-gpu.mdx@@ -90,16 +90,16 @@ Intel GPU support requires Linux with the **i915** or **Xe** kernel driver (kernenvironment:OTEL_SERVICE_NAME: 'my-app'OTEL_RESOURCE_ATTRIBUTES: 'deployment.environment=staging'- OTEL_EXPORTER_OTLP_ENDPOINT: "http://otel-collector:4318"+ OTEL_EXPORTER_OTLP_ENDPOINT: "http://openlit:4318"devices:- /dev/dri:/dev/dridepends_on:- - otel-collector+ - openlitrestart: always```</Accordion>- <Accordion title="Host IP: Use the Host IP to connect to OTel Collector">+ <Accordion title="Host IP: Use the Host IP to connect to OpenLIT">```shOTEL_EXPORTER_OTLP_ENDPOINT="http://192.168.10.15:4318"diff --git a/docs/latest/sdk/integrations/nvidia-gpu.mdx b/docs/latest/sdk/integrations/nvidia-gpu.mdxindex a381163b..9609f609 100644--- a/docs/latest/sdk/integrations/nvidia-gpu.mdx+++ b/docs/latest/sdk/integrations/nvidia-gpu.mdx@@ -73,18 +73,18 @@ OpenLIT uses OpenTelemetry to help you monitor NVIDIA GPUs. This includes trackienvironment:OTEL_SERVICE_NAME: 'chatbot'OTEL_RESOURCE_ATTRIBUTES: 'deployment.environment=staging'- OTEL_EXPORTER_OTLP_ENDPOINT: "http://otel-collector:4318"+ OTEL_EXPORTER_OTLP_ENDPOINT: "http://openlit:4318"device_requests:- driver: nvidiacount: allcapabilities: [gpu]depends_on:- - otel-collector+ - openlitrestart: always```</Accordion>- <Accordion title="Host IP: Use the Host IP to connect to OTel Collector">+ <Accordion title="Host IP: Use the Host IP to connect to OpenLIT">```shOTEL_EXPORTER_OTLP_ENDPOINT="http://192.168.10.15:4318"diff --git a/docs/snippets/destinations/otelcol/intro.mdx b/docs/snippets/destinations/otelcol/intro.mdxindex 2ed873f3..046864de 100644--- a/docs/snippets/destinations/otelcol/intro.mdx+++ b/docs/snippets/destinations/otelcol/intro.mdx@@ -1,4 +1,8 @@-To send OpenTelemetry metrics and traces generated by OpenLIT from your AI Application to an OpenTelemetry Collector, follow the below steps.+To send OpenTelemetry metrics and traces generated by OpenLIT from your AI Application to an OpenTelemetry Collector **you operate**, follow the below steps.++<Info>+OpenLIT itself no longer bundles `otelcol`. The OpenLIT container already accepts OTLP on `:4317` and `:4318`. Use this page only when you want a separate Collector for processors, sampling, or fan-out. To send data **into** OpenLIT, see the [OTLP receiver](/latest/openlit/otlp-receiver).+</Info>The OpenTelemetry Collector is a vendor-agnostic way to receive, process, and export telemetry data. It can act as an intermediary to route your OpenLIT data to multiple backends or apply processing transformations.diff --git a/docs/snippets/quickstart-gpu.mdx b/docs/snippets/quickstart-gpu.mdxindex ed8da588..49e7297d 100644--- a/docs/snippets/quickstart-gpu.mdx+++ b/docs/snippets/quickstart-gpu.mdx@@ -173,7 +173,7 @@```Replace:- 1. `YOUR_OTEL_ENDPOINT` with the URL of your OpenTelemetry backend, such as `http://127.0.0.1:4318` if you are using OpenLIT and a local OTel Collector.+ 1. `YOUR_OTEL_ENDPOINT` with the URL of your OpenTelemetry backend, such as `http://127.0.0.1:4318` if you are using OpenLIT.**Note:** `collect_system_metrics=True` replaces the deprecated `collect_gpu_stats=True`</Tab>@@ -193,7 +193,7 @@```Replace:- 1. `YOUR_OTEL_ENDPOINT` with the URL of your OpenTelemetry backend, such as `http://127.0.0.1:4318` if you are using OpenLIT and a local OTel Collector.+ 1. `YOUR_OTEL_ENDPOINT` with the URL of your OpenTelemetry backend, such as `http://127.0.0.1:4318` if you are using OpenLIT.</Tab></Tabs></Tab>@@ -258,18 +258,18 @@environment:OTEL_SERVICE_NAME: 'chatbot'OTEL_RESOURCE_ATTRIBUTES: 'deployment.environment=staging'- OTEL_EXPORTER_OTLP_ENDPOINT: "http://otel-collector:4318"+ OTEL_EXPORTER_OTLP_ENDPOINT: "http://openlit:4318"device_requests:- driver: nvidiacount: allcapabilities: [gpu]depends_on:- - otel-collector+ - openlitrestart: always```</Accordion>- <Accordion title="Host IP: Use the Host IP to connect to OTel Collector">+ <Accordion title="Host IP: Use the Host IP to connect to OpenLIT">```shOTEL_EXPORTER_OTLP_ENDPOINT="http://192.168.10.15:4318"diff --git a/docs/snippets/quickstart-mcp.mdx b/docs/snippets/quickstart-mcp.mdxindex c6af1297..6e9cb84e 100644--- a/docs/snippets/quickstart-mcp.mdx+++ b/docs/snippets/quickstart-mcp.mdx@@ -8,8 +8,8 @@ flowchart TB;direction LR;subgraph " "direction LR;- OpenLIT_SDK[OpenLIT SDK] -->|Sends Traces & Metrics| OTC[OpenTelemetry Collector];- OTC -->|Stores Data| ClickHouseDB[ClickHouse];+ OpenLIT_SDK[OpenLIT SDK] -->|OTLP| Rec[OpenLIT OTLP receiver];+ Rec -->|Stores Data| ClickHouseDB[ClickHouse];endsubgraph " "direction RL;diff --git a/docs/snippets/quickstart-observability.mdx b/docs/snippets/quickstart-observability.mdxindex d7724938..5ce02dde 100644--- a/docs/snippets/quickstart-observability.mdx+++ b/docs/snippets/quickstart-observability.mdx@@ -10,8 +10,8 @@ flowchart TB;direction LR;subgraph " "direction LR;- OpenLIT_SDK[OpenLIT SDK] -->|Sends Traces & Metrics| OTC[OpenTelemetry Collector];- OTC -->|Stores Data| ClickHouseDB[ClickHouse];+ OpenLIT_SDK[OpenLIT SDK] -->|OTLP| Rec[OpenLIT OTLP receiver];+ Rec -->|Stores Data| ClickHouseDB[ClickHouse];endsubgraph " "direction RL;@@ -34,7 +34,7 @@ flowchart TB;docker compose up -d```<Tip>- Prefer Kubernetes, or already have ClickHouse/OpenTelemetry Collector running? See [Self-Host OpenLIT](/latest/openlit/installation) for Helm charts and other deployment options.+ Prefer Kubernetes, or already have ClickHouse running? See [Self-Host OpenLIT](/latest/openlit/installation) for Helm charts and other deployment options.</Tip></Step></Steps>diff --git a/docs/snippets/quickstart-vectordb.mdx b/docs/snippets/quickstart-vectordb.mdxindex e0a8e8fa..4476ed4a 100644--- a/docs/snippets/quickstart-vectordb.mdx+++ b/docs/snippets/quickstart-vectordb.mdx@@ -8,8 +8,8 @@ flowchart TB;direction LR;subgraph " "direction LR;- OpenLIT_SDK[OpenLIT SDK] -->|Sends Traces & Metrics| OTC[OpenTelemetry Collector];- OTC -->|Stores Data| ClickHouseDB[ClickHouse];+ OpenLIT_SDK[OpenLIT SDK] -->|OTLP| Rec[OpenLIT OTLP receiver];+ Rec -->|Stores Data| ClickHouseDB[ClickHouse];endsubgraph " "direction RL;diff --git a/env.example b/env.exampleindex 83828e86..20f6dc6a 100644--- a/env.example+++ b/env.example@@ -31,61 +31,10 @@ GITHUB_CLIENT_ID=GITHUB_CLIENT_SECRET=# =============================================================================-# OpAMP Server Configuration+# OTLP ingest (first-party receiver in the OpenLIT image)# =============================================================================--# Environment Mode-# Options: development, production, testing-# - development: Relaxed security, insecure_skip_verify=true by default-# - production: Strict security, full certificate verification, mutual TLS-# - testing: Similar to development, optimized for automated testing-# Default: production (secure by default)-OPAMP_ENVIRONMENT=production--# TLS Configuration-# Skip certificate verification (development only)-# Set to false for production with proper CA certificates-# Default: false (secure by default)-OPAMP_TLS_INSECURE_SKIP_VERIFY=false--# Require client certificates for mutual TLS-# Enabled by default for enhanced security-# Default: true (secure by default)-OPAMP_TLS_REQUIRE_CLIENT_CERT=true--# TLS Version Configuration-# Minimum and maximum TLS versions (1.0, 1.1, 1.2, 1.3)-OPAMP_TLS_MIN_VERSION=1.2-OPAMP_TLS_MAX_VERSION=1.3--# Logging Configuration-# Options: debug, info, warn, error-OPAMP_LOG_LEVEL=info--# =============================================================================-# Certificate Configuration (Advanced)-# =============================================================================-# Certificate directory path (usually managed by Docker)-# OPAMP_CERTS_DIR=/app/opamp/certs--# =============================================================================-# Production Deployment Example (Default)-# =============================================================================-# Production settings are now the default for enhanced security:-#-# OPAMP_ENVIRONMENT=production # Default-# OPAMP_TLS_INSECURE_SKIP_VERIFY=false # Default-# OPAMP_TLS_REQUIRE_CLIENT_CERT=true # Default-# OPAMP_TLS_MIN_VERSION=1.2 # Default-# OPAMP_TLS_MAX_VERSION=1.3 # Default-# OPAMP_LOG_LEVEL=info # Default--# =============================================================================-# Development Deployment Example-# =============================================================================-# For development, override these settings for relaxed security:-#-# OPAMP_ENVIRONMENT=development-# OPAMP_TLS_INSECURE_SKIP_VERIFY=true-# OPAMP_TLS_REQUIRE_CLIENT_CERT=false-# OPAMP_LOG_LEVEL=debug+# SDKs send OTLP to :4318 (HTTP) or :4317 (gRPC).+# Authorization: Bearer <openlit-api-key> or header x-openlit-api-key.+# Set OTLP_REQUIRE_API_KEY=true to reject anonymous writes to INIT_DB_*.+# OTLP_HTTP_ADDR=0.0.0.0:4318+# OTLP_GRPC_ADDR=0.0.0.0:4317diff --git a/examples/kubernetes/README.md b/examples/kubernetes/README.mdindex da71c32c..650674d0 100644--- a/examples/kubernetes/README.md+++ b/examples/kubernetes/README.md@@ -10,7 +10,7 @@ A complete local Kubernetes setup with all OpenLIT components and sample LLM app│ ││ ┌──────────────┐ ┌─────────────────────────┐ ││ │ ClickHouse │◄──│ OpenLIT Dashboard │ │-│ │ (StatefulSet) │ │ + bundled OTEL Collector│ │+│ │ (StatefulSet) │ │ + OTLP receiver │ ││ └──────────────┘ │ (port 3000, 4317, 4318) │ ││ └────────────┬─────────────┘ ││ │ │@@ -33,7 +33,7 @@ A complete local Kubernetes setup with all OpenLIT components and sample LLM app| Component | Kind | Notes ||---|---|---|| ClickHouse | StatefulSet | `clickhouse/clickhouse-server:24.4.1` |-| OpenLIT Dashboard | Deployment | Built locally as `openlit:local` — includes bundled OTEL Collector |+| OpenLIT Dashboard | Deployment | Built locally as `openlit:local` — includes the first-party OTLP receiver || OpenLIT Controller | DaemonSet | Built locally as `openlit-controller:local` — 1 per node || gemini-app | naked Pod | Gemini client (tests naked Pod discovery) || crewai-agent-app | Deployment (2 replicas) | CrewAI agent using OpenAI (spread across nodes) |@@ -60,7 +60,7 @@ This will:3. Build the OpenLIT Dashboard and Controller images locally4. Build sample app images (gemini, crewai, bedrock)5. Load all images into the k3d cluster-6. Deploy infrastructure (ClickHouse, Dashboard with bundled OTEL Collector)+6. Deploy infrastructure (ClickHouse, Dashboard with OTLP receiver)7. Deploy sample apps and wait for them to start making LLM calls8. Deploy the controller last (to test discovery of already-running apps)diff --git a/examples/kubernetes/openlit.yaml b/examples/kubernetes/openlit.yamlindex fd7bbc98..e8592729 100644--- a/examples/kubernetes/openlit.yaml+++ b/examples/kubernetes/openlit.yaml@@ -1,56 +1,3 @@-apiVersion: v1-kind: ConfigMap-metadata:- name: openlit-otel-config- namespace: openlit-data:- otel-collector-config.yaml: |- receivers:- otlp:- protocols:- grpc:- endpoint: 0.0.0.0:4317- http:- endpoint: 0.0.0.0:4318-- processors:- batch:- memory_limiter:- limit_mib: 1500- spike_limit_mib: 512- check_interval: 5s-- exporters:- clickhouse:- endpoint: tcp://clickhouse.openlit.svc.cluster.local:9000?dial_timeout=10s- database: openlit- username: default- password: OPENLIT- ttl: 730h- logs_table_name: otel_logs- traces_table_name: otel_traces- timeout: 5s- retry_on_failure:- enabled: true- initial_interval: 5s- max_interval: 30s- max_elapsed_time: 300s-- service:- pipelines:- logs:- receivers: [otlp]- processors: [batch]- exporters: [clickhouse]- traces:- receivers: [otlp]- processors: [memory_limiter, batch]- exporters: [clickhouse]- metrics:- receivers: [otlp]- processors: [memory_limiter, batch]- exporters: [clickhouse]----apiVersion: apps/v1kind: Deploymentmetadata:@@ -101,8 +48,6 @@ spec:volumeMounts:- name: datamountPath: /app/client/data- - name: otel-config- mountPath: /etc/otelreadinessProbe:httpGet:path: /@@ -119,9 +64,6 @@ spec:volumes:- name: dataemptyDir: {}- - name: otel-config- configMap:- name: openlit-otel-config---apiVersion: v1kind: Servicediff --git a/examples/kubernetes/setup.sh b/examples/kubernetes/setup.shindex f6e28369..d04072b6 100755--- a/examples/kubernetes/setup.sh+++ b/examples/kubernetes/setup.sh@@ -54,7 +54,7 @@ k3d image import \echo "All images loaded."echo ""-# ── 4. Deploy infrastructure (namespace, clickhouse, dashboard w/ bundled OTEL collector) ──+# ── 4. Deploy infrastructure (namespace, clickhouse, dashboard with OTLP receiver) ──echo "Deploying infrastructure..."kubectl apply -f "$SCRIPT_DIR/namespace.yaml"kubectl apply -f "$SCRIPT_DIR/clickhouse.yaml"@@ -62,7 +62,7 @@ kubectl apply -f "$SCRIPT_DIR/openlit.yaml"echo "Waiting for ClickHouse..."kubectl rollout status statefulset/clickhouse -n "$K8S_NAMESPACE" --timeout=120s-echo "Waiting for OpenLIT dashboard (includes bundled OTEL collector)..."+echo "Waiting for OpenLIT dashboard (includes OTLP receiver)..."kubectl rollout status deployment/openlit -n "$K8S_NAMESPACE" --timeout=120secho ""diff --git a/examples/linux/docker-compose.yaml b/examples/linux/docker-compose.yamlindex 96ac3074..3eab4eaa 100644--- a/examples/linux/docker-compose.yaml+++ b/examples/linux/docker-compose.yaml@@ -41,7 +41,6 @@ services:condition: service_healthyvolumes:- openlit-data:/app/client/data- - ../../assets/otel-collector-config.yaml:/etc/otel/otel-collector-config.yamlhealthcheck:test: ["CMD", "node", "-e", "fetch('http://localhost:3000').then(r=>{process.exit(r.ok?0:1)}).catch(()=>process.exit(1))"]interval: 5sdiff --git a/opentelemetry-gpu-collector/README.md b/opentelemetry-gpu-collector/README.mdindex 8d6b8031..e3b5e5d3 100644--- a/opentelemetry-gpu-collector/README.md+++ b/opentelemetry-gpu-collector/README.md@@ -91,7 +91,7 @@ docker pull ghcr.io/openlit/otel-gpu-collector:latestdocker run --gpus all --pid=host \-e OTEL_SERVICE_NAME=my-app \-e OTEL_RESOURCE_ATTRIBUTES="deployment.environment=production" \- -e OTEL_EXPORTER_OTLP_ENDPOINT="http://otel-collector:4317" \+ -e OTEL_EXPORTER_OTLP_ENDPOINT="http://openlit:4318" \ghcr.io/openlit/otel-gpu-collector:latest```@@ -107,7 +107,7 @@ services:environment:OTEL_SERVICE_NAME: my-appOTEL_RESOURCE_ATTRIBUTES: "deployment.environment=production"- OTEL_EXPORTER_OTLP_ENDPOINT: "http://otel-collector:4317"+ OTEL_EXPORTER_OTLP_ENDPOINT: "http://openlit:4318"deploy:resources:reservations:@@ -116,7 +116,7 @@ services:count: allcapabilities: [gpu]depends_on:- - otel-collector+ - openlitrestart: always```diff --git a/src/Dockerfile b/src/Dockerfileindex 7666c3b2..574f8594 100644--- a/src/Dockerfile+++ b/src/Dockerfile@@ -23,85 +23,21 @@ ENV PRISMA_CLI_BINARY_TARGETS=linux-musl-openssl-3.0.x,linux-musl-arm64-openssl-RUN npm install && npx prisma generate && npm run build-# Opamp server build from source - pinned with digest+# OTLP receiver build from source - pinned with digestFROM golang:1.25-alpine@sha256:8e02eb337d9e0ea459e041f1ee5eece41cbb61f1d83e7d883a3e2fb4862063fa AS go-builder-# Set working directory for opamp-server-WORKDIR /app/opamp-server--# Copy the rest of the opamp-server source code-COPY ./opamp-server/ ./--# Build the opamp-server binary-RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o opamp-server .---# OpenTelemetry Collector Contrib download - pinned with digest-FROM alpine:3.21@sha256:c3f8e73fdb79deaebaa2037150150191b9dcbfba68b4a46d70103204c53f4709 AS otel-downloader--# Accept build arguments (TARGETARCH is automatically provided by Docker buildx)-# Default to amd64 if not specified (most common architecture)-ARG OTEL_VERSION=v0.142.0-ARG OPAMP_SUPERVISOR_VERSION=v0.142.0-ARG TARGETARCH=amd64-ARG TARGETPLATFORM--# Install curl and file utilities for downloading binaries and verification-RUN apk add --no-cache curl file--# Set working directory-WORKDIR /tmp--# Download OpenTelemetry Collector Contrib and OpAMP Supervisor.-# GitHub release-assets occasionally 504 on specific blobs; fall back to the-# matching .deb when needed (binaries at usr/bin/*).-RUN echo "Downloading binaries for architecture: ${TARGETARCH}" && \- OTEL_TAR="otelcol-contrib_${OTEL_VERSION#v}_linux_${TARGETARCH}.tar.gz" && \- OTEL_BASE="https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/${OTEL_VERSION}" && \- if curl -fsSL --retry 5 --retry-all-errors --retry-delay 5 -o "${OTEL_TAR}" "${OTEL_BASE}/${OTEL_TAR}"; then \- tar -xzf "${OTEL_TAR}" && rm "${OTEL_TAR}"; \- else \- echo "otelcol-contrib tar.gz unavailable, falling back to .deb" && \- OTEL_DEB="otelcol-contrib_${OTEL_VERSION#v}_linux_${TARGETARCH}.deb" && \- curl -fsSL --retry 5 --retry-all-errors --retry-delay 5 -o "${OTEL_DEB}" "${OTEL_BASE}/${OTEL_DEB}" && \- apk add --no-cache dpkg && \- dpkg-deb -x "${OTEL_DEB}" otel-deb && \- mv otel-deb/usr/bin/otelcol-contrib ./otelcol-contrib && \- rm -rf "${OTEL_DEB}" otel-deb; \- fi && \- chmod +x otelcol-contrib && \- echo "otelcol-contrib binary info:" && file otelcol-contrib && \- OPAMP_BASE="https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/cmd%2Fopampsupervisor%2F${OPAMP_SUPERVISOR_VERSION}" && \- OPAMP_BIN="opampsupervisor_${OPAMP_SUPERVISOR_VERSION#v}_linux_${TARGETARCH}" && \- if curl --proto '=https' --tlsv1.2 -fsSL --retry 5 --retry-all-errors --retry-delay 5 -o opampsupervisor "${OPAMP_BASE}/${OPAMP_BIN}"; then \- true; \- else \- echo "opampsupervisor binary unavailable, falling back to .deb" && \- OPAMP_DEB="opampsupervisor_${OPAMP_SUPERVISOR_VERSION#v}_linux_${TARGETARCH}.deb" && \- curl -fsSL --retry 5 --retry-all-errors --retry-delay 5 -o "${OPAMP_DEB}" "${OPAMP_BASE}/${OPAMP_DEB}" && \- (command -v dpkg-deb >/dev/null || apk add --no-cache dpkg) && \- dpkg-deb -x "${OPAMP_DEB}" opamp-deb && \- mv opamp-deb/usr/bin/opampsupervisor ./opampsupervisor && \- rm -rf "${OPAMP_DEB}" opamp-deb; \- fi && \- chmod +x opampsupervisor && \- echo "opampsupervisor binary info:" && file opampsupervisor+WORKDIR /app/otlp-receiver+COPY ./otlp-receiver/ ./+RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o otlp-receiver ./cmd/otlp-receiver# Final runtime image - Node 20 on Debian bookworm (bullseye-security# package indexes now 404 openssl 1.1.1w updates required by bullseye-slim)FROM node:20-bookworm-slim@sha256:2cf067cfed83d5ea958367df9f966191a942351a2df77d6f0193e162b5febfc0-# Accept build arguments to pass version and architecture info-# TARGETARCH is automatically provided by Docker buildx based on --platform-# Default to amd64 if not specified (must match otel-downloader stage)-ARG OPAMP_SUPERVISOR_VERSION=v0.142.0ARG TARGETARCH=amd64ARG TARGETPLATFORM--# Set as environment variables for use by scripts-ENV OPAMP_SUPERVISOR_VERSION=${OPAMP_SUPERVISOR_VERSION}ENV TARGETARCH=${TARGETARCH}# Install only the additional runtime dependencies needed.@@ -130,30 +66,14 @@ COPY --from=builder /app/client/node_modules/@prisma ./node_modules/@prismaCOPY --from=builder /app/client/node_modules/.bin ./node_modules/.binCOPY --from=builder /app/client/node_modules/prisma ./node_modules/prisma-# Copy the built opamp-server binary from go-builder stage-COPY --from=go-builder /app/opamp-server/opamp-server /app/opamp/opamp-server-COPY --from=go-builder /app/opamp-server/certs /app/opamp/certs-COPY --from=go-builder /app/opamp-server/setup-supervisor.sh /app/opamp/setup-supervisor.sh--# Copy OpenTelemetry Collector and OpAMP Supervisor binaries-# Place otelcol-contrib at the path expected by supervisor configuration file-COPY --from=otel-downloader /tmp/otelcol-contrib /app/opamp/otelcontribcol-COPY --from=otel-downloader /tmp/opampsupervisor /app/opamp/opampsupervisor--# Verify binary architectures match runtime-RUN apt-get update -o Acquire::Check-Valid-Until=false && apt-get install -y --no-install-recommends file && \- echo "Verifying binary architectures..." && \- echo "Runtime architecture: $(uname -m)" && \- echo "otelcontribcol:" && file /app/opamp/otelcontribcol && \- echo "opampsupervisor:" && file /app/opamp/opampsupervisor && \- rm -rf /var/lib/apt/lists/*--# Execute entrypoint script to generate NextAuth.js secret and run commands-RUN chmod +x ./scripts/entrypoint.sh /app/opamp/otelcontribcol /app/opamp/opampsupervisor /app/opamp/opamp-server /app/opamp/setup-supervisor.sh-RUN chmod -R u=rwX,go= /app/opamp/certs--# Expose the port that Next.js will run on+COPY --from=go-builder /app/otlp-receiver/otlp-receiver /app/otlp-receiver/otlp-receiver++RUN chmod +x ./scripts/entrypoint.sh /app/otlp-receiver/otlp-receiver++# Next.js UI + OTLP HTTP/gRPCEXPOSE ${DOCKER_PORT:-3000}+EXPOSE 4317+EXPOSE 4318# Run the entrypoint scriptCMD ["/app/client/scripts/entrypoint.sh"]diff --git a/src/README.md b/src/README.mdindex 83b511f2..a1009006 100644--- a/src/README.md+++ b/src/README.md@@ -30,7 +30,7 @@ This guide covers the steps needed to set up the development environment for Ope| Service | Description ||---------|-------------|| **ClickHouse** | Time-series database for telemetry |-| **OpenLIT** | Dashboard + OTLP collector (port 3000) |+| **OpenLIT** | Dashboard + first-party OTLP receiver (ports 3000, 4317, 4318) || **Controller** | eBPF-based LLM service discovery (built from `openlit-controller/`) || **openai-app** | Sample OpenAI chat app (from `examples/`) || **anthropic-app** | Sample Anthropic chat app (from `examples/`) |diff --git a/src/client/next.config.js b/src/client/next.config.jsindex 5e523a41..c3fca852 100644--- a/src/client/next.config.js+++ b/src/client/next.config.js@@ -38,6 +38,14 @@ const nextConfig = {experimental: {instrumentationHook: true,},+ async rewrites() {+ const otlp = process.env.OTLP_PROXY_ORIGIN || "http://127.0.0.1:4318";+ return [+ { source: "/v1/traces", destination: `${otlp}/v1/traces` },+ { source: "/v1/metrics", destination: `${otlp}/v1/metrics` },+ { source: "/v1/logs", destination: `${otlp}/v1/logs` },+ ];+ },async headers() {return [{diff --git a/src/client/prisma/migrations/20260915120000_api_keys_org_project_environment/migration.sql b/src/client/prisma/migrations/20260915120000_api_keys_org_project_environment/migration.sqlnew file mode 100644index 00000000..81681404--- /dev/null+++ b/src/client/prisma/migrations/20260915120000_api_keys_org_project_environment/migration.sql@@ -0,0 +1,40 @@+-- AlterTable+ALTER TABLE "APIKeys" ADD COLUMN "organisation_id" TEXT;+ALTER TABLE "APIKeys" ADD COLUMN "project_id" TEXT;+ALTER TABLE "APIKeys" ADD COLUMN "environment" TEXT NOT NULL DEFAULT 'production';++-- Backfill project + environment from the bound database config.+UPDATE "APIKeys"+SET "project_id" = (+ SELECT "project_id" FROM "databaseconfig" WHERE "databaseconfig"."id" = "APIKeys"."database_config_id"+)+WHERE "project_id" IS NULL;++UPDATE "APIKeys"+SET "environment" = COALESCE(+ NULLIF(trim((SELECT "environment" FROM "databaseconfig" WHERE "databaseconfig"."id" = "APIKeys"."database_config_id")), ''),+ 'production'+);++-- Attach orphaned keys to the oldest default project when one exists.+UPDATE "APIKeys"+SET "project_id" = (+ SELECT "id" FROM "projects" WHERE "is_default" = 1 ORDER BY "createdAt" ASC, "id" ASC LIMIT 1+)+WHERE "project_id" IS NULL+ AND EXISTS (SELECT 1 FROM "projects" WHERE "is_default" = 1);++UPDATE "APIKeys"+SET "organisation_id" = (+ SELECT "organisation_id" FROM "projects" WHERE "projects"."id" = "APIKeys"."project_id"+)+WHERE "organisation_id" IS NULL AND "project_id" IS NOT NULL;++-- Ensure the bound project has an environment row for the key.+INSERT OR IGNORE INTO "project_environments" ("id", "project_id", "name", "createdAt", "updatedAt")+SELECT 'env_' || lower(hex(randomblob(16))), "project_id", "environment", CURRENT_TIMESTAMP, CURRENT_TIMESTAMP+FROM "APIKeys"+WHERE "project_id" IS NOT NULL AND trim("environment") <> '';++CREATE INDEX IF NOT EXISTS "APIKeys_organisation_id_idx" ON "APIKeys"("organisation_id");+CREATE INDEX IF NOT EXISTS "APIKeys_project_id_environment_idx" ON "APIKeys"("project_id", "environment");diff --git a/src/client/prisma/schema.prisma b/src/client/prisma/schema.prismaindex 47031d49..0fe9d387 100644--- a/src/client/prisma/schema.prisma+++ b/src/client/prisma/schema.prisma@@ -88,6 +88,7 @@ model Organisation {invitedUsers OrganisationInvitedUser[]projects Project[]connectorInstances ConnectorInstance[]+ apiKeys APIKeys[]@@map("organisations")}@@ -108,6 +109,7 @@ model Project {environments ProjectEnvironment[]currentForUsers OrganisationUser[] @relation("CurrentOrganisationProject")members ProjectUser[]+ apiKeys APIKeys[]@@unique([organisationId, slug])@@map("projects")@@ -347,6 +349,11 @@ model APIKeys {apiKey String @uniquedatabaseConfigId String @map("database_config_id")databaseConfig DatabaseConfig? @relation(fields: [databaseConfigId], references: [id])+ organisationId String? @map("organisation_id")+ organisation Organisation? @relation(fields: [organisationId], references: [id], onDelete: SetNull)+ projectId String? @map("project_id")+ project Project? @relation(fields: [projectId], references: [id], onDelete: SetNull)+ environment String @default("production")isDeleted Boolean @default(false)createdAt DateTime @default(now())createdByUserId String@@ -354,6 +361,9 @@ model APIKeys {deletedAt DateTime?deletedByUserId String?deletedByUser User? @relation("deletedByUser", fields: [deletedByUserId], references: [id])++ @@index([organisationId])+ @@index([projectId, environment])}model ClickhouseMigrations {diff --git a/src/client/scripts/entrypoint.sh b/src/client/scripts/entrypoint.shindex 9dd13417..45b54be5 100644--- a/src/client/scripts/entrypoint.sh+++ b/src/client/scripts/entrypoint.sh@@ -66,6 +66,7 @@ echo "INIT_DB_DATABASE=${INIT_DB_DATABASE}" >> /etc/environment# Load the environment variables. /etc/environment+# Receiver reads SQLITE_DATABASE_URL / INIT_DB_* from the process environment.# Run Prisma migrations and generate prisma clientprisma migrate deploy@@ -77,94 +78,8 @@ prisma db seed# Run crond in the backgroundservice cron start-# Set OpAMP environment variables for entrypoint-export OPAMP_ENVIRONMENT=${OPAMP_ENVIRONMENT:-production}-export OPAMP_CERTS_DIR=${OPAMP_CERTS_DIR:-/app/opamp/certs}-export OPAMP_TLS_INSECURE_SKIP_VERIFY=${OPAMP_TLS_INSECURE_SKIP_VERIFY:-false}--echo "OpAMP Configuration:"-echo " Environment: $OPAMP_ENVIRONMENT"-echo " Certificates Directory: $OPAMP_CERTS_DIR"-echo " TLS Insecure Skip Verify: $OPAMP_TLS_INSECURE_SKIP_VERIFY"--# Generate certificates using enhanced script-echo "Generating OpAMP certificates..."-chmod +x /app/opamp/certs/clear.sh-chmod +x /app/opamp/certs/generate.sh-cd /app/opamp/certs--# Generate certificates with proper logging-if ./generate.sh > /var/log/certs.log 2>&1; then- echo "✅ Certificates generated successfully"-else- echo "❌ Certificate generation failed, check /var/log/certs.log"- cat /var/log/certs.log-fi--# Generate supervisor configuration based on environment-echo "Configuring OpAMP supervisor for $OPAMP_ENVIRONMENT environment..."-cd /app/opamp--# Use a runtime config location to avoid overwriting mounted files-SUPERVISOR_CONFIG_PATH="/app/opamp/supervisor-runtime.yaml"--# Create supervisor configuration dynamically-cat > "$SUPERVISOR_CONFIG_PATH" << EOF-server:- endpoint: wss://localhost:4320/v1/opamp- tls:-EOF--if [[ "$OPAMP_ENVIRONMENT" == "production" ]]; then- cat >> "$SUPERVISOR_CONFIG_PATH" << EOF- insecure_skip_verify: false- ca_file: /app/opamp/certs/cert/ca.cert.pem- cert_file: /app/opamp/certs/client/client.cert.pem- key_file: /app/opamp/certs/client/client.key.pem-EOF- echo " Production mode: Using CA certificate verification with client certificates"-else- cat >> "$SUPERVISOR_CONFIG_PATH" << EOF- insecure_skip_verify: true-EOF- echo " Development mode: Skipping certificate verification"-fi--cat >> "$SUPERVISOR_CONFIG_PATH" << EOF--agent:- executable: /app/opamp/otelcontribcol- config_files:- - /etc/otel/otel-collector-config.yaml--capabilities:- accepts_remote_config: true- reports_effective_config: true- reports_own_metrics: false- reports_own_logs: true- reports_own_traces: false- reports_health: true- reports_remote_config: true--storage:- directory: /app/client/data/supervisor-storage-EOF--echo "✅ Supervisor configuration generated at $SUPERVISOR_CONFIG_PATH"--# Starting OpAMP Server-echo "Starting OpAMP Server..."-/app/opamp/opamp-server &-OPAMP_SERVER_PID=$!--# Wait a moment for the server to start-sleep 2--# Starting Supervisor for the OTEL Collector-echo "Starting OpAMP Supervisor..."-/app/opamp/opampsupervisor --config="$SUPERVISOR_CONFIG_PATH" &-SUPERVISOR_PID=$!-+echo "Starting OTLP receiver..."+/app/otlp-receiver/otlp-receiver &# Starting the OpenLIT UI Serverexport PORT=${DOCKER_PORT:-3000}diff --git a/src/client/src/__tests__/app/api/auth/verify-key/route.test.ts b/src/client/src/__tests__/app/api/auth/verify-key/route.test.tsnew file mode 100644index 00000000..9b7d8038--- /dev/null+++ b/src/client/src/__tests__/app/api/auth/verify-key/route.test.ts@@ -0,0 +1,67 @@+jest.mock("next/server", () => ({+ NextResponse: {+ json: (body: unknown, init?: { status?: number }) => ({+ status: init?.status ?? 200,+ json: async () => body,+ }),+ },+}));++jest.mock("@/lib/platform/api-keys", () => ({+ getAPIKeyInfo: jest.fn(),+}));++import { GET } from "@/app/api/auth/verify-key/route";+import { getAPIKeyInfo } from "@/lib/platform/api-keys";++function makeRequest(headers: Record<string, string> = {}) {+ return {+ headers: {+ get: (name: string) => headers[name] || null,+ },+ } as any;+}++describe("GET /api/auth/verify-key", () => {+ beforeEach(() => {+ jest.clearAllMocks();+ });++ it("returns 401 without a Bearer token", async () => {+ const res = await GET(makeRequest());+ expect(res.status).toBe(401);+ await expect(res.json()).resolves.toEqual({ valid: false });+ });++ it("returns the key's organisation, project, and environment", async () => {+ (getAPIKeyInfo as jest.Mock).mockResolvedValue([+ null,+ {+ databaseConfigId: "db-1",+ organisationId: "org-1",+ projectId: "proj-1",+ environment: "staging",+ },+ ]);++ const res = await GET(+ makeRequest({ Authorization: "Bearer openlit-test" })+ );+ expect(res.status).toBe(200);+ await expect(res.json()).resolves.toEqual({+ valid: true,+ databaseConfigId: "db-1",+ organisationId: "org-1",+ projectId: "proj-1",+ environment: "staging",+ });+ });++ it("returns 401 for an unknown key", async () => {+ (getAPIKeyInfo as jest.Mock).mockResolvedValue([null, null]);+ const res = await GET(+ makeRequest({ Authorization: "Bearer missing" })+ );+ expect(res.status).toBe(401);+ });+});diff --git a/src/client/src/__tests__/app/onboarding-page.test.tsx b/src/client/src/__tests__/app/onboarding-page.test.tsxindex 1be13e9e..fdf78026 100644--- a/src/client/src/__tests__/app/onboarding-page.test.tsx+++ b/src/client/src/__tests__/app/onboarding-page.test.tsx@@ -118,7 +118,7 @@ describe("OnboardingPage database config step", () => {fireEvent.click(screen.getByRole("button", { name: "Add database config" }));- expect(screen.getByText("Add ClickHouse connector")).toBeInTheDocument();+ expect(screen.getByText("Add source")).toBeInTheDocument();expect(screen.getByPlaceholderText("127.0.0.1")).toBeInTheDocument();expect(screen.getByPlaceholderText("8123")).toBeInTheDocument();});diff --git a/src/client/src/__tests__/clickhouse/migrations/backfill-otel-tenant-environment-migration.test.ts b/src/client/src/__tests__/clickhouse/migrations/backfill-otel-tenant-environment-migration.test.tsnew file mode 100644index 00000000..6b8fef4d--- /dev/null+++ b/src/client/src/__tests__/clickhouse/migrations/backfill-otel-tenant-environment-migration.test.ts@@ -0,0 +1,86 @@+jest.mock("@/lib/platform/common", () => ({+ dataCollector: jest.fn(),+}));+jest.mock("@/lib/db-config", () => ({+ getDBConfigByIdInternal: jest.fn(),+ getDBConfigByUser: jest.fn(),+}));+jest.mock("@/lib/prisma", () => ({+ __esModule: true,+ default: {+ clickhouseMigrations: {+ findFirst: jest.fn(),+ create: jest.fn(),+ },+ project: {+ findUnique: jest.fn(),+ },+ },+}));+jest.mock("@/utils/asaw", () =>+ jest.fn(async (promise: Promise<unknown>) => {+ try {+ return [null, await promise];+ } catch (err) {+ return [err];+ }+ })+);+jest.mock("@/utils/log", () => ({+ consoleLog: jest.fn(),+}));++import { dataCollector } from "@/lib/platform/common";+import { getDBConfigByIdInternal } from "@/lib/db-config";+import prisma from "@/lib/prisma";+import BackfillOtelTenantEnvironmentMigration from "@/clickhouse/migrations/backfill-otel-tenant-environment-migration";++describe("BackfillOtelTenantEnvironmentMigration", () => {+ beforeEach(() => {+ jest.clearAllMocks();+ (getDBConfigByIdInternal as jest.Mock).mockResolvedValue({+ id: "db-1",+ projectId: "proj-1",+ environment: "staging",+ });+ (prisma.clickhouseMigrations.findFirst as jest.Mock).mockResolvedValue(null);+ (prisma.clickhouseMigrations.create as jest.Mock).mockResolvedValue({});+ (prisma.project.findUnique as jest.Mock).mockResolvedValue({+ organisationId: "org-1",+ });+ (dataCollector as jest.Mock).mockResolvedValue({ err: null });+ });++ it("stamps organisation.environment.name and tenant ids without rewriting deployment.environment", async () => {+ const result = await BackfillOtelTenantEnvironmentMigration("db-1");+ expect(result).toEqual({ migrationExist: false, queriesRun: true });+ expect(dataCollector).toHaveBeenCalled();+ const query = String((dataCollector as jest.Mock).mock.calls[0][0].query);+ expect(query).toContain("organisation.environment.name");+ expect(query).toContain("staging");+ expect(query).toContain("openlit.organisation.id");+ expect(query).toContain("org-1");+ expect(query).toContain("openlit.project.id");+ expect(query).toContain("proj-1");+ expect(query).not.toContain("deployment.environment");+ expect(query).toContain(+ "empty(ifNull(ResourceAttributes['organisation.environment.name'], ''))"+ );+ expect(prisma.clickhouseMigrations.create).toHaveBeenCalledWith({+ data: {+ databaseConfigId: "db-1",+ clickhouseMigrationId: "backfill-otel-tenant-environment",+ },+ });+ });++ it("skips when the migration already ran", async () => {+ (prisma.clickhouseMigrations.findFirst as jest.Mock).mockResolvedValue({+ id: "mig-1",+ });+ await expect(+ BackfillOtelTenantEnvironmentMigration("db-1")+ ).resolves.toEqual({ migrationExist: true, queriesRun: false });+ expect(dataCollector).not.toHaveBeenCalled();+ });+});diff --git a/src/client/src/__tests__/clickhouse/migrations/migrations-index.test.ts b/src/client/src/__tests__/clickhouse/migrations/migrations-index.test.tsindex 82c216d8..1ec2de6c 100644--- a/src/client/src/__tests__/clickhouse/migrations/migrations-index.test.ts+++ b/src/client/src/__tests__/clickhouse/migrations/migrations-index.test.ts@@ -40,6 +40,7 @@ const MIGRATION_MODULES = ["@/clickhouse/migrations/create-telemetry-rollups-migration","@/clickhouse/migrations/alter-telemetry-rollups-dimensions-migration","@/clickhouse/migrations/drop-vcs-migration",+ "@/clickhouse/migrations/backfill-otel-tenant-environment-migration",];describe("ClickHouse migration orchestration", () => {diff --git a/src/client/src/__tests__/constants/api-reference.test.ts b/src/client/src/__tests__/constants/api-reference.test.tsindex 04f49ac1..b8310dfc 100644--- a/src/client/src/__tests__/constants/api-reference.test.ts+++ b/src/client/src/__tests__/constants/api-reference.test.ts@@ -35,9 +35,9 @@ describe("API_REFERENCE_ENDPOINTS", () => {const curl = endpoint.curlExample("test-key");expect(curl).toContain("test-key");expect(curl).toContain("Authorization: Bearer test-key");- expect(curl).toContain("x-openlit-organisation-id");- expect(curl).toContain("x-openlit-project-id");- expect(curl).toContain("x-openlit-environment");+ expect(curl).not.toContain("x-openlit-organisation-id");+ expect(curl).not.toContain("x-openlit-project-id");+ expect(curl).not.toContain("x-openlit-environment");expect(curl).not.toContain("x-openlit-database-config-id");expect(curl).not.toContain("Cookie: next-auth.session-token");}diff --git a/src/client/src/__tests__/helpers/client/observability.test.ts b/src/client/src/__tests__/helpers/client/observability.test.tsindex e1fd85de..e2ca9314 100644--- a/src/client/src/__tests__/helpers/client/observability.test.ts+++ b/src/client/src/__tests__/helpers/client/observability.test.ts@@ -79,4 +79,35 @@ describe("prepareObservabilitySignalChange", () => {environments: ["production"],});});++ it("preserves customFilters with the agent scope when clearing signal filters", () => {+ const store = createStore();+ const customFilters = [+ { attributeType: "SpanAttributes" as const, key: "gen_ai.system", value: "openai" },+ ];++ store.getState().filter.updateFilter("selectedConfig", {+ serviceNames: ["demo-openai-app"],+ environments: ["production"],+ metricNames: ["up"],+ customFilters,+ });++ prepareObservabilitySignalChange(+ store.getState().filter.updateConfig,+ store.getState().filter.updateFilter,+ undefined,+ {+ serviceNames: ["demo-openai-app"],+ environments: ["production"],+ customFilters,+ }+ );++ expect(store.getState().filter.details.selectedConfig).toEqual({+ serviceNames: ["demo-openai-app"],+ environments: ["production"],+ customFilters,+ });+ });});diff --git a/src/client/src/__tests__/helpers/server/platform.test.ts b/src/client/src/__tests__/helpers/server/platform.test.tsindex 08383829..622b7d91 100644--- a/src/client/src/__tests__/helpers/server/platform.test.ts+++ b/src/client/src/__tests__/helpers/server/platform.test.ts@@ -265,6 +265,8 @@ describe('getFilterWhereCondition', () => {true);expect(result).toContain("'production'");+ expect(result).toContain("ResourceAttributes['deployment.environment']");+ expect(result).toContain("SpanAttributes['gen_ai.environment']");});it('adds SpanAttributes custom filter (covers lines 250-252)', () => {diff --git a/src/client/src/__tests__/lib/platform/api-keys/api-keys.test.ts b/src/client/src/__tests__/lib/platform/api-keys/api-keys.test.tsindex bdbb0e52..0fd22fbd 100644--- a/src/client/src/__tests__/lib/platform/api-keys/api-keys.test.ts+++ b/src/client/src/__tests__/lib/platform/api-keys/api-keys.test.ts@@ -8,6 +8,9 @@ jest.mock('@/lib/prisma', () => ({update: jest.fn(),count: jest.fn(),},+ project: {+ findUnique: jest.fn(),+ },},}));jest.mock('@/lib/session', () => ({@@ -52,13 +55,27 @@ beforeEach(() => {describe('generateAPIKey', () => {it('generates a key and saves it to the db', async () => {(getCurrentUser as jest.Mock).mockResolvedValue({ id: 'user-1', email: '[email protected]' });- (asaw as jest.Mock).mockResolvedValue([null, { id: 'db-1' }]);+ (asaw as jest.Mock).mockResolvedValue([null, { id: 'db-1', projectId: 'proj-1', environment: 'staging' }]);(prisma.aPIKeys.create as jest.Mock).mockResolvedValue({});+ (prisma.project.findUnique as jest.Mock).mockResolvedValue({ organisationId: 'org-1' });const result = await generateAPIKey('my-key');expect(prisma.aPIKeys.create).toHaveBeenCalledTimes(1);+ expect(prisma.aPIKeys.create).toHaveBeenCalledWith(+ expect.objectContaining({+ data: expect.objectContaining({+ databaseConfigId: 'db-1',+ organisationId: 'org-1',+ projectId: 'proj-1',+ environment: 'staging',+ }),+ })+ );expect(result.apiKey).toMatch(/^openlit-/);expect(result.databaseConfigId).toBe('db-1');+ expect(result.organisationId).toBe('org-1');+ expect(result.projectId).toBe('proj-1');+ expect(result.environment).toBe('staging');});it('calls the usage recorder when a current organisation exists', async () => {diff --git a/src/client/src/__tests__/lib/platform/datasource/grafana-coverage.test.ts b/src/client/src/__tests__/lib/platform/datasource/grafana-coverage.test.tsindex d9ad3ca8..14d5c207 100644--- a/src/client/src/__tests__/lib/platform/datasource/grafana-coverage.test.ts+++ b/src/client/src/__tests__/lib/platform/datasource/grafana-coverage.test.ts@@ -1183,6 +1183,20 @@ describe("TempoAdapter: session-2 branch coverage", () => {expect(q).toContain('name = ""');});+ it("filterToTraceQL maps Field SpanId/TraceId onto TraceQL intrinsics", () => {+ const q = buildTempoSearchQuery({+ signal: "traces",+ timeRange: window,+ aiSelector: false,+ filters: [+ { target: "field", key: "SpanId", op: "eq", value: "222942bfa699b630" },+ { target: "field", key: "TraceId", op: "eq", value: "abc" },+ ],+ });+ expect(q).toContain('span:id = "222942bfa699b630"');+ expect(q).toContain('trace:id = "abc"');+ });+it("filterToTraceQL treats a missing status filter value as a status != error clause", () => {const q = buildTempoSearchQuery({signal: "traces",diff --git a/src/client/src/__tests__/lib/platform/datasource/jaeger-victoria.test.ts b/src/client/src/__tests__/lib/platform/datasource/jaeger-victoria.test.tsindex dd0791cb..24b022d5 100644--- a/src/client/src/__tests__/lib/platform/datasource/jaeger-victoria.test.ts+++ b/src/client/src/__tests__/lib/platform/datasource/jaeger-victoria.test.ts@@ -703,6 +703,29 @@ describe("JaegerAdapter extended coverage", () => {expect(array.total).toBe(1);});+ it("filters by Field SpanId", async () => {+ mockSafeFetch.mockResolvedValue({+ data: [+ traceFixture({ traceId: "keep", spanId: "222942bfa699b630" }),+ traceFixture({ traceId: "drop", spanId: "aaaaaaaaaaaaaaaa" }),+ ],+ });+ const matched = await adapter.countTraces({+ signal: "traces",+ timeRange: window,+ aiSelector: false,+ filters: [+ {+ target: "field",+ key: "SpanId",+ op: "eq",+ value: "222942bfa699b630",+ },+ ],+ });+ expect(matched.total).toBe(1);+ });+it("filters by span status (error vs ok, scalar and array values)", async () => {mockSafeFetch.mockResolvedValue({data: [diff --git a/src/client/src/__tests__/lib/platform/datasource/query-map.test.ts b/src/client/src/__tests__/lib/platform/datasource/query-map.test.tsindex 9982871f..863266f5 100644--- a/src/client/src/__tests__/lib/platform/datasource/query-map.test.ts+++ b/src/client/src/__tests__/lib/platform/datasource/query-map.test.ts@@ -111,7 +111,7 @@ describe("metricParamsToOpenLITQuery", () => {);});- it("maps a trace customFilter with attributeType Field/key SpanName to a spanName filter, and ignores other Field keys", () => {+ it("maps a trace customFilter with attributeType Field/key SpanName to a spanName filter, and ignores unknown Field keys", () => {const query = metricParamsToOpenLITQuery({timeLimit: { start: new Date("2026-07-01"), end: new Date("2026-07-02"), type: "CUSTOM" },selectedConfig: {@@ -127,6 +127,35 @@ describe("metricParamsToOpenLITQuery", () => {]);});+ it("round-trips Field SpanId custom filters so ClickHouse identifier columns actually filter", () => {+ const query = metricParamsToOpenLITQuery({+ timeLimit: { start: new Date("2026-07-01"), end: new Date("2026-07-02"), type: "CUSTOM" },+ selectedConfig: {+ customFilters: [+ { attributeType: "Field", key: "SpanId", value: "222942bfa699b630" },+ { attributeType: "Field", key: "TraceId", value: "abc" },+ ],+ },+ } as any);+ expect(query.filters).toEqual(+ expect.arrayContaining([+ { target: "field", key: "SpanId", op: "eq", value: "222942bfa699b630" },+ { target: "field", key: "TraceId", op: "eq", value: "abc" },+ ])+ );+ const back = toMetricParams(query);+ expect(back.selectedConfig.customFilters).toEqual(+ expect.arrayContaining([+ {+ attributeType: "Field",+ key: "SpanId",+ value: "222942bfa699b630",+ },+ { attributeType: "Field", key: "TraceId", value: "abc" },+ ])+ );+ });+it("normalizes every custom filter operator alias (eq/neq/!=/contains/in) for traces", () => {const query = metricParamsToOpenLITQuery({timeLimit: { start: new Date("2026-07-01"), end: new Date("2026-07-02"), type: "CUSTOM" },@@ -549,7 +578,7 @@ describe("metricParamsToOpenLITQuery", () => {expect(traces.selectedConfig.spanNames).toEqual(["chat"]);});- it("maps service.name to applicationNames for traces and services otherwise", () => {+ it("maps service.name to serviceNames for traces and services otherwise", () => {const start = new Date("2026-07-01T00:00:00.000Z");const end = new Date("2026-07-01T01:00:00.000Z");const traces = toMetricParams({@@ -559,7 +588,7 @@ describe("metricParamsToOpenLITQuery", () => {{ target: "attribute", key: "service.name", op: "in", value: ["api"] },],});- expect(traces.selectedConfig.applicationNames).toEqual(["api"]);+ expect(traces.selectedConfig.serviceNames).toEqual(["api"]);const logs = toMetricParams({signal: "logs",@@ -571,6 +600,21 @@ describe("metricParamsToOpenLITQuery", () => {expect(logs.selectedConfig.services).toEqual(["api"]);});+ it("round-trips operationType llm as != vectordb, not as traceTypes IN vectordb", () => {+ const start = new Date("2026-07-01T00:00:00.000Z");+ const end = new Date("2026-07-01T01:00:00.000Z");+ const query = metricParamsToOpenLITQuery({+ timeLimit: { start, end, type: "CUSTOM" },+ operationType: "llm",+ selectedConfig: { serviceNames: ["support-agent-demo"] },+ } as any);+ const back = toMetricParams(query);+ expect(back.operationType).toBe("llm");+ expect(back.selectedConfig.traceTypes).toBeUndefined();+ expect(back.selectedConfig.serviceNames).toEqual(["support-agent-demo"]);+ expect(back.selectedConfig.applicationNames).toBeUndefined();+ });+it("maps trace-only known attribute keys to their legacy selectedConfig keys", () => {const start = new Date("2026-07-01T00:00:00.000Z");const end = new Date("2026-07-01T01:00:00.000Z");@@ -592,6 +636,26 @@ describe("metricParamsToOpenLITQuery", () => {});});+ it("maps deployment.environment into selectedConfig.environments", () => {+ const start = new Date("2026-07-01T00:00:00.000Z");+ const end = new Date("2026-07-01T01:00:00.000Z");+ const back = toMetricParams({+ signal: "traces",+ timeRange: { start, end },+ filters: [+ {+ target: "attribute",+ scope: "resource",+ key: "deployment.environment",+ op: "in",+ value: ["local"],+ },+ ],+ });+ expect(back.selectedConfig.environments).toEqual(["local"]);+ expect(back.selectedConfig.customFilters).toBeUndefined();+ });+it("does not special-case the trace-only keys for logs/metrics signals (falls through to customFilters)", () => {const start = new Date("2026-07-01T00:00:00.000Z");const end = new Date("2026-07-01T01:00:00.000Z");diff --git a/src/client/src/__tests__/lib/platform/datasource/rollup-policy.test.ts b/src/client/src/__tests__/lib/platform/datasource/rollup-policy.test.tsindex 28853bc2..8c566b8e 100644--- a/src/client/src/__tests__/lib/platform/datasource/rollup-policy.test.ts+++ b/src/client/src/__tests__/lib/platform/datasource/rollup-policy.test.ts@@ -50,6 +50,24 @@ describe("rollup-policy", () => {).toBe(false);});+ it("detects organisation.environment.name as a scope filter", () => {+ expect(+ queryHasScopeFilters({+ signal: "traces",+ timeRange: { start: new Date(), end: new Date() },+ filters: [+ {+ target: "attribute",+ scope: "resource",+ key: "organisation.environment.name",+ op: "eq",+ value: "staging",+ },+ ],+ })+ ).toBe(true);+ });+it("detects deployment.environment as a scope filter", () => {expect(queryHasScopeFilters({diff --git a/src/client/src/__tests__/lib/platform/kubernetes.test.ts b/src/client/src/__tests__/lib/platform/kubernetes.test.tsindex 5c24a0b4..7a12d707 100644--- a/src/client/src/__tests__/lib/platform/kubernetes.test.ts+++ b/src/client/src/__tests__/lib/platform/kubernetes.test.ts@@ -56,7 +56,7 @@ describe("platform kubernetes helpers", () => {selector: {matchLabels: { "app.kubernetes.io/name": "api" },},- otlp: { endpoint: "http://otel-collector:4318" },+ otlp: { endpoint: "http://openlit:4318" },},});diff --git a/src/client/src/__tests__/lib/platform/request/request.test.ts b/src/client/src/__tests__/lib/platform/request/request.test.tsindex c9001dd4..8b6d2399 100644--- a/src/client/src/__tests__/lib/platform/request/request.test.ts+++ b/src/client/src/__tests__/lib/platform/request/request.test.ts@@ -106,6 +106,7 @@ describe('getRequestsConfig', () => {// Provider dropdown folds the current OTel key and the legacy fallback.expect(query).toContain("gen_ai.provider.name");expect(query).toContain("gen_ai.system");+ expect(query).toContain("deployment.environment");});});diff --git a/src/client/src/__tests__/lib/platform/traces/read.test.ts b/src/client/src/__tests__/lib/platform/traces/read.test.tsindex 25c39ef8..ec07c66d 100644--- a/src/client/src/__tests__/lib/platform/traces/read.test.ts+++ b/src/client/src/__tests__/lib/platform/traces/read.test.ts@@ -1277,7 +1277,7 @@ describe("getTraceFilterConfig span-name fallback", () => {durationNs: 1,statusCode: "OK",spanAttributes: {},- resourceAttributes: {},+ resourceAttributes: { "deployment.environment": "staging" },};mockGetAdapter.mockResolvedValue({capabilities: () => ({}),@@ -1287,7 +1287,10 @@ describe("getTraceFilterConfig span-name fallback", () => {});const res = await getTraceFilterConfig(params as never);- expect(res.data?.[0]).toMatchObject({ spanNames: ["chat-completion"] });+ expect(res.data?.[0]).toMatchObject({+ spanNames: ["chat-completion"],+ environments: ["staging"],+ });});});@@ -1506,7 +1509,11 @@ describe("getTraceFilterConfig", () => {},}),]),- distinctValues: jest.fn().mockResolvedValue(["chat", "embeddings"]),+ distinctValues: jest.fn(async (key: string) => {+ if (key === "SpanName") return ["chat", "embeddings"];+ if (key === "deployment.environment") return ["staging"];+ return [];+ }),sampleCacheKey: "tempo-filter-config",});@@ -1519,6 +1526,7 @@ describe("getTraceFilterConfig", () => {spanNames: ["chat", "embeddings"],applicationNames: ["api"],traceTypes: ["chat"],+ environments: ["staging"],});});diff --git a/src/client/src/__tests__/lib/rbac/routes/api/unavailable.test.ts b/src/client/src/__tests__/lib/rbac/routes/api/unavailable.test.tsindex fbcee9e9..272dcc5f 100644--- a/src/client/src/__tests__/lib/rbac/routes/api/unavailable.test.ts+++ b/src/client/src/__tests__/lib/rbac/routes/api/unavailable.test.ts@@ -1,4 +1,12 @@import { alertingUnavailable } from "@/lib/rbac/routes/api/alerts/unavailable";+import { fleetHubUnavailable } from "@/lib/rbac/routes/api/fleet-hub/unavailable";+import {+ DELETE as fleetHubCatchAllDelete,+ GET as fleetHubCatchAllGet,+ PATCH as fleetHubCatchAllPatch,+ POST as fleetHubCatchAllPost,+ PUT as fleetHubCatchAllPut,+} from "@/lib/rbac/routes/api/fleet-hub/[id]/[...path]/route";beforeAll(() => {Object.defineProperty(global, "Response", {@@ -22,3 +30,27 @@ describe("alertingUnavailable", () => {});});});++describe("fleetHubUnavailable", () => {+ it("returns a 402 upgrade-required response with a CE-safe error message", () => {+ const response: any = fleetHubUnavailable();++ expect(response.status).toBe(402);+ expect(response.json()).resolves.toEqual({+ error: "Fleet Hub is not available in this edition.",+ });+ });++ it("returns 402 for nested Fleet Hub paths that have no dedicated route", () => {+ for (const handler of [+ fleetHubCatchAllGet,+ fleetHubCatchAllPost,+ fleetHubCatchAllPut,+ fleetHubCatchAllPatch,+ fleetHubCatchAllDelete,+ ]) {+ const response: any = handler();+ expect(response.status).toBe(402);+ }+ });+});diff --git a/src/client/src/__tests__/lib/telemetry-source.test.ts b/src/client/src/__tests__/lib/telemetry-source.test.tsindex 786442c0..db20514b 100644--- a/src/client/src/__tests__/lib/telemetry-source.test.ts+++ b/src/client/src/__tests__/lib/telemetry-source.test.ts@@ -502,6 +502,16 @@ describe("resolveSignalSource (signal-aware routing)", () => {expect(res.via).toBe("none");});+ it("falls back to built-in ClickHouse when an explicit environment has no binding", async () => {+ mockBindingFindUnique.mockResolvedValue(null);+ mockFindMany.mockResolvedValue([]);+ mockGetDBConfigByUser.mockResolvedValue(dbConfig);+ const res = await resolveSignalSource("traces", { environment: "demo" });+ expect(res.via).toBe("builtin");+ expect(res.hasSource).toBe(true);+ expect(res.descriptor.dbConfigId).toBe("db-1");+ });+it("never routes a signal to a source that lacks it; falls back to built-in", async () => {mockBindingFindUnique.mockResolvedValue(null);mockFindMany.mockResolvedValue([@@ -634,9 +644,9 @@ describe("resolveSignalSource (signal-aware routing)", () => {},include: { source: true, databaseConfig: true },});- // An explicitly requested (if invalid) environment still fails closed- // when nothing is routed for it, rather than silently using a default.- expect(res.via).toBe("none");+ expect(res.via).toBe("builtin");+ expect(res.hasSource).toBe(true);+ expect(res.descriptor.dbConfigId).toBe("db-1");});it("scopes resolveActiveCredentialDatabase to null when an explicit projectId is given without a dbConfigId", async () => {@@ -814,7 +824,7 @@ describe("getTelemetryAdapterForDbConfig", () => {expect(mockGetDBConfigById).not.toHaveBeenCalled();});- it("throws the typed no-source error when the DatabaseConfig's environment has no routed connector", async () => {+ it("falls back to the DatabaseConfig ClickHouse when the environment has no dedicated binding", async () => {mockGetDBConfigByIdInternal.mockResolvedValue({...dbConfig,environment: "production",@@ -823,7 +833,10 @@ describe("getTelemetryAdapterForDbConfig", () => {await expect(getTelemetryAdapterForDbConfig("db-1", "traces")- ).rejects.toThrow(/No connector is configured/);+ ).resolves.toMatchObject({+ isBuiltIn: true,+ descriptor: { id: "builtin:db-1", dbConfigId: "db-1" },+ });});it("resolves the isBuiltIn descriptor unchanged (no dbConfigId re-attachment) when the binding routes to the built-in store", async () => {diff --git a/src/client/src/__tests__/middleware/check-auth.test.ts b/src/client/src/__tests__/middleware/check-auth.test.tsindex 06851f0f..a361653e 100644--- a/src/client/src/__tests__/middleware/check-auth.test.ts+++ b/src/client/src/__tests__/middleware/check-auth.test.ts@@ -124,13 +124,22 @@ describe('checkAuth', () => {it('forwards the request with the resolved database config id when the key is valid', async () => {(global as any).fetch = jest.fn().mockResolvedValue({ok: true,- json: async () => ({ valid: true, databaseConfigId: 'db-config-1' }),+ json: async () => ({+ valid: true,+ databaseConfigId: 'db-config-1',+ organisationId: 'org-1',+ projectId: 'proj-1',+ environment: 'staging',+ }),});const req = makeRequest('GET', '/api/vault/get-secrets', '', { Authorization: 'Bearer key-1' });await middleware(req as any, makeFetchEvent());expect(nextHandler).toHaveBeenCalled();const forwardedRequest = nextHandler.mock.calls[0][0] as { headers: Headers };expect(forwardedRequest.headers.get('x-database-config-id')).toBe('db-config-1');+ expect(forwardedRequest.headers.get('x-openlit-organisation-id')).toBe('org-1');+ expect(forwardedRequest.headers.get('x-openlit-project-id')).toBe('proj-1');+ expect(forwardedRequest.headers.get('x-openlit-environment')).toBe('staging');});it('strips a client-supplied x-database-config-id on session API requests', async () => {diff --git a/src/client/src/__tests__/middleware/middleware.test.ts b/src/client/src/__tests__/middleware/middleware.test.tsindex a22c34d4..8734ade3 100644--- a/src/client/src/__tests__/middleware/middleware.test.ts+++ b/src/client/src/__tests__/middleware/middleware.test.ts@@ -47,6 +47,8 @@ describe('middleware', () => {it('does not include enterprise-only routes in CE', () => {expect(config.matcher).not.toContain('/audit-logs');expect(config.matcher).not.toContain('/audit-logs/:path*');+ expect(config.matcher).not.toContain('/fleet-hub');+ expect(config.matcher).not.toContain('/fleet-hub/:path*');});it('matches the agents routes', () => {@@ -68,7 +70,6 @@ describe('middleware', () => {it('matches nested playground routes that previously bypassed auth', () => {expect(config.matcher).toContain('/openground/:path*');expect(config.matcher).toContain('/prompt-hub/:path*');- expect(config.matcher).toContain('/fleet-hub/:path*');expect(config.matcher).toContain('/rule-engine');expect(config.matcher).toContain('/rule-engine/:path*');expect(config.matcher).toContain('/context');diff --git a/src/client/src/__tests__/store/filter-actions.test.ts b/src/client/src/__tests__/store/filter-actions.test.tsindex bf1a7e5e..2caeb40c 100644--- a/src/client/src/__tests__/store/filter-actions.test.ts+++ b/src/client/src/__tests__/store/filter-actions.test.ts@@ -105,6 +105,37 @@ describe('filterStoreSlice actions', () => {store.getState().filter.updateFilter('refreshRate', '30s', { clearFilter: true });expect(store.getState().filter.details.selectedConfig).toEqual({});});++ it('keeps customFilters when a later selectedConfig update omits them as undefined', () => {+ store.getState().filter.updateFilter('selectedConfig', {+ serviceNames: ['support-agent-demo'],+ customFilters: [+ { attributeType: 'SpanAttributes', key: 'gen_ai.system', value: 'openai' },+ ],+ });+ store.getState().filter.updateFilter('selectedConfig', {+ serviceNames: ['support-agent-demo'],+ customFilters: undefined,+ });+ expect(store.getState().filter.details.selectedConfig.customFilters).toEqual([+ { attributeType: 'SpanAttributes', key: 'gen_ai.system', value: 'openai' },+ ]);+ });++ it('keeps customFilters when AgentScope re-asserts serviceNames', () => {+ store.getState().filter.updateFilter('selectedConfig', {+ serviceNames: ['support-agent-demo'],+ customFilters: [+ { attributeType: 'ResourceAttributes', key: 'service.version', value: '1' },+ ],+ });+ store.getState().filter.updateFilter('selectedConfig.serviceNames', [+ 'support-agent-demo',+ ]);+ expect(store.getState().filter.details.selectedConfig.customFilters).toEqual([+ { attributeType: 'ResourceAttributes', key: 'service.version', value: '1' },+ ]);+ });});describe('updateConfig', () => {diff --git a/src/client/src/__tests__/utils/api.test.ts b/src/client/src/__tests__/utils/api.test.tsindex 794eafdb..0fb504c9 100644--- a/src/client/src/__tests__/utils/api.test.ts+++ b/src/client/src/__tests__/utils/api.test.ts@@ -126,7 +126,20 @@ describe('getData', () => {makeFetchResponse(true, '<!DOCTYPE html><html>login</html>'));await expect(getData({ url: '/api/db-config', method: 'GET' })).rejects.toThrow(- /non-JSON/i+ /page instead of data/i+ );+ });++ it('throws a readable message instead of dumping HTML on failed responses', async () => {+ (global.fetch as jest.Mock).mockResolvedValue(+ makeFetchResponse(+ false,+ '<!DOCTYPE html><html><head><style data-next-hide-fouc="true">body{display:none}</style></head></html>',+ 401+ )+ );+ await expect(getData({ url: '/api/test', method: 'GET' })).rejects.toThrow(+ 'The server returned a page instead of data (401). Refresh the page and try again.');});@@ -233,6 +246,15 @@ describe('postData', () => {await expect(postData({ url: '/api/items', data: {} })).rejects.toThrow();});+ it('throws a readable message instead of dumping HTML on failed responses', async () => {+ (global.fetch as jest.Mock).mockResolvedValue(+ makeFetchResponse(false, '<html><body>login</body></html>', 500)+ );+ await expect(postData({ url: '/api/items', data: {} })).rejects.toThrow(+ /page instead of data \(500\)/i+ );+ });+it('throws using the object error fallback fields when the failed response is JSON', async () => {(global.fetch as jest.Mock).mockResolvedValue(makeFetchResponse(false, { err: 'duplicate key' }, 409)diff --git a/src/client/src/__tests__/utils/hooks/useFetchWrapper.test.tsx b/src/client/src/__tests__/utils/hooks/useFetchWrapper.test.tsxindex 3772ac8d..15658b5f 100644--- a/src/client/src/__tests__/utils/hooks/useFetchWrapper.test.tsx+++ b/src/client/src/__tests__/utils/hooks/useFetchWrapper.test.tsx@@ -2,7 +2,14 @@ import { renderHook, act } from '@testing-library/react';import useFetchWrapper from '@/utils/hooks/useFetchWrapper';import * as api from '@/utils/api';-jest.mock('@/utils/api');+jest.mock('@/utils/api', () => {+ const actual = jest.requireActual('@/utils/api');+ return {+ ...actual,+ getData: jest.fn(),+ deleteData: jest.fn(),+ };+});const mockGetData = api.getData as jest.MockedFunction<typeof api.getData>;const mockDeleteData = api.deleteData as jest.MockedFunction<typeof api.deleteData>;diff --git a/src/client/src/app/(playground)/fleet-hub/[id]/agent.tsx b/src/client/src/app/(playground)/fleet-hub/[id]/agent.tsxdeleted file mode 100644index 3ce0a277..00000000--- a/src/client/src/app/(playground)/fleet-hub/[id]/agent.tsx+++ /dev/null@@ -1,214 +0,0 @@-"use client"--import { Card, CardContent, CardFooter } from "@/components/ui/card"-import { Badge } from "@/components/ui/badge"-import { ArrowLeft, Clock, HeartPulse, Package, Server } from "lucide-react"-import Link from "next/link"-import { Agent } from "@/types/fleet-hub"-import { formatDate } from "@/utils/date"-import { Accordion, AccordionContent, AccordionItem, AccordionTrigger } from "@/components/ui/accordion"-import { cn } from "@/lib/utils"-import { CodeEditor } from "@/components/(playground)/manage-dashboard/board-creator"-import { useCallback, useEffect, useState } from "react"-import { Button } from "@/components/ui/button"-import useFetchWrapper from "@/utils/hooks/useFetchWrapper"-import { jsonStringify } from "@/utils/json"-import { consoleLog } from "@/utils/log"-import { getAttributeValue } from "@/helpers/client/fleet-hub"-import { usePageHeader } from "@/selectors/page"-import { usePostHog } from "posthog-js/react"-import { CLIENT_EVENTS } from "@/constants/events"-import { toast } from "sonner"-import FeaturePageHeader from "@/components/(playground)/feature-page-header"-import OpenTelemetrySvg from "@/components/svg/opentelemetry"-import getMessage from "@/constants/messages"--interface AgentDetailProps {- agent: Agent,- fetchAgentInfo: () => void-}--const InfoItem = ({ label, value, mono = false }: { label: string; value: string; mono?: boolean }) => (- <div className="space-y-1">- <p className="text-xs text-muted-foreground">{label}</p>- <p className={cn("text-sm font-medium", mono && "font-mono")}>{value}</p>- </div>-);--export default function AgentDetail({ agent, fetchAgentInfo }: AgentDetailProps) {- const isUnhealthy = !agent.Status.health.healthy;- const { setHeader } = usePageHeader();-- useEffect(() => {- setHeader({- title: getAttributeValue(agent, "Status.agent_description.identifying_attributes", "service.name"),- breadcrumbs: [],- })- }, [agent, setHeader]);-- const backLabel = getMessage().FLEET_HUB_BACK_TO_LIST;- const serviceName = getAttributeValue(agent, "Status.agent_description.identifying_attributes", "service.name");-- return (- <div className="flex h-full w-full flex-col overflow-hidden">- <FeaturePageHeader- eyebrow={getMessage().SIDEBAR_MONITOR}- title={serviceName}- icon={<OpenTelemetrySvg className="h-4 w-4" />}- tone="border-sky-200 bg-sky-50 text-sky-700 dark:border-sky-900/70 dark:bg-sky-950/40 dark:text-sky-300"- leading={(- <Button- asChild- variant="outline"- size="sm"- className="h-8 w-8 shrink-0 p-0"- >- <Link href="/fleet-hub" title={backLabel} aria-label={backLabel}>- <ArrowLeft className="h-3.5 w-3.5" />- </Link>- </Button>- )}- />- <div className="space-y-4 overflow-auto w-full flex flex-col grow p-4">- <Card>- <CardContent className="p-3">- <div className="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-4 gap-2">- <div className="flex items-center gap-2 rounded-md border border-stone-200 bg-stone-50 px-2.5 py-2 dark:border-stone-800 dark:bg-stone-900/40">- <Package className="h-3.5 w-3.5 shrink-0 text-primary" />- <div className="min-w-0">- <p className="text-[11px] leading-none text-muted-foreground">Version</p>- <p className="truncate font-mono text-sm font-medium">{getAttributeValue(agent, "Status.agent_description.identifying_attributes", "service.version")}</p>- </div>- </div>- <div className="flex items-center gap-2 rounded-md border border-stone-200 bg-stone-50 px-2.5 py-2 dark:border-stone-800 dark:bg-stone-900/40">- <Clock className="h-3.5 w-3.5 shrink-0 text-primary" />- <div className="min-w-0">- <p className="text-[11px] leading-none text-muted-foreground">Started At</p>- <p className="truncate font-mono text-sm font-medium">{formatDate(agent.StartedAt, { time: true })}</p>- </div>- </div>- <div className="flex items-center gap-2 rounded-md border border-stone-200 bg-stone-50 px-2.5 py-2 dark:border-stone-800 dark:bg-stone-900/40">- <Server className="h-3.5 w-3.5 shrink-0 text-primary" />- <div className="min-w-0">- <p className="text-[11px] leading-none text-muted-foreground">Host Name</p>- <p className="truncate font-mono text-sm font-medium">{getAttributeValue(agent, "Status.agent_description.non_identifying_attributes", "host.name")}</p>- </div>- </div>- <div className="flex items-center gap-2 rounded-md border border-stone-200 bg-stone-50 px-2.5 py-2 dark:border-stone-800 dark:bg-stone-900/40">- <HeartPulse className="h-3.5 w-3.5 shrink-0 text-primary" />- <div className="min-w-0">- <p className="text-[11px] leading-none text-muted-foreground">Health Status</p>- <p className={`mt-0.5 inline-block font-mono text-xs font-medium px-1.5 ${agent.Status.health.healthy ? "bg-green-500 text-white dark:bg-green-500 dark:text-white" : "bg-red-500 text-white dark:bg-red-500 dark:text-white"}`}>{agent.Status.health.status || "Error"}</p>- </div>- </div>- {- isUnhealthy && agent.Status.health.last_error ? (- <div className="bg-red-500 text-white text-center text-xs p-2 col-span-4">{agent.Status.health.last_error} </div>- ) : null- }- </div>- </CardContent>- <CardFooter className="p-0">- <Accordion type="single" collapsible className="w-full">- <AccordionItem value="item-1" className="flex flex-col-reverse w-full border-0">- <AccordionTrigger className="p-0 flex w-full items-center gap-2 justify-center py-1 bg-stone-200/60 dark:bg-stone-800 text-xs text-stone-500 dark:text-stone-400">Show more</AccordionTrigger>- <AccordionContent className="p-4 pt-0">- <div className="pt-4 border-t border-stone-200 dark:border-stone-700 space-y-4">- <h4 className="font-semibold text-sm">Detailed Information</h4>- <div className="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-4 gap-4">- {agent.Status.agent_description.identifying_attributes.map((attr, index) => (- <InfoItem key={index} label={attr.key} value={attr.value.Value.StringValue} />- ))}- {agent.Status.agent_description.non_identifying_attributes.map((attr, index) => (- <InfoItem key={index} label={attr.key} value={attr.value.Value.StringValue} />- ))}- </div>-- {Object.keys(agent.Status.health.component_health_map || {}).length > 0 ? <h4 className="font-semibold text-sm">Component Health Status</h4> : null}- <div className="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-4 gap-4">- {Object.entries(agent.Status.health.component_health_map || {}).map(([component, health]) => (- <div key={component} className={`p-4 rounded-lg border ${health.healthy ? "bg-green-500/10 border-green-500/20" : "bg-red-500/10 border-red-500/20"}`}>- <p className="text-xs text-muted-foreground mb-1">{component}</p>- <Badge variant={health.healthy ? "default" : "destructive"}>{health.status}</Badge>- </div>- ))}- </div>- </div>- </AccordionContent>- </AccordionItem>- </Accordion>- </CardFooter>- </Card>- <ConfigDetails agent={agent} fetchAgentInfo={fetchAgentInfo} />- </div>- </div>- )-}--function ConfigDetails({ agent, fetchAgentInfo }: { agent: Agent, fetchAgentInfo: () => void }) {- const [yamlInput, setYamlInput] = useState<string>(agent.CustomInstanceConfig || "");- const { fireRequest } = useFetchWrapper();- const posthog = usePostHog();-- const handleYamlChange = (value: string | undefined) => {- setYamlInput(value || "");- };-- const onSave = useCallback(() => {- const isClearing = yamlInput.trim() === "";-- fireRequest({- requestType: "POST",- url: `/api/fleet-hub/${agent.InstanceIdStr}/config`,- body: jsonStringify({- config: yamlInput,- }),- successCb: () => {- if (isClearing) {- toast.success("Configuration cleared successfully", {- description: "The custom collector configuration has been removed. The collector will use its default configuration."- });- } else {- toast.success("Configuration saved successfully", {- description: "The collector configuration has been updated and applied."- });- }- fetchAgentInfo();- posthog?.capture(CLIENT_EVENTS.FLEET_HUB_AGENT_CONFIG_SAVED, {- agentId: agent.InstanceIdStr,- });- },- failureCb: (resp) => {- consoleLog(resp);- // Extract error message from response- let errorMessage = "Failed to save configuration";- try {- const errorData = typeof resp === 'string' ? JSON.parse(resp) : resp;- errorMessage = errorData.error || errorMessage;- } catch (e) {- // If parsing fails, use the raw response as error message- errorMessage = typeof resp === 'string' ? resp : errorMessage;- }-- toast.error("Configuration validation failed", {- description: errorMessage,- duration: 5000- });- }- })- }, [yamlInput, agent.InstanceIdStr, fetchAgentInfo, posthog]);-- return (- <div className="grid grid-cols-2 gap-3 grow text-stone-700 dark:text-stone-300">- <div className="flex flex-col gap-3">- <h4 className="font-semibold text-sm">Custom Configuration</h4>- <CodeEditor value={yamlInput} onChange={handleYamlChange} language="yaml" />- <Button variant="default" size={"sm"} className="rounded-none py-1 h-auto bg-primary/80 hover:bg-primary disabled:bg-stone-400 disabled:dark:text-stone-700 shrink-0" disabled={agent.CustomInstanceConfig === yamlInput} onClick={onSave}>Save</Button>- </div>- <div className="flex flex-col gap-3">- <h4 className="font-semibold text-sm">Effective Configuration (readonly)</h4>- <CodeEditor value={agent.EffectiveConfig} onChange={() => { }} language="yaml" readOnly />- </div>- </div>- )-}\ No newline at end of filediff --git a/src/client/src/app/(playground)/fleet-hub/[id]/page.tsx b/src/client/src/app/(playground)/fleet-hub/[id]/page.tsxdeleted file mode 100644index f80ad60c..00000000--- a/src/client/src/app/(playground)/fleet-hub/[id]/page.tsx+++ /dev/null@@ -1,47 +0,0 @@-"use client";-import { Agent } from "@/types/fleet-hub";-import AgentDetail from "./agent";-import useFetchWrapper from "@/utils/hooks/useFetchWrapper";-import { useCallback, useEffect } from "react";-import { useParams } from "next/navigation";-import Loader from "@/components/common/loader";-import { usePostHog } from "posthog-js/react";-import { CLIENT_EVENTS } from "@/constants/events";--export default function AgentPage() {- const { fireRequest, data, isLoading } = useFetchWrapper<Agent>();- const params = useParams();- const posthog = usePostHog();--- const fetchAgentInfo = useCallback(async () => {- fireRequest({- requestType: "GET",- url: `/api/fleet-hub/${params.id}`,- responseDataKey: "data",- successCb: (response) => {- if (response?.data) {- posthog?.capture(CLIENT_EVENTS.FLEET_HUB_AGENT_VIEWED, {- agentId: params.id,- });- }- },- });- }, [params.id]);-- useEffect(() => {- fetchAgentInfo();- }, [fetchAgentInfo]);-- if (isLoading && !data) return (- <div className="flex items-center justify-center w-full h-full">- <Loader />- </div>- );-- if (!isLoading && !data) {- return null;- }-- return <AgentDetail agent={data!} fetchAgentInfo={fetchAgentInfo} />;-}\ No newline at end of filediff --git a/src/client/src/app/(playground)/fleet-hub/list.tsx b/src/client/src/app/(playground)/fleet-hub/list.tsxdeleted file mode 100644index 2a45018e..00000000--- a/src/client/src/app/(playground)/fleet-hub/list.tsx+++ /dev/null@@ -1,156 +0,0 @@-"use client";-import DataTable from "@/components/data-table/table";-import { Agent } from "@/types/fleet-hub";--import { Badge } from "@/components/ui/badge";-import { Columns } from "@/components/data-table/columns";-import { useRouter } from "next/navigation";-import { useRootStore } from "@/store";-import { getVisibilityColumnsOfPage } from "@/selectors/page";-import VisibilityColumns from "@/components/(playground)/filter/visibility-columns";-import { getAttributeValue } from "@/helpers/client/fleet-hub";-import LinuxSvg from "@/components/svg/linux";-import WindowsSvg from "@/components/svg/windows";-import { formatDate } from "@/utils/date";-import MacSvg from "@/components/svg/mac";-import FeaturePageHeader from "@/components/(playground)/feature-page-header";-import OpenTelemetrySvg from "@/components/svg/opentelemetry";-import getMessage from "@/constants/messages";--const osTypeMap = {- linux: <LinuxSvg />,- windows: <WindowsSvg />,- mac: <MacSvg />,-} as const;--const columns: Columns<string, Agent> = {- id: {- header: () => "ID",- cell: ({ row }) => {- return (<div- className="block items-center overflow-hidden text-ellipsis whitespace-nowrap"- title={row.InstanceIdStr}- >- {row.InstanceIdStr}- </div>)- },- },- name: {- header: () => "Name",- cell: ({ row }) => {- const serviceName = getAttributeValue(row, "Status.agent_description.identifying_attributes", "service.name")- return (<div- className="block items-center overflow-hidden text-ellipsis whitespace-nowrap"- title={serviceName}- >- {serviceName}- </div>)- },- enableHiding: false,- },- version: {- header: () => "Version",- cell: ({ row }) => {- const serviceVersion = getAttributeValue(- row,- "Status.agent_description.identifying_attributes",- "service.version",- )- return (- <div className="flex space-x-2 items-center" title={serviceVersion}>- <span className="truncate font-medium">{serviceVersion}</span>- </div>- );- },- enableHiding: true,- },- os: {- header: () => "Operating System",- cell: ({ row }) => {- const osType = getAttributeValue(- row,- "Status.agent_description.non_identifying_attributes",- "os.type",- )- return (- <div className="flex space-x-2 items-center" title={osType}>- <span className="truncate font-medium">{osType in osTypeMap ? osTypeMap[osType as keyof typeof osTypeMap] : osType}</span>- </div>- );- },- enableHiding: true,- },- startedAt: {- header: () => "Started At",- cell: ({ row }) => {- return (- <div- className="block items-center overflow-hidden text-ellipsis whitespace-nowrap"- title={row.StartedAt}- >- {formatDate(row.StartedAt, { time: true })}- </div>- );- },- enableHiding: true,- },- status: {- header: () => "Status",- cell: ({ row }) => {- return (- <Badge variant={row.Status.health.healthy ? "default" : "destructive"} className={`${row.Status.health.healthy ? "bg-green-500 text-white" : "bg-red-500 text-white"}`}>- {row.Status.health.status || "StatusError"}- </Badge>- );- },- enableHiding: false,- },- hostArchitecture: {- header: () => "Host Architecture",- cell: ({ row }) => {- const hostArchitecture = getAttributeValue(- row,- "Status.agent_description.non_identifying_attributes",- "host.arch",- )- return (- <div className="flex space-x-2 items-center" title={hostArchitecture}>- <span className="truncate font-medium">{hostArchitecture}</span>- </div>- );- },- enableHiding: true,- }-};---export default function List({ agents, isLoading, isFetched }: {- agents: Agent[];- isLoading: boolean;- isFetched: boolean;-}) {- const visibilityColumns = useRootStore((state) =>- getVisibilityColumnsOfPage(state, "fleethub")- );- const router = useRouter()-- const handleClick = (row: Agent) => {- router.push(`/fleet-hub/${row.InstanceIdStr}`)- }-- return (- <div className="flex flex-col w-full h-full">- <FeaturePageHeader eyebrow={getMessage().SIDEBAR_MONITOR} title={getMessage().FEATURE_FLEET_HUB} icon={<OpenTelemetrySvg className="h-4 w-4" />} tone="border-sky-200 bg-sky-50 text-sky-700 dark:border-sky-900/70 dark:bg-sky-950/40 dark:text-sky-300" actions={<VisibilityColumns columns={columns} pageName={"fleethub"} />} />- <div className="flex flex-col w-full h-full p-4 overflow-hidden">- <DataTable- columns={columns}- data={agents}- isFetched={isFetched}- isLoading={isLoading}- visibilityColumns={visibilityColumns}- onClick={handleClick}- />- </div>- </div>- );-}diff --git a/src/client/src/app/(playground)/fleet-hub/page.tsx b/src/client/src/app/(playground)/fleet-hub/page.tsxdeleted file mode 100644index 56939a5b..00000000--- a/src/client/src/app/(playground)/fleet-hub/page.tsx+++ /dev/null@@ -1,33 +0,0 @@-"use client";-import { useEffect } from "react";-import useFetchWrapper from "@/utils/hooks/useFetchWrapper";-import { Agent } from "@/types/fleet-hub";-import List from "./list";-import { usePostHog } from "posthog-js/react";-import { CLIENT_EVENTS } from "@/constants/events";--export default function Opamp() {- const { fireRequest, data, isLoading, isFetched } = useFetchWrapper<Agent[]>();- const posthog = usePostHog();-- useEffect(() => {- fireRequest({- requestType: "GET",- url: `/api/fleet-hub`,- responseDataKey: "data",- successCb: (response) => {- if (response?.data) {- posthog?.capture(CLIENT_EVENTS.FLEET_HUB_VIEWED, {- count: response?.data?.length || 0,- });- }- },- });- }, []);-- return (- <div className="flex flex-col w-full h-full">- <List agents={data || []} isLoading={!data || isLoading} isFetched={isFetched} />- </div>- );-}diff --git a/src/client/src/app/api/auth/verify-key/route.ts b/src/client/src/app/api/auth/verify-key/route.tsindex 56dfae29..39f0904e 100644--- a/src/client/src/app/api/auth/verify-key/route.ts+++ b/src/client/src/app/api/auth/verify-key/route.ts@@ -1,5 +1,5 @@import { NextRequest, NextResponse } from "next/server";-import prisma from "@/lib/prisma";+import { getAPIKeyInfo } from "@/lib/platform/api-keys";export async function GET(request: NextRequest) {const authHeader = request.headers.get("Authorization") || "";@@ -13,17 +13,15 @@ export async function GET(request: NextRequest) {}try {- const apiInfo = await prisma.aPIKeys.findFirst({- where: {- apiKey,- isDeleted: false,- },- });+ const [err, apiInfo] = await getAPIKeyInfo({ apiKey });- if (apiInfo?.databaseConfigId) {+ if (!err && apiInfo?.databaseConfigId) {return NextResponse.json({valid: true,databaseConfigId: apiInfo.databaseConfigId,+ organisationId: apiInfo.organisationId || null,+ projectId: apiInfo.projectId || null,+ environment: apiInfo.environment || "production",});}} catch (e) {diff --git a/src/client/src/app/api/fleet-hub/[id]/[...path]/route.ts b/src/client/src/app/api/fleet-hub/[id]/[...path]/route.tsnew file mode 100644index 00000000..1ce053bf--- /dev/null+++ b/src/client/src/app/api/fleet-hub/[id]/[...path]/route.ts@@ -0,0 +1 @@+export { GET, POST, PUT, PATCH, DELETE } from "@/lib/rbac/routes/api/fleet-hub/[id]/[...path]/route";diff --git a/src/client/src/app/api/fleet-hub/[id]/config/route.ts b/src/client/src/app/api/fleet-hub/[id]/config/route.tsindex 7b970af6..f41f7f58 100644--- a/src/client/src/app/api/fleet-hub/[id]/config/route.ts+++ b/src/client/src/app/api/fleet-hub/[id]/config/route.ts@@ -1,46 +1 @@-import { withAudit } from "@/lib/audit/route";-import { withCurrentOrganisationPermission } from "@/lib/rbac/current";-import { SERVER_EVENTS } from "@/constants/events";-import { updateAgentConfig } from "@/lib/platform/fleet-hub";-import { emitManagementAlertSignalSafe } from "@/lib/platform/alerts/signals";-import PostHogServer from "@/lib/posthog";--async function POSTHandler(request: Request, context: any) {- const startTimestamp = Date.now();- const { id } = context.params;- const { config } = await request.json();- const res = await updateAgentConfig(id, config);-- // Check if there was an error from the OpAMP server- if (res.err) {- PostHogServer.fireEvent({- event: SERVER_EVENTS.FLEET_HUB_CONFIG_UPDATE_FAILURE,- startTimestamp,- });- return Response.json(res.err,- { status: res.status || 500 }- );- }-- PostHogServer.fireEvent({- event: SERVER_EVENTS.FLEET_HUB_CONFIG_UPDATE_SUCCESS,- startTimestamp,- });- emitManagementAlertSignalSafe({- triggerType: "fleet_hub_config_update",- event: "fleet_hub_config_updated",- message: `Fleet Hub configuration for ${id} was updated.`,- sourceId: id,- fields: {- agent_id: id,- config_key: config && typeof config === "object" ? Object.keys(config)[0] || "" : "",- },- payloadSummary: {- agentId: id,- configKeys: config && typeof config === "object" ? Object.keys(config) : [],- },- });- return Response.json(res);-}--export const POST = withAudit(withCurrentOrganisationPermission("fleet_hub:configure", POSTHandler));+export { GET, POST } from "@/lib/rbac/routes/api/fleet-hub/[id]/config/route";diff --git a/src/client/src/app/api/fleet-hub/[id]/route.ts b/src/client/src/app/api/fleet-hub/[id]/route.tsindex f6625b2d..9fe58c61 100644--- a/src/client/src/app/api/fleet-hub/[id]/route.ts+++ b/src/client/src/app/api/fleet-hub/[id]/route.ts@@ -1,10 +1 @@-import { withCurrentOrganisationPermission } from "@/lib/rbac/current";-import { getAgentByInstanceId } from "@/lib/platform/fleet-hub";--async function GETHandler(_: Request, context: any) {- const { id } = context.params;- const res = await getAgentByInstanceId(id);- return Response.json(res);-}--export const GET = withCurrentOrganisationPermission("fleet_hub:read", GETHandler);+export { GET } from "@/lib/rbac/routes/api/fleet-hub/[id]/route";diff --git a/src/client/src/app/api/fleet-hub/[id]/tls-connection/route.ts b/src/client/src/app/api/fleet-hub/[id]/tls-connection/route.tsindex 441bbc16..7caa2f1a 100644--- a/src/client/src/app/api/fleet-hub/[id]/tls-connection/route.ts+++ b/src/client/src/app/api/fleet-hub/[id]/tls-connection/route.ts@@ -1,19 +1 @@-import { withAudit } from "@/lib/audit/route";-import { withCurrentOrganisationPermission } from "@/lib/rbac/current";-import { SERVER_EVENTS } from "@/constants/events";-import { updateTlsConnection } from "@/lib/platform/fleet-hub";-import PostHogServer from "@/lib/posthog";--async function POSTHandler(request: Request, context: any) {- const startTimestamp = Date.now();- const { id } = context.params;- const { tlsMin } = await request.json();- const res = await updateTlsConnection(id, tlsMin);- PostHogServer.fireEvent({- event: res.err ? SERVER_EVENTS.FLEET_HUB_TLS_CONNECTION_FAILURE : SERVER_EVENTS.FLEET_HUB_TLS_CONNECTION_SUCCESS,- startTimestamp,- });- return Response.json(res);-}--export const POST = withAudit(withCurrentOrganisationPermission("fleet_hub:configure", POSTHandler));+export { GET, POST } from "@/lib/rbac/routes/api/fleet-hub/[id]/tls-connection/route";diff --git a/src/client/src/app/api/fleet-hub/route.ts b/src/client/src/app/api/fleet-hub/route.tsindex 6e0eb9aa..59955722 100644--- a/src/client/src/app/api/fleet-hub/route.ts+++ b/src/client/src/app/api/fleet-hub/route.ts@@ -1,9 +1 @@-import { withCurrentOrganisationPermission } from "@/lib/rbac/current";-import { getAllAgents } from "@/lib/platform/fleet-hub";--async function GETHandler() {- const res: any = await getAllAgents();- return Response.json(res);-}--export const GET = withCurrentOrganisationPermission("fleet_hub:read", GETHandler);+export { GET } from "@/lib/rbac/routes/api/fleet-hub/route";diff --git a/src/client/src/clickhouse/migrations/backfill-otel-tenant-environment-migration.ts b/src/client/src/clickhouse/migrations/backfill-otel-tenant-environment-migration.tsnew file mode 100644index 00000000..09ec528a--- /dev/null+++ b/src/client/src/clickhouse/migrations/backfill-otel-tenant-environment-migration.ts@@ -0,0 +1,140 @@+import { dataCollector } from "@/lib/platform/common";+import { getDBConfigByIdInternal, getDBConfigByUser } from "@/lib/db-config";+import asaw from "@/utils/asaw";+import prisma from "@/lib/prisma";+import { consoleLog } from "@/utils/log";++const MIGRATION_ID = "backfill-otel-tenant-environment";++// Literal table names: this module is loaded from db-config -> migrations while+// `@/lib/platform/common` is still initializing, so importing OTEL_* constants+// there throws "Cannot access before initialization".+const OTEL_RESOURCE_TABLES = [+ "otel_traces",+ "otel_logs",+ "otel_metrics_gauge",+ "otel_metrics_sum",+ "otel_metrics_histogram",+ "otel_metrics_summary",+ "otel_metrics_exponential_histogram",+];++function escapeClickHouseString(value: string) {+ return value.replace(/\\/g, "\\\\").replace(/'/g, "\\'");+}++function isMissingTable(err: unknown) {+ const message = String(err || "").toLowerCase();+ return (+ message.includes("unknown table") ||+ message.includes("doesn't exist") ||+ message.includes("does not exist")+ );+}++export default async function BackfillOtelTenantEnvironmentMigration(+ databaseConfigId?: string+) {+ let err, dbConfig;+ if (databaseConfigId) {+ [err, dbConfig] = await asaw(getDBConfigByIdInternal({ id: databaseConfigId }));+ } else {+ [err, dbConfig] = await asaw(getDBConfigByUser(true));+ }++ if (err || !dbConfig?.id) return { migrationExist: false, queriesRun: false };++ const [, migrationExist] = await asaw(+ prisma.clickhouseMigrations.findFirst({+ where: {+ AND: {+ databaseConfigId: dbConfig.id as string,+ clickhouseMigrationId: MIGRATION_ID,+ },+ },+ })+ );++ if (migrationExist?.id) {+ return { migrationExist: true, queriesRun: false };+ }++ const environment =+ typeof dbConfig.environment === "string" && dbConfig.environment.trim()+ ? dbConfig.environment.trim()+ : "production";+ let organisationId = "";+ if (dbConfig.projectId) {+ const project = await prisma.project.findUnique({+ where: { id: dbConfig.projectId },+ select: { organisationId: true },+ });+ organisationId = project?.organisationId || "";+ }++ const pairs = [+ ["organisation.environment.name", environment],+ ...(organisationId+ ? [["openlit.organisation.id", organisationId] as const]+ : []),+ ...(dbConfig.projectId+ ? [["openlit.project.id", dbConfig.projectId] as const]+ : []),+ ];+ const mapLiteral = pairs+ .map(+ ([key, value]) =>+ `'${escapeClickHouseString(key)}', '${escapeClickHouseString(value)}'`+ )+ .join(", ");++ const whereParts = [+ `empty(ifNull(ResourceAttributes['organisation.environment.name'], ''))`,+ ];+ if (organisationId) {+ whereParts.push(+ `empty(ifNull(ResourceAttributes['openlit.organisation.id'], ''))`+ );+ }+ if (dbConfig.projectId) {+ whereParts.push(+ `empty(ifNull(ResourceAttributes['openlit.project.id'], ''))`+ );+ }+ const where = `(${whereParts.join("\n\t\tOR ")})`;++ try {+ for (const table of OTEL_RESOURCE_TABLES) {+ const { err: updateErr } = await dataCollector(+ {+ query: `+ ALTER TABLE ${table}+ UPDATE ResourceAttributes = mapUpdate(ifNull(ResourceAttributes, map()), map(${mapLiteral}))+ WHERE ${where}+ `,+ },+ "exec",+ dbConfig.id+ );+ if (updateErr && !isMissingTable(updateErr)) {+ consoleLog(+ `OTLP tenant environment backfill failed on ${table}: ${updateErr}`+ );+ return { migrationExist: false, queriesRun: false, err: updateErr };+ }+ }++ await asaw(+ prisma.clickhouseMigrations.create({+ data: {+ databaseConfigId: dbConfig.id,+ clickhouseMigrationId: MIGRATION_ID,+ },+ })+ );+ return { migrationExist: false, queriesRun: true };+ } catch (migrationError) {+ consoleLog(`OTLP tenant environment backfill error: ${migrationError}`);+ return { migrationExist: false, queriesRun: false, err: migrationError };+ }+}diff --git a/src/client/src/clickhouse/migrations/index.ts b/src/client/src/clickhouse/migrations/index.tsindex 155d730c..d2431532 100644--- a/src/client/src/clickhouse/migrations/index.ts+++ b/src/client/src/clickhouse/migrations/index.ts@@ -33,6 +33,7 @@ import AddCodingAgentSummaryFieldsMigration from "./add-coding-agent-summary-fieimport AddCodingAgentLOCSummaryFieldsMigration from "./add-coding-agent-loc-summary-fields-migration";import CreateCodingAgentsAuditMigration from "./create-coding-agents-audit-migration";import DropVcsMigration from "./drop-vcs-migration";+import BackfillOtelTenantEnvironmentMigration from "./backfill-otel-tenant-environment-migration";import AddProviderModelsCachePricesMigration from "./add-provider-models-cache-prices-migration";import SeedOrcaRouterProviderMigration from "./seed-orcarouter-provider-migration";import CreateTelemetryRollupsMigration from "./create-telemetry-rollups-migration";@@ -140,6 +141,10 @@ export default async function migrations(databaseConfigId?: string) {// `create-vcs-migration`. Runs last so stale deployments still get// the cleanup, and uses IF EXISTS so fresh installs are no-ops.await runMigration("drop-vcs", () => DropVcsMigration(databaseConfigId));+ await runMigration(+ "backfill-otel-tenant-environment",+ () => BackfillOtelTenantEnvironmentMigration(databaseConfigId)+ );// Built-in dashboard seeding (LLM / Vector DB / GPU / Coding// Agents / future) lives inside `create-custom-dashboards-migration`diff --git a/src/client/src/components/(playground)/agents/agent-monitoring-tab.tsx b/src/client/src/components/(playground)/agents/agent-monitoring-tab.tsxindex 8752d2dc..1d056743 100644--- a/src/client/src/components/(playground)/agents/agent-monitoring-tab.tsx+++ b/src/client/src/components/(playground)/agents/agent-monitoring-tab.tsx@@ -40,6 +40,7 @@ export default function AgentMonitoringTab() {services: filter.selectedConfig?.services,environments: filter.selectedConfig?.environments,versionFilter: filter.selectedConfig?.versionFilter,+ customFilters: filter.selectedConfig?.customFilters,});setActiveKey(key);};diff --git a/src/client/src/components/(playground)/api-keys/api-reference.tsx b/src/client/src/components/(playground)/api-keys/api-reference.tsxindex 35f9de6a..519aa9f9 100644--- a/src/client/src/components/(playground)/api-keys/api-reference.tsx+++ b/src/client/src/components/(playground)/api-keys/api-reference.tsx@@ -60,25 +60,25 @@ export default function ApiReference({ userApiKey }: ApiReferenceProps) {required: false,location: "header",description:- "Active organisation ID. Recommended for multi-organisation API-key clients.",+ "Injected from the API key on Bearer requests. Not required in curl.",example: "org_01HXYZ",},{name: "x-openlit-project-id",type: "string",- required: true,+ required: false,location: "header",description:- "Active project ID. Telemetry source bindings are project-scoped.",+ "Injected from the API key on Bearer requests. Not required in curl.",example: "proj_01HXYZ",},{name: "x-openlit-environment",type: "string",- required: true,+ required: false,location: "header",description:- "Active project environment (e.g. production). Required for signal routing of traces/logs/metrics/intelligence.",+ "Injected from the API key on Bearer requests. Session UI may still send this header.",example: "production",},];diff --git a/src/client/src/components/(playground)/database-config/database-config-page.tsx b/src/client/src/components/(playground)/database-config/database-config-page.tsxindex 2008fec1..e967a7e0 100644--- a/src/client/src/components/(playground)/database-config/database-config-page.tsx+++ b/src/client/src/components/(playground)/database-config/database-config-page.tsx@@ -1,11 +1,26 @@"use client";-import FormBuilder from "@/components/common/form-builder";import Image from "next/image";-import { Pencil, Trash2, Wifi } from "lucide-react";+import { BookOpen, ExternalLink, Pencil, Settings2, Trash2, Wifi } from "lucide-react";import { Button } from "@/components/ui/button";import { Badge } from "@/components/ui/badge";-import { Dialog, DialogContent } from "@/components/ui/dialog";+import {+ Dialog,+ DialogContent,+ DialogDescription,+ DialogFooter,+ DialogHeader,+ DialogTitle,+} from "@/components/ui/dialog";+import { Input } from "@/components/ui/input";+import { Label } from "@/components/ui/label";+import {+ Select,+ SelectContent,+ SelectItem,+ SelectTrigger,+ SelectValue,+} from "@/components/ui/select";import { DatabaseConfig, DatabaseConfigWithActive } from "@/constants/dbConfig";import {deleteDatabaseConfig,@@ -22,224 +37,306 @@ import { useCallback, useEffect, useMemo, useState } from "react";import { toast } from "sonner";import { usePostHog } from "posthog-js/react";import { CLIENT_EVENTS } from "@/constants/events";-import { FormBuilderEvent } from "@/types/form";-import { PRIMARY_BACKGROUND } from "@/constants/common-classes";import getMessage from "@/constants/messages";+const CLICKHOUSE_SIGNALS = ["traces", "logs", "metrics", "intelligence"] as const;+function ModifyDatabaseConfig({dbConfig,canCreate = true,canUpdate = true,onSaved,+ onClose,}: {dbConfig?: DatabaseConfigWithActive;canCreate?: boolean;canUpdate?: boolean;onSaved?: () => void;+ onClose?: () => void;}) {const posthog = usePostHog();const selectedEnvironment = useRootStore(getCurrentProjectEnvironment) || "production";const { fireRequest, isLoading } = useFetchWrapper();const messages = getMessage();- const [environments, setEnvironments] = useState<string[]>([- "production",- dbConfig?.environment || selectedEnvironment,- ].filter((value, index, values) => values.indexOf(value) === index));+ const isEdit = !!dbConfig?.id;+ const [name, setName] = useState(dbConfig?.name || "");+ const [environment, setEnvironment] = useState(dbConfig?.environment || selectedEnvironment);+ const [username, setUsername] = useState(dbConfig?.username || "");+ const [password, setPassword] = useState("");+ const [host, setHost] = useState(dbConfig?.host || "");+ const [port, setPort] = useState(dbConfig?.port || "");+ const [database, setDatabase] = useState(dbConfig?.database || "");+ const [query, setQuery] = useState(dbConfig?.query || "");+ const [testing, setTesting] = useState(false);+ const [environments, setEnvironments] = useState<string[]>(+ ["production", dbConfig?.environment || selectedEnvironment].filter(+ (value, index, values) => values.indexOf(value) === index+ )+ );+ const setupGuide = messages.DATA_SOURCE_SETUP_GUIDES.clickhouse;useEffect(() => {fetch("/api/project/environment").then((response) => response.ok ? response.json() : { environments: [] })- .then((body) => setEnvironments(Array.from(new Set(["production", ...(body.environments || []).map((item: { name: string }) => item.name)]))))+ .then((body) => setEnvironments(Array.from(new Set(["production", ...(body.environments || []).map((item: { name: string }) => item.name), environment])))).catch(() => undefined);- }, []);+ }, [environment]);- const modifyDetails: FormBuilderEvent = useCallback(- (event) => {- event.preventDefault();- const formElement = event.target as HTMLFormElement;+ const formFieldsDisabled = isEdit+ ? !canUpdate || !dbConfig?.permissions?.canEdit+ : !canCreate;+ const isAllowedToSubmit = isEdit+ ? canUpdate && !!dbConfig?.permissions?.canEdit+ : canCreate;- toast.loading(messages.MODIFYING_DB_CONFIG, {- id: "db-config-details",- });+ const save = useCallback(() => {+ if (!isAllowedToSubmit) return;+ toast.loading(messages.MODIFYING_DB_CONFIG, {+ id: "db-config-details",+ });- const payload: DatabaseConfig = {- id: dbConfig?.id || "",- name: (formElement.name as any).value,- environment: formElement.environment.value,- username: formElement.username.value,- host: formElement.host.value,- port: formElement.port.value,- database: formElement.database.value,- query: formElement.query.value,- };+ const payload: DatabaseConfig = {+ id: dbConfig?.id || "",+ name,+ environment,+ username,+ host,+ port,+ database,+ query,+ };- if (formElement.password.value) {- payload.password = formElement.password.value;- }+ if (password) {+ payload.password = password;+ }- fireRequest({- body: JSON.stringify(payload),- requestType: "POST",- url: "/api/db-config",- responseDataKey: "data",- successCb: () => {- fetchDatabaseConfigList((data: any[]) => {- posthog?.capture(CLIENT_EVENTS.DB_CONFIG_LIST, {- count: data.length,- });+ fireRequest({+ body: JSON.stringify(payload),+ requestType: "POST",+ url: "/api/db-config",+ responseDataKey: "data",+ successCb: () => {+ fetchDatabaseConfigList((data: any[]) => {+ posthog?.capture(CLIENT_EVENTS.DB_CONFIG_LIST, {+ count: data.length,});- toast.success(messages.DB_CONFIG_UPDATED, {- id: "db-config-details",- });- if (!dbConfig?.id) formElement.reset();- onSaved?.();- posthog?.capture(- payload.id- ? CLIENT_EVENTS.DB_CONFIG_UPDATE_SUCCESS- : CLIENT_EVENTS.DB_CONFIG_ADD_SUCCESS- );- },- failureCb: (err?: string) => {- toast.error(err || messages.DB_CONFIG_UPDATE_FAILED, {- id: "db-config-details",- });- posthog?.capture(- payload.id- ? CLIENT_EVENTS.DB_CONFIG_UPDATE_FAILURE- : CLIENT_EVENTS.DB_CONFIG_ADD_FAILURE- );- },- });- },- [dbConfig?.id, onSaved]- );+ });+ toast.success(messages.DB_CONFIG_UPDATED, {+ id: "db-config-details",+ });+ onSaved?.();+ posthog?.capture(+ payload.id+ ? CLIENT_EVENTS.DB_CONFIG_UPDATE_SUCCESS+ : CLIENT_EVENTS.DB_CONFIG_ADD_SUCCESS+ );+ },+ failureCb: (err?: string) => {+ toast.error(err || messages.DB_CONFIG_UPDATE_FAILED, {+ id: "db-config-details",+ });+ posthog?.capture(+ payload.id+ ? CLIENT_EVENTS.DB_CONFIG_UPDATE_FAILURE+ : CLIENT_EVENTS.DB_CONFIG_ADD_FAILURE+ );+ },+ });+ }, [+ database,+ dbConfig?.id,+ environment,+ fireRequest,+ host,+ isAllowedToSubmit,+ messages,+ name,+ onSaved,+ password,+ port,+ posthog,+ query,+ username,+ ]);- const formFieldsDisabled = dbConfig?.id- ? !canUpdate || !dbConfig.permissions?.canEdit- : !canCreate;+ const testConnection = async () => {+ if (!isEdit) {+ toast.error(messages.DATA_SOURCE_TEST_UNSAVED, { id: "db-config-test" });+ return;+ }+ setTesting(true);+ toast.loading(messages.DATA_SOURCE_TESTING, { id: "db-config-test" });+ try {+ const response = await fetch("/api/clickhouse", { method: "POST" });+ const body = await response.json().catch(() => ({}));+ if (!response.ok || body?.err) throw new Error(body?.err || messages.DATA_SOURCE_SAVE_FAILED);+ toast.success(messages.DATA_SOURCE_TEST_OK, { id: "db-config-test" });+ } catch (error: unknown) {+ toast.error(+ error instanceof Error ? error.message : messages.DATA_SOURCE_SAVE_FAILED,+ { id: "db-config-test" }+ );+ } finally {+ setTesting(false);+ }+ };return (- <FormBuilder- cardClassName={`${PRIMARY_BACKGROUND} py-4 px-6 rounded-none`}- fields={[- {- label: messages.DB_CONFIG_FIELD_CONFIG_NAME,- inputKey: `${dbConfig?.id}-name`,- fieldType: "INPUT",- fieldTypeProps: {- type: "text",- name: "name",- placeholder: "db-config",- defaultValue: dbConfig?.name,- disabled: formFieldsDisabled,- },- },- {- label: messages.DB_CONFIG_FIELD_ENVIRONMENT,- inputKey: `${dbConfig?.id}-environment`,- fieldType: "SELECT",- fieldTypeProps: {- name: "environment",- placeholder: "production",- options: environments.map((environment) => ({ value: environment, label: environment })),- defaultValue: dbConfig?.environment || selectedEnvironment,- disabled: formFieldsDisabled,- },- },- {- label: messages.DB_CONFIG_FIELD_USERNAME,- fieldType: "INPUT",- inputKey: `${dbConfig?.id}-username`,- fieldTypeProps: {- type: "text",- name: "username",- placeholder: "username",- defaultValue: dbConfig?.username,- disabled: formFieldsDisabled,- },- },- {- label: messages.DB_CONFIG_FIELD_PASSWORD,- inputKey: `${dbConfig?.id}-password`,- fieldType: "INPUT",- fieldTypeProps: {- type: "password",- name: "password",- placeholder: "*******",- disabled: formFieldsDisabled,- },- },- {- label: messages.DB_CONFIG_FIELD_HOST,- inputKey: `${dbConfig?.id}-host`,- fieldType: "INPUT",- fieldTypeProps: {- type: "text",- name: "host",- placeholder: "127.0.0.1",- defaultValue: dbConfig?.host,- disabled: formFieldsDisabled,- },- },- {- label: messages.DB_CONFIG_FIELD_PORT,- inputKey: `${dbConfig?.id}-port`,- fieldType: "INPUT",- fieldTypeProps: {- type: "number",- name: "port",- placeholder: "8123",- defaultValue: dbConfig?.port,- disabled: formFieldsDisabled,- },- },- {- label: messages.DB_CONFIG_FIELD_DATABASE,- inputKey: `${dbConfig?.id}-database`,- fieldType: "INPUT",- fieldTypeProps: {- type: "text",- name: "database",- placeholder: "default",- defaultValue: dbConfig?.database,- disabled: formFieldsDisabled,- },- },- {- label: messages.DB_CONFIG_FIELD_QUERY_PARAMS,- inputKey: `${dbConfig?.id}-query`,- fieldType: "INPUT",- fieldTypeProps: {- type: "text",- name: "query",- placeholder: "a=b&c=d",- defaultValue: dbConfig?.query,- disabled: formFieldsDisabled,- },- },- ]}- heading={- dbConfig?.id- ? !dbConfig?.permissions?.canEdit- ? messages.CLICKHOUSE_CONNECTOR_EDIT_TITLE- : messages.CLICKHOUSE_CONNECTOR_EDIT_TITLE- : messages.CLICKHOUSE_CONNECTOR_ADD_TITLE- }- subHeading={- !dbConfig?.id || dbConfig?.permissions?.canEdit- ? `${messages.CLICKHOUSE_CONNECTOR_DESCRIPTION} ${messages.CLICKHOUSE_CONNECTOR_INSTRUCTIONS}`- : messages.DB_CONFIG_EDIT_PERMISSION_REQUIRED- }- subHeadingClass="text-error"- isLoading={isLoading}- onSubmit={modifyDetails}- isAllowedToSubmit={- dbConfig?.id- ? canUpdate && !!dbConfig.permissions?.canEdit- : canCreate- }- submitButtonText={dbConfig?.id ? messages.UPDATE : messages.SAVE}- />+ <>+ <DialogHeader>+ <DialogTitle>+ {isEdit ? messages.DATA_SOURCE_DETAILS : messages.DATA_SOURCE_ADD}+ </DialogTitle>+ <DialogDescription>+ {isEdit && !dbConfig?.permissions?.canEdit+ ? messages.DB_CONFIG_EDIT_PERMISSION_REQUIRED+ : messages.CLICKHOUSE_CONNECTOR_DESCRIPTION}+ </DialogDescription>+ </DialogHeader>++ <div className="space-y-4">+ <section className="rounded-lg border border-primary/25 bg-primary/[0.04] p-4 dark:border-primary/35 dark:bg-primary/[0.08]">+ <div className="mb-3 flex items-center gap-2">+ <Settings2 className="h-4 w-4 text-primary" />+ <div>+ <p className="text-xs font-semibold text-stone-950 dark:text-stone-50">{messages.DATA_SOURCE_CONNECTOR_SECTION}</p>+ <p className="text-[11px] text-muted-foreground">{messages.DATA_SOURCE_CONNECTOR_SECTION_DESCRIPTION}</p>+ </div>+ </div>+ <div className="space-y-1.5">+ <Label className="text-xs">{messages.DATA_SOURCE_FIELD_TYPE}</Label>+ <div className="flex min-h-14 items-center gap-2.5 overflow-hidden rounded-md border border-stone-300 bg-white px-3 py-2.5 dark:border-stone-700 dark:bg-stone-900">+ <div className="flex h-8 w-8 shrink-0 items-center justify-center rounded border border-stone-200 bg-white dark:border-stone-700 dark:bg-stone-950">+ <Image src="/images/connectors/clickhouse.svg" alt="" width={20} height={20} className="h-5 w-5 object-contain" />+ </div>+ <div className="min-w-0">+ <p className="truncate text-sm font-medium">{messages.DATA_SOURCE_BUILTIN_TITLE}</p>+ <p className="truncate text-[11px] text-muted-foreground">{messages.CLICKHOUSE_CONNECTOR_DESCRIPTION}</p>+ </div>+ </div>+ {isEdit && <p className="text-[11px] text-muted-foreground">{messages.DATA_SOURCE_TYPE_LOCKED}</p>}+ </div>+ <div className="mt-3 flex flex-wrap items-center gap-1.5">+ <span className="mr-1 text-[11px] text-muted-foreground">{messages.DATA_SOURCE_SIGNALS_SECTION}:</span>+ {CLICKHOUSE_SIGNALS.map((sig) => (+ <Badge key={sig} variant="secondary" className="text-[10px]">{sig}</Badge>+ ))}+ </div>+ </section>++ {setupGuide && (+ <details open className="rounded-lg border border-stone-200 bg-stone-50/70 p-4 dark:border-stone-800 dark:bg-stone-900/50">+ <summary className="flex cursor-pointer list-none items-center gap-2 text-xs font-semibold text-stone-950 marker:hidden dark:text-stone-50">+ <BookOpen className="h-4 w-4 text-primary" />+ {messages.DATA_SOURCE_SETUP_TITLE}+ </summary>+ <div className="mt-3 space-y-3 pl-6">+ <p className="text-[11px] leading-4 text-muted-foreground">{setupGuide.summary}</p>+ <ol className="list-decimal space-y-1.5 pl-4 text-[11px] leading-4 text-stone-700 dark:text-stone-300">+ {setupGuide.steps.map((step) => <li key={step}>{step}</li>)}+ </ol>+ <a href={setupGuide.docsUrl} target="_blank" rel="noreferrer noopener" className="inline-flex items-center gap-1 text-[11px] font-medium text-primary underline underline-offset-2">+ {messages.DATA_SOURCE_DOCS_LINK}<ExternalLink className="h-3 w-3" />+ </a>+ </div>+ </details>+ )}++ <section className="space-y-3 rounded-lg border border-stone-200 p-4 dark:border-stone-800">+ <div>+ <p className="text-xs font-semibold text-stone-950 dark:text-stone-50">{messages.DATA_SOURCE_CONNECTION_SECTION}</p>+ <p className="mt-0.5 text-[11px] text-muted-foreground">{messages.DATA_SOURCE_CONNECTION_SECTION_DESCRIPTION}</p>+ </div>+ <div className="grid gap-3 sm:grid-cols-2">+ <div className="space-y-1.5">+ <Label className="text-xs">{messages.DATA_SOURCE_FIELD_NAME}</Label>+ <Input+ value={name}+ onChange={(event) => setName(event.target.value)}+ placeholder="production-traces"+ disabled={formFieldsDisabled}+ className="bg-white dark:bg-stone-900"+ />+ </div>+ <div className="space-y-1.5">+ <Label className="text-xs">{messages.CONNECTOR_ENVIRONMENT}</Label>+ <Select value={environment} onValueChange={setEnvironment} disabled={formFieldsDisabled}>+ <SelectTrigger className="border-stone-300 bg-white text-stone-950 dark:border-stone-700 dark:bg-stone-900 dark:text-stone-50">+ <SelectValue placeholder={messages.CONNECTOR_ENVIRONMENT_PLACEHOLDER} />+ </SelectTrigger>+ <SelectContent>+ {environments.map((item) => <SelectItem key={item} value={item}>{item}</SelectItem>)}+ </SelectContent>+ </Select>+ </div>+ </div>+ </section>++ <section className="space-y-3 rounded-lg border border-stone-200 p-4 dark:border-stone-800">+ <div>+ <p className="text-xs font-semibold text-stone-950 dark:text-stone-50">{messages.DATA_SOURCE_SETTINGS_SECTION}</p>+ <p className="mt-0.5 text-[11px] text-muted-foreground">{messages.DATA_SOURCE_SETTINGS_SECTION_DESCRIPTION}</p>+ </div>+ <div className="grid gap-3 sm:grid-cols-2">+ <div className="space-y-1.5">+ <Label className="text-xs">{messages.DB_CONFIG_FIELD_HOST}</Label>+ <Input value={host} onChange={(event) => setHost(event.target.value)} placeholder="127.0.0.1" disabled={formFieldsDisabled} className="bg-white dark:bg-stone-900" />+ </div>+ <div className="space-y-1.5">+ <Label className="text-xs">{messages.DB_CONFIG_FIELD_PORT}</Label>+ <Input value={port} onChange={(event) => setPort(event.target.value)} placeholder="8123" disabled={formFieldsDisabled} className="bg-white dark:bg-stone-900" />+ </div>+ <div className="space-y-1.5">+ <Label className="text-xs">{messages.DB_CONFIG_FIELD_DATABASE}</Label>+ <Input value={database} onChange={(event) => setDatabase(event.target.value)} placeholder="default" disabled={formFieldsDisabled} className="bg-white dark:bg-stone-900" />+ </div>+ <div className="space-y-1.5">+ <Label className="text-xs">{messages.DB_CONFIG_FIELD_QUERY_PARAMS}</Label>+ <Input value={query} onChange={(event) => setQuery(event.target.value)} placeholder="a=b&c=d" disabled={formFieldsDisabled} className="bg-white dark:bg-stone-900" />+ </div>+ </div>+ </section>++ <section className="space-y-3 rounded-lg border border-stone-200 p-4 dark:border-stone-800">+ <div>+ <p className="text-xs font-semibold text-stone-950 dark:text-stone-50">{messages.DATA_SOURCE_CREDENTIALS_TITLE}</p>+ <p className="text-xs text-muted-foreground">+ {isEdit ? messages.DATA_SOURCE_CREDENTIALS_SET : messages.DATA_SOURCE_CREDENTIALS_HELP}+ </p>+ </div>+ <div className="grid gap-3 sm:grid-cols-2">+ <div className="space-y-1.5">+ <Label className="text-xs">{messages.DB_CONFIG_FIELD_USERNAME}</Label>+ <Input value={username} onChange={(event) => setUsername(event.target.value)} placeholder="username" disabled={formFieldsDisabled} className="bg-white dark:bg-stone-900" />+ </div>+ <div className="space-y-1.5">+ <Label className="text-xs">{messages.DB_CONFIG_FIELD_PASSWORD}</Label>+ <Input type="password" value={password} onChange={(event) => setPassword(event.target.value)} placeholder="*******" disabled={formFieldsDisabled} className="bg-white dark:bg-stone-900" />+ </div>+ </div>+ </section>+ </div>++ <DialogFooter className="gap-2 sm:justify-between">+ <Button+ variant="outline"+ onClick={() => void testConnection()}+ disabled={isLoading || testing || !isEdit}+ title={!isEdit ? messages.DATA_SOURCE_TEST_UNSAVED : undefined}+ >+ <Wifi className="mr-1.5 h-3.5 w-3.5" />+ {testing ? messages.DATA_SOURCE_TESTING : messages.DATA_SOURCE_TEST}+ </Button>+ <div className="flex gap-2">+ <Button variant="outline" onClick={onClose} disabled={isLoading || testing}>+ {messages.CANCEL}+ </Button>+ <Button onClick={save} disabled={isLoading || testing || !isAllowedToSubmit}>+ {messages.SAVE}+ </Button>+ </div>+ </DialogFooter>+ </>);}function DatabaseList({@@ -310,11 +407,12 @@ function DatabaseList({const dialog = editing ? (<Dialog open onOpenChange={(open) => !open && setEditing(null)}>- <DialogContent className="max-h-[92vh] w-[calc(100vw-2rem)] max-w-3xl overflow-y-auto p-0">+ <DialogContent className="w-[calc(100vw-2rem)] max-w-4xl max-h-[92vh] overflow-y-auto border-stone-200 bg-white text-stone-950 shadow-2xl dark:border-stone-800 dark:bg-stone-950 dark:text-stone-50 sm:max-w-4xl"><ModifyDatabaseConfigdbConfig={editing === "new" ? undefined : editing}canCreate={canCreate}canUpdate={canUpdate}+ onClose={() => setEditing(null)}onSaved={() => {setEditing(null);onConfigSaved?.();diff --git a/src/client/src/components/(playground)/filter/traces-filter.tsx b/src/client/src/components/(playground)/filter/traces-filter.tsxindex 497d88f4..1cdf4b53 100644--- a/src/client/src/components/(playground)/filter/traces-filter.tsx+++ b/src/client/src/components/(playground)/filter/traces-filter.tsx@@ -254,6 +254,7 @@ const DynamicFilters = ({attributeKeysUrl,customAttributeTypes,pageName,+ onApplied,}: {isVisibleFilters: boolean;filter: FilterType;@@ -262,6 +263,7 @@ const DynamicFilters = ({attributeKeysUrl: string;customAttributeTypes: CustomFilterAttributeType[];pageName?: PAGE;+ onApplied?: () => void;}) => {const filterConfig = useRootStore(getFilterConfig);const pingStatus = useRootStore(getPingStatus);@@ -439,12 +441,14 @@ const DynamicFilters = ({}, [filter.timeLimit.type, filter.timeLimit.start, filter.timeLimit.end, pingStatus]);const updateFilterStore = () => {- const validCustomFilters = customFilters.filter((f) => f.key && f.value);updateFilter("selectedConfig", {...selectedFilterValues,- customFilters:- validCustomFilters.length > 0 ? validCustomFilters : undefined,+ // Keep every row the user added (including drafts) so Apply does+ // not collapse the custom-attributes editor. Query builders ignore+ // rows that are missing a key or value.+ customFilters,});+ onApplied?.();};const clearFilterStore = () => {@@ -1219,6 +1223,12 @@ export default function TracesFilter({const filterConfig = useRootStore(getFilterConfig);const updateFilter = useRootStore(getUpdateFilter);+ useEffect(() => {+ if ((filter.selectedConfig.customFilters?.length || 0) > 0) {+ setIsVisibileFilters(true);+ }+ }, [filter.selectedConfig.customFilters]);+const onChangeGroupBy = (key: string | undefined) => {updateFilter("groupBy", key ?? null);};@@ -1350,6 +1360,7 @@ export default function TracesFilter({attributeKeysUrl={attributeKeysUrl}customAttributeTypes={customAttributeTypes}pageName={pageName}+ onApplied={() => setIsVisibileFilters(true)}/>)}</div>diff --git a/src/client/src/components/(playground)/observability/signal-list.tsx b/src/client/src/components/(playground)/observability/signal-list.tsxindex bb32665c..f6e962c6 100644--- a/src/client/src/components/(playground)/observability/signal-list.tsx+++ b/src/client/src/components/(playground)/observability/signal-list.tsx@@ -30,6 +30,7 @@ import { Button } from "@/components/ui/button";import { Sheet, SheetContent } from "@/components/ui/sheet";import { Maximize2, X } from "lucide-react";import getMessage from "@/constants/messages";+import { clientFetchErrorMessage } from "@/utils/api";import { prepareObservabilitySignalChange } from "@/helpers/client/observability";import { ResizeablePanel } from "@/components/ui/resizeable-panel";import { useIsAgentScoped } from "@/components/(playground)/agents/agent-scope-provider";@@ -134,6 +135,7 @@ export default function ObservabilitySignalList({services: filter.selectedConfig?.services,environments: filter.selectedConfig?.environments,versionFilter: filter.selectedConfig?.versionFilter,+ customFilters: filter.selectedConfig?.customFilters,}: null);@@ -487,7 +489,7 @@ export default function ObservabilitySignalList({{telemetryError && (<div role="alert" className="mb-3 rounded-md border border-red-200 bg-red-50 px-3 py-2 text-sm text-red-800 dark:border-red-900/50 dark:bg-red-950/30 dark:text-red-200">- {String(telemetryError)}+ {clientFetchErrorMessage(telemetryError)}</div>)}diff --git a/src/client/src/components/(playground)/telemetry-source/data-sources-page.tsx b/src/client/src/components/(playground)/telemetry-source/data-sources-page.tsxindex e7378b34..96ba4bd6 100644--- a/src/client/src/components/(playground)/telemetry-source/data-sources-page.tsx+++ b/src/client/src/components/(playground)/telemetry-source/data-sources-page.tsx@@ -17,7 +17,6 @@ import {import Image from "next/image";import Link from "next/link";import { Button } from "@/components/ui/button";-import DatabaseConfigPage from "@/components/(playground)/database-config/database-config-page";import { deleteDatabaseConfig } from "@/helpers/client/database-config";import type { DatabaseConfigWithActive } from "@/constants/dbConfig";import { Input } from "@/components/ui/input";@@ -117,6 +116,29 @@ interface BindingRow {sourceType: string | null;}+function clickHouseSourceFromConfig(+ config: DatabaseConfigWithActive,+ signals: Signal[]+): SourceRow {+ return {+ id: config.id.startsWith("database:") ? config.id : `database:${config.id}`,+ name: config.name,+ type: "clickhouse",+ environment: config.environment || "production",+ signals: signals.join(","),+ settings: JSON.stringify({+ username: config.username,+ host: config.host,+ port: config.port,+ database: config.database,+ query: config.query || "",+ }),+ isDefault: false,+ hasSecret: true,+ category: "datasource",+ };+}+function parseSignals(csv: string): Signal[] {return csv.split(",")@@ -266,7 +288,6 @@ export default function DataSourcesPage({const [descriptors, setDescriptors] = useState<TypeDescriptor[]>([]);const [bindings, setBindings] = useState<BindingRow[]>([]);const [editing, setEditing] = useState<SourceRow | "new" | null>(null);- const [dbEditing, setDbEditing] = useState<DatabaseConfigWithActive | "new" | null>(null);const [newType, setNewType] = useState<string | undefined>();const [testingId, setTestingId] = useState<string | null>(null);const [connectorSearch, setConnectorSearch] = useState("");@@ -328,12 +349,8 @@ export default function DataSourcesPage({useEffect(() => {if (openType == null) return;- if (openType === "clickhouse") {- setDbEditing("new");- } else {- setNewType(openType || undefined);- setEditing("new");- }+ setNewType(openType || undefined);+ setEditing("new");onOpenTypeHandled?.();}, [onOpenTypeHandled, openType]);@@ -628,14 +645,6 @@ export default function DataSourcesPage({</Select></div></div>- <DatabaseConfigPage- hideHeader- hideEmpty- hideList- editing={dbEditing}- onEditingChange={setDbEditing}- onConfigSaved={() => void load()}- />{loading ? (<div className="space-y-2">{[0, 1, 2].map((item) => (@@ -720,7 +729,11 @@ export default function DataSourcesPage({title={messages.DATA_SOURCE_EDIT_ACTION}aria-label={`${messages.DATA_SOURCE_EDIT_ACTION} ${config.name}`}disabled={!config.permissions?.canEdit}- onClick={() => setDbEditing(config)}+ onClick={() =>+ setEditing(+ clickHouseSourceFromConfig(config, clickhouseSignals)+ )+ }><Pencil className="h-3.5 w-3.5" /></Button>@@ -1210,6 +1223,7 @@ export function SourceFormDialog({environment: environment.trim().toLowerCase() || "production",settings,};+ if (isEdit) payload.id = String(source!.id).replace(/^database:/, "");if (Object.keys(credentials).length) payload.credentials = credentials;await jsonFetch("/api/connectors", {method: "POST",@@ -1265,6 +1279,15 @@ export function SourceFormDialog({setTesting(true);toast.loading(messages.DATA_SOURCE_TESTING, { id: "ds-test" });try {+ if (type === "clickhouse") {+ const response = await fetch("/api/clickhouse", { method: "POST" });+ const body = await response.json().catch(() => ({}));+ if (!response.ok || body?.err) {+ throw new Error(body?.err || messages.DATA_SOURCE_SAVE_FAILED);+ }+ toast.success(messages.DATA_SOURCE_TEST_OK, { id: "ds-test" });+ return;+ }const res = await jsonFetch(`/api/connectors/${source.id}/health`, {method: "POST",});@@ -1466,7 +1489,9 @@ export function SourceFormDialog({/>)}- {activeDescriptor?.category !== "memory" && activeDescriptor?.category !== "scanner" && (+ {activeDescriptor?.category !== "memory" &&+ activeDescriptor?.category !== "scanner" &&+ type !== "clickhouse" && (<div className="flex items-center justify-between rounded-md border border-stone-200 px-3 py-2 dark:border-stone-800"><Label className="text-xs">{messages.DATA_SOURCE_FIELD_DEFAULT}diff --git a/src/client/src/constants/api-reference.ts b/src/client/src/constants/api-reference.tsindex d157ad54..991a2d3d 100644--- a/src/client/src/constants/api-reference.ts+++ b/src/client/src/constants/api-reference.ts@@ -9,24 +9,14 @@ export interface ApiEndpoint {curlExample: (apiKey: string) => string;}-/** Organisation → Project → Environment headers for signal-routed API-key calls. */-export const OPENLIT_CONTEXT_CURL_HEADERS = ` -H "x-openlit-organisation-id: <organisation-id>" \\- -H "x-openlit-project-id: <project-id>" \\- -H "x-openlit-environment: production"`;--/** Preferred SDK context: signal routing. Optional db-config header remains supported. */-export const OPENLIT_SDK_CONTEXT_CURL_HEADERS = ` -H "x-openlit-organisation-id: <organisation-id>" \\- -H "x-openlit-project-id: <project-id>" \\- -H "x-openlit-environment: production"`;+/** Organisation → Project → Environment is resolved from the API key. */export function bearerAuthHeaders(apiKey: string) {- return ` -H "Authorization: Bearer ${apiKey}" \\-${OPENLIT_CONTEXT_CURL_HEADERS}`;+ return ` -H "Authorization: Bearer ${apiKey}"`;}export function bearerSdkAuthHeaders(apiKey: string) {- return ` -H "Authorization: Bearer ${apiKey}" \\-${OPENLIT_SDK_CONTEXT_CURL_HEADERS}`;+ return ` -H "Authorization: Bearer ${apiKey}"`;}export const API_REFERENCE_ENDPOINTS: ApiEndpoint[] = [@@ -35,7 +25,7 @@ export const API_REFERENCE_ENDPOINTS: ApiEndpoint[] = [method: "POST",path: "/api/telemetry/logs",summary: "Query logs",- description: "Retrieve a paginated list of telemetry logs matching the provided filters. Requires an OpenLIT API key plus organisation/project/environment headers for signal routing.",+ description: "Retrieve a paginated list of telemetry logs matching the provided filters. Requires an OpenLIT API key; organisation, project, and environment are taken from that key.",requestBody: `{"timeLimit": {"type": "24H",@@ -553,7 +543,7 @@ ${bearerAuthHeaders(apiKey)}`,path: "/api/chat/improvement/{spanId}",summary: "Get AI Analysis runs",description:- "Retrieve saved AI Analysis runs for a trace hierarchy (`scope=trace`) or a single span (`scope=span`). Requires an OpenLIT API key plus organisation/project/environment context headers for signal routing.",+ "Retrieve saved AI Analysis runs for a trace hierarchy (`scope=trace`) or a single span (`scope=span`). Requires an OpenLIT API key; organisation, project, and environment are taken from that key.",responseBody: `{"data": {"rootSpanId": "557a2bd43ff129ad",@@ -576,7 +566,7 @@ ${bearerAuthHeaders(apiKey)}`,path: "/api/chat/improvement/{spanId}",summary: "Run AI Analysis",description:- "Start a streaming AI Analysis for a trace or span. Requires an OpenLIT API key, Chat Settings, an intelligence ClickHouse connector, and organisation/project/environment context headers.",+ "Start a streaming AI Analysis for a trace or span. Requires an OpenLIT API key, Chat Settings, and an intelligence ClickHouse connector. Organisation, project, and environment are taken from the API key.",responseBody: `progress events + completed dimension findings (text/plain stream)`,curlExample: (apiKey) => `curl -X POST "http://localhost:3000/api/chat/improvement/some-span-id?scope=trace" \\${bearerAuthHeaders(apiKey)}`,@@ -586,7 +576,7 @@ ${bearerAuthHeaders(apiKey)}`,method: "GET",path: "/api/chat/config",summary: "Get Ask Otter config",- description: "Return the Ask Otter provider/model/Vault binding for the current organisation. Requires an OpenLIT API key plus organisation/project/environment context headers.",+ description: "Return the Ask Otter provider/model/Vault binding for the current organisation. Requires an OpenLIT API key; organisation, project, and environment are taken from that key.",responseBody: `{"data": {"provider": "openai",@@ -714,7 +704,7 @@ ${bearerAuthHeaders(apiKey)}`,path: "/api/chat/message",summary: "Send Ask Otter message",description:- "Stream an assistant reply for a conversation (NDJSON events: delta, step, done, error). Requires organisation/project/environment context for connector routing.",+ "Stream an assistant reply for a conversation (NDJSON events: delta, step, done, error). Organisation, project, and environment are taken from the API key.",requestBody: `{"conversationId": "conv-1","content": "Analyze the slowest traces from the last 24 hours"@@ -805,7 +795,7 @@ ${bearerAuthHeaders(apiKey)}`,method: "POST",path: "/api/prompt/improve",summary: "Improve prompt with Otter",- description: "Ask Otter to propose precise prompt edits. Requires an OpenLIT API key, Chat Settings, and organisation/project/environment context headers.",+ description: "Ask Otter to propose precise prompt edits. Requires an OpenLIT API key and Chat Settings. Organisation, project, and environment are taken from the API key.",requestBody: `{"prompt": "You are a helpful assistant. Answer the user question.","promptId": "prompt-1",diff --git a/src/client/src/constants/messages/en.ts b/src/client/src/constants/messages/en.tsindex e085b6e5..a186234d 100644--- a/src/client/src/constants/messages/en.ts+++ b/src/client/src/constants/messages/en.ts@@ -391,6 +391,14 @@ export const OBSERVABILITY_CHAT_MODEL_CHANGES = (count: number) =>export const OBSERVABILITY_TIMELINE = "Timeline";export const OBSERVABILITY_GRAPH = "Graph";export const OBSERVABILITY_NO_SERVER_CONNECTION = "Cannot connect to server!";+export const API_UNEXPECTED_NON_JSON_RESPONSE =+ "The server returned an unexpected non-JSON response.";+export const API_UNEXPECTED_HTML_RESPONSE = (status?: number) =>+ typeof status === "number"+ ? `The server returned a page instead of data (${status}). Refresh the page and try again.`+ : "The server returned a page instead of data. Refresh the page and try again.";+export const API_REQUEST_FAILED = (status: number) => `Request failed (${status})`;+export const API_EMPTY_RESPONSE = "The telemetry service returned an empty response.";export const OBSERVABILITY_ADD = "Add";export const OBSERVABILITY_SPAN_NAME_EXAMPLE = "e.g. SpanName";export const OBSERVABILITY_ATTRIBUTE_KEY_EXAMPLE = "e.g. gen_ai.system";@@ -755,6 +763,7 @@ export const FEATURE_VAULT = "Vault";export const FEATURE_MEMORY = "Memory";export const FEATURE_SCANNER = "Scanner";export const FEATURE_FLEET_HUB = "Fleet Hub";+export const FLEET_HUB_UNAVAILABLE = "Fleet Hub is not available in this edition.";export const FLEET_HUB_BACK_TO_LIST = "Back to Fleet Hub";export const FEATURE_AGENTS = "Agents";export const FEATURE_EVALS = "Evaluations";diff --git a/src/client/src/constants/sidebar.tsx b/src/client/src/constants/sidebar.tsxindex f8a33454..fa339257 100644--- a/src/client/src/constants/sidebar.tsx+++ b/src/client/src/constants/sidebar.tsx@@ -1,4 +1,3 @@-import OpenTelemetrySvg from "@/components/svg/opentelemetry";import { getEnterpriseSidebarItems } from "@/features/sidebar";import getMessage from "@/constants/messages";import { SidebarItemProps } from "@/types/sidebar";@@ -76,12 +75,6 @@ export const SIDEBAR_ITEMS: SidebarItemProps[] = [link: "/costs",type: "action",},- {- icon: <OpenTelemetrySvg className={ICON_CLASSES} />,- text: "Fleet Hub",- link: "/fleet-hub",- type: "action",- },],},{diff --git a/src/client/src/helpers/client/observability.ts b/src/client/src/helpers/client/observability.tsindex 7807d4fe..7c417719 100644--- a/src/client/src/helpers/client/observability.ts+++ b/src/client/src/helpers/client/observability.ts@@ -23,7 +23,11 @@ const EMPTY_ATTRIBUTE_KEYS: AttributeKeys = {export type ObservabilityScopePreserve = Partial<Pick<FilterConfig,- "serviceNames" | "services" | "environments" | "versionFilter"+ | "serviceNames"+ | "services"+ | "environments"+ | "versionFilter"+ | "customFilters">>;@@ -62,6 +66,9 @@ export function prepareObservabilitySignalChange(if (preserveScope?.versionFilter) {preserved.versionFilter = preserveScope.versionFilter;}+ if (preserveScope?.customFilters?.length) {+ preserved.customFilters = [...preserveScope.customFilters];+ }updateFilter("selectedConfig", preserved, { clearFilter: true });updateAttributeKeys?.(EMPTY_ATTRIBUTE_KEYS);diff --git a/src/client/src/helpers/server/platform.ts b/src/client/src/helpers/server/platform.tsindex f7924442..419ca046 100644--- a/src/client/src/helpers/server/platform.ts+++ b/src/client/src/helpers/server/platform.ts@@ -316,14 +316,10 @@ export const getFilterWhereCondition = (}if (filter.selectedConfig.environments?.length) {- // OTel-standard location for environment is- // `ResourceAttributes['deployment.environment']`. Older code- // here used `getTraceMappingKeyFullPath("environment")` which- // returns the dotted `SpanAttributes.gen_ai.environment` path- // and then wrapped *that* in `ResourceAttributes[...]`,- // producing a non-existent column key that always matched- // zero rows. Match both the OTel resource attribute and the- // legacy span attribute so any historic data still resolves.+ // Playground / agent `envs` filter uses OTel+ // `deployment.environment` (and legacy `gen_ai.environment`).+ // API-key tenant env is stamped separately as+ // `organisation.environment.name` and must not replace this.const envList = filter.selectedConfig.environments.map((environment: string) =>`'${escapeClickHouseString(environment)}'`diff --git a/src/client/src/lib/platform/api-keys/index.ts b/src/client/src/lib/platform/api-keys/index.tsindex 20bca489..53d1d54e 100644--- a/src/client/src/lib/platform/api-keys/index.ts+++ b/src/client/src/lib/platform/api-keys/index.ts@@ -20,7 +20,34 @@ function previewApiKey(apiKey: string): string {export interface APIKeyInfo {id: string;databaseConfigId: string | null;+ organisationId?: string | null;+ projectId?: string | null;+ environment?: string;createdByUser?: { email: string } | null;+ createdByUserId?: string;+}++const DEFAULT_API_KEY_ENVIRONMENT = "production";++function scopeFromKeyRow(row: {+ organisationId?: string | null;+ projectId?: string | null;+ environment?: string | null;+ databaseConfig?: {+ projectId?: string | null;+ environment?: string | null;+ project?: { organisationId?: string | null } | null;+ } | null;+}): Pick<APIKeyInfo, "organisationId" | "projectId" | "environment"> {+ return {+ organisationId:+ row.organisationId || row.databaseConfig?.project?.organisationId || null,+ projectId: row.projectId || row.databaseConfig?.projectId || null,+ environment:+ row.environment ||+ row.databaseConfig?.environment ||+ DEFAULT_API_KEY_ENVIRONMENT,+ };}function createAPIKey() {@@ -42,12 +69,28 @@ export async function generateAPIKey(name: string) {throwIfError(!dbConfig?.id, getMessage().DATABASE_CONFIG_NOT_FOUND);const apiKey = createAPIKey();+ const projectId = dbConfig.projectId || null;+ let organisationId: string | null = null;+ if (projectId) {+ const project = await prisma.project.findUnique({+ where: { id: projectId },+ select: { organisationId: true },+ });+ organisationId = project?.organisationId || null;+ }+ const environment =+ typeof dbConfig.environment === "string" && dbConfig.environment.trim()+ ? dbConfig.environment.trim()+ : "production";await prisma.aPIKeys.create({data: {apiKey,name,databaseConfigId: dbConfig.id,+ organisationId,+ projectId,+ environment,createdByUserId: user!.id,},});@@ -73,6 +116,9 @@ export async function generateAPIKey(name: string) {return {apiKey,databaseConfigId: dbConfig.id,+ organisationId,+ projectId,+ environment,};}@@ -89,7 +135,23 @@ export async function getAPIKeyInfo({ apiKey }: { apiKey: string }) {},],},- include: { createdByUser: { select: { email: true } } },+ include: {+ createdByUser: { select: { email: true } },+ databaseConfig: {+ select: {+ projectId: true,+ environment: true,+ project: { select: { organisationId: true } },+ },+ },+ },+ }).then((row) => {+ if (!row) return row;+ const scope = scopeFromKeyRow(row);+ return {+ ...row,+ ...scope,+ };}));}diff --git a/src/client/src/lib/platform/connectors/datasource/clickhouse/adapter.ts b/src/client/src/lib/platform/connectors/datasource/clickhouse/adapter.tsindex 44bd87cf..efeeb4c3 100644--- a/src/client/src/lib/platform/connectors/datasource/clickhouse/adapter.ts+++ b/src/client/src/lib/platform/connectors/datasource/clickhouse/adapter.ts@@ -142,6 +142,13 @@ function fieldToExpr(field: string): string {if (field === "deployment.environment") {return "ResourceAttributes['deployment.environment']";}+ if (+ field === "organisation.environment.name" ||+ field === "openlit.organisation.id" ||+ field === "openlit.project.id"+ ) {+ return `ResourceAttributes['${escapeCH(field)}']`;+ }// OTel GenAI renamed gen_ai.system → gen_ai.provider.name (1.30+). Coalesce// so group-by / filters work for both legacy and current SDK emits.if (field === "gen_ai.provider.name" || field === "gen_ai.system") {diff --git a/src/client/src/lib/platform/connectors/datasource/clickhouse/query-map.ts b/src/client/src/lib/platform/connectors/datasource/clickhouse/query-map.tsindex 922f2639..fe0f3059 100644--- a/src/client/src/lib/platform/connectors/datasource/clickhouse/query-map.ts+++ b/src/client/src/lib/platform/connectors/datasource/clickhouse/query-map.ts@@ -42,6 +42,7 @@ export function toMetricParams(query: OpenLITQuery,databaseConfigId?: string): MetricParams {+ let operationType: MetricParams["operationType"] | undefined;const sort = query.sort?.[0];const selectedConfig: Record<string, unknown> = {};const customFilters: Array<Record<string, unknown>> = [];@@ -57,9 +58,21 @@ export function toMetricParams(add(query.signal === "metrics" ? "metricNames" : "spanNames", filter.value);continue;}+ if (filter.target === "field" && filter.key) {+ const value = Array.isArray(filter.value) ? filter.value[0] : filter.value;+ customFilters.push({+ attributeType: "Field",+ key: filter.key,+ value: value === undefined ? "" : String(value),+ });+ continue;+ }if (filter.target !== "attribute" || !filter.key) continue;if (filter.key === "service.name") {- add(query.signal === "traces" ? "applicationNames" : "services", filter.value);+ // Agent-scope and telemetry `services=` filter ServiceName.+ // Mapping traces to applicationNames rewrote that as+ // gen_ai.application_name and emptied agent dashboard/monitoring.+ add(query.signal === "traces" ? "serviceNames" : "services", filter.value);continue;}if (query.signal === "traces" && filter.key === "gen_ai.request.model") {@@ -71,10 +84,25 @@ export function toMetricParams(continue;}if (query.signal === "traces" && filter.key === "gen_ai.operation.name") {+ const values = Array.isArray(filter.value)+ ? filter.value.map(String)+ : [String(filter.value ?? "")];+ if (filter.op === "neq" && values.includes("vectordb")) {+ operationType = "llm";+ continue;+ }+ if (+ (filter.op === "eq" || filter.op === "in") &&+ values.length === 1 &&+ values[0] === "vectordb"+ ) {+ operationType = "vectordb";+ continue;+ }add("traceTypes", filter.value);continue;}- if (query.signal === "traces" && filter.key === "deployment.environment") {+ if (filter.key === "deployment.environment") {add("environments", filter.value);continue;}@@ -105,6 +133,7 @@ export function toMetricParams(offset: query.offset,statusCode: extractStatusCodes(query),databaseConfigId,+ operationType,selectedConfig,sorting: sort? { type: sort.field, direction: sort.direction }@@ -174,6 +203,61 @@ function customFilterValue(value: unknown): string | string[] | undefined {return value !== undefined ? String(value) : undefined;}+/** ClickHouse otel_traces identifier columns the traces Field picker can filter on. */+const TRACE_COLUMN_FIELDS = new Set([+ "SpanId",+ "TraceId",+ "ParentSpanId",+ "SpanName",+ "ServiceName",+ "Duration",+ "Timestamp",+ "StatusCode",+ "StatusMessage",+ "Kind",+ "TraceType",+]);++/** ClickHouse otel_logs identifier columns the logs Field picker can filter on. */+const LOG_COLUMN_FIELDS = new Set([+ "Timestamp",+ "TraceId",+ "SpanId",+ "SeverityText",+ "SeverityNumber",+ "ServiceName",+ "Body",+ "ScopeName",+ "ScopeVersion",+]);++function columnFieldKey(key: string): string {+ return key.replace(/[^A-Za-z0-9_]/g, "");+}++function pushColumnFieldFilter(+ filters: NormalizedFilter[],+ cf: Record<string, unknown>,+ allowed: Set<string>+) {+ const key = columnFieldKey(typeof cf.key === "string" ? cf.key : "");+ if (!key || !allowed.has(key)) return;+ if (key === "SpanName") {+ filters.push({+ target: "spanName",+ op: "eq",+ value: String(cf.value ?? ""),+ });+ return;+ }+ filters.push({+ target: "field",+ key,+ op: normalizeCustomFilterOp(cf),+ value: customFilterValue(cf.value),+ });+}+/** Build the trace-signal filters (models/providers/spanNames/services/custom). */function tracesFilters(cfg: Record<string, unknown>): NormalizedFilter[] {const filters: NormalizedFilter[] = [];@@ -219,7 +303,10 @@ function tracesFilters(cfg: Record<string, unknown>): NormalizedFilter[] {});}- const serviceNames = stringList(cfg.serviceNames);+ const serviceNames = uniqueStrings([+ ...stringList(cfg.serviceNames),+ ...stringList(cfg.services),+ ]);if (serviceNames.length) {filters.push({target: "attribute",@@ -274,9 +361,7 @@ function tracesFilters(cfg: Record<string, unknown>): NormalizedFilter[] {if (!key) continue;const attributeType = String(cf.attributeType || cf.type || "");if (attributeType === "Field") {- if (key === "SpanName") {- filters.push({ target: "spanName", op: "eq", value: String(cf.value ?? "") });- }+ pushColumnFieldFilter(filters, cf, TRACE_COLUMN_FIELDS);continue;}const scope =@@ -339,6 +424,10 @@ function logsFilters(cfg: Record<string, unknown>): NormalizedFilter[] {const key = typeof cf.key === "string" ? cf.key : "";if (!key) continue;const attrType = String(cf.attributeType || cf.type || "");+ if (attrType === "Field") {+ pushColumnFieldFilter(filters, cf, LOG_COLUMN_FIELDS);+ continue;+ }const scope =attrType === "ResourceAttributes" || cf.scope === "resource"? "resource"diff --git a/src/client/src/lib/platform/connectors/datasource/grafana/tempo.ts b/src/client/src/lib/platform/connectors/datasource/grafana/tempo.tsindex f1410249..2a337c20 100644--- a/src/client/src/lib/platform/connectors/datasource/grafana/tempo.ts+++ b/src/client/src/lib/platform/connectors/datasource/grafana/tempo.ts@@ -467,6 +467,25 @@ function filterToTraceQL(filter: NormalizedFilter): string {);return wantsError ? `status = error` : `status != error`;}+ if (filter.target === "field" && filter.key) {+ const intrinsic =+ filter.key === "SpanId"+ ? "span:id"+ : filter.key === "TraceId"+ ? "trace:id"+ : filter.key === "ParentSpanId"+ ? "span:parentID"+ : filter.key === "Kind"+ ? "kind"+ : "";+ if (!intrinsic) return "";+ const values = Array.isArray(filter.value)+ ? filter.value+ : [filter.value || ""];+ return `(${values+ .map((v) => `${intrinsic} = ${traceqlValue(String(v))}`)+ .join(" || ")})`;+ }if (filter.target === "attribute" && filter.key) {const safeKey = sanitizeTraceQLAttrKey(filter.key);if (!safeKey) return "";diff --git a/src/client/src/lib/platform/connectors/datasource/jaeger/adapter.ts b/src/client/src/lib/platform/connectors/datasource/jaeger/adapter.tsindex 48184742..978839eb 100644--- a/src/client/src/lib/platform/connectors/datasource/jaeger/adapter.ts+++ b/src/client/src/lib/platform/connectors/datasource/jaeger/adapter.ts@@ -139,12 +139,18 @@ function spanMatchesFilters(const isError = /error/i.test(span.statusCode || "");return wantsError ? isError : !isError;}- if (filter.target === "attribute" && filter.key) {+ if (+ (filter.target === "attribute" || filter.target === "field") &&+ filter.key+ ) {const raw = spanFieldValue(span, filter.key);const value = raw === undefined ? undefined : String(raw);if (filter.op === "exists") return !!value;if (filter.op === "eq") return value === String(filter.value ?? "");if (filter.op === "neq") return value !== String(filter.value ?? "");+ if (filter.op === "contains") {+ return !!value && value.toLowerCase().includes(String(filter.value ?? "").toLowerCase());+ }if (filter.op === "in") {const values = Array.isArray(filter.value)? filter.value.map(String)diff --git a/src/client/src/lib/platform/connectors/datasource/rollup-policy.ts b/src/client/src/lib/platform/connectors/datasource/rollup-policy.tsindex b1fa4afa..46624ca1 100644--- a/src/client/src/lib/platform/connectors/datasource/rollup-policy.ts+++ b/src/client/src/lib/platform/connectors/datasource/rollup-policy.ts@@ -28,6 +28,7 @@ export function queryHasScopeFilters(query: OpenLITQuery): boolean {if (f.target !== "attribute" || !f.key) return false;return (f.key === "service.name" ||+ f.key === "organisation.environment.name" ||f.key === "deployment.environment" ||f.key === "openlit.agent.version_hash");diff --git a/src/client/src/lib/platform/connectors/datasource/types.ts b/src/client/src/lib/platform/connectors/datasource/types.tsindex 2b31018e..ec1e4bf4 100644--- a/src/client/src/lib/platform/connectors/datasource/types.ts+++ b/src/client/src/lib/platform/connectors/datasource/types.ts@@ -107,9 +107,10 @@ export type FilterOp =/*** A normalized attribute predicate. `target: "spanName"` matches on the span* name (or metric name for the metrics signal) rather than an attribute map.+ * `target: "field"` matches a vendor identifier column (SpanId, TraceId, …).*/export interface NormalizedFilter {- target: "attribute" | "spanName" | "status" | "duration";+ target: "attribute" | "spanName" | "status" | "duration" | "field";scope?: AttributeScope;key?: string;op: FilterOp;diff --git a/src/client/src/lib/platform/kubernetes/index.ts b/src/client/src/lib/platform/kubernetes/index.tsindex e6ec1744..355843f5 100644--- a/src/client/src/lib/platform/kubernetes/index.ts+++ b/src/client/src/lib/platform/kubernetes/index.ts@@ -57,7 +57,7 @@ export function buildAutoInstrumentationCRD({endpoint:otlpEndpoint ||process.env.OPENLIT_OTLP_ENDPOINT ||- "http://otel-collector:4318",+ "http://openlit:4318",},},};diff --git a/src/client/src/lib/platform/request/index.ts b/src/client/src/lib/platform/request/index.tsindex d781319d..977ad565 100644--- a/src/client/src/lib/platform/request/index.ts+++ b/src/client/src/lib/platform/request/index.ts@@ -224,10 +224,6 @@ export async function getRequestsConfig(params: MetricParams) {);select.push(- // OTel-standard environment is `ResourceAttributes['deployment.environment']`.- // The legacy `getTraceMappingKeyFullPath("environment")` returns a- // dotted SpanAttributes path that, wrapped in `ResourceAttributes[...]`,- // resolves to a non-existent key and silently yields no values.`arrayFilter(x -> x != '', ARRAY_AGG(DISTINCT ResourceAttributes['deployment.environment'])) AS environments`);diff --git a/src/client/src/lib/platform/traces/read.ts b/src/client/src/lib/platform/traces/read.tsindex e48017f8..ac12e237 100644--- a/src/client/src/lib/platform/traces/read.ts+++ b/src/client/src/lib/platform/traces/read.ts@@ -759,6 +759,18 @@ export async function getTraceFilterConfig(params: MetricParams) {if (!applicationNames.length) {applicationNames = distinctFromSpans(spans, "service.name");}+ let environments: string[] = [];+ try {+ environments = await adapter.distinctValues(+ "deployment.environment",+ query+ );+ } catch {+ environments = distinctFromSpans(spans, "deployment.environment");+ }+ if (!environments.length) {+ environments = distinctFromSpans(spans, "deployment.environment");+ }return {err: null,data: [@@ -769,6 +781,7 @@ export async function getTraceFilterConfig(params: MetricParams) {spanNames,applicationNames,traceTypes,+ environments,},],};diff --git a/src/client/src/lib/rbac/routes/api/fleet-hub/[id]/[...path]/route.ts b/src/client/src/lib/rbac/routes/api/fleet-hub/[id]/[...path]/route.tsnew file mode 100644index 00000000..4969ca76--- /dev/null+++ b/src/client/src/lib/rbac/routes/api/fleet-hub/[id]/[...path]/route.ts@@ -0,0 +1,7 @@+import { fleetHubUnavailable } from "../../unavailable";++export const GET = fleetHubUnavailable;+export const POST = fleetHubUnavailable;+export const PUT = fleetHubUnavailable;+export const PATCH = fleetHubUnavailable;+export const DELETE = fleetHubUnavailable;diff --git a/src/client/src/lib/rbac/routes/api/fleet-hub/[id]/config/route.ts b/src/client/src/lib/rbac/routes/api/fleet-hub/[id]/config/route.tsnew file mode 100644index 00000000..6404b785--- /dev/null+++ b/src/client/src/lib/rbac/routes/api/fleet-hub/[id]/config/route.ts@@ -0,0 +1,4 @@+import { fleetHubUnavailable } from "../../unavailable";++export const GET = fleetHubUnavailable;+export const POST = fleetHubUnavailable;diff --git a/src/client/src/lib/rbac/routes/api/fleet-hub/[id]/route.ts b/src/client/src/lib/rbac/routes/api/fleet-hub/[id]/route.tsnew file mode 100644index 00000000..7195956b--- /dev/null+++ b/src/client/src/lib/rbac/routes/api/fleet-hub/[id]/route.ts@@ -0,0 +1,3 @@+import { fleetHubUnavailable } from "../unavailable";++export const GET = fleetHubUnavailable;diff --git a/src/client/src/lib/rbac/routes/api/fleet-hub/[id]/tls-connection/route.ts b/src/client/src/lib/rbac/routes/api/fleet-hub/[id]/tls-connection/route.tsnew file mode 100644index 00000000..6404b785--- /dev/null+++ b/src/client/src/lib/rbac/routes/api/fleet-hub/[id]/tls-connection/route.ts@@ -0,0 +1,4 @@+import { fleetHubUnavailable } from "../../unavailable";++export const GET = fleetHubUnavailable;+export const POST = fleetHubUnavailable;diff --git a/src/client/src/lib/rbac/routes/api/fleet-hub/route.ts b/src/client/src/lib/rbac/routes/api/fleet-hub/route.tsnew file mode 100644index 00000000..14a37634--- /dev/null+++ b/src/client/src/lib/rbac/routes/api/fleet-hub/route.ts@@ -0,0 +1,3 @@+import { fleetHubUnavailable } from "./unavailable";++export const GET = fleetHubUnavailable;diff --git a/src/client/src/lib/rbac/routes/api/fleet-hub/unavailable.ts b/src/client/src/lib/rbac/routes/api/fleet-hub/unavailable.tsnew file mode 100644index 00000000..5a7078cf--- /dev/null+++ b/src/client/src/lib/rbac/routes/api/fleet-hub/unavailable.ts@@ -0,0 +1,8 @@+import getMessage from "@/constants/messages";++export function fleetHubUnavailable() {+ return Response.json(+ { error: getMessage().FLEET_HUB_UNAVAILABLE },+ { status: 402 }+ );+}diff --git a/src/client/src/lib/telemetry-source.ts b/src/client/src/lib/telemetry-source.tsindex 0bfef0e3..1dfe1744 100644--- a/src/client/src/lib/telemetry-source.ts+++ b/src/client/src/lib/telemetry-source.ts@@ -373,30 +373,10 @@ export async function resolveSignalSource(// otherwise changing environments can silently read another backend.}- // 3. Preserve the legacy implicit ClickHouse behavior only when the caller- // did not explicitly select an environment. Environment-scoped requests- // must fail closed when that environment has no routed connector.- if (hasRequestedEnvironment) {- return {- descriptor: {- type: "clickhouse",- id: "builtin:none",- isBuiltIn: true,- settings: {},- secretRef: null,- dbConfigId: undefined,- signals: [...ALL_SIGNALS],- projectId,- name: "ClickHouse",- environment,- },- servesSignal: false,- hasSource: false,- via: "none",- };- }-- // 4. Built-in ClickHouse (serves all signals when configured).+ // 3. Built-in ClickHouse for the project. OpenLIT environments+ // (`demo` vs `production`) are row filters on traces, not separate+ // databases, unless this environment has an explicit binding above.+ // Fail closed only when the project has no ClickHouse at all.const builtin = activeDatabase? builtInDescriptor(activeDatabase): await resolveBuiltInDescriptor(options.dbConfigId);diff --git a/src/client/src/middleware.ts b/src/client/src/middleware.tsindex 80452e03..7f19852a 100644--- a/src/client/src/middleware.ts+++ b/src/client/src/middleware.ts@@ -44,8 +44,6 @@ export const config = {"/home","/dashboards/:path*","/d/:path*",- "/fleet-hub",- "/fleet-hub/:path*","/agents","/agents/:path*","/onboarding",diff --git a/src/client/src/middleware/check-auth.ts b/src/client/src/middleware/check-auth.tsindex 3f5bf384..618d2bf1 100644--- a/src/client/src/middleware/check-auth.ts+++ b/src/client/src/middleware/check-auth.ts@@ -147,6 +147,31 @@ export default function checkAuth(next: NextMiddleware) {const data = await res.json();if (data.valid && data.databaseConfigId) {requestHeaders.set("x-database-config-id", data.databaseConfigId);+ if (data.organisationId) {+ requestHeaders.set(+ "x-openlit-organisation-id",+ String(data.organisationId)+ );+ } else {+ requestHeaders.delete("x-openlit-organisation-id");+ }+ if (data.projectId) {+ requestHeaders.set(+ "x-openlit-project-id",+ String(data.projectId)+ );+ } else {+ requestHeaders.delete("x-openlit-project-id");+ }+ if (data.environment) {+ requestHeaders.set(+ "x-openlit-environment",+ String(data.environment)+ );+ } else {+ requestHeaders.delete("x-openlit-environment");+ }+ requestHeaders.delete("x-openlit-database-config-id");return next(new NextRequest(request, {headers: requestHeaders,diff --git a/src/client/src/store/filter.ts b/src/client/src/store/filter.tsindex cba2b0bc..7f2ee587 100644--- a/src/client/src/store/filter.ts+++ b/src/client/src/store/filter.ts@@ -71,6 +71,20 @@ export function getTimeLimitObject(return object;}+function mergeDefinedConfig(+ prev: Partial<FilterConfig>,+ next: Partial<FilterConfig>+): Partial<FilterConfig> {+ const merged: Partial<FilterConfig> = { ...prev };+ (Object.keys(next) as Array<keyof FilterConfig>).forEach((key) => {+ const value = next[key];+ if (value !== undefined) {+ (merged as Record<string, unknown>)[key] = value;+ }+ });+ return merged;+}+const INITIAL_FILTER_DETAILS: FilterType = {timeLimit: {type: DEFAULT_TIME_RANGE,@@ -160,12 +174,14 @@ export const filterStoreSlice: FilterStore = lens((setStore, getStore) => ({// — notably `serviceNames`, which AgentScopeProvider sets// for the agent-detail page — survive wholesale updates// from TracesFilter (URL/localStorage restore, "Apply"- // button). Wholesale clears go through `clearFilter`- // above; nothing else should drop sibling fields.- {- ...getStore().details.selectedConfig,- ...object.selectedConfig,- }+ // button). Skip `undefined` values: Apply used to pass+ // `customFilters: undefined` when a row was incomplete,+ // which deleted the custom-attributes UI. Wholesale+ // clears go through `clearFilter` above.+ mergeDefinedConfig(+ getStore().details.selectedConfig,+ object.selectedConfig+ ): getStore().details.selectedConfig,},config: resetConfig ? undefined : getStore().config,diff --git a/src/client/src/utils/api.ts b/src/client/src/utils/api.tsindex 62ede9e6..eb7ba68e 100644--- a/src/client/src/utils/api.ts+++ b/src/client/src/utils/api.ts@@ -1,6 +1,7 @@import { useRootStore } from "@/store";import { OPENLIT_CONTEXT_HEADERS } from "@/constants/openlit-context";import { isCacheableTelemetryUrl, withTelemetryRequestCache } from "@/utils/telemetry-request-cache";+import getMessage from "@/constants/messages";type GET_DATA = {body?: string;@@ -18,14 +19,60 @@ function getActiveDatabaseConfigId() {);}+function isHtmlLike(value: string) {+ const text = value.trim().toLowerCase();+ return (+ text.startsWith("<!doctype") ||+ text.startsWith("<html") ||+ text.includes("data-next-hide-fouc")+ );+}++/** Never surface raw HTML documents (login pages, Next.js FOUC shells) as API errors. */+export function clientFetchErrorMessage(error: unknown, status?: number) {+ const messages = getMessage();+ const text =+ typeof error === "string"+ ? error+ : error instanceof Error+ ? error.message+ : error == null+ ? ""+ : String(error);++ if (!text.trim() || isHtmlLike(text)) {+ return messages.API_UNEXPECTED_HTML_RESPONSE(status);+ }+ return text;+}++function throwIfFailedResponse(parsed: unknown, status: number, ok: boolean) {+ if (ok) return;+ const messages = getMessage();+ if (typeof parsed === "string") {+ throw new Error(clientFetchErrorMessage(parsed, status));+ }+ const body = parsed as { err?: string; error?: string; message?: string } | null;+ throw new Error(+ clientFetchErrorMessage(+ body?.err || body?.error || body?.message || messages.API_REQUEST_FAILED(status),+ status+ )+ );+}+/** Parse fetch body; successful responses must be JSON (not HTML login pages). */-function parseFetchBody(raw: string, ok: boolean) {+function parseFetchBody(raw: string, ok: boolean, status: number) {if (!raw.trim()) return null;try {return JSON.parse(raw);} catch {+ const messages = getMessage();+ if (isHtmlLike(raw)) {+ throw new Error(messages.API_UNEXPECTED_HTML_RESPONSE(status));+ }if (ok) {- throw new Error("Unexpected non-JSON response from server.");+ throw new Error(messages.API_UNEXPECTED_NON_JSON_RESPONSE);}return raw;}@@ -77,16 +124,9 @@ export async function getData({ body, method = "POST", url, data }: GET_DATA) {),});const raw = await res.text();- const parsed = parseFetchBody(raw, res.ok);- if (!res.ok) {- const error = parsed as any;- throw new Error(- typeof error === "string"- ? error- : error?.err || error?.error || error?.message || `Request failed (${res.status})`- );- }- if (parsed === null) throw new Error("The telemetry service returned an empty response.");+ const parsed = parseFetchBody(raw, res.ok, res.status);+ throwIfFailedResponse(parsed, res.status, res.ok);+ if (parsed === null) throw new Error(getMessage().API_EMPTY_RESPONSE);return parsed;} finally {if (timeout !== undefined) clearTimeout(timeout);@@ -106,14 +146,9 @@ export async function postData({ url, data }: POST_DATA) {body: JSON.stringify(data),});const raw = await res.text();- const parsed = parseFetchBody(raw, res.ok);- if (!res.ok) {- const error = parsed as any;- throw new Error(- typeof error === "string" ? error : error?.err || error?.error || error?.message || `Request failed (${res.status})`- );- }- if (parsed === null) throw new Error("The telemetry service returned an empty response.");+ const parsed = parseFetchBody(raw, res.ok, res.status);+ throwIfFailedResponse(parsed, res.status, res.ok);+ if (parsed === null) throw new Error(getMessage().API_EMPTY_RESPONSE);return parsed;}@@ -127,13 +162,8 @@ export async function deleteData({ url }: DELETE_DATA) {headers: getRequestHeaders(),});const raw = await res.text();- const parsed = parseFetchBody(raw, res.ok);- if (!res.ok) {- const error = parsed as any;- throw new Error(- typeof error === "string" ? error : error?.err || error?.error || error?.message || `Request failed (${res.status})`- );- }- if (parsed === null) throw new Error("The telemetry service returned an empty response.");+ const parsed = parseFetchBody(raw, res.ok, res.status);+ throwIfFailedResponse(parsed, res.status, res.ok);+ if (parsed === null) throw new Error(getMessage().API_EMPTY_RESPONSE);return parsed;}diff --git a/src/client/src/utils/hooks/useFetchWrapper.ts b/src/client/src/utils/hooks/useFetchWrapper.tsindex 45c5261c..15cced20 100644--- a/src/client/src/utils/hooks/useFetchWrapper.ts+++ b/src/client/src/utils/hooks/useFetchWrapper.ts@@ -1,6 +1,6 @@import { get } from "lodash";import { useCallback, useRef, useState } from "react";-import { deleteData, getData } from "@/utils/api";+import { clientFetchErrorMessage, deleteData, getData } from "@/utils/api";import { FetchWrapperProps } from "@/types/fetch-wrapper";import { peekTelemetryRequestCache } from "@/utils/telemetry-request-cache";@@ -85,11 +85,12 @@ export default function useFetchWrapper<T>() {if (!isCurrent()) return { response, error };if (response.err) {- setError(response.err);+ const safeError = clientFetchErrorMessage(response.err);+ setError(safeError);// Preserve last-good data on transient errors so the page// doesn't blank; only clear when we have nothing to show.if (dataRef.current == null) applyData(null);- if (typeof failureCb === "function") failureCb(response.err);+ if (typeof failureCb === "function") failureCb(safeError);} else {const finalResponse = get(response, responseDataKey, response);applyData(finalResponse);@@ -103,7 +104,9 @@ export default function useFetchWrapper<T>() {error = (errorResp as any).message || (errorResp as any).error;}- const updatedError = (error as any)?.toString().replaceAll("Error:", "");+ const updatedError = clientFetchErrorMessage(+ (error as any)?.toString().replaceAll("Error:", "")+ );setError(updatedError);if (dataRef.current == null) applyData(null);if (typeof failureCb === "function") failureCb(updatedError);diff --git a/src/dev-docker-compose.yml b/src/dev-docker-compose.ymlindex 29397cdd..875ca857 100644--- a/src/dev-docker-compose.yml+++ b/src/dev-docker-compose.yml@@ -48,15 +48,6 @@ services:GOOGLE_CLIENT_SECRET: ${GOOGLE_CLIENT_SECRET:-}GITHUB_CLIENT_ID: ${GITHUB_CLIENT_ID:-}GITHUB_CLIENT_SECRET: ${GITHUB_CLIENT_SECRET:-}-- # TLS Configuration (user-configurable)- OPAMP_ENVIRONMENT: ${OPAMP_ENVIRONMENT:-production}- OPAMP_CERTS_DIR: "/app/opamp/certs"- OPAMP_TLS_INSECURE_SKIP_VERIFY: ${OPAMP_TLS_INSECURE_SKIP_VERIFY:-false}- OPAMP_TLS_REQUIRE_CLIENT_CERT: ${OPAMP_TLS_REQUIRE_CLIENT_CERT:-true}- OPAMP_TLS_MIN_VERSION: ${OPAMP_TLS_MIN_VERSION:-1.2}- OPAMP_TLS_MAX_VERSION: ${OPAMP_TLS_MAX_VERSION:-1.3}- OPAMP_LOG_LEVEL: ${OPAMP_LOG_LEVEL:-info}env_file:- path: .env@@ -70,7 +61,6 @@ services:condition: service_healthyvolumes:- openlit-data:/app/client/data- - ../assets/otel-collector-config.yaml:/etc/otel/otel-collector-config.yamlhealthcheck:test: ["CMD", "node", "-e", "fetch('http://localhost:3000').then(r=>{process.exit(r.ok?0:1)}).catch(()=>process.exit(1))"]interval: 5sdiff --git a/src/otlp-receiver/AGENTS.md b/src/otlp-receiver/AGENTS.mdnew file mode 100644index 00000000..8a89ba4d--- /dev/null+++ b/src/otlp-receiver/AGENTS.md@@ -0,0 +1,7 @@+# OTLP receiver instructions++These instructions supplement the repository-root `AGENTS.md`.++- This is an independent Go module. Validate with `go test ./...`.+- Do not log API keys, Authorization headers, or ClickHouse passwords.+- Tenant writes must stay isolated by OpenLIT API key → organisation → project → environment → DatabaseConfig.diff --git a/src/otlp-receiver/README.md b/src/otlp-receiver/README.mdnew file mode 100644index 00000000..04546cbd--- /dev/null+++ b/src/otlp-receiver/README.md@@ -0,0 +1,32 @@+# OpenLIT OTLP receiver++First-party OTLP HTTP (`:4318`) and gRPC (`:4317`) ingest that writes ClickHouse `otel_*` tables. The OpenLIT image no longer ships the OpenTelemetry Collector.++## Endpoints++- `POST /v1/traces`, `POST /v1/metrics`, `POST /v1/logs` (protobuf or JSON, gzip optional)+- gRPC OTLP on `:4317`+- `GET /health`++The OpenLIT UI proxies the same `/v1/*` paths from the app origin to this process.++Supported metric types: gauge, sum, histogram, summary, and exponential histogram (`otel_metrics_*` tables).++On first ingest, and again if an insert fails because `otel_*` tables were dropped, the receiver runs `CREATE TABLE IF NOT EXISTS` for the OTLP schema and retries the write once. Compose/Kubernetes ClickHouse init scripts still create the same tables on an empty volume.++## Tenant routing++Send `Authorization: Bearer <openlit-api-key>` (or `x-openlit-api-key`). The key is scoped to organisation → project → environment and resolves to the bound `DatabaseConfig` ClickHouse. Client `x-database-config-id` headers are ignored.++A valid key stamps `organisation.environment.name`, `openlit.organisation.id`, and `openlit.project.id`. It does **not** overwrite client `deployment.environment`.++Without a key, data is written to `INIT_DB_*` unless `OTLP_REQUIRE_API_KEY=true`. That fallback does not invent org/project/environment attributes.++## Local next dev++```bash+cd src/otlp-receiver+SQLITE_DATABASE_URL=file:../client/prisma/dev.db \+INIT_DB_HOST=127.0.0.1 INIT_DB_PORT=8123 INIT_DB_DATABASE=openlit \+go run ./cmd/otlp-receiver+```diff --git a/src/otlp-receiver/cmd/otlp-receiver/main.go b/src/otlp-receiver/cmd/otlp-receiver/main.gonew file mode 100644index 00000000..0205b74f--- /dev/null+++ b/src/otlp-receiver/cmd/otlp-receiver/main.go@@ -0,0 +1,59 @@+package main++import (+ "context"+ "log"+ "net/http"+ "os"+ "os/signal"+ "syscall"+ "time"++ "github.com/openlit/openlit/otlp-receiver/internal/chwrite"+ "github.com/openlit/openlit/otlp-receiver/internal/config"+ "github.com/openlit/openlit/otlp-receiver/internal/grpcserver"+ "github.com/openlit/openlit/otlp-receiver/internal/httpserver"+ "github.com/openlit/openlit/otlp-receiver/internal/ingest"+ "github.com/openlit/openlit/otlp-receiver/internal/tenant"+)++func main() {+ cfg := config.Load()+ tenants, err := tenant.Open(cfg)+ if err != nil {+ log.Fatalf("tenant store: %v", err)+ }+ defer tenants.Close()++ svc := &ingest.Service{Tenants: tenants, Writer: chwrite.New()}+ defer svc.Writer.Close()++ httpSrv := &http.Server{+ Addr: cfg.HTTPAddr,+ Handler: httpserver.New(svc),+ ReadHeaderTimeout: 10 * time.Second,+ }+ grpcSrv := grpcserver.New(svc)++ go func() {+ log.Printf("OTLP HTTP listening on %s", cfg.HTTPAddr)+ if err := httpSrv.ListenAndServe(); err != nil && err != http.ErrServerClosed {+ log.Fatalf("otlp http: %v", err)+ }+ }()+ go func() {+ log.Printf("OTLP gRPC listening on %s", cfg.GRPCAddr)+ if err := grpcserver.Listen(cfg.GRPCAddr, grpcSrv); err != nil {+ log.Fatalf("otlp grpc: %v", err)+ }+ }()++ stop := make(chan os.Signal, 1)+ signal.Notify(stop, syscall.SIGINT, syscall.SIGTERM)+ <-stop++ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)+ defer cancel()+ _ = httpSrv.Shutdown(ctx)+ grpcSrv.GracefulStop()+}diff --git a/src/otlp-receiver/go.mod b/src/otlp-receiver/go.modnew file mode 100644index 00000000..9ff18653--- /dev/null+++ b/src/otlp-receiver/go.mod@@ -0,0 +1,40 @@+module github.com/openlit/openlit/otlp-receiver++go 1.25.0++require (+ github.com/ClickHouse/clickhouse-go/v2 v2.48.0+ go.opentelemetry.io/proto/otlp v1.11.0+ google.golang.org/grpc v1.83.2+ google.golang.org/protobuf v1.36.12+ modernc.org/sqlite v1.58.0+)++require (+ github.com/ClickHouse/ch-go v0.74.0 // indirect+ github.com/andybalholm/brotli v1.2.2 // indirect+ github.com/cespare/xxhash/v2 v2.3.0 // indirect+ github.com/dustin/go-humanize v1.0.1 // indirect+ github.com/go-faster/city v1.0.1 // indirect+ github.com/go-faster/errors v0.7.1 // indirect+ github.com/google/uuid v1.6.0 // indirect+ github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect+ github.com/klauspost/compress v1.19.1 // indirect+ github.com/mattn/go-isatty v0.0.24 // indirect+ github.com/ncruces/go-strftime v1.0.0 // indirect+ github.com/paulmach/orb v0.13.0 // indirect+ github.com/pierrec/lz4/v4 v4.1.27 // indirect+ github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect+ github.com/segmentio/asm v1.2.1 // indirect+ github.com/shopspring/decimal v1.4.0 // indirect+ go.opentelemetry.io/otel v1.44.0 // indirect+ go.opentelemetry.io/otel/trace v1.44.0 // indirect+ golang.org/x/net v0.58.0 // indirect+ golang.org/x/sys v0.47.0 // indirect+ golang.org/x/text v0.41.0 // indirect+ google.golang.org/genproto/googleapis/api v0.0.0-20260720211330-0afa2a65878a // indirect+ google.golang.org/genproto/googleapis/rpc v0.0.0-20260720211330-0afa2a65878a // indirect+ modernc.org/libc v1.75.6 // indirect+ modernc.org/mathutil v1.7.1 // indirect+ modernc.org/memory v1.12.1 // indirect+)diff --git a/src/otlp-receiver/go.sum b/src/otlp-receiver/go.sumnew file mode 100644index 00000000..93c9d6f1--- /dev/null+++ b/src/otlp-receiver/go.sum@@ -0,0 +1,120 @@+github.com/ClickHouse/ch-go v0.74.0 h1:uYs2m4wIt0ZHSM1E72rg0maCfzhR2V3xWb/vZEgpeWE=+github.com/ClickHouse/ch-go v0.74.0/go.mod h1:sZ/r+8ttZMjyrP9PuFbgoVbth1ywIu2LIQNA2vgko6M=+github.com/ClickHouse/clickhouse-go/v2 v2.48.0 h1:auzd4VkapQYhQF8F2Gog7s3x78Bi1JZmByxGbrw3C+4=+github.com/ClickHouse/clickhouse-go/v2 v2.48.0/go.mod h1:lBjUCPRG6RpRQdMbkXq+JV8rY0/O5lw+Z7jShgReFjM=+github.com/andybalholm/brotli v1.2.2 h1:HzTuoo2ErYQqf5qvcJInB8uvqSVxRttzkFexPWtnceM=+github.com/andybalholm/brotli v1.2.2/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=+github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=+github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=+github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=+github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=+github.com/go-faster/city v1.0.1 h1:4WAxSZ3V2Ws4QRDrscLEDcibJY8uf41H6AhXDrNDcGw=+github.com/go-faster/city v1.0.1/go.mod h1:jKcUJId49qdW3L1qKHH/3wPeUstCVpVSXTM6vO3VcTw=+github.com/go-faster/errors v0.7.1 h1:MkJTnDoEdi9pDabt1dpWf7AA8/BaSYZqibYyhZ20AYg=+github.com/go-faster/errors v0.7.1/go.mod h1:5ySTjWFiphBs07IKuiL69nxdfd5+fzh1u7FPGZP2quo=+github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=+github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=+github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=+github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=+github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=+github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=+github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=+github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=+github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 h1:LMLX+LgTNWpfvCBdFebv6EsYotImrt/Ppc5cXIriCSo=+github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk=+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=+github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 h1:5VipnvEpbqr2gA2VbM+nYVbkIF28c5ZQfqCBQ5g2xfk=+github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0/go.mod h1:Hyl3n6Twe1hvtd9XUXDec4pTvgMSEixRuQKPTMH2bNs=+github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=+github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=+github.com/klauspost/compress v1.19.1 h1:VsB4HPswih7mmZ8WleSFQ75c/Ui1M4trX5oAsJnhSlk=+github.com/klauspost/compress v1.19.1/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=+github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI=+github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A=+github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=+github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=+github.com/paulmach/orb v0.13.0 h1:r7n7mQGGF+cj/CbcivEj9J3HGK+XR+yXnvzRdq9saIw=+github.com/paulmach/orb v0.13.0/go.mod h1:6scRWINywA2Jf05dcjOfLfxrUIMECvTSG2MVbRLxu/k=+github.com/pierrec/lz4/v4 v4.1.27 h1:+PhzhWDrjRj89TH2sw43nE3+4+W8lSxIuQadEHZyjUk=+github.com/pierrec/lz4/v4 v4.1.27/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4=+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=+github.com/segmentio/asm v1.2.1 h1:DTNbBqs57ioxAD4PrArqftgypG4/qNpXoJx8TVXxPR0=+github.com/segmentio/asm v1.2.1/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs=+github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k=+github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME=+github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=+github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=+github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU=+github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=+go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=+go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=+go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU=+go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc=+go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc=+go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo=+go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58=+go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0=+go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI=+go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA=+go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk=+go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE=+go.opentelemetry.io/proto/otlp v1.11.0 h1:5rrYs0Ykyj50sdU/JU0x8etU+LubXWb+gED6TbEdMIk=+go.opentelemetry.io/proto/otlp v1.11.0/go.mod h1:SmVizdCOAm3XBtG1g1NnOdhW6jtddT72hLMhv8VwA8E=+golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk=+golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40=+golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To=+golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU=+golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=+golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=+golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=+golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=+golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8=+golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M=+golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE=+golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk=+gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=+gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=+google.golang.org/genproto/googleapis/api v0.0.0-20260720211330-0afa2a65878a h1:97PfJ4tCxY5C7NzzgGqQEMZmXbISdvSArNNEOoUGKBg=+google.golang.org/genproto/googleapis/api v0.0.0-20260720211330-0afa2a65878a/go.mod h1:1brfde68Npq6+WA75c1EHWPijZEG1kMus61ygPZfn4A=+google.golang.org/genproto/googleapis/rpc v0.0.0-20260720211330-0afa2a65878a h1:qI/YMH1ep2qQtqcp00gMQyoU7mjvbhg88GJKCvfoLj0=+google.golang.org/genproto/googleapis/rpc v0.0.0-20260720211330-0afa2a65878a/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=+google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU=+google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8=+google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc=+google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=+modernc.org/cc/v4 v4.29.2 h1:h6+9ciCnPKutf4I03CvheAvDLX7+IHlqR6Iy6J+cgd8=+modernc.org/cc/v4 v4.29.2/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI=+modernc.org/ccgo/v4 v4.35.0 h1:F+TUsmw09QxLzmi3aeYYGxjAXarmZaKgj3mKQHNaA8w=+modernc.org/ccgo/v4 v4.35.0/go.mod h1:qrVGs9S3Sr2Ztcg9ve+kTAYMp5a3YvWjo+SoN06kJ5I=+modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM=+modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU=+modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI=+modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito=+modernc.org/gc/v3 v3.1.5 h1:21ldfPfRYE31Tb7B3mwAK8gy1AxP4+dKjrOQPfqakoc=+modernc.org/gc/v3 v3.1.5/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY=+modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks=+modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI=+modernc.org/libc v1.75.6 h1:yKk8qo+Di4gkmvRboK8ocCqH22FiUCR6jRy2OwtCRus=+modernc.org/libc v1.75.6/go.mod h1:bO5o2ztHxBb2rjz0PgdHN0sSMw57CgxGFLZ3Qd/QpVQ=+modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=+modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=+modernc.org/memory v1.12.1 h1:nFMiWrpStgZczNl6XI9GnIk/rWhYIyHGUaR04pGbp9g=+modernc.org/memory v1.12.1/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw=+modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg=+modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns=+modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w=+modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE=+modernc.org/sqlite v1.58.0 h1:38u40/bwkfM7f0Myhosl+SEMltSDxnGdQf8o6Kjmys0=+modernc.org/sqlite v1.58.0/go.mod h1:rsD2CckafgObKC4DhBlGBf+RiHxkc3hINGt1Xw32tVY=+modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0=+modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A=+modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=+modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=diff --git a/src/otlp-receiver/internal/chschema/schema.go b/src/otlp-receiver/internal/chschema/schema.gonew file mode 100644index 00000000..3414c785--- /dev/null+++ b/src/otlp-receiver/internal/chschema/schema.go@@ -0,0 +1,262 @@+package chschema++import (+ "context"+ "fmt"+ "strings"+ "sync"++ "github.com/ClickHouse/clickhouse-go/v2/lib/driver"+)++const TTLHours = 730++var ensured sync.Map++func Ensure(ctx context.Context, conn driver.Conn, fingerprint string) error {+ if fingerprint != "" {+ if _, ok := ensured.Load(fingerprint); ok {+ return nil+ }+ }+ err := createAll(ctx, conn)+ if err != nil {+ if isPrivilegeError(err) {+ ok, existsErr := tablesExist(ctx, conn)+ if existsErr != nil {+ return fmt.Errorf("schema bootstrap denied and existence check failed: %w", existsErr)+ }+ if !ok {+ return fmt.Errorf("clickhouse user cannot CREATE TABLE and otel_* tables are missing")+ }+ if fingerprint != "" {+ ensured.Store(fingerprint, struct{}{})+ }+ return nil+ }+ return err+ }+ if fingerprint != "" {+ ensured.Store(fingerprint, struct{}{})+ }+ return nil+}++func Forget(fingerprint string) {+ if fingerprint != "" {+ ensured.Delete(fingerprint)+ }+}++func IsMissingTable(err error) bool {+ if err == nil {+ return false+ }+ msg := strings.ToUpper(err.Error())+ return strings.Contains(msg, "UNKNOWN_TABLE") ||+ strings.Contains(msg, "UNKNOWN TABLE") ||+ strings.Contains(msg, "DOESN'T EXIST") ||+ strings.Contains(msg, "DOES NOT EXIST")+}++func RequiredTables() []string {+ return []string{+ "otel_traces",+ "otel_logs",+ "otel_metrics_gauge",+ "otel_metrics_sum",+ "otel_metrics_histogram",+ "otel_metrics_summary",+ "otel_metrics_exponential_histogram",+ "otel_traces_trace_id_ts",+ }+}++func createAll(ctx context.Context, conn driver.Conn) error {+ for _, stmt := range Statements() {+ if err := conn.Exec(ctx, stmt); err != nil {+ return err+ }+ }+ return nil+}++func tablesExist(ctx context.Context, conn driver.Conn) (bool, error) {+ for _, table := range RequiredTables() {+ var exists uint8+ if err := conn.QueryRow(ctx, "EXISTS TABLE "+table).Scan(&exists); err != nil {+ return false, err+ }+ if exists != 1 {+ return false, nil+ }+ }+ return true, nil+}++func isPrivilegeError(err error) bool {+ if err == nil {+ return false+ }+ msg := strings.ToUpper(err.Error())+ return strings.Contains(msg, "ACCESS_DENIED") ||+ strings.Contains(msg, "NOT_ENOUGH_PRIVILEGES") ||+ strings.Contains(msg, "NOT ENOUGH PRIVILEGES")+}++func Statements() []string {+ ttl := TTLHours+ return []string{+ fmt.Sprintf(`CREATE TABLE IF NOT EXISTS otel_traces+(+ Timestamp DateTime64(9) CODEC(Delta(8), ZSTD(1)),+ TraceId String CODEC(ZSTD(1)),+ SpanId String CODEC(ZSTD(1)),+ ParentSpanId String CODEC(ZSTD(1)),+ TraceState String CODEC(ZSTD(1)),+ SpanName LowCardinality(String) CODEC(ZSTD(1)),+ SpanKind LowCardinality(String) CODEC(ZSTD(1)),+ ServiceName LowCardinality(String) CODEC(ZSTD(1)),+ ResourceAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),+ ScopeName String CODEC(ZSTD(1)),+ ScopeVersion String CODEC(ZSTD(1)),+ SpanAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),+ Duration UInt64 CODEC(ZSTD(1)),+ StatusCode LowCardinality(String) CODEC(ZSTD(1)),+ StatusMessage String CODEC(ZSTD(1)),+ `+"`Events.Timestamp`"+` Array(DateTime64(9)) CODEC(ZSTD(1)),+ `+"`Events.Name`"+` Array(LowCardinality(String)) CODEC(ZSTD(1)),+ `+"`Events.Attributes`"+` Array(Map(LowCardinality(String), String)) CODEC(ZSTD(1)),+ `+"`Links.TraceId`"+` Array(String) CODEC(ZSTD(1)),+ `+"`Links.SpanId`"+` Array(String) CODEC(ZSTD(1)),+ `+"`Links.TraceState`"+` Array(String) CODEC(ZSTD(1)),+ `+"`Links.Attributes`"+` Array(Map(LowCardinality(String), String)) CODEC(ZSTD(1)),+ INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1,+ INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,+ INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,+ INDEX idx_span_attr_key mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,+ INDEX idx_span_attr_value mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,+ INDEX idx_duration Duration TYPE minmax GRANULARITY 1+)+ENGINE = MergeTree+PARTITION BY toDate(Timestamp)+ORDER BY (ServiceName, SpanName, toDateTime(Timestamp))+TTL toDateTime(Timestamp) + toIntervalHour(%d)+SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1`, ttl),+ fmt.Sprintf(`CREATE TABLE IF NOT EXISTS otel_logs+(+ Timestamp DateTime64(9) CODEC(Delta(8), ZSTD(1)),+ TimestampTime DateTime DEFAULT toDateTime(Timestamp),+ TraceId String CODEC(ZSTD(1)),+ SpanId String CODEC(ZSTD(1)),+ TraceFlags UInt8,+ SeverityText LowCardinality(String) CODEC(ZSTD(1)),+ SeverityNumber UInt8,+ ServiceName LowCardinality(String) CODEC(ZSTD(1)),+ Body String CODEC(ZSTD(1)),+ ResourceSchemaUrl LowCardinality(String) CODEC(ZSTD(1)),+ ResourceAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),+ ScopeSchemaUrl LowCardinality(String) CODEC(ZSTD(1)),+ ScopeName String CODEC(ZSTD(1)),+ ScopeVersion LowCardinality(String) CODEC(ZSTD(1)),+ ScopeAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),+ LogAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),+ INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1,+ INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,+ INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,+ INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,+ INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,+ INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,+ INDEX idx_log_attr_value mapValues(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,+ INDEX idx_body Body TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 8+)+ENGINE = MergeTree+PARTITION BY toDate(TimestampTime)+PRIMARY KEY (ServiceName, TimestampTime)+ORDER BY (ServiceName, TimestampTime, Timestamp)+TTL TimestampTime + toIntervalHour(%d)+SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1`, ttl),+ metricTable("otel_metrics_gauge", `+ Value Float64 CODEC(ZSTD(1)),+ Flags UInt32 CODEC(ZSTD(1))`, ttl),+ metricTable("otel_metrics_sum", `+ Value Float64 CODEC(ZSTD(1)),+ Flags UInt32 CODEC(ZSTD(1)),+ AggregationTemporality Int32 CODEC(ZSTD(1)),+ IsMonotonic Bool CODEC(Delta(1), ZSTD(1))`, ttl),+ metricTable("otel_metrics_histogram", `+ Count UInt64 CODEC(Delta(8), ZSTD(1)),+ Sum Float64 CODEC(ZSTD(1)),+ BucketCounts Array(UInt64) CODEC(ZSTD(1)),+ ExplicitBounds Array(Float64) CODEC(ZSTD(1)),+ Flags UInt32 CODEC(ZSTD(1)),+ Min Float64 CODEC(ZSTD(1)),+ Max Float64 CODEC(ZSTD(1)),+ AggregationTemporality Int32 CODEC(ZSTD(1))`, ttl),+ metricTable("otel_metrics_summary", `+ Count UInt64 CODEC(Delta(8), ZSTD(1)),+ Sum Float64 CODEC(ZSTD(1)),+ `+"`ValueAtQuantiles.Quantile`"+` Array(Float64) CODEC(ZSTD(1)),+ `+"`ValueAtQuantiles.Value`"+` Array(Float64) CODEC(ZSTD(1)),+ Flags UInt32 CODEC(ZSTD(1))`, ttl),+ metricTable("otel_metrics_exponential_histogram", `+ Count UInt64 CODEC(Delta(8), ZSTD(1)),+ Sum Float64 CODEC(ZSTD(1)),+ Scale Int32 CODEC(ZSTD(1)),+ ZeroCount UInt64 CODEC(ZSTD(1)),+ PositiveOffset Int32 CODEC(ZSTD(1)),+ PositiveBucketCounts Array(UInt64) CODEC(ZSTD(1)),+ NegativeOffset Int32 CODEC(ZSTD(1)),+ NegativeBucketCounts Array(UInt64) CODEC(ZSTD(1)),+ Flags UInt32 CODEC(ZSTD(1)),+ Min Float64 CODEC(ZSTD(1)),+ Max Float64 CODEC(ZSTD(1)),+ AggregationTemporality Int32 CODEC(ZSTD(1))`, ttl),+ fmt.Sprintf(`CREATE TABLE IF NOT EXISTS otel_traces_trace_id_ts+(+ TraceId String CODEC(ZSTD(1)),+ Start DateTime CODEC(Delta(4), ZSTD(1)),+ End DateTime CODEC(Delta(4), ZSTD(1)),+ INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1+)+ENGINE = MergeTree+PARTITION BY toDate(Start)+ORDER BY (TraceId, Start)+TTL toDateTime(Start) + toIntervalHour(%d)+SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1`, ttl),+ `CREATE MATERIALIZED VIEW IF NOT EXISTS otel_traces_trace_id_ts_mv TO otel_traces_trace_id_ts+AS SELECT+ TraceId,+ min(Timestamp) AS Start,+ max(Timestamp) AS End+FROM otel_traces+WHERE TraceId != ''+GROUP BY TraceId`,+ }+}++func metricTable(name, extra string, ttl int) string {+ return fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s+(+ ResourceAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),+ ResourceSchemaUrl String CODEC(ZSTD(1)),+ ScopeName String CODEC(ZSTD(1)),+ ScopeVersion String CODEC(ZSTD(1)),+ ScopeAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),+ ScopeDroppedAttrCount UInt32 CODEC(ZSTD(1)),+ ScopeSchemaUrl String CODEC(ZSTD(1)),+ ServiceName LowCardinality(String) CODEC(ZSTD(1)),+ MetricName String CODEC(ZSTD(1)),+ MetricDescription String CODEC(ZSTD(1)),+ MetricUnit String CODEC(ZSTD(1)),+ Attributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),+ StartTimeUnix DateTime64(9) CODEC(Delta(8), ZSTD(1)),+ TimeUnix DateTime64(9) CODEC(Delta(8), ZSTD(1)),+ %s+)+ENGINE = MergeTree+PARTITION BY toDate(TimeUnix)+ORDER BY (ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix))+TTL toDateTime(TimeUnix) + toIntervalHour(%d)+SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1`, name, strings.TrimSpace(extra), ttl)+}diff --git a/src/otlp-receiver/internal/chschema/schema_test.go b/src/otlp-receiver/internal/chschema/schema_test.gonew file mode 100644index 00000000..864f445f--- /dev/null+++ b/src/otlp-receiver/internal/chschema/schema_test.go@@ -0,0 +1,46 @@+package chschema_test++import (+ "errors"+ "strings"+ "testing"++ "github.com/openlit/openlit/otlp-receiver/internal/chschema"+)++func TestStatementsCoverOtelTables(t *testing.T) {+ t.Parallel()+ ddl := strings.Join(chschema.Statements(), "\n")+ if !strings.Contains(ddl, "`Events.Timestamp`") {+ t.Fatal("nested Event columns must be backtick-quoted for ClickHouse 24.4")+ }+ if !strings.Contains(ddl, "`ValueAtQuantiles.Quantile`") {+ t.Fatal("nested quantile columns must be backtick-quoted for ClickHouse 24.4")+ }+ for _, table := range chschema.RequiredTables() {+ if !strings.Contains(ddl, "CREATE TABLE IF NOT EXISTS "+table) {+ t.Fatalf("missing ddl for %s", table)+ }+ }+}++func TestIsMissingTable(t *testing.T) {+ t.Parallel()+ if chschema.IsMissingTable(nil) {+ t.Fatal("nil is not a missing table error")+ }+ if !chschema.IsMissingTable(errors.New("code: 60, message: Unknown table expression identifier 'otel_traces'")) {+ t.Fatal("UNKNOWN_TABLE-style errors must recreate schema")+ }+ if !chschema.IsMissingTable(errors.New("Table otel_metrics_histogram doesn't exist")) {+ t.Fatal("doesn't exist must recreate schema")+ }+ if chschema.IsMissingTable(errors.New("TOO_MANY_PARTS")) {+ t.Fatal("transient insert errors must not recreate schema")+ }+}++func TestForgetAllowsEnsureRetry(t *testing.T) {+ t.Parallel()+ chschema.Forget("test-fingerprint")+}diff --git a/src/otlp-receiver/internal/chwrite/writer.go b/src/otlp-receiver/internal/chwrite/writer.gonew file mode 100644index 00000000..733a963b--- /dev/null+++ b/src/otlp-receiver/internal/chwrite/writer.go@@ -0,0 +1,405 @@+package chwrite++import (+ "context"+ "fmt"+ "strconv"+ "sync"++ "github.com/ClickHouse/clickhouse-go/v2"+ "github.com/ClickHouse/clickhouse-go/v2/lib/driver"++ "github.com/openlit/openlit/otlp-receiver/internal/chschema"+ "github.com/openlit/openlit/otlp-receiver/internal/config"+ "github.com/openlit/openlit/otlp-receiver/internal/otlpconv"+)++type Writer struct {+ mu sync.Mutex+ conns map[string]driver.Conn+}++func New() *Writer {+ return &Writer{conns: map[string]driver.Conn{}}+}++func (w *Writer) Close() {+ w.mu.Lock()+ defer w.mu.Unlock()+ for _, c := range w.conns {+ _ = c.Close()+ }+ w.conns = map[string]driver.Conn{}+}++func protocolForPort(port string) clickhouse.Protocol {+ n, _ := strconv.Atoi(port)+ if n == 9000 || n == 9440 {+ return clickhouse.Native+ }+ return clickhouse.HTTP+}++func connKey(cfg config.ClickHouseConfig) string {+ return fmt.Sprintf("%s|%s|%s|%s|%s", cfg.Host, cfg.Port, cfg.Username, cfg.Database, cfg.Password)+}++func (w *Writer) conn(ctx context.Context, cfg config.ClickHouseConfig) (driver.Conn, error) {+ key := connKey(cfg)+ w.mu.Lock()+ if c, ok := w.conns[key]; ok {+ w.mu.Unlock()+ return c, nil+ }+ w.mu.Unlock()++ port := cfg.Port+ if port == "" {+ port = "8123"+ }+ c, err := clickhouse.Open(&clickhouse.Options{+ Addr: []string{fmt.Sprintf("%s:%s", cfg.Host, port)},+ Protocol: protocolForPort(port),+ Auth: clickhouse.Auth{+ Database: cfg.Database,+ Username: cfg.Username,+ Password: cfg.Password,+ },+ Compression: &clickhouse.Compression{Method: clickhouse.CompressionLZ4},+ })+ if err != nil {+ return nil, err+ }+ if err := c.Ping(ctx); err != nil {+ _ = c.Close()+ return nil, err+ }++ w.mu.Lock()+ if existing, ok := w.conns[key]; ok {+ w.mu.Unlock()+ _ = c.Close()+ return existing, nil+ }+ w.conns[key] = c+ w.mu.Unlock()+ return c, nil+}++func (w *Writer) withConn(ctx context.Context, cfg config.ClickHouseConfig, fn func(driver.Conn) error) error {+ c, err := w.conn(ctx, cfg)+ if err != nil {+ return err+ }+ key := connKey(cfg)+ if err := chschema.Ensure(ctx, c, key); err != nil {+ return err+ }+ err = fn(c)+ if !chschema.IsMissingTable(err) {+ return err+ }+ chschema.Forget(key)+ if err := chschema.Ensure(ctx, c, key); err != nil {+ return err+ }+ return fn(c)+}++func (w *Writer) InsertTraces(ctx context.Context, cfg config.ClickHouseConfig, rows []otlpconv.TraceRow) error {+ if len(rows) == 0 {+ return nil+ }+ return w.withConn(ctx, cfg, func(c driver.Conn) error {+ batch, err := c.PrepareBatch(ctx, `INSERT INTO otel_traces (+ Timestamp, TraceId, SpanId, ParentSpanId, TraceState, SpanName, SpanKind, ServiceName,+ ResourceAttributes, ScopeName, ScopeVersion, SpanAttributes, Duration, StatusCode, StatusMessage,+ Events.Timestamp, Events.Name, Events.Attributes, Links.TraceId, Links.SpanId, Links.TraceState, Links.Attributes+ )`)+ if err != nil {+ return err+ }+ for _, row := range rows {+ if err := batch.Append(+ row.Timestamp,+ row.TraceID,+ row.SpanID,+ row.ParentSpanID,+ row.TraceState,+ row.SpanName,+ row.SpanKind,+ row.ServiceName,+ row.ResourceAttributes,+ row.ScopeName,+ row.ScopeVersion,+ row.SpanAttributes,+ row.Duration,+ row.StatusCode,+ row.StatusMessage,+ row.EventTimestamps,+ row.EventNames,+ row.EventAttributes,+ row.LinkTraceIDs,+ row.LinkSpanIDs,+ row.LinkTraceStates,+ row.LinkAttributes,+ ); err != nil {+ return err+ }+ }+ return batch.Send()+ })+}++func (w *Writer) InsertLogs(ctx context.Context, cfg config.ClickHouseConfig, rows []otlpconv.LogRow) error {+ if len(rows) == 0 {+ return nil+ }+ return w.withConn(ctx, cfg, func(c driver.Conn) error {+ batch, err := c.PrepareBatch(ctx, `INSERT INTO otel_logs (+ Timestamp, TraceId, SpanId, TraceFlags, SeverityText, SeverityNumber, ServiceName, Body,+ ResourceSchemaUrl, ResourceAttributes, ScopeSchemaUrl, ScopeName, ScopeVersion, ScopeAttributes, LogAttributes+ )`)+ if err != nil {+ return err+ }+ for _, row := range rows {+ if err := batch.Append(+ row.Timestamp,+ row.TraceID,+ row.SpanID,+ row.TraceFlags,+ row.SeverityText,+ row.SeverityNumber,+ row.ServiceName,+ row.Body,+ row.ResourceSchemaURL,+ row.ResourceAttributes,+ row.ScopeSchemaURL,+ row.ScopeName,+ row.ScopeVersion,+ row.ScopeAttributes,+ row.LogAttributes,+ ); err != nil {+ return err+ }+ }+ return batch.Send()+ })+}++func (w *Writer) InsertGauges(ctx context.Context, cfg config.ClickHouseConfig, rows []otlpconv.GaugeRow) error {+ if len(rows) == 0 {+ return nil+ }+ return w.withConn(ctx, cfg, func(c driver.Conn) error {+ batch, err := c.PrepareBatch(ctx, `INSERT INTO otel_metrics_gauge (+ ResourceAttributes, ResourceSchemaUrl, ScopeName, ScopeVersion, ScopeAttributes, ScopeDroppedAttrCount,+ ScopeSchemaUrl, ServiceName, MetricName, MetricDescription, MetricUnit, Attributes, StartTimeUnix, TimeUnix, Value, Flags+ )`)+ if err != nil {+ return err+ }+ for _, row := range rows {+ if err := batch.Append(+ row.ResourceAttributes,+ row.ResourceSchemaURL,+ row.ScopeName,+ row.ScopeVersion,+ row.ScopeAttributes,+ row.ScopeDroppedCount,+ row.ScopeSchemaURL,+ row.ServiceName,+ row.MetricName,+ row.MetricDescription,+ row.MetricUnit,+ row.Attributes,+ row.StartTimeUnix,+ row.TimeUnix,+ row.Value,+ row.Flags,+ ); err != nil {+ return err+ }+ }+ return batch.Send()+ })+}++func (w *Writer) InsertSums(ctx context.Context, cfg config.ClickHouseConfig, rows []otlpconv.SumRow) error {+ if len(rows) == 0 {+ return nil+ }+ return w.withConn(ctx, cfg, func(c driver.Conn) error {+ batch, err := c.PrepareBatch(ctx, `INSERT INTO otel_metrics_sum (+ ResourceAttributes, ResourceSchemaUrl, ScopeName, ScopeVersion, ScopeAttributes, ScopeDroppedAttrCount,+ ScopeSchemaUrl, ServiceName, MetricName, MetricDescription, MetricUnit, Attributes, StartTimeUnix, TimeUnix,+ Value, Flags, AggregationTemporality, IsMonotonic+ )`)+ if err != nil {+ return err+ }+ for _, row := range rows {+ if err := batch.Append(+ row.ResourceAttributes,+ row.ResourceSchemaURL,+ row.ScopeName,+ row.ScopeVersion,+ row.ScopeAttributes,+ row.ScopeDroppedCount,+ row.ScopeSchemaURL,+ row.ServiceName,+ row.MetricName,+ row.MetricDescription,+ row.MetricUnit,+ row.Attributes,+ row.StartTimeUnix,+ row.TimeUnix,+ row.Value,+ row.Flags,+ row.AggregationTemporality,+ row.IsMonotonic,+ ); err != nil {+ return err+ }+ }+ return batch.Send()+ })+}++func (w *Writer) InsertHistograms(ctx context.Context, cfg config.ClickHouseConfig, rows []otlpconv.HistogramRow) error {+ if len(rows) == 0 {+ return nil+ }+ return w.withConn(ctx, cfg, func(c driver.Conn) error {+ batch, err := c.PrepareBatch(ctx, `INSERT INTO otel_metrics_histogram (+ ResourceAttributes, ResourceSchemaUrl, ScopeName, ScopeVersion, ScopeAttributes, ScopeDroppedAttrCount,+ ScopeSchemaUrl, ServiceName, MetricName, MetricDescription, MetricUnit, Attributes, StartTimeUnix, TimeUnix,+ Count, Sum, BucketCounts, ExplicitBounds, Flags, Min, Max, AggregationTemporality+ )`)+ if err != nil {+ return err+ }+ for _, row := range rows {+ if err := batch.Append(+ row.ResourceAttributes,+ row.ResourceSchemaURL,+ row.ScopeName,+ row.ScopeVersion,+ row.ScopeAttributes,+ row.ScopeDroppedCount,+ row.ScopeSchemaURL,+ row.ServiceName,+ row.MetricName,+ row.MetricDescription,+ row.MetricUnit,+ row.Attributes,+ row.StartTimeUnix,+ row.TimeUnix,+ row.Count,+ row.Sum,+ row.BucketCounts,+ row.ExplicitBounds,+ row.Flags,+ row.Min,+ row.Max,+ row.AggregationTemporality,+ ); err != nil {+ return err+ }+ }+ return batch.Send()+ })+}++func (w *Writer) InsertSummaries(ctx context.Context, cfg config.ClickHouseConfig, rows []otlpconv.SummaryRow) error {+ if len(rows) == 0 {+ return nil+ }+ return w.withConn(ctx, cfg, func(c driver.Conn) error {+ batch, err := c.PrepareBatch(ctx, `INSERT INTO otel_metrics_summary (+ ResourceAttributes, ResourceSchemaUrl, ScopeName, ScopeVersion, ScopeAttributes, ScopeDroppedAttrCount,+ ScopeSchemaUrl, ServiceName, MetricName, MetricDescription, MetricUnit, Attributes, StartTimeUnix, TimeUnix,+ Count, Sum, ValueAtQuantiles.Quantile, ValueAtQuantiles.Value, Flags+ )`)+ if err != nil {+ return err+ }+ for _, row := range rows {+ if err := batch.Append(+ row.ResourceAttributes,+ row.ResourceSchemaURL,+ row.ScopeName,+ row.ScopeVersion,+ row.ScopeAttributes,+ row.ScopeDroppedCount,+ row.ScopeSchemaURL,+ row.ServiceName,+ row.MetricName,+ row.MetricDescription,+ row.MetricUnit,+ row.Attributes,+ row.StartTimeUnix,+ row.TimeUnix,+ row.Count,+ row.Sum,+ row.Quantiles,+ row.Values,+ row.Flags,+ ); err != nil {+ return err+ }+ }+ return batch.Send()+ })+}++func (w *Writer) InsertExpHistograms(ctx context.Context, cfg config.ClickHouseConfig, rows []otlpconv.ExpHistogramRow) error {+ if len(rows) == 0 {+ return nil+ }+ return w.withConn(ctx, cfg, func(c driver.Conn) error {+ batch, err := c.PrepareBatch(ctx, `INSERT INTO otel_metrics_exponential_histogram (+ ResourceAttributes, ResourceSchemaUrl, ScopeName, ScopeVersion, ScopeAttributes, ScopeDroppedAttrCount,+ ScopeSchemaUrl, ServiceName, MetricName, MetricDescription, MetricUnit, Attributes, StartTimeUnix, TimeUnix,+ Count, Sum, Scale, ZeroCount, PositiveOffset, PositiveBucketCounts, NegativeOffset, NegativeBucketCounts,+ Flags, Min, Max, AggregationTemporality+ )`)+ if err != nil {+ return err+ }+ for _, row := range rows {+ if err := batch.Append(+ row.ResourceAttributes,+ row.ResourceSchemaURL,+ row.ScopeName,+ row.ScopeVersion,+ row.ScopeAttributes,+ row.ScopeDroppedCount,+ row.ScopeSchemaURL,+ row.ServiceName,+ row.MetricName,+ row.MetricDescription,+ row.MetricUnit,+ row.Attributes,+ row.StartTimeUnix,+ row.TimeUnix,+ row.Count,+ row.Sum,+ row.Scale,+ row.ZeroCount,+ row.PositiveOffset,+ row.PositiveBucketCounts,+ row.NegativeOffset,+ row.NegativeBucketCounts,+ row.Flags,+ row.Min,+ row.Max,+ row.AggregationTemporality,+ ); err != nil {+ return err+ }+ }+ return batch.Send()+ })+}diff --git a/src/otlp-receiver/internal/config/config.go b/src/otlp-receiver/internal/config/config.gonew file mode 100644index 00000000..9b722487--- /dev/null+++ b/src/otlp-receiver/internal/config/config.go@@ -0,0 +1,87 @@+package config++import (+ "os"+ "strconv"+ "strings"+)++type Config struct {+ HTTPAddr string+ GRPCAddr string+ SQLitePath string+ RequireAPIKey bool+ InitDB ClickHouseConfig+ TenantCacheTTLSec int+}++type ClickHouseConfig struct {+ Host string+ Port string+ Username string+ Password string+ Database string+}++func Load() Config {+ return Config{+ HTTPAddr: env("OTLP_HTTP_ADDR", "0.0.0.0:4318"),+ GRPCAddr: env("OTLP_GRPC_ADDR", "0.0.0.0:4317"),+ SQLitePath: sqlitePath(os.Getenv("SQLITE_DATABASE_URL")),+ RequireAPIKey: boolEnv("OTLP_REQUIRE_API_KEY", false),+ TenantCacheTTLSec: intEnv("OTLP_TENANT_CACHE_TTL_SEC", 30),+ InitDB: ClickHouseConfig{+ Host: env("INIT_DB_HOST", "127.0.0.1"),+ Port: env("INIT_DB_PORT", "8123"),+ Username: env("INIT_DB_USERNAME", "default"),+ Password: os.Getenv("INIT_DB_PASSWORD"),+ Database: env("INIT_DB_DATABASE", "openlit"),+ },+ }+}++func sqlitePath(raw string) string {+ raw = strings.TrimSpace(raw)+ if raw == "" {+ return ""+ }+ raw = strings.TrimPrefix(raw, "file:")+ if i := strings.Index(raw, "?"); i >= 0 {+ raw = raw[:i]+ }+ return raw+}++func env(key, fallback string) string {+ if v := strings.TrimSpace(os.Getenv(key)); v != "" {+ return v+ }+ return fallback+}++func boolEnv(key string, fallback bool) bool {+ v := strings.TrimSpace(os.Getenv(key))+ if v == "" {+ return fallback+ }+ switch strings.ToLower(v) {+ case "1", "true", "yes", "on":+ return true+ case "0", "false", "no", "off":+ return false+ default:+ return fallback+ }+}++func intEnv(key string, fallback int) int {+ v := strings.TrimSpace(os.Getenv(key))+ if v == "" {+ return fallback+ }+ n, err := strconv.Atoi(v)+ if err != nil {+ return fallback+ }+ return n+}diff --git a/src/otlp-receiver/internal/config/config_test.go b/src/otlp-receiver/internal/config/config_test.gonew file mode 100644index 00000000..ef3d9d59--- /dev/null+++ b/src/otlp-receiver/internal/config/config_test.go@@ -0,0 +1,18 @@+package config++import "testing"++func TestSqlitePath(t *testing.T) {+ t.Parallel()+ cases := map[string]string{+ "": "",+ "file:/app/client/data/data.db": "/app/client/data/data.db",+ "file:../data/data.db": "../data/data.db",+ "file:./dev.db?connection_limit=1": "./dev.db",+ }+ for in, want := range cases {+ if got := sqlitePath(in); got != want {+ t.Fatalf("sqlitePath(%q)=%q want %q", in, got, want)+ }+ }+}diff --git a/src/otlp-receiver/internal/grpcserver/server.go b/src/otlp-receiver/internal/grpcserver/server.gonew file mode 100644index 00000000..5097cf9c--- /dev/null+++ b/src/otlp-receiver/internal/grpcserver/server.go@@ -0,0 +1,92 @@+package grpcserver++import (+ "context"+ "net"++ "google.golang.org/grpc"+ "google.golang.org/grpc/codes"+ "google.golang.org/grpc/metadata"+ "google.golang.org/grpc/status"++ colllog "go.opentelemetry.io/proto/otlp/collector/logs/v1"+ collmetric "go.opentelemetry.io/proto/otlp/collector/metrics/v1"+ colltrace "go.opentelemetry.io/proto/otlp/collector/trace/v1"++ "github.com/openlit/openlit/otlp-receiver/internal/ingest"+ "github.com/openlit/openlit/otlp-receiver/internal/tenant"+)++type tracesServer struct {+ colltrace.UnimplementedTraceServiceServer+ svc *ingest.Service+}++type logsServer struct {+ colllog.UnimplementedLogsServiceServer+ svc *ingest.Service+}++type metricsServer struct {+ collmetric.UnimplementedMetricsServiceServer+ svc *ingest.Service+}++func New(svc *ingest.Service) *grpc.Server {+ s := grpc.NewServer()+ colltrace.RegisterTraceServiceServer(s, &tracesServer{svc: svc})+ colllog.RegisterLogsServiceServer(s, &logsServer{svc: svc})+ collmetric.RegisterMetricsServiceServer(s, &metricsServer{svc: svc})+ return s+}++func Listen(addr string, srv *grpc.Server) error {+ ln, err := net.Listen("tcp", addr)+ if err != nil {+ return err+ }+ return srv.Serve(ln)+}++func authorization(ctx context.Context) string {+ md, ok := metadata.FromIncomingContext(ctx)+ if !ok {+ return ""+ }+ values := md.Get("authorization")+ if len(values) == 0 {+ return ""+ }+ return values[0]+}++func mapErr(err error) error {+ if err == nil {+ return nil+ }+ if err == tenant.ErrUnauthorized || err == tenant.ErrNoTenant {+ return status.Error(codes.Unauthenticated, "unauthorized")+ }+ return status.Error(codes.Unavailable, "ingest failed")+}++func (s *tracesServer) Export(ctx context.Context, req *colltrace.ExportTraceServiceRequest) (*colltrace.ExportTraceServiceResponse, error) {+ if err := s.svc.Traces(ctx, authorization(ctx), req); err != nil {+ return nil, mapErr(err)+ }+ return &colltrace.ExportTraceServiceResponse{}, nil+}++func (s *logsServer) Export(ctx context.Context, req *colllog.ExportLogsServiceRequest) (*colllog.ExportLogsServiceResponse, error) {+ if err := s.svc.Logs(ctx, authorization(ctx), req); err != nil {+ return nil, mapErr(err)+ }+ return &colllog.ExportLogsServiceResponse{}, nil+}++func (s *metricsServer) Export(ctx context.Context, req *collmetric.ExportMetricsServiceRequest) (*collmetric.ExportMetricsServiceResponse, error) {+ if err := s.svc.Metrics(ctx, authorization(ctx), req); err != nil {+ return nil, mapErr(err)+ }+ return &collmetric.ExportMetricsServiceResponse{}, nil+}diff --git a/src/otlp-receiver/internal/httpserver/server.go b/src/otlp-receiver/internal/httpserver/server.gonew file mode 100644index 00000000..86572439--- /dev/null+++ b/src/otlp-receiver/internal/httpserver/server.go@@ -0,0 +1,139 @@+package httpserver++import (+ "compress/gzip"+ "errors"+ "io"+ "net/http"+ "strings"++ "google.golang.org/protobuf/encoding/protojson"+ "google.golang.org/protobuf/proto"++ colllog "go.opentelemetry.io/proto/otlp/collector/logs/v1"+ collmetric "go.opentelemetry.io/proto/otlp/collector/metrics/v1"+ colltrace "go.opentelemetry.io/proto/otlp/collector/trace/v1"++ "github.com/openlit/openlit/otlp-receiver/internal/ingest"+ "github.com/openlit/openlit/otlp-receiver/internal/tenant"+)++func New(svc *ingest.Service) http.Handler {+ mux := http.NewServeMux()+ mux.HandleFunc("/health", func(w http.ResponseWriter, _ *http.Request) {+ w.WriteHeader(http.StatusOK)+ _, _ = w.Write([]byte("ok"))+ })+ mux.HandleFunc("/v1/traces", func(w http.ResponseWriter, r *http.Request) {+ handle(w, r, func(body []byte, contentType string) error {+ var req colltrace.ExportTraceServiceRequest+ if err := unmarshal(body, contentType, &req); err != nil {+ return err+ }+ return svc.Traces(r.Context(), r.Header.Get("Authorization"), &req)+ })+ })+ mux.HandleFunc("/v1/logs", func(w http.ResponseWriter, r *http.Request) {+ handle(w, r, func(body []byte, contentType string) error {+ var req colllog.ExportLogsServiceRequest+ if err := unmarshal(body, contentType, &req); err != nil {+ return err+ }+ return svc.Logs(r.Context(), r.Header.Get("Authorization"), &req)+ })+ })+ mux.HandleFunc("/v1/metrics", func(w http.ResponseWriter, r *http.Request) {+ handle(w, r, func(body []byte, contentType string) error {+ var req collmetric.ExportMetricsServiceRequest+ if err := unmarshal(body, contentType, &req); err != nil {+ return err+ }+ return svc.Metrics(r.Context(), r.Header.Get("Authorization"), &req)+ })+ })+ return headerGuard(mux)+}++func headerGuard(next http.Handler) http.Handler {+ return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ r.Header.Del("x-database-config-id")+ r.Header.Del("X-Database-Config-Id")+ r.Header.Del("x-openlit-database-config-id")+ next.ServeHTTP(w, r)+ })+}++type decodeError struct{ error }++var errTooLarge = errors.New("payload too large")++const maxOTLPBody = 32 << 20++func handle(w http.ResponseWriter, r *http.Request, fn func([]byte, string) error) {+ if r.Method != http.MethodPost {+ w.WriteHeader(http.StatusMethodNotAllowed)+ return+ }+ body, err := readBody(r)+ if err != nil {+ if errors.Is(err, errTooLarge) {+ http.Error(w, "payload too large", http.StatusRequestEntityTooLarge)+ return+ }+ http.Error(w, "invalid body", http.StatusBadRequest)+ return+ }+ if err := fn(body, r.Header.Get("Content-Type")); err != nil {+ if _, ok := err.(*decodeError); ok {+ http.Error(w, "invalid otlp payload", http.StatusBadRequest)+ return+ }+ status := ingest.HTTPStatus(err)+ if status == http.StatusUnauthorized {+ http.Error(w, "unauthorized", http.StatusUnauthorized)+ return+ }+ http.Error(w, "ingest failed", status)+ return+ }+ w.WriteHeader(http.StatusOK)+}++func readBody(r *http.Request) ([]byte, error) {+ reader := r.Body+ if strings.EqualFold(r.Header.Get("Content-Encoding"), "gzip") {+ gz, err := gzip.NewReader(r.Body)+ if err != nil {+ return nil, err+ }+ defer gz.Close()+ reader = gz+ }+ limited := io.LimitReader(reader, int64(maxOTLPBody)+1)+ body, err := io.ReadAll(limited)+ if err != nil {+ return nil, err+ }+ if len(body) > maxOTLPBody {+ return nil, errTooLarge+ }+ return body, nil+}++func unmarshal(body []byte, contentType string, msg proto.Message) error {+ ct := strings.ToLower(contentType)+ if strings.Contains(ct, "json") {+ if err := protojson.Unmarshal(body, msg); err != nil {+ return &decodeError{err}+ }+ return nil+ }+ if err := proto.Unmarshal(body, msg); err != nil {+ return &decodeError{err}+ }+ return nil+}++func UnauthorizedFrom(err error) bool {+ return err == tenant.ErrUnauthorized+}diff --git a/src/otlp-receiver/internal/httpserver/server_test.go b/src/otlp-receiver/internal/httpserver/server_test.gonew file mode 100644index 00000000..706a2606--- /dev/null+++ b/src/otlp-receiver/internal/httpserver/server_test.go@@ -0,0 +1,126 @@+package httpserver++import (+ "bytes"+ "compress/gzip"+ "net/http"+ "net/http/httptest"+ "testing"++ colltrace "go.opentelemetry.io/proto/otlp/collector/trace/v1"+ commonpb "go.opentelemetry.io/proto/otlp/common/v1"+ resourcepb "go.opentelemetry.io/proto/otlp/resource/v1"+ tracepb "go.opentelemetry.io/proto/otlp/trace/v1"+ "google.golang.org/protobuf/encoding/protojson"+ "google.golang.org/protobuf/proto"++ "github.com/openlit/openlit/otlp-receiver/internal/ingest"+ "github.com/openlit/openlit/otlp-receiver/internal/tenant"+)++func TestHealthAndAuthAndJSON(t *testing.T) {+ require := &tenant.Store{RequireKey: true}+ svc := &ingest.Service{Tenants: require}+ h := New(svc)++ req := httptest.NewRequest(http.MethodGet, "/health", nil)+ rr := httptest.NewRecorder()+ h.ServeHTTP(rr, req)+ if rr.Code != 200 {+ t.Fatalf("health %d", rr.Code)+ }++ body := []byte(`{"resourceSpans":[]}`)+ post := httptest.NewRequest(http.MethodPost, "/v1/traces", bytes.NewReader(body))+ post.Header.Set("Content-Type", "application/json")+ rr = httptest.NewRecorder()+ h.ServeHTTP(rr, post)+ if rr.Code != http.StatusUnauthorized {+ t.Fatalf("missing key status %d", rr.Code)+ }++}++func TestJSONAndProtobufDecode(t *testing.T) {+ req := &colltrace.ExportTraceServiceRequest{+ ResourceSpans: []*tracepb.ResourceSpans{{+ Resource: &resourcepb.Resource{Attributes: []*commonpb.KeyValue{+ {Key: "service.name", Value: &commonpb.AnyValue{Value: &commonpb.AnyValue_StringValue{StringValue: "svc"}}},+ }},+ }},+ }+ jsonBody, err := protojson.Marshal(req)+ if err != nil {+ t.Fatal(err)+ }+ var decoded colltrace.ExportTraceServiceRequest+ if err := unmarshal(jsonBody, "application/json", &decoded); err != nil {+ t.Fatal(err)+ }+ pb, err := proto.Marshal(req)+ if err != nil {+ t.Fatal(err)+ }+ decoded = colltrace.ExportTraceServiceRequest{}+ if err := unmarshal(pb, "application/x-protobuf", &decoded); err != nil {+ t.Fatal(err)+ }+ if len(decoded.ResourceSpans) != 1 {+ t.Fatal("protobuf decode")+ }++ var gz bytes.Buffer+ w := gzip.NewWriter(&gz)+ _, _ = w.Write(jsonBody)+ _ = w.Close()+ httpReq := httptest.NewRequest(http.MethodPost, "/v1/traces", bytes.NewReader(gz.Bytes()))+ httpReq.Header.Set("Content-Encoding", "gzip")+ got, err := readBody(httpReq)+ if err != nil {+ t.Fatal(err)+ }+ if !bytes.Equal(got, jsonBody) {+ t.Fatal("gzip roundtrip")+ }+}++func TestInvalidJSON(t *testing.T) {+ svc := &ingest.Service{Tenants: &tenant.Store{RequireKey: true}}+ h := New(svc)+ post := httptest.NewRequest(http.MethodPost, "/v1/traces", bytes.NewReader([]byte("not-json")))+ post.Header.Set("Content-Type", "application/json")+ rr := httptest.NewRecorder()+ h.ServeHTTP(rr, post)+ if rr.Code != http.StatusBadRequest {+ t.Fatalf("got %d", rr.Code)+ }+}++func TestStripsTenantHeaders(t *testing.T) {+ svc := &ingest.Service{Tenants: &tenant.Store{RequireKey: true}}+ h := New(svc)+ post := httptest.NewRequest(http.MethodPost, "/v1/traces", bytes.NewReader([]byte("{}")))+ post.Header.Set("Content-Type", "application/json")+ post.Header.Set("x-database-config-id", "forged")+ rr := httptest.NewRecorder()+ h.ServeHTTP(rr, post)+ if post.Header.Get("x-database-config-id") != "forged" {+ // handler clones? ServeHTTP receives the same request; headerGuard deletes on the request.+ }+ if rr.Code != http.StatusUnauthorized {+ t.Fatalf("got %d", rr.Code)+ }+}++func TestRejectsOversizedBody(t *testing.T) {+ svc := &ingest.Service{Tenants: &tenant.Store{RequireKey: true}}+ h := New(svc)+ body := make([]byte, maxOTLPBody+1)+ post := httptest.NewRequest(http.MethodPost, "/v1/traces", bytes.NewReader(body))+ post.Header.Set("Content-Type", "application/json")+ rr := httptest.NewRecorder()+ h.ServeHTTP(rr, post)+ if rr.Code != http.StatusRequestEntityTooLarge {+ t.Fatalf("got %d", rr.Code)+ }+}diff --git a/src/otlp-receiver/internal/ingest/service.go b/src/otlp-receiver/internal/ingest/service.gonew file mode 100644index 00000000..58500b0e--- /dev/null+++ b/src/otlp-receiver/internal/ingest/service.go@@ -0,0 +1,74 @@+package ingest++import (+ "context"+ "errors"++ colllog "go.opentelemetry.io/proto/otlp/collector/logs/v1"+ collmetric "go.opentelemetry.io/proto/otlp/collector/metrics/v1"+ colltrace "go.opentelemetry.io/proto/otlp/collector/trace/v1"+ logspb "go.opentelemetry.io/proto/otlp/logs/v1"+ metricspb "go.opentelemetry.io/proto/otlp/metrics/v1"+ tracepb "go.opentelemetry.io/proto/otlp/trace/v1"++ "github.com/openlit/openlit/otlp-receiver/internal/chwrite"+ "github.com/openlit/openlit/otlp-receiver/internal/otlpconv"+ "github.com/openlit/openlit/otlp-receiver/internal/tenant"+)++type Service struct {+ Tenants *tenant.Store+ Writer *chwrite.Writer+}++func (s *Service) Traces(ctx context.Context, authorization string, req *colltrace.ExportTraceServiceRequest) error {+ resolved, err := s.Tenants.Resolve(ctx, authorization)+ if err != nil {+ return err+ }+ rows := otlpconv.Traces(&tracepb.TracesData{ResourceSpans: req.GetResourceSpans()})+ otlpconv.StampTraces(rows, resolved.Resource())+ return s.Writer.InsertTraces(ctx, resolved.ClickHouse, rows)+}++func (s *Service) Logs(ctx context.Context, authorization string, req *colllog.ExportLogsServiceRequest) error {+ resolved, err := s.Tenants.Resolve(ctx, authorization)+ if err != nil {+ return err+ }+ rows := otlpconv.Logs(&logspb.LogsData{ResourceLogs: req.GetResourceLogs()})+ otlpconv.StampLogs(rows, resolved.Resource())+ return s.Writer.InsertLogs(ctx, resolved.ClickHouse, rows)+}++func (s *Service) Metrics(ctx context.Context, authorization string, req *collmetric.ExportMetricsServiceRequest) error {+ resolved, err := s.Tenants.Resolve(ctx, authorization)+ if err != nil {+ return err+ }+ metrics := otlpconv.Metrics(&metricspb.MetricsData{ResourceMetrics: req.GetResourceMetrics()})+ otlpconv.StampMetrics(&metrics, resolved.Resource())+ if err := s.Writer.InsertGauges(ctx, resolved.ClickHouse, metrics.Gauges); err != nil {+ return err+ }+ if err := s.Writer.InsertSums(ctx, resolved.ClickHouse, metrics.Sums); err != nil {+ return err+ }+ if err := s.Writer.InsertHistograms(ctx, resolved.ClickHouse, metrics.Histograms); err != nil {+ return err+ }+ if err := s.Writer.InsertSummaries(ctx, resolved.ClickHouse, metrics.Summaries); err != nil {+ return err+ }+ return s.Writer.InsertExpHistograms(ctx, resolved.ClickHouse, metrics.ExpHistograms)+}++func HTTPStatus(err error) int {+ if err == nil {+ return 200+ }+ if errors.Is(err, tenant.ErrUnauthorized) || errors.Is(err, tenant.ErrNoTenant) {+ return 401+ }+ return 503+}diff --git a/src/otlp-receiver/internal/otlpconv/convert.go b/src/otlp-receiver/internal/otlpconv/convert.gonew file mode 100644index 00000000..243966e1--- /dev/null+++ b/src/otlp-receiver/internal/otlpconv/convert.go@@ -0,0 +1,676 @@+package otlpconv++import (+ "encoding/hex"+ "encoding/json"+ "strconv"+ "time"++ commonpb "go.opentelemetry.io/proto/otlp/common/v1"+ logspb "go.opentelemetry.io/proto/otlp/logs/v1"+ metricspb "go.opentelemetry.io/proto/otlp/metrics/v1"+ resourcepb "go.opentelemetry.io/proto/otlp/resource/v1"+ tracepb "go.opentelemetry.io/proto/otlp/trace/v1"+)++const maxEncodedAnyValue = 1 << 20++var spanKinds = map[tracepb.Span_SpanKind]string{+ tracepb.Span_SPAN_KIND_UNSPECIFIED: "SPAN_KIND_UNSPECIFIED",+ tracepb.Span_SPAN_KIND_INTERNAL: "SPAN_KIND_INTERNAL",+ tracepb.Span_SPAN_KIND_SERVER: "SPAN_KIND_SERVER",+ tracepb.Span_SPAN_KIND_CLIENT: "SPAN_KIND_CLIENT",+ tracepb.Span_SPAN_KIND_PRODUCER: "SPAN_KIND_PRODUCER",+ tracepb.Span_SPAN_KIND_CONSUMER: "SPAN_KIND_CONSUMER",+}++var statusCodes = map[tracepb.Status_StatusCode]string{+ tracepb.Status_STATUS_CODE_UNSET: "STATUS_CODE_UNSET",+ tracepb.Status_STATUS_CODE_OK: "STATUS_CODE_OK",+ tracepb.Status_STATUS_CODE_ERROR: "STATUS_CODE_ERROR",+}++func spanKindName(kind tracepb.Span_SpanKind) string {+ if name, ok := spanKinds[kind]; ok {+ return name+ }+ return "SPAN_KIND_UNSPECIFIED"+}++type TraceRow struct {+ Timestamp time.Time+ TraceID string+ SpanID string+ ParentSpanID string+ TraceState string+ SpanName string+ SpanKind string+ ServiceName string+ ResourceAttributes map[string]string+ ScopeName string+ ScopeVersion string+ SpanAttributes map[string]string+ Duration uint64+ StatusCode string+ StatusMessage string+ EventTimestamps []time.Time+ EventNames []string+ EventAttributes []map[string]string+ LinkTraceIDs []string+ LinkSpanIDs []string+ LinkTraceStates []string+ LinkAttributes []map[string]string+}++type LogRow struct {+ Timestamp time.Time+ TraceID string+ SpanID string+ TraceFlags uint8+ SeverityText string+ SeverityNumber uint8+ ServiceName string+ Body string+ ResourceSchemaURL string+ ResourceAttributes map[string]string+ ScopeSchemaURL string+ ScopeName string+ ScopeVersion string+ ScopeAttributes map[string]string+ LogAttributes map[string]string+}++type GaugeRow struct {+ ResourceAttributes map[string]string+ ResourceSchemaURL string+ ScopeName string+ ScopeVersion string+ ScopeAttributes map[string]string+ ScopeDroppedCount uint32+ ScopeSchemaURL string+ ServiceName string+ MetricName string+ MetricDescription string+ MetricUnit string+ Attributes map[string]string+ StartTimeUnix time.Time+ TimeUnix time.Time+ Value float64+ Flags uint32+}++type SumRow struct {+ GaugeRow+ AggregationTemporality int32+ IsMonotonic bool+}++type HistogramRow struct {+ GaugeRow+ Count uint64+ Sum float64+ BucketCounts []uint64+ ExplicitBounds []float64+ Min float64+ Max float64+ AggregationTemporality int32+}++type SummaryRow struct {+ GaugeRow+ Count uint64+ Sum float64+ Quantiles []float64+ Values []float64+}++type ExpHistogramRow struct {+ GaugeRow+ Count uint64+ Sum float64+ Scale int32+ ZeroCount uint64+ PositiveOffset int32+ PositiveBucketCounts []uint64+ NegativeOffset int32+ NegativeBucketCounts []uint64+ Min float64+ Max float64+ AggregationTemporality int32+}++type MetricRows struct {+ Gauges []GaugeRow+ Sums []SumRow+ Histograms []HistogramRow+ Summaries []SummaryRow+ ExpHistograms []ExpHistogramRow+}++func metricBase(resAttrs map[string]string, resURL, scopeName, scopeVersion string, scopeAttrs map[string]string, dropped uint32, scopeURL, service string, metric *metricspb.Metric, attrs []*commonpb.KeyValue, start, unix uint64, flags uint32) GaugeRow {+ name, desc, unit := "", "", ""+ if metric != nil {+ name, desc, unit = metric.Name, metric.Description, metric.Unit+ }+ return GaugeRow{+ ResourceAttributes: resAttrs,+ ResourceSchemaURL: resURL,+ ScopeName: scopeName,+ ScopeVersion: scopeVersion,+ ScopeAttributes: scopeAttrs,+ ScopeDroppedCount: dropped,+ ScopeSchemaURL: scopeURL,+ ServiceName: service,+ MetricName: name,+ MetricDescription: desc,+ MetricUnit: unit,+ Attributes: AttrMap(attrs),+ StartTimeUnix: NanoTime(start),+ TimeUnix: NanoTime(unix),+ Flags: flags,+ }+}++func AttrMap(kvs []*commonpb.KeyValue) map[string]string {+ out := make(map[string]string, len(kvs))+ for _, kv := range kvs {+ if kv == nil {+ continue+ }+ out[kv.Key] = AnyValue(kv.Value)+ }+ return out+}++func AnyValue(v *commonpb.AnyValue) string {+ if v == nil {+ return ""+ }+ switch x := v.Value.(type) {+ case *commonpb.AnyValue_StringValue:+ return x.StringValue+ case *commonpb.AnyValue_BoolValue:+ return strconv.FormatBool(x.BoolValue)+ case *commonpb.AnyValue_IntValue:+ return strconv.FormatInt(x.IntValue, 10)+ case *commonpb.AnyValue_DoubleValue:+ return strconv.FormatFloat(x.DoubleValue, 'f', -1, 64)+ case *commonpb.AnyValue_BytesValue:+ return hex.EncodeToString(x.BytesValue)+ case *commonpb.AnyValue_ArrayValue, *commonpb.AnyValue_KvlistValue:+ return marshalJSON(anyValueJSON(v))+ default:+ return ""+ }+}++func marshalJSON(v any) string {+ b, err := json.Marshal(v)+ if err != nil || len(b) > maxEncodedAnyValue {+ return ""+ }+ return string(b)+}++func anyValueJSON(v *commonpb.AnyValue) any {+ if v == nil {+ return nil+ }+ switch x := v.Value.(type) {+ case *commonpb.AnyValue_StringValue:+ return x.StringValue+ case *commonpb.AnyValue_BoolValue:+ return x.BoolValue+ case *commonpb.AnyValue_IntValue:+ return x.IntValue+ case *commonpb.AnyValue_DoubleValue:+ return x.DoubleValue+ case *commonpb.AnyValue_BytesValue:+ return hex.EncodeToString(x.BytesValue)+ case *commonpb.AnyValue_ArrayValue:+ values := []*commonpb.AnyValue{}+ if x.ArrayValue != nil {+ values = x.ArrayValue.Values+ }+ out := make([]any, 0, len(values))+ for _, item := range values {+ out = append(out, anyValueJSON(item))+ }+ return out+ case *commonpb.AnyValue_KvlistValue:+ out := map[string]any{}+ if x.KvlistValue == nil {+ return out+ }+ for _, kv := range x.KvlistValue.Values {+ if kv == nil {+ continue+ }+ out[kv.Key] = anyValueJSON(kv.Value)+ }+ return out+ default:+ return nil+ }+}++func ServiceName(res *resourcepb.Resource) string {+ if res == nil {+ return ""+ }+ for _, kv := range res.Attributes {+ if kv != nil && kv.Key == "service.name" {+ return AnyValue(kv.Value)+ }+ }+ return ""+}++func IDHex(b []byte) string {+ if len(b) == 0 {+ return ""+ }+ return hex.EncodeToString(b)+}++func NanoTime(ts uint64) time.Time {+ if ts == 0 {+ return time.Unix(0, 0).UTC()+ }+ return time.Unix(0, int64(ts)).UTC()+}++func Traces(req *tracepb.TracesData) []TraceRow {+ if req == nil {+ return nil+ }+ var rows []TraceRow+ for _, rs := range req.ResourceSpans {+ if rs == nil {+ continue+ }+ resAttrs := AttrMap(nil)+ service := ""+ if rs.Resource != nil {+ resAttrs = AttrMap(rs.Resource.Attributes)+ service = ServiceName(rs.Resource)+ }+ for _, ss := range rs.ScopeSpans {+ if ss == nil {+ continue+ }+ scopeName, scopeVersion := "", ""+ if ss.Scope != nil {+ scopeName = ss.Scope.Name+ scopeVersion = ss.Scope.Version+ }+ for _, span := range ss.Spans {+ if span == nil {+ continue+ }+ statusCode, statusMsg := "STATUS_CODE_UNSET", ""+ if span.Status != nil {+ if code, ok := statusCodes[span.Status.Code]; ok {+ statusCode = code+ }+ statusMsg = span.Status.Message+ }+ start := NanoTime(span.StartTimeUnixNano)+ end := NanoTime(span.EndTimeUnixNano)+ var duration uint64+ if span.EndTimeUnixNano > span.StartTimeUnixNano {+ duration = span.EndTimeUnixNano - span.StartTimeUnixNano+ }+ row := TraceRow{+ Timestamp: start,+ TraceID: IDHex(span.TraceId),+ SpanID: IDHex(span.SpanId),+ ParentSpanID: IDHex(span.ParentSpanId),+ TraceState: span.TraceState,+ SpanName: span.Name,+ SpanKind: spanKindName(span.Kind),+ ServiceName: service,+ ResourceAttributes: resAttrs,+ ScopeName: scopeName,+ ScopeVersion: scopeVersion,+ SpanAttributes: AttrMap(span.Attributes),+ Duration: duration,+ StatusCode: statusCode,+ StatusMessage: statusMsg,+ EventTimestamps: make([]time.Time, 0, len(span.Events)),+ EventNames: make([]string, 0, len(span.Events)),+ EventAttributes: make([]map[string]string, 0, len(span.Events)),+ LinkTraceIDs: make([]string, 0, len(span.Links)),+ LinkSpanIDs: make([]string, 0, len(span.Links)),+ LinkTraceStates: make([]string, 0, len(span.Links)),+ LinkAttributes: make([]map[string]string, 0, len(span.Links)),+ }+ _ = end+ for _, ev := range span.Events {+ if ev == nil {+ continue+ }+ row.EventTimestamps = append(row.EventTimestamps, NanoTime(ev.TimeUnixNano))+ row.EventNames = append(row.EventNames, ev.Name)+ row.EventAttributes = append(row.EventAttributes, AttrMap(ev.Attributes))+ }+ for _, link := range span.Links {+ if link == nil {+ continue+ }+ row.LinkTraceIDs = append(row.LinkTraceIDs, IDHex(link.TraceId))+ row.LinkSpanIDs = append(row.LinkSpanIDs, IDHex(link.SpanId))+ row.LinkTraceStates = append(row.LinkTraceStates, link.TraceState)+ row.LinkAttributes = append(row.LinkAttributes, AttrMap(link.Attributes))+ }+ rows = append(rows, row)+ }+ }+ }+ return rows+}++func Logs(req *logspb.LogsData) []LogRow {+ if req == nil {+ return nil+ }+ var rows []LogRow+ for _, rl := range req.ResourceLogs {+ if rl == nil {+ continue+ }+ resAttrs := AttrMap(nil)+ service := ""+ if rl.Resource != nil {+ resAttrs = AttrMap(rl.Resource.Attributes)+ service = ServiceName(rl.Resource)+ }+ for _, sl := range rl.ScopeLogs {+ if sl == nil {+ continue+ }+ scopeName, scopeVersion := "", ""+ scopeAttrs := AttrMap(nil)+ if sl.Scope != nil {+ scopeName = sl.Scope.Name+ scopeVersion = sl.Scope.Version+ scopeAttrs = AttrMap(sl.Scope.Attributes)+ }+ for _, rec := range sl.LogRecords {+ if rec == nil {+ continue+ }+ rows = append(rows, LogRow{+ Timestamp: NanoTime(rec.TimeUnixNano),+ TraceID: IDHex(rec.TraceId),+ SpanID: IDHex(rec.SpanId),+ TraceFlags: uint8(rec.Flags),+ SeverityText: rec.SeverityText,+ SeverityNumber: uint8(rec.SeverityNumber),+ ServiceName: service,+ Body: AnyValue(rec.Body),+ ResourceSchemaURL: rl.SchemaUrl,+ ResourceAttributes: resAttrs,+ ScopeSchemaURL: sl.SchemaUrl,+ ScopeName: scopeName,+ ScopeVersion: scopeVersion,+ ScopeAttributes: scopeAttrs,+ LogAttributes: AttrMap(rec.Attributes),+ })+ }+ }+ }+ return rows+}++func Metrics(req *metricspb.MetricsData) MetricRows {+ var out MetricRows+ if req == nil {+ return out+ }+ for _, rm := range req.ResourceMetrics {+ if rm == nil {+ continue+ }+ resAttrs := AttrMap(nil)+ service := ""+ resURL := rm.SchemaUrl+ if rm.Resource != nil {+ resAttrs = AttrMap(rm.Resource.Attributes)+ service = ServiceName(rm.Resource)+ }+ for _, sm := range rm.ScopeMetrics {+ if sm == nil {+ continue+ }+ scopeName, scopeVersion, dropped := "", "", uint32(0)+ scopeAttrs := AttrMap(nil)+ if sm.Scope != nil {+ scopeName = sm.Scope.Name+ scopeVersion = sm.Scope.Version+ dropped = sm.Scope.DroppedAttributesCount+ scopeAttrs = AttrMap(sm.Scope.Attributes)+ }+ for _, metric := range sm.Metrics {+ if metric == nil {+ continue+ }+ switch d := metric.Data.(type) {+ case *metricspb.Metric_Gauge:+ if d.Gauge == nil {+ continue+ }+ for _, pt := range d.Gauge.DataPoints {+ if pt == nil {+ continue+ }+ row := metricBase(resAttrs, resURL, scopeName, scopeVersion, scopeAttrs, dropped, sm.SchemaUrl, service, metric, pt.Attributes, pt.StartTimeUnixNano, pt.TimeUnixNano, uint32(pt.Flags))+ row.Value = numberValue(pt)+ out.Gauges = append(out.Gauges, row)+ }+ case *metricspb.Metric_Sum:+ if d.Sum == nil {+ continue+ }+ for _, pt := range d.Sum.DataPoints {+ if pt == nil {+ continue+ }+ base := metricBase(resAttrs, resURL, scopeName, scopeVersion, scopeAttrs, dropped, sm.SchemaUrl, service, metric, pt.Attributes, pt.StartTimeUnixNano, pt.TimeUnixNano, uint32(pt.Flags))+ base.Value = numberValue(pt)+ out.Sums = append(out.Sums, SumRow{+ GaugeRow: base,+ AggregationTemporality: int32(d.Sum.AggregationTemporality),+ IsMonotonic: d.Sum.IsMonotonic,+ })+ }+ case *metricspb.Metric_Histogram:+ if d.Histogram == nil {+ continue+ }+ for _, pt := range d.Histogram.DataPoints {+ if pt == nil {+ continue+ }+ base := metricBase(resAttrs, resURL, scopeName, scopeVersion, scopeAttrs, dropped, sm.SchemaUrl, service, metric, pt.Attributes, pt.StartTimeUnixNano, pt.TimeUnixNano, uint32(pt.Flags))+ out.Histograms = append(out.Histograms, HistogramRow{+ GaugeRow: base,+ Count: pt.Count,+ Sum: pt.GetSum(),+ BucketCounts: nonemptyUint64(pt.BucketCounts),+ ExplicitBounds: nonemptyFloat64(pt.ExplicitBounds),+ Min: pt.GetMin(),+ Max: pt.GetMax(),+ AggregationTemporality: int32(d.Histogram.AggregationTemporality),+ })+ }+ case *metricspb.Metric_Summary:+ if d.Summary == nil {+ continue+ }+ for _, pt := range d.Summary.DataPoints {+ if pt == nil {+ continue+ }+ base := metricBase(resAttrs, resURL, scopeName, scopeVersion, scopeAttrs, dropped, sm.SchemaUrl, service, metric, pt.Attributes, pt.StartTimeUnixNano, pt.TimeUnixNano, uint32(pt.Flags))+ qs := make([]float64, 0, len(pt.QuantileValues))+ vs := make([]float64, 0, len(pt.QuantileValues))+ for _, qv := range pt.QuantileValues {+ if qv == nil {+ continue+ }+ qs = append(qs, qv.Quantile)+ vs = append(vs, qv.Value)+ }+ out.Summaries = append(out.Summaries, SummaryRow{+ GaugeRow: base,+ Count: pt.Count,+ Sum: pt.Sum,+ Quantiles: qs,+ Values: vs,+ })+ }+ case *metricspb.Metric_ExponentialHistogram:+ if d.ExponentialHistogram == nil {+ continue+ }+ for _, pt := range d.ExponentialHistogram.DataPoints {+ if pt == nil {+ continue+ }+ base := metricBase(resAttrs, resURL, scopeName, scopeVersion, scopeAttrs, dropped, sm.SchemaUrl, service, metric, pt.Attributes, pt.StartTimeUnixNano, pt.TimeUnixNano, uint32(pt.Flags))+ posOff, posCounts := int32(0), []uint64{}+ negOff, negCounts := int32(0), []uint64{}+ if pt.Positive != nil {+ posOff = pt.Positive.Offset+ posCounts = nonemptyUint64(pt.Positive.BucketCounts)+ }+ if pt.Negative != nil {+ negOff = pt.Negative.Offset+ negCounts = nonemptyUint64(pt.Negative.BucketCounts)+ }+ out.ExpHistograms = append(out.ExpHistograms, ExpHistogramRow{+ GaugeRow: base,+ Count: pt.Count,+ Sum: pt.GetSum(),+ Scale: pt.Scale,+ ZeroCount: pt.ZeroCount,+ PositiveOffset: posOff,+ PositiveBucketCounts: posCounts,+ NegativeOffset: negOff,+ NegativeBucketCounts: negCounts,+ Min: pt.GetMin(),+ Max: pt.GetMax(),+ AggregationTemporality: int32(d.ExponentialHistogram.AggregationTemporality),+ })+ }+ }+ }+ }+ }+ return out+}++func nonemptyUint64(in []uint64) []uint64 {+ if in == nil {+ return []uint64{}+ }+ return in+}++func nonemptyFloat64(in []float64) []float64 {+ if in == nil {+ return []float64{}+ }+ return in+}++type ResourceTenant struct {+ OrganisationID string+ ProjectID string+ Environment string+}++func StampResource(attrs map[string]string, tenant ResourceTenant) map[string]string {+ out := make(map[string]string, len(attrs))+ for k, v := range attrs {+ out[k] = v+ }+ if tenant.Environment != "" {+ out["organisation.environment.name"] = tenant.Environment+ }+ if tenant.OrganisationID != "" {+ out["openlit.organisation.id"] = tenant.OrganisationID+ }+ if tenant.ProjectID != "" {+ out["openlit.project.id"] = tenant.ProjectID+ }+ return out+}++func StampTraces(rows []TraceRow, tenant ResourceTenant) {+ for i := range rows {+ rows[i].ResourceAttributes = StampResource(rows[i].ResourceAttributes, tenant)+ }+}++func StampLogs(rows []LogRow, tenant ResourceTenant) {+ for i := range rows {+ rows[i].ResourceAttributes = StampResource(rows[i].ResourceAttributes, tenant)+ }+}++func StampGauges(rows []GaugeRow, tenant ResourceTenant) {+ for i := range rows {+ rows[i].ResourceAttributes = StampResource(rows[i].ResourceAttributes, tenant)+ }+}++func StampSums(rows []SumRow, tenant ResourceTenant) {+ for i := range rows {+ rows[i].ResourceAttributes = StampResource(rows[i].ResourceAttributes, tenant)+ }+}++func StampHistograms(rows []HistogramRow, tenant ResourceTenant) {+ for i := range rows {+ rows[i].ResourceAttributes = StampResource(rows[i].ResourceAttributes, tenant)+ }+}++func StampSummaries(rows []SummaryRow, tenant ResourceTenant) {+ for i := range rows {+ rows[i].ResourceAttributes = StampResource(rows[i].ResourceAttributes, tenant)+ }+}++func StampExpHistograms(rows []ExpHistogramRow, tenant ResourceTenant) {+ for i := range rows {+ rows[i].ResourceAttributes = StampResource(rows[i].ResourceAttributes, tenant)+ }+}++func StampMetrics(rows *MetricRows, tenant ResourceTenant) {+ if rows == nil {+ return+ }+ StampGauges(rows.Gauges, tenant)+ StampSums(rows.Sums, tenant)+ StampHistograms(rows.Histograms, tenant)+ StampSummaries(rows.Summaries, tenant)+ StampExpHistograms(rows.ExpHistograms, tenant)+}++func numberValue(pt *metricspb.NumberDataPoint) float64 {+ if pt == nil {+ return 0+ }+ switch v := pt.Value.(type) {+ case *metricspb.NumberDataPoint_AsDouble:+ return v.AsDouble+ case *metricspb.NumberDataPoint_AsInt:+ return float64(v.AsInt)+ default:+ return 0+ }+}diff --git a/src/otlp-receiver/internal/otlpconv/convert_test.go b/src/otlp-receiver/internal/otlpconv/convert_test.gonew file mode 100644index 00000000..22207b2d--- /dev/null+++ b/src/otlp-receiver/internal/otlpconv/convert_test.go@@ -0,0 +1,210 @@+package otlpconv++import (+ "encoding/hex"+ "testing"+ "time"++ commonpb "go.opentelemetry.io/proto/otlp/common/v1"+ metricspb "go.opentelemetry.io/proto/otlp/metrics/v1"+ resourcepb "go.opentelemetry.io/proto/otlp/resource/v1"+ tracepb "go.opentelemetry.io/proto/otlp/trace/v1"+)++func TestTracesMapping(t *testing.T) {+ t.Parallel()+ traceID, _ := hex.DecodeString("0af7651916cd43dd8448eb211c80319c")+ spanID, _ := hex.DecodeString("b7ad6b7169203331")+ start := uint64(time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC).UnixNano())+ end := start + 1_500_000+ rows := Traces(&tracepb.TracesData{+ ResourceSpans: []*tracepb.ResourceSpans{{+ Resource: &resourcepb.Resource{Attributes: []*commonpb.KeyValue{+ {Key: "service.name", Value: &commonpb.AnyValue{Value: &commonpb.AnyValue_StringValue{StringValue: "demo"}}},+ }},+ ScopeSpans: []*tracepb.ScopeSpans{{+ Scope: &commonpb.InstrumentationScope{Name: "lib", Version: "1"},+ Spans: []*tracepb.Span{{+ TraceId: traceID,+ SpanId: spanID,+ Name: "chat",+ Kind: tracepb.Span_SPAN_KIND_CLIENT,+ StartTimeUnixNano: start,+ EndTimeUnixNano: end,+ Attributes: []*commonpb.KeyValue{+ {Key: "gen_ai.system", Value: &commonpb.AnyValue{Value: &commonpb.AnyValue_StringValue{StringValue: "openai"}}},+ },+ Status: &tracepb.Status{Code: tracepb.Status_STATUS_CODE_OK},+ }},+ }},+ }},+ })+ if len(rows) != 1 {+ t.Fatalf("len=%d", len(rows))+ }+ row := rows[0]+ if row.ServiceName != "demo" || row.SpanName != "chat" || row.SpanKind != "SPAN_KIND_CLIENT" {+ t.Fatalf("row=%+v", row)+ }+ if row.Duration != 1_500_000 || row.StatusCode != "STATUS_CODE_OK" {+ t.Fatalf("duration/status %+v", row)+ }+ if row.TraceID != "0af7651916cd43dd8448eb211c80319c" || row.SpanAttributes["gen_ai.system"] != "openai" {+ t.Fatalf("ids/attrs %+v", row)+ }+}++func TestStampResourceKeepsDeploymentEnvironment(t *testing.T) {+ t.Parallel()+ stamped := StampResource(map[string]string{+ "service.name": "demo",+ "deployment.environment": "local",+ }, ResourceTenant{+ OrganisationID: "org-1",+ ProjectID: "proj-1",+ Environment: "staging",+ })+ if stamped["deployment.environment"] != "local" {+ t.Fatalf("env=%s", stamped["deployment.environment"])+ }+ if stamped["gen_ai.environment"] != "" {+ t.Fatalf("gen_ai env=%s", stamped["gen_ai.environment"])+ }+ if stamped["organisation.environment.name"] != "staging" {+ t.Fatalf("org env=%s", stamped["organisation.environment.name"])+ }+ if stamped["openlit.organisation.id"] != "org-1" || stamped["openlit.project.id"] != "proj-1" {+ t.Fatalf("tenant attrs %+v", stamped)+ }+ if stamped["service.name"] != "demo" {+ t.Fatal("lost original attr")+ }+}++func TestAnyValueStructuredJSON(t *testing.T) {+ t.Parallel()+ array := AnyValue(&commonpb.AnyValue{Value: &commonpb.AnyValue_ArrayValue{+ ArrayValue: &commonpb.ArrayValue{Values: []*commonpb.AnyValue{+ {Value: &commonpb.AnyValue_StringValue{StringValue: "a"}},+ {Value: &commonpb.AnyValue_IntValue{IntValue: 2}},+ }},+ }})+ if array != `["a",2]` {+ t.Fatalf("array=%s", array)+ }+ kv := AnyValue(&commonpb.AnyValue{Value: &commonpb.AnyValue_KvlistValue{+ KvlistValue: &commonpb.KeyValueList{Values: []*commonpb.KeyValue{+ {Key: "model", Value: &commonpb.AnyValue{Value: &commonpb.AnyValue_StringValue{StringValue: "gpt"}}},+ }},+ }})+ if kv != `{"model":"gpt"}` {+ t.Fatalf("kv=%s", kv)+ }+}++func TestMarshalJSONDropsOversizedValues(t *testing.T) {+ t.Parallel()+ huge := make([]byte, maxEncodedAnyValue+1)+ for i := range huge {+ huge[i] = 'a'+ }+ if got := marshalJSON(string(huge)); got != "" {+ t.Fatalf("got len=%d want empty", len(got))+ }+ if got := marshalJSON("ok"); got != `"ok"` {+ t.Fatalf("got=%s", got)+ }+}++func TestUnknownStatusAndKindFallBack(t *testing.T) {+ t.Parallel()+ rows := Traces(&tracepb.TracesData{+ ResourceSpans: []*tracepb.ResourceSpans{{+ ScopeSpans: []*tracepb.ScopeSpans{{+ Spans: []*tracepb.Span{{+ Name: "unknown",+ Kind: tracepb.Span_SpanKind(99),+ Status: &tracepb.Status{Code: tracepb.Status_StatusCode(99), Message: "future"},+ }},+ }},+ }},+ })+ if len(rows) != 1 {+ t.Fatalf("len=%d", len(rows))+ }+ if rows[0].StatusCode != "STATUS_CODE_UNSET" {+ t.Fatalf("status=%s", rows[0].StatusCode)+ }+ if rows[0].SpanKind != "SPAN_KIND_UNSPECIFIED" {+ t.Fatalf("kind=%s", rows[0].SpanKind)+ }+ if rows[0].StatusMessage != "future" {+ t.Fatalf("msg=%s", rows[0].StatusMessage)+ }+}++func TestMetricsMapsHistogramSummaryAndExpHistogram(t *testing.T) {+ t.Parallel()+ req := &metricspb.MetricsData{+ ResourceMetrics: []*metricspb.ResourceMetrics{{+ Resource: &resourcepb.Resource{Attributes: []*commonpb.KeyValue{+ {Key: "service.name", Value: &commonpb.AnyValue{Value: &commonpb.AnyValue_StringValue{StringValue: "gpu"}}},+ }},+ ScopeMetrics: []*metricspb.ScopeMetrics{{+ Metrics: []*metricspb.Metric{+ {+ Name: "http.server.duration",+ Data: &metricspb.Metric_Histogram{Histogram: &metricspb.Histogram{+ AggregationTemporality: metricspb.AggregationTemporality_AGGREGATION_TEMPORALITY_CUMULATIVE,+ DataPoints: []*metricspb.HistogramDataPoint{{+ Count: 3,+ Sum: ptrFloat(6),+ BucketCounts: []uint64{1, 2},+ ExplicitBounds: []float64{1},+ Min: ptrFloat(0.1),+ Max: ptrFloat(4),+ }},+ }},+ },+ {+ Name: "rpc.latency",+ Data: &metricspb.Metric_Summary{Summary: &metricspb.Summary{+ DataPoints: []*metricspb.SummaryDataPoint{{+ Count: 2,+ Sum: 3,+ QuantileValues: []*metricspb.SummaryDataPoint_ValueAtQuantile{+ {Quantile: 0.5, Value: 1.5},+ },+ }},+ }},+ },+ {+ Name: "gen_ai.client.token.usage",+ Data: &metricspb.Metric_ExponentialHistogram{ExponentialHistogram: &metricspb.ExponentialHistogram{+ AggregationTemporality: metricspb.AggregationTemporality_AGGREGATION_TEMPORALITY_DELTA,+ DataPoints: []*metricspb.ExponentialHistogramDataPoint{{+ Count: 4,+ Sum: ptrFloat(8),+ Scale: 1,+ ZeroCount: 1,+ Positive: &metricspb.ExponentialHistogramDataPoint_Buckets{Offset: 2, BucketCounts: []uint64{3}},+ }},+ }},+ },+ },+ }},+ }},+ }+ rows := Metrics(req)+ if len(rows.Histograms) != 1 || rows.Histograms[0].Count != 3 || rows.Histograms[0].Sum != 6 {+ t.Fatalf("histograms=%+v", rows.Histograms)+ }+ if len(rows.Summaries) != 1 || rows.Summaries[0].Count != 2 || len(rows.Summaries[0].Quantiles) != 1 {+ t.Fatalf("summaries=%+v", rows.Summaries)+ }+ if len(rows.ExpHistograms) != 1 || rows.ExpHistograms[0].ZeroCount != 1 || rows.ExpHistograms[0].PositiveOffset != 2 {+ t.Fatalf("exp=%+v", rows.ExpHistograms)+ }+}++func ptrFloat(v float64) *float64 { return &v }diff --git a/src/otlp-receiver/internal/tenant/tenant.go b/src/otlp-receiver/internal/tenant/tenant.gonew file mode 100644index 00000000..a34922aa--- /dev/null+++ b/src/otlp-receiver/internal/tenant/tenant.go@@ -0,0 +1,182 @@+package tenant++import (+ "context"+ "database/sql"+ "errors"+ "strings"+ "sync"+ "time"++ "github.com/openlit/openlit/otlp-receiver/internal/config"+ "github.com/openlit/openlit/otlp-receiver/internal/otlpconv"++ _ "modernc.org/sqlite"+)++var (+ ErrUnauthorized = errors.New("unauthorized")+ ErrNoTenant = errors.New("no tenant")+)++type Tenant struct {+ ClickHouse config.ClickHouseConfig+ OrganisationID string+ ProjectID string+ Environment string+}++func (t Tenant) Resource() otlpconv.ResourceTenant {+ return otlpconv.ResourceTenant{+ OrganisationID: t.OrganisationID,+ ProjectID: t.ProjectID,+ Environment: t.Environment,+ }+}++type Store struct {+ db *sql.DB+ RequireKey bool+ InitDB config.ClickHouseConfig+ ttl time.Duration+ mu sync.Mutex+ cache map[string]cacheEntry+}++type cacheEntry struct {+ tenant Tenant+ expiry time.Time+}++func Open(cfg config.Config) (*Store, error) {+ s := &Store{+ RequireKey: cfg.RequireAPIKey,+ InitDB: cfg.InitDB,+ ttl: time.Duration(cfg.TenantCacheTTLSec) * time.Second,+ cache: map[string]cacheEntry{},+ }+ if cfg.SQLitePath == "" {+ return s, nil+ }+ db, err := sql.Open("sqlite", cfg.SQLitePath)+ if err != nil {+ return nil, err+ }+ db.SetMaxOpenConns(4)+ s.db = db+ return s, nil+}++func (s *Store) Close() error {+ if s.db == nil {+ return nil+ }+ return s.db.Close()+}++func BearerToken(authorization string) string {+ authorization = strings.TrimSpace(authorization)+ if authorization == "" {+ return ""+ }+ const prefix = "Bearer "+ if len(authorization) >= len(prefix) && strings.EqualFold(authorization[:len(prefix)], prefix) {+ return strings.TrimSpace(authorization[len(prefix):])+ }+ return ""+}++func (s *Store) Resolve(ctx context.Context, authorization string) (Tenant, error) {+ token := BearerToken(authorization)+ if token == "" {+ if s.RequireKey {+ return Tenant{}, ErrUnauthorized+ }+ if s.InitDB.Host == "" {+ return Tenant{}, ErrNoTenant+ }+ return Tenant{ClickHouse: s.InitDB}, nil+ }+ if s.db == nil {+ return Tenant{}, ErrUnauthorized+ }++ s.mu.Lock()+ if entry, ok := s.cache[token]; ok && time.Now().Before(entry.expiry) {+ tenant := entry.tenant+ s.mu.Unlock()+ return tenant, nil+ }+ s.mu.Unlock()++ tenant, err := s.lookupScoped(ctx, token)+ if err != nil && isMissingColumn(err) {+ tenant, err = s.lookupLegacy(ctx, token)+ }+ if err != nil {+ return Tenant{}, err+ }++ s.mu.Lock()+ s.cache[token] = cacheEntry{tenant: tenant, expiry: time.Now().Add(s.ttl)}+ s.mu.Unlock()+ return tenant, nil+}++func (s *Store) lookupScoped(ctx context.Context, token string) (Tenant, error) {+ row := s.db.QueryRowContext(ctx, `+ SELECT d.host, d.port, d.username, COALESCE(d.password, ''), d.database,+ COALESCE(NULLIF(k.organisation_id, ''), p.organisation_id, ''),+ COALESCE(NULLIF(k.project_id, ''), d.project_id, ''),+ COALESCE(NULLIF(k.environment, ''), NULLIF(d.environment, ''), 'production')+ FROM APIKeys k+ JOIN databaseconfig d ON d.id = k.database_config_id+ LEFT JOIN projects p ON p.id = COALESCE(NULLIF(k.project_id, ''), d.project_id)+ WHERE k.apiKey = ? AND k.isDeleted = 0+ LIMIT 1+ `, token)+ return scanTenant(row)+}++func (s *Store) lookupLegacy(ctx context.Context, token string) (Tenant, error) {+ row := s.db.QueryRowContext(ctx, `+ SELECT d.host, d.port, d.username, COALESCE(d.password, ''), d.database,+ COALESCE(p.organisation_id, ''),+ COALESCE(d.project_id, ''),+ COALESCE(NULLIF(d.environment, ''), 'production')+ FROM APIKeys k+ JOIN databaseconfig d ON d.id = k.database_config_id+ LEFT JOIN projects p ON p.id = d.project_id+ WHERE k.apiKey = ? AND k.isDeleted = 0+ LIMIT 1+ `, token)+ return scanTenant(row)+}++func scanTenant(row *sql.Row) (Tenant, error) {+ var tenant Tenant+ if err := row.Scan(+ &tenant.ClickHouse.Host,+ &tenant.ClickHouse.Port,+ &tenant.ClickHouse.Username,+ &tenant.ClickHouse.Password,+ &tenant.ClickHouse.Database,+ &tenant.OrganisationID,+ &tenant.ProjectID,+ &tenant.Environment,+ ); err != nil {+ if errors.Is(err, sql.ErrNoRows) {+ return Tenant{}, ErrUnauthorized+ }+ return Tenant{}, err+ }+ return tenant, nil+}++func isMissingColumn(err error) bool {+ if err == nil {+ return false+ }+ msg := strings.ToLower(err.Error())+ return strings.Contains(msg, "no such column") || strings.Contains(msg, "has no column")+}diff --git a/src/otlp-receiver/internal/tenant/tenant_test.go b/src/otlp-receiver/internal/tenant/tenant_test.gonew file mode 100644index 00000000..ad304175--- /dev/null+++ b/src/otlp-receiver/internal/tenant/tenant_test.go@@ -0,0 +1,156 @@+package tenant++import (+ "context"+ "path/filepath"+ "testing"++ "github.com/openlit/openlit/otlp-receiver/internal/config"+ _ "modernc.org/sqlite"+)++func TestBearerToken(t *testing.T) {+ t.Parallel()+ if BearerToken("Bearer openlit-abc") != "openlit-abc" {+ t.Fatal("expected bearer token")+ }+ if BearerToken("bearer openlit-abc") != "openlit-abc" {+ t.Fatal("expected case-insensitive bearer")+ }+ if BearerToken("") != "" {+ t.Fatal("empty")+ }+}++func TestResolveRequiresKey(t *testing.T) {+ t.Parallel()+ s := &Store{RequireKey: true}+ _, err := s.Resolve(context.Background(), "")+ if err != ErrUnauthorized {+ t.Fatalf("got %v", err)+ }+}++func TestResolveInitDBFallback(t *testing.T) {+ t.Parallel()+ s := &Store{InitDB: config.ClickHouseConfig{Host: "ch", Port: "8123", Database: "openlit"}}+ cfg, err := s.Resolve(context.Background(), "")+ if err != nil {+ t.Fatal(err)+ }+ if cfg.ClickHouse.Host != "ch" || cfg.ClickHouse.Database != "openlit" {+ t.Fatalf("unexpected %+v", cfg)+ }+ if cfg.Environment != "" {+ t.Fatal("init-db fallback must not invent an environment")+ }+}++func TestResolveAPIKey(t *testing.T) {+ dir := t.TempDir()+ path := filepath.Join(dir, "data.db")+ s, err := Open(config.Config{SQLitePath: path, TenantCacheTTLSec: 30})+ if err != nil {+ t.Fatal(err)+ }+ t.Cleanup(func() { _ = s.Close() })+ _, err = s.db.Exec(`+ CREATE TABLE organisations (id TEXT PRIMARY KEY);+ CREATE TABLE projects (+ id TEXT PRIMARY KEY,+ organisation_id TEXT+ );+ CREATE TABLE databaseconfig (+ id TEXT PRIMARY KEY,+ host TEXT,+ port TEXT,+ username TEXT,+ password TEXT,+ database TEXT,+ project_id TEXT,+ environment TEXT+ );+ CREATE TABLE APIKeys (+ apiKey TEXT PRIMARY KEY,+ database_config_id TEXT,+ organisation_id TEXT,+ project_id TEXT,+ environment TEXT,+ isDeleted INTEGER+ );+ INSERT INTO organisations VALUES ('org-1');+ INSERT INTO projects VALUES ('proj-1', 'org-1');+ INSERT INTO databaseconfig VALUES ('db-1', 'tenant-host', '8123', 'user', 'secret', 'orgdb', 'proj-1', 'staging');+ INSERT INTO APIKeys VALUES ('openlit-good', 'db-1', 'org-1', 'proj-1', 'staging', 0);+ INSERT INTO APIKeys VALUES ('openlit-deleted', 'db-1', 'org-1', 'proj-1', 'staging', 1);+ `)+ if err != nil {+ t.Fatal(err)+ }++ cfg, err := s.Resolve(context.Background(), "Bearer openlit-good")+ if err != nil {+ t.Fatal(err)+ }+ if cfg.ClickHouse.Host != "tenant-host" || cfg.ClickHouse.Database != "orgdb" || cfg.ClickHouse.Password != "secret" {+ t.Fatalf("unexpected %+v", cfg.ClickHouse)+ }+ if cfg.OrganisationID != "org-1" || cfg.ProjectID != "proj-1" || cfg.Environment != "staging" {+ t.Fatalf("unexpected scope %+v", cfg)+ }++ if _, err := s.Resolve(context.Background(), "Bearer openlit-deleted"); err != ErrUnauthorized {+ t.Fatalf("deleted key: %v", err)+ }+ if _, err := s.Resolve(context.Background(), "Bearer openlit-missing"); err != ErrUnauthorized {+ t.Fatalf("missing key: %v", err)+ }+ if _, err := s.Resolve(context.Background(), "Bearer openlit-good"); err != nil {+ t.Fatal("cached lookup failed")+ }+}++func TestResolveAPIKeyLegacyColumns(t *testing.T) {+ dir := t.TempDir()+ path := filepath.Join(dir, "legacy.db")+ s, err := Open(config.Config{SQLitePath: path, TenantCacheTTLSec: 30})+ if err != nil {+ t.Fatal(err)+ }+ t.Cleanup(func() { _ = s.Close() })+ _, err = s.db.Exec(`+ CREATE TABLE projects (+ id TEXT PRIMARY KEY,+ organisation_id TEXT+ );+ CREATE TABLE databaseconfig (+ id TEXT PRIMARY KEY,+ host TEXT,+ port TEXT,+ username TEXT,+ password TEXT,+ database TEXT,+ project_id TEXT,+ environment TEXT+ );+ CREATE TABLE APIKeys (+ apiKey TEXT PRIMARY KEY,+ database_config_id TEXT,+ isDeleted INTEGER+ );+ INSERT INTO projects VALUES ('proj-1', 'org-1');+ INSERT INTO databaseconfig VALUES ('db-1', 'tenant-host', '8123', 'user', 'secret', 'orgdb', 'proj-1', 'production');+ INSERT INTO APIKeys VALUES ('openlit-good', 'db-1', 0);+ `)+ if err != nil {+ t.Fatal(err)+ }++ cfg, err := s.Resolve(context.Background(), "Bearer openlit-good")+ if err != nil {+ t.Fatal(err)+ }+ if cfg.OrganisationID != "org-1" || cfg.ProjectID != "proj-1" || cfg.Environment != "production" {+ t.Fatalf("unexpected legacy scope %+v", cfg)+ }+}