Skip to content

Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.

feat: add VictoriaMetrics stack as destination (#1067)

Roman Khavronenkobdaeeb411e16
diff --git a/docs/images/vm-dashboard-1.png b/docs/images/vm-dashboard-1.png
new file mode 100644
index 00000000..c2324b27
Binary files /dev/null and b/docs/images/vm-dashboard-1.png differ
diff --git a/docs/images/vm-dashboard-2.png b/docs/images/vm-dashboard-2.png
new file mode 100644
index 00000000..20738414
Binary files /dev/null and b/docs/images/vm-dashboard-2.png differ
diff --git a/docs/latest/sdk/destinations/victoriametrics-stack.mdx b/docs/latest/sdk/destinations/victoriametrics-stack.mdx
new file mode 100644
index 00000000..1e009ad0
--- /dev/null
+++ b/docs/latest/sdk/destinations/victoriametrics-stack.mdx
@@ -0,0 +1,12 @@
+---
+title: 'VictoriaMetrics stack'
+description: 'LLM Observability with VictoriaMetrics stack using OpenLIT'
+---
+
+import Intro from '/snippets/destinations/victoriametrics-stack/intro.mdx';
+import SDK from '/snippets/destinations/victoriametrics-stack/sdk.mdx';
+import Conclusion from '/snippets/destinations/victoriametrics-stack/conclusion.mdx';
+
+<Intro />
+<SDK />
+<Conclusion />
diff --git a/docs/snippets/destinations/victoriametrics-stack/conclusion.mdx b/docs/snippets/destinations/victoriametrics-stack/conclusion.mdx
new file mode 100644
index 00000000..a2c6fcef
--- /dev/null
+++ b/docs/snippets/destinations/victoriametrics-stack/conclusion.mdx
@@ -0,0 +1,463 @@
+### 3. Import the pre-built Dashboard
+
+1. **Log into your Grafana Instance**. To install Grafana, refer to the [Official documentation](https://grafana.com/docs/grafana/latest/#installing-grafana).
+1. **Add Data Sources**: Make sure Prometheus (VictoriaMetrics), Jaeger (VictoriaTraces) and VictoriaLogs are added as data sources in Grafana. To add a new data source, follow the steps in the [Official documentation](https://grafana.com/docs/grafana/latest/datasources/#add-a-data-source).
+1. **Import Dashboard**: Once datasources are available in Grafana:
+ - Click **Dashboards** in the primary menu
+ - Click **New** and select **Import** in the drop-down menu
+ - Copy the dashboard JSON provided in the accordion below
+ - Paste the dashboard JSON text directly into the text area
+ - Click **Import**
+ - Save the dashboard
+
+<Accordion title="Dashboard">
+```json
+{
+ "__inputs": [
+ {
+ "name": "DS_VICTORIAMETRICS",
+ "label": "VictoriaMetrics",
+ "description": "",
+ "type": "datasource",
+ "pluginId": "prometheus",
+ "pluginName": "Prometheus"
+ },
+ {
+ "name": "DS_VICTORIALOGS",
+ "label": "VictoriaLogs",
+ "description": "",
+ "type": "datasource",
+ "pluginId": "victoriametrics-logs-datasource",
+ "pluginName": "VictoriaLogs"
+ },
+ {
+ "name": "DS_VICTORIATRACES",
+ "label": "VictoriaTraces",
+ "description": "",
+ "type": "datasource",
+ "pluginId": "jaeger",
+ "pluginName": "Jaeger"
+ }
+ ],
+ "__elements": {},
+ "__requires": [
+ {
+ "type": "grafana",
+ "id": "grafana",
+ "name": "Grafana",
+ "version": "12.3.3"
+ },
+ {
+ "type": "panel",
+ "id": "heatmap",
+ "name": "Heatmap",
+ "version": ""
+ },
+ {
+ "type": "datasource",
+ "id": "jaeger",
+ "name": "Jaeger",
+ "version": "12.3.3"
+ },
+ {
+ "type": "panel",
+ "id": "logs",
+ "name": "Logs",
+ "version": ""
+ },
+ {
+ "type": "datasource",
+ "id": "prometheus",
+ "name": "Prometheus",
+ "version": "1.0.0"
+ },
+ {
+ "type": "panel",
+ "id": "table",
+ "name": "Table",
+ "version": ""
+ },
+ {
+ "type": "panel",
+ "id": "timeseries",
+ "name": "Time series",
+ "version": ""
+ },
+ {
+ "type": "datasource",
+ "id": "victoriametrics-logs-datasource",
+ "name": "VictoriaLogs",
+ "version": "0.26.2"
+ }
+ ],
+ "annotations": {
+ "list": [
+ {
+ "builtIn": 1,
+ "datasource": {
+ "type": "grafana",
+ "uid": "-- Grafana --"
+ },
+ "enable": true,
+ "hide": true,
+ "iconColor": "rgba(0, 211, 255, 1)",
+ "name": "Annotations & Alerts",
+ "type": "dashboard"
+ }
+ ]
+ },
+ "editable": true,
+ "fiscalYearStartMonth": 0,
+ "graphTooltip": 0,
+ "id": null,
+ "links": [],
+ "panels": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "showValues": false,
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": 0
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 0
+ },
+ "id": 1,
+ "options": {
+ "legend": {
+ "calcs": [
+ "last",
+ "mean",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "12.3.3",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate({\"gen_ai.client.token.usage_count\"}[5m])) by(\"gen_ai.operation.name\",\"gen_ai.provider.name\",\"gen_ai.response.model\",\"gen_ai.token.type\")",
+ "instant": false,
+ "legendFormat": "__auto",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Tokens usage rate",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "scaleDistribution": {
+ "type": "linear"
+ }
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 0
+ },
+ "id": 2,
+ "options": {
+ "calculate": false,
+ "cellGap": 1,
+ "color": {
+ "exponent": 0.5,
+ "fill": "dark-orange",
+ "mode": "scheme",
+ "reverse": false,
+ "scale": "exponential",
+ "scheme": "Oranges",
+ "steps": 58
+ },
+ "exemplars": {
+ "color": "rgba(255,0,255,0.7)"
+ },
+ "filterValues": {
+ "le": 1e-9
+ },
+ "legend": {
+ "show": true
+ },
+ "rowsFrame": {
+ "layout": "auto"
+ },
+ "tooltip": {
+ "mode": "single",
+ "showColorScale": false,
+ "yHistogram": false
+ },
+ "yAxis": {
+ "axisPlacement": "hidden",
+ "decimals": -2,
+ "reverse": false,
+ "unit": "s"
+ }
+ },
+ "pluginVersion": "12.3.3",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(increase({__name__=\"gen_ai.server.request.duration_bucket\"}[5m])) by(le)",
+ "format": "heatmap",
+ "instant": false,
+ "legendFormat": "request latency",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Server request latency",
+ "type": "heatmap"
+ },
+ {
+ "datasource": {
+ "type": "victoriametrics-logs-datasource",
+ "uid": "${logs}"
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 8
+ },
+ "id": 3,
+ "options": {
+ "dedupStrategy": "none",
+ "enableInfiniteScrolling": false,
+ "enableLogDetails": true,
+ "showControls": false,
+ "showTime": false,
+ "sortOrder": "Descending",
+ "wrapLogMessage": false
+ },
+ "pluginVersion": "12.3.3",
+ "targets": [
+ {
+ "datasource": {
+ "type": "victoriametrics-logs-datasource",
+ "uid": "${logs}"
+ },
+ "direction": "desc",
+ "editorMode": "code",
+ "expr": "*",
+ "queryType": "instant",
+ "refId": "A"
+ }
+ ],
+ "title": "Logs",
+ "type": "logs"
+ },
+ {
+ "datasource": {
+ "type": "jaeger",
+ "uid": "${traces}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "custom": {
+ "align": "auto",
+ "cellOptions": {
+ "type": "auto"
+ },
+ "footer": {
+ "reducers": []
+ },
+ "inspect": false
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": 0
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 8
+ },
+ "id": 4,
+ "options": {
+ "cellHeight": "sm",
+ "showHeader": true
+ },
+ "pluginVersion": "12.3.3",
+ "targets": [
+ {
+ "datasource": {
+ "type": "jaeger",
+ "uid": "${traces}"
+ },
+ "hide": false,
+ "maxDuration": "",
+ "minDuration": "",
+ "queryType": "search",
+ "refId": "A",
+ "service": "random_joke_generator"
+ }
+ ],
+ "title": "Traces",
+ "type": "table"
+ }
+ ],
+ "preload": false,
+ "schemaVersion": 42,
+ "tags": [],
+ "templating": {
+ "list": [
+ {
+ "current": {
+ "text": "",
+ "value": "${DS_VICTORIAMETRICS}",
+ "selected": true
+ },
+ "name": "metrics",
+ "options": [],
+ "query": "prometheus",
+ "refresh": 1,
+ "regex": "",
+ "type": "datasource"
+ },
+ {
+ "current": {
+ "text": "",
+ "value": "${DS_VICTORIALOGS}",
+ "selected": true
+ },
+ "name": "logs",
+ "options": [],
+ "query": "victoriametrics-logs-datasource",
+ "refresh": 1,
+ "regex": "",
+ "type": "datasource"
+ },
+ {
+ "current": {
+ "text": "",
+ "value": "${DS_VICTORIATRACES}",
+ "selected": true
+ },
+ "name": "traces",
+ "options": [],
+ "query": "jaeger",
+ "refresh": 1,
+ "regex": "",
+ "type": "datasource"
+ }
+ ]
+ },
+ "time": {
+ "from": "now-30m",
+ "to": "now"
+ },
+ "timepicker": {},
+ "timezone": "browser",
+ "title": "VictoriaMetrics Openlit Gen AI demo",
+ "uid": "adw6krf",
+ "version": 6,
+ "weekStart": ""
+}
+```
+</Accordion>
diff --git a/docs/snippets/destinations/victoriametrics-stack/intro.mdx b/docs/snippets/destinations/victoriametrics-stack/intro.mdx
new file mode 100644
index 00000000..87783b78
--- /dev/null
+++ b/docs/snippets/destinations/victoriametrics-stack/intro.mdx
@@ -0,0 +1,52 @@
+<Frame>
+ <img src="/images/vm-dashboard-1.png" />
+ <img src="/images/vm-dashboard-2.png" />
+</Frame>
+
+See practical guide [AI Agents Observability with OpenTelemetry and the VictoriaMetrics Stack](https://victoriametrics.com/blog/ai-agents-observability)
+for detailed overview.
+
+To send OpenTelemetry metrics, logs and traces generated by OpenLIT from your AI Application to VictoriaMetrics stack, follow the below steps.
+
+### 1. Configure OpenTelemetry Collector
+
+VictoriaMetrics stack requires an OpenTelemetry Collector to route metrics, logs and traces to VictoriaMetrics components.
+
+**Install OpenTelemetry Collector (if not already running)**
+
+For detailed installation instructions, refer to the [OpenTelemetry Collector Documentation](https://opentelemetry.io/docs/collector/installation/).
+
+**Configure the Collector**
+
+1. **Configure OTLP Receiver**: Set up receiver on `0.0.0.0:4318` for HTTP and `0.0.0.0:4317` for gRPC.
+2. **Define Exporters**: `otlphttp` for metrics, logs and traces.
+3. **Assign to Pipelines**: Route metrics, logs and traces to appropriate backends.
+
+<Accordion title="Complete Collector Configuration">
+```yaml
+receivers:
+ otlp:
+ protocols:
+ grpc:
+ endpoint: 0.0.0.0:4317
+ http:
+ endpoint: 0.0.0.0:4318
+
+exporters:
+ otlphttp/victoriametrics:
+ endpoint: "http://<VictoriaMetrics>/opentelemetry"
+ otlphttp/victorialogs:
+ endpoint: "http://<VictoriaLogs>/insert/opentelemetry"
+ otlphttp/victoriatraces:
+ endpoint: "http://<VictoriaTraces>/insert/opentelemetry"
+
+service:
+ pipelines:
+ traces: { receivers: [ otlp ], exporters: [ otlphttp/victoriatraces ] }
+ metrics: { receivers: [ otlp ], exporters: [ otlphttp/victoriametrics ] }
+ logs: { receivers: [ otlp ], exporters: [ otlphttp/victorialogs ] }
+```
+
+See examples of configuring VictoriaMetrics stack and OpenTelemetry Collector via [docker](https://victoriametrics.com/blog/ai-agents-observability/#installation-with-docker-compose)
+or [helm](https://victoriametrics.com/blog/ai-agents-observability/#installation-with-helm).
+</Accordion>
diff --git a/docs/snippets/destinations/victoriametrics-stack/sdk.mdx b/docs/snippets/destinations/victoriametrics-stack/sdk.mdx
new file mode 100644
index 00000000..eb1a2c42
--- /dev/null
+++ b/docs/snippets/destinations/victoriametrics-stack/sdk.mdx
@@ -0,0 +1,84 @@
+### 2. Instrument your application
+ <Tabs>
+ <Tab title="SDK">
+ **For direct integration into your Python applications:**
+
+ <Tabs>
+ <Tab title="Function Arguments">
+ ```python
+ import openlit
+
+ openlit.init(
+ otlp_endpoint="YOUR_OTELCOL_URL:4318"
+ )
+ ```
+
+ Replace:
+ 1. `YOUR_OTELCOL_URL:4318` with the HTTP endpoint of your OpenTelemetry Collector.
+ * Example: `http://127.0.0.1:4318` (for local collector)
+ * Example: `http://otel-collector.monitoring.svc.cluster.local:4318` (for Kubernetes)
+ </Tab>
+
+ <Tab title="Environment Variables">
+ ```python
+ import openlit
+
+ openlit.init()
+ ```
+
+ Set these environment variables:
+ ```shell
+ export OTEL_EXPORTER_OTLP_ENDPOINT="YOUR_OTELCOL_URL:4318"
+ ```
+
+ Replace:
+ 1. `YOUR_OTELCOL_URL:4318` with the HTTP endpoint of your OpenTelemetry Collector.
+ * Example: `http://127.0.0.1:4318` (for local collector)
+ * Example: `http://otel-collector.monitoring.svc.cluster.local:4318` (for Kubernetes)
+ </Tab>
+ </Tabs>
+
+ Refer to the OpenLIT [Python SDK repository](https://github.com/openlit/openlit/tree/main/sdk/python) for more advanced configurations and use cases.
+ </Tab>
+
+ <Tab title="CLI">
+ **For zero-code auto-instrumentation via command line:**
+
+ <Tabs>
+ <Tab title="CLI Arguments">
+ ```shell
+ # Using CLI arguments
+ openlit-instrument \
+ --otlp-endpoint "YOUR_OTELCOL_URL:4318" \
+ --service-name "my-ai-service" \
+ --deployment-environment "production" \
+ python app.py
+ ```
+
+ Replace:
+ 1. `YOUR_OTELCOL_URL:4318` with the HTTP endpoint of your OpenTelemetry Collector.
+ * Example: `http://127.0.0.1:4318` (for local collector)
+ * Example: `http://otel-collector.monitoring.svc.cluster.local:4318` (for Kubernetes)
+ </Tab>
+
+ <Tab title="Environment Variables">
+ ```shell
+ # Set environment variables (takes precedence over CLI args)
+ export OTEL_EXPORTER_OTLP_ENDPOINT="YOUR_OTELCOL_URL:4318"
+ export OTEL_SERVICE_NAME="my-ai-service"
+ export OTEL_DEPLOYMENT_ENVIRONMENT="production"
+
+ # Run your application
+ openlit-instrument python app.py
+ ```
+
+ Replace:
+ 1. `YOUR_OTELCOL_URL:4318` with the HTTP endpoint of your OpenTelemetry Collector.
+ * Example: `http://127.0.0.1:4318` (for local collector)
+ * Example: `http://otel-collector.monitoring.svc.cluster.local:4318` (for Kubernetes)
+ </Tab>
+ </Tabs>
+
+ Refer to the OpenLIT [Python SDK repository](https://github.com/openlit/openlit/tree/main/sdk/python) for more advanced configurations and use cases.
+ </Tab>
+ </Tabs>