Skip to content

Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.

[Feat]: Add OpenLIT library (#167)

patcher9d4eadc6f007f
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
index 83b43162..e163beef 100644
--- a/.github/dependabot.yml
+++ b/.github/dependabot.yml
@@ -4,11 +4,15 @@ updates:
directory: "/"
schedule:
interval: "weekly"
- - package-ecosystem: "gomod"
- directory: "/src/ingester/"
- schedule:
- interval: "weekly"
- package-ecosystem: "npm"
directory: "/src/client/"
schedule:
interval: "weekly"
+ - package-ecosystem: "pip"
+ directory: "/sdk/python"
+ schedule:
+ interval: "weekly"
+ - package-ecosystem: "pip"
+ directory: "/sdk/python/tests/"
+ schedule:
+ interval: "weekly"
diff --git a/.github/workflows/doku-release.yml b/.github/workflows/doku-release.yml
new file mode 100644
index 00000000..a0eadc8a
--- /dev/null
+++ b/.github/workflows/doku-release.yml
@@ -0,0 +1,89 @@
+name: Release Doku
+
+on:
+ push:
+ tags: [ '*.*.*' ]
+
+env:
+ REGISTRY: ghcr.io
+
+permissions:
+ contents: write
+
+jobs:
+ release:
+ name: Client Docker Image
+ runs-on: ubuntu-latest
+ outputs:
+ version: ${{ steps.set-version-output.outputs.version }}
+ permissions:
+ contents: write
+ packages: write
+ id-token: write
+ env:
+ IMAGE_NAME: ${{ github.repository }}-client
+
+ steps:
+ - name: Get current date and time
+ id: datetime
+ run: echo "::set-output name=datetime::$(date -u +'%Y-%m-%dT%H:%M:%SZ')"
+
+ - name: Checkout repository
+ uses: actions/checkout@v4
+
+ - name: Install cosign
+ uses: sigstore/cosign-installer@e1523de7571e31dbe865fd2e80c5c7c23ae71eb4 #v3.4.0
+ with:
+ cosign-release: 'v2.2.2'
+
+ - name: Set up Docker Buildx
+ uses: docker/setup-buildx-action@2b51285047da1547ffb1b2203d8be4c0af6b1f20 # v3.2.0
+
+ - name: Code Format Check (gofmt)
+ run: |
+ if [ -n "$(gofmt -l .)" ]; then
+ echo "Go code is not formatted:"
+ gofmt -d .
+ exit 1
+ fi
+
+ - name: Log into registry ${{ env.REGISTRY }}
+ uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v3.1.0
+ with:
+ registry: ${{ env.REGISTRY }}
+ username: ${{ github.actor }}
+ password: ${{ secrets.GITHUB_TOKEN }}
+
+ - name: Extract Docker metadata
+ id: meta
+ uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1
+ with:
+ images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+ env:
+ DOCKER_METADATA_ANNOTATIONS_LEVELS: manifest,index
+
+ - name: Build and push Docker image
+ id: build-and-push
+ uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5.3.0
+ with:
+ context: ./src/client/
+ provenance: false
+ file: ./src/client/Dockerfile
+ push: ${{ github.event_name != 'pull_request' }}
+ tags: ${{ steps.meta.outputs.tags }}
+ labels: ${{ steps.meta.outputs.labels }}
+ platforms: linux/amd64,linux/arm64/v8
+ annotations: |
+ ${{ steps.meta.outputs.annotations }},
+ io.artifacthub.package.logo-url=https://github.com/dokulabs/.github/blob/main/profile/assets/logo-orange-bg.jpg?raw=true,
+ io.artifacthub.package.readme-url=https://raw.githubusercontent.com/dokulabs/doku/main/README.md,
+ io.artifacthub.package.license=Apache-2.0,
+ org.opencontainers.image.vendor=Doku Labs,
+ io.artifacthub.package.maintainers=[{'name':'Doku Labs','email':'[email protected]'}],
+ io.artifacthub.package.keywords='Monitoring, Observability, LLMs, OpenAI'
+
+ - name: Sign the published Docker image
+ env:
+ TAGS: ${{ steps.meta.outputs.tags }}
+ DIGEST: ${{ steps.build-and-push.outputs.digest }}
+ run: echo "${TAGS}" | xargs -I {} cosign sign --yes {}@${DIGEST}
\ No newline at end of file
diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml
new file mode 100644
index 00000000..e1c69b9d
--- /dev/null
+++ b/.github/workflows/pylint.yml
@@ -0,0 +1,28 @@
+name: Pylint
+
+on: [push]
+
+jobs:
+ build:
+ runs-on: ubuntu-latest
+ strategy:
+ matrix:
+ python-version: ["3.9", "3.10", "3.11"]
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Set up Python ${{ matrix.python-version }}
+ uses: actions/setup-python@v3
+ with:
+ python-version: ${{ matrix.python-version }}
+
+ - name: Install dependencies
+ working-directory: sdk/python
+ run: |
+ python -m pip install --upgrade pip
+ python -m pip install -r tests/requirements.txt
+
+ - name: Analysing the code with pylint
+ working-directory: sdk/python
+ run: |
+ pylint $(git ls-files '*.py')
diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml
new file mode 100644
index 00000000..204d37f3
--- /dev/null
+++ b/.github/workflows/pypi-release.yml
@@ -0,0 +1,37 @@
+name: Release on PYPI
+
+on:
+ push:
+ tags: [ '*.*.*' ]
+
+permissions:
+ contents: write
+
+jobs:
+ release:
+ runs-on: ubuntu-latest
+
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Set up Python
+ uses: actions/setup-python@v3
+ with:
+ python-version: '3.x'
+
+ - name: Install dependencies
+ working-directory: sdk/python
+ run: |
+ python -m pip install --upgrade pip
+ pip install build
+
+ - name: Build package
+ working-directory: sdk/python
+ run: python -m build
+
+ - name: Publish package
+ uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29
+ with:
+ user: __token__
+ password: ${{ secrets.PYPI_API_TOKEN }}
+ packages-dir: sdk/python
diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml
new file mode 100644
index 00000000..210a0f23
--- /dev/null
+++ b/.github/workflows/python-tests.yml
@@ -0,0 +1,47 @@
+name: Python Tests
+
+on:
+ push:
+ branches: [ "main" ]
+ pull_request:
+ branches: [ "main" ]
+ workflow_dispatch:
+ schedule:
+ - cron: '0 0 1-31/15 * *'
+
+env:
+ OPENAI_API_TOKEN: ${{ secrets.OPENAI_API_TOKEN }}
+ COHERE_API_TOKEN: ${{ secrets.COHERE_API_TOKEN }}
+ ANTHROPIC_API_TOKEN: ${{ secrets.ANTHROPIC_API_TOKEN }}
+ MISTRAL_API_TOKEN: ${{ secrets.MISTRAL_API_TOKEN }}
+ PINECONE_API_TOKEN: ${{ secrets.PINECONE_API_TOKEN }}
+
+jobs:
+ build:
+ runs-on: ubuntu-latest
+ strategy:
+ fail-fast: false
+ matrix:
+ python-version: ["3.9", "3.10", "3.11"]
+
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Set up Docker Buildx
+ uses: docker/setup-buildx-action@0d103c3126aa41d772a8362f6aa67afac040f80c # v3.1.0
+
+ - name: Set up Python ${{ matrix.python-version }}
+ uses: actions/setup-python@v3
+ with:
+ python-version: ${{ matrix.python-version }}
+
+ - name: Install dependencies
+ working-directory: sdk/python
+ run: |
+ python -m pip install --upgrade pip
+ python -m pip install -r tests/requirements.txt
+
+ - name: PyTest
+ working-directory: sdk/python
+ run: |
+ pytest
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 54b16681..3e0c15b3 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -203,4 +203,4 @@ jobs:
file: src/ingester/ingester-${{ matrix.platform.os }}-${{ matrix.platform.arch }}
asset_name: ingester-${{ matrix.platform.os }}-${{ matrix.platform.arch }}
tag: ${{ needs.ingester-docker.outputs.version }}
- overwrite: true
+ overwrite: true
\ No newline at end of file
diff --git a/.github/workflows/test-pypi-release.yml b/.github/workflows/test-pypi-release.yml
new file mode 100644
index 00000000..87a4aecd
--- /dev/null
+++ b/.github/workflows/test-pypi-release.yml
@@ -0,0 +1,69 @@
+name: Release on Test PYPI
+
+on:
+ workflow_dispatch:
+
+permissions:
+ contents: read
+
+jobs:
+ release:
+ runs-on: ubuntu-latest
+
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Set up Python
+ uses: actions/setup-python@v3
+ with:
+ python-version: '3.x'
+
+ - name: Install dependencies
+ working-directory: sdk/python
+ run: |
+ python -m pip install --upgrade pip
+ pip install build
+
+ - name: Remove Main pyproject.toml from repository
+ working-directory: sdk/python
+ run: rm pyproject.toml
+
+ - name: Create a new pyproject.toml
+ working-directory: sdk/python
+ run: |
+ cat <<EOF > pyproject.toml
+ [tool.poetry]
+ name = "openlit"
+ version = "0.0.${{ github.run_number }}"
+ description = "OpenTelemetry-native Auto instrumentation library for monitoring LLM Applications, facilitating the integration of observability into your GenAI-driven projects"
+ authors = ["OpenLIT"]
+ repository = "https://github.com/open-lit/openlit/tree/main/openlit/python"
+ readme = "README.md"
+ homepage = "https://github.com/open-lit/openlit/tree/main/openlit/python"
+ keywords = ["OpenTelemetry", "otel", "otlp","llm", "tracing", "openai", "anthropic", "claude", "cohere", "llm monitoring", "observability", "monitoring", "gpt", "Generative AI", "chatGPT"]
+
+ [tool.poetry.dependencies]
+ python = "^3.7.1"
+ requests = "^2.26.0"
+ tiktoken = "^0.1.1"
+ opentelemetry-api = "^1.24.0"
+ opentelemetry-sdk = "^1.24.0"
+ opentelemetry-exporter-otlp = "^1.24.0"
+ opentelemetry-instrumentation = "^0.45b0"
+
+ [build-system]
+ requires = ["poetry-core>=1.1.0"]
+ build-backend = "poetry.core.masonry.api"
+ EOF
+
+ - name: Build package
+ working-directory: sdk/python
+ run: python -m build
+
+ - name: Publish package
+ uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29
+ with:
+ user: __token__
+ repository_url: https://test.pypi.org/legacy/
+ password: ${{ secrets.TEST_PYPI_API_TOKEN }}
+ packages-dir: sdk/python
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
deleted file mode 100644
index 09d8fd22..00000000
--- a/.github/workflows/tests.yml
+++ /dev/null
@@ -1,35 +0,0 @@
-name: Tests
-
-on:
- push:
- branches: [ "main" ]
- pull_request:
- branches: [ "main" ]
-
-jobs:
- ingester:
- runs-on: ubuntu-latest
- steps:
- - uses: actions/checkout@v4
-
- - name: Set up Go
- uses: actions/setup-go@v5
- with:
- go-version: '1.21.1'
-
- - name: Build
- working-directory: ./src/ingester
- run: go build -v ./...
-
- - name: Code Format Check (gofmt)
- working-directory: ./src/ingester
- run: |
- if [ -n "$(gofmt -l .)" ]; then
- echo "Go code is not formatted:"
- gofmt -d .
- exit 1
- fi
-
- - name: Test
- working-directory: ./src/ingester
- run: go test -v ./...
diff --git a/.gitignore b/.gitignore
index 46e4f065..c65d5648 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,4 +1,7 @@
node_modules
test.py
test.go
-.DS_Store
\ No newline at end of file
+.DS_Store
+yala.py
+yala.yml
+speech.mp3
\ No newline at end of file
diff --git a/assets/otel-collector-config.yaml b/assets/otel-collector-config.yaml
new file mode 100644
index 00000000..ed967881
--- /dev/null
+++ b/assets/otel-collector-config.yaml
@@ -0,0 +1,47 @@
+receivers:
+ otlp:
+ protocols:
+ grpc:
+ endpoint: 0.0.0.0:4317
+ http:
+ endpoint: 0.0.0.0:4318
+processors:
+ batch:
+ memory_limiter:
+ # 80% of maximum memory up to 2G
+ limit_mib: 1500
+ # 25% of limit up to 2G
+ spike_limit_mib: 512
+ check_interval: 5s
+extensions:
+ zpages: {}
+exporters:
+ clickhouse:
+ endpoint: tcp://146.190.11.107:9000?dial_timeout=10s
+ database: openlit
+ username: default
+ password: DOKU
+ ttl: 72h
+ logs_table_name: otel_logs
+ traces_table_name: otel_traces
+ metrics_table_name: otel_metrics
+ timeout: 5s
+ retry_on_failure:
+ enabled: true
+ initial_interval: 5s
+ max_interval: 30s
+ max_elapsed_time: 300s
+service:
+ pipelines:
+ logs:
+ receivers: [ otlp ]
+ processors: [ batch ]
+ exporters: [ clickhouse ]
+ traces:
+ receivers: [ otlp ]
+ processors: [memory_limiter, batch]
+ exporters: [ clickhouse ]
+ metrics:
+ receivers: [ otlp ]
+ processors: [memory_limiter, batch]
+ exporters: [ clickhouse ]
diff --git a/ch.py b/ch.py
new file mode 100644
index 00000000..c55c89b1
--- /dev/null
+++ b/ch.py
@@ -0,0 +1,28 @@
+# Display Tables in Clickhouse
+from clickhouse_driver import Client
+
+# Connect to Clickhouse
+client = Client(host='146.190.11.107', user='default', password='DOKU', database='openlmt')
+
+# Getting the list of existing tables
+tables = client.execute('SHOW TABLES')
+
+# Print the existing tables in the database
+print("Existing tables in the database:")
+for table in tables:
+ print(table[0])
+
+
+# # Display data from table in Clickhouse
+# from clickhouse_driver import Client
+
+# client = Client(host='146.190.11.107, user='default', password='DOKU', database='openlmt')
+
+# # Replace your_table_name with the name of your table
+# query = "SELECT * FROM DOKU_APIKEYS"
+
+# result = client.execute(query)
+
+# # Print each row
+# for row in result:
+# print(row)
\ No newline at end of file
diff --git a/docker-compose.yml b/docker-compose.yml
index 5c6f95ac..17ba550b 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -1,52 +1,62 @@
version: '3.8'
services:
- clickhouse:
- image: clickhouse/clickhouse-server:24.2.2
- container_name: clickhouse
- environment:
- CLICKHOUSE_PASSWORD: ${DOKU_DB_PASSWORD:-DOKU}
- CLICKHOUSE_USER: ${DOKU_DB_USER:-default}
- volumes:
- - clickhouse-data:/var/lib/clickhouse
- ports:
- - "9000:9000"
- - "8123:8123"
- restart: always
+ # clickhouse:
+ # image: clickhouse/clickhouse-server:24.2.2
+ # container_name: clickhouse
+ # environment:
+ # CLICKHOUSE_PASSWORD: ${DOKU_DB_PASSWORD:-DOKU}
+ # CLICKHOUSE_USER: ${DOKU_DB_USER:-default}
+ # volumes:
+ # - clickhouse-data:/var/lib/clickhouse
+ # ports:
+ # - "9000:9000"
+ # - "8123:8123"
+ # restart: always
- doku-ingester:
- image: ghcr.io/dokulabs/doku-ingester:latest
- container_name: doku-ingester
- environment:
- DOKU_DB_HOST: clickhouse
- DOKU_DB_PORT: 9000
- DOKU_DB_NAME: ${DOKU_DB_NAME:-default}
- DOKU_DB_USER: ${DOKU_DB_USER:-default}
- DOKU_DB_PASSWORD: ${DOKU_DB_PASSWORD:-DOKU}
- ports:
- - "9044:9044"
- depends_on:
- - clickhouse
- restart: always
+ # doku-ingester:
+ # image: ghcr.io/dokulabs/doku-ingester:latest
+ # container_name: doku-ingester
+ # environment:
+ # DOKU_DB_HOST: clickhouse
+ # DOKU_DB_PORT: 9000
+ # DOKU_DB_NAME: ${DOKU_DB_NAME:-default}
+ # DOKU_DB_USER: ${DOKU_DB_USER:-default}
+ # DOKU_DB_PASSWORD: ${DOKU_DB_PASSWORD:-DOKU}
+ # ports:
+ # - "9044:9044"
+ # depends_on:
+ # - clickhouse
+ # restart: always
+
+ # doku-client:
+ # image: ghcr.io/dokulabs/doku-client:latest
+ # container_name: doku-client
+ # environment:
+ # INIT_DB_HOST: clickhouse
+ # INIT_DB_PORT: 8123
+ # INIT_DB_DATABASE: ${DOKU_DB_NAME:-default}
+ # INIT_DB_USERNAME: ${DOKU_DB_USER:-default}
+ # INIT_DB_PASSWORD: ${DOKU_DB_PASSWORD:-DOKU}
+ # SQLITE_DATABASE_URL: file:/app/client/data/data.db
+ # ports:
+ # - "3000:3000"
+ # depends_on:
+ # - clickhouse
+ # volumes:
+ # - doku-client-data:/app/client/data
+ # restart: always
- doku-client:
- image: ghcr.io/dokulabs/doku-client:latest
- container_name: doku-client
- environment:
- INIT_DB_HOST: clickhouse
- INIT_DB_PORT: 8123
- INIT_DB_DATABASE: ${DOKU_DB_NAME:-default}
- INIT_DB_USERNAME: ${DOKU_DB_USER:-default}
- INIT_DB_PASSWORD: ${DOKU_DB_PASSWORD:-DOKU}
- SQLITE_DATABASE_URL: file:/app/client/data/data.db
+ otel-collector:
+ image: otel/opentelemetry-collector-contrib:0.94.0
ports:
- - "3000:3000"
- depends_on:
- - clickhouse
+ - "4317:4317"
+ - "4318:4318"
+ - "8888:8888"
+ - "55679:55679"
volumes:
- - doku-client-data:/app/client/data
- restart: always
+ - ./assets/otel-collector-config.yaml:/etc/otelcol-contrib/config.yaml
volumes:
clickhouse-data:
- doku-client-data:
+ doku-client-data:
\ No newline at end of file
diff --git a/docs/latest/connections/grafanacloud.mdx b/docs/latest/connections/grafanacloud.mdx
index c95403b9..72251ed4 100644
--- a/docs/latest/connections/grafanacloud.mdx
+++ b/docs/latest/connections/grafanacloud.mdx
@@ -1,6 +1,6 @@
---
title: 'Grafana Cloud'
-description: 'LLM Observability with Grafana Cloud and Doku'
+description: 'LLM Observability with Grafana Cloud and OpenLIT'
---
<Frame>
@@ -8,9 +8,28 @@ description: 'LLM Observability with Grafana Cloud and Doku'
<img src="https://github.com/dokulabs/doku/blob/main/docs/images/grafana-dashboard-2.jpg?raw=true" />
</Frame>
-Doku can automatically send LLM usage data from Doku to Grafana Cloud using the [Grafana Cloud Influx Telegraf](https://grafana.com/docs/grafana-cloud/send-data/metrics/metrics-influxdb/push-from-telegraf/) and [Loki Push HTTP API](https://grafana.com/docs/loki/latest/reference/api/#push-log-entries-to-loki)
+You can [send data using OpenTelemetry Protocol (OTLP)](https://grafana.com/docs/grafana-cloud/send-data/otlp/send-data-otlp/) to Grafana Cloud drectly from the OpenLIT SDK.
-Make the following curl request to Doku to add a connection for Grafana Cloud. See more on the [Connections API](/latest/api-reference/endpoint/connections/create)
+To start sending Traces and Metrics, Follow the below steps
+1. Create an Access Policy token with the metrics:write and traces:write scopes, depending on the data you will be sending.
+2. Open the Cloud Portal, and click Configure on the OpenTelemetry card.
+3. Click on Generate Now in the Password / API Token section. As a result, the Environment Variable section will be populated with environment variables necessary to send data directly to Grafana Cloud.
+
+4. Configure the OpenLIT SDK to use environment variables to push directly to Grafana Cloud
+
+```
+export OTEL_EXPORTER_OTLP_ENDPOINT=<OTLP Endpoint URL>
+export OTEL_EXPORTER_OTLP_HEADERS=Authorization=Basic%20<base64 instanceID:token>
+```
+
+<Tip>
+Make sure to replace the space after `Basic` with `%20`:
+
+`OTEL_EXPORTER_OTLP_HEADERS=Authorization=Basic%20<base64 instanceID:token>`
+</Tip>
+
+You can automatically obtain these values from the Grafana Cloud Portal.
+click on Generate Now in the Password / API Token section. As a result, the Environment Variable section will be populated with environment variables necessary to send data directly to Grafana Cloud.
```shell
curl --request POST \
@@ -51,125 +70,35 @@ curl --request POST \
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
- "id": 14,
- "links": [
- {
- "icon": "info",
- "tags": [],
- "targetBlank": true,
- "title": "Doku Documentation",
- "tooltip": "",
- "type": "link",
- "url": "https://docs.dokulabs.com/"
- },
- {
- "icon": "info",
- "tags": [],
- "targetBlank": true,
- "title": "Doku Community",
- "tooltip": "",
- "type": "link",
- "url": "https://dokulabs.slack.com/"
- },
- {
- "icon": "info",
- "tags": [],
- "targetBlank": true,
- "title": "Dokulabs GitHub",
- "tooltip": "",
- "type": "link",
- "url": "https://github.com/dokulabs/"
- }
- ],
- "liveNow": false,
+ "id": 12,
+ "links": [],
"panels": [
{
- "datasource": {
- "type": "prometheus",
- "uid": "grafanacloud-prom"
- },
+ "collapsed": false,
"gridPos": {
- "h": 4,
- "w": 11,
+ "h": 1,
+ "w": 24,
"x": 0,
"y": 0
},
- "id": 23,
- "options": {
- "code": {
- "language": "plaintext",
- "showLineNumbers": false,
- "showMiniMap": false
- },
- "content": "---\n---\n---\n---\n---\n---",
- "mode": "markdown"
- },
- "pluginVersion": "10.4.0-65875",
- "transparent": true,
- "type": "text"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "grafanacloud-prom"
- },
- "gridPos": {
- "h": 4,
- "w": 2,
- "x": 11,
- "y": 0
- },
- "id": 2,
- "options": {
- "code": {
- "language": "plaintext",
- "showLineNumbers": false,
- "showMiniMap": false
- },
- "content": "![Doku Logo](https://github.com/dokulabs/.github/blob/main/profile/assets/favicon.png?raw=true)",
- "mode": "markdown"
- },
- "pluginVersion": "10.4.0-65875",
- "transparent": true,
- "type": "text"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "grafanacloud-prom"
- },
- "gridPos": {
- "h": 4,
- "w": 11,
- "x": 13,
- "y": 0
- },
- "id": 22,
- "options": {
- "code": {
- "language": "plaintext",
- "showLineNumbers": false,
- "showMiniMap": false
- },
- "content": "---\n---\n---\n---\n---\n---",
- "mode": "markdown"
- },
- "pluginVersion": "10.4.0-65875",
- "transparent": true,
- "type": "text"
+ "id": 11,
+ "panels": [],
+ "title": "Row title",
+ "type": "row"
},
{
"datasource": {
"type": "prometheus",
- "uid": "grafanacloud-prom"
+ "uid": "${metricsDatasource}"
},
- "description": "This panel shows the total number of requests sent to Large Language Models (LLM) Platforms in the selected time interval.",
+ "description": "This panel displays the total number of successful requests made to the GenAI system. A successful request is one that is completed without errors, indicating seamless operation and effective utilization of the GenAI service. Tracking this helps in understanding the reliability and performance of the GenAI system.",
"fieldConfig": {
"defaults": {
"color": {
- "mode": "thresholds"
+ "mode": "palette-classic-by-name"
},
"mappings": [],
+ "min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
@@ -183,17 +112,17 @@ curl --request POST \
}
]
},
- "unitScale": true
+ "unit": "none"
},
"overrides": []
},
"gridPos": {
- "h": 3,
- "w": 6,
+ "h": 4,
+ "w": 12,
"x": 0,
- "y": 4
+ "y": 1
},
- "id": 3,
+ "id": 1,
"options": {
"colorMode": "background",
"graphMode": "area",
@@ -210,80 +139,34 @@ curl --request POST \
"textMode": "auto",
"wideLayout": true
},
- "pluginVersion": "10.4.0-65875",
+ "pluginVersion": "11.1.0-69372",
"targets": [
{
"datasource": {
"type": "prometheus",
- "uid": "grafanacloud-prom"
+ "uid": "${metricsDatasource}"
},
"disableTextWrap": false,
"editorMode": "code",
- "expr": "count_over_time(doku_llm_requestDuration{job=\"doku\", applicationName=~\"$Application\", environment=~\"$Environment\"}[$__interval])",
- "format": "table",
+ "expr": "sum by() (gen_ai_requests_total{telemetry_sdk_name=\"openlit\", gen_ai_application_name=~\"$application\", gen_ai_environment=~\"$environment\"})",
"fullMetaSearch": false,
"includeNullMetadata": true,
+ "instant": false,
"legendFormat": "__auto",
"range": true,
- "rawQuery": true,
- "rawSql": "SELECT count(*) FROM DOKU\nWHERE $__timeFilter(time)",
"refId": "A",
- "sql": {
- "columns": [
- {
- "name": "COUNT",
- "parameters": [
- {
- "name": "environment",
- "type": "functionParameter"
- }
- ],
- "type": "function"
- }
- ],
- "groupBy": [
- {
- "property": {
- "type": "string"
- },
- "type": "groupBy"
- }
- ],
- "limit": 50
- },
- "table": "doku",
"useBackend": false
}
],
- "title": "Total Requests",
- "transformations": [
- {
- "id": "groupBy",
- "options": {
- "fields": {
- "Value": {
- "aggregations": [
- "sum"
- ],
- "operation": "aggregate"
- },
- "job": {
- "aggregations": [],
- "operation": "groupby"
- }
- }
- }
- }
- ],
- "transparent": true,
+ "title": "Total Successful GenAI Requests",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
- "uid": "grafanacloud-prom"
+ "uid": "${metricsDatasource}"
},
- "description": "This panel shows the average time taken to process each request by the Large Language Models (LLM), offering a snapshot of system performance and efficiency.",
+ "description": "This panel displays the total count of successful VectorDB requests. It indicates requests that have been processed completely without any errors, showcasing VectorDB's effectiveness and reliability. Watching this helps gauge how well VectorDB is performing and its success in handling operations.",
"fieldConfig": {
"defaults": {
"color": {
@@ -299,22 +182,21 @@ curl --request POST \
},
{
"color": "red",
- "value": 5
+ "value": 80
}
]
},
- "unit": "s",
- "unitScale": true
+ "unit": "none"
},
"overrides": []
},
"gridPos": {
- "h": 3,
- "w": 4,
- "x": 6,
- "y": 4
+ "h": 4,
+ "w": 12,
+ "x": 12,
+ "y": 1
},
- "id": 4,
+ "id": 7,
"options": {
"colorMode": "background",
"graphMode": "area",
@@ -331,84 +213,38 @@ curl --request POST \
"textMode": "auto",
"wideLayout": true
},
- "pluginVersion": "10.4.0-65875",
+ "pluginVersion": "11.1.0-69372",
"targets": [
{
"datasource": {
"type": "prometheus",
- "uid": "grafanacloud-prom"
+ "uid": "${metricsDatasource}"
},
"disableTextWrap": false,
"editorMode": "code",
- "expr": "doku_llm_requestDuration{job=\"doku\", applicationName=~\"$Application\", environment=~\"$Environment\"}",
- "format": "table",
+ "expr": "sum(db_requests_total{telemetry_sdk_name=\"openlit\", gen_ai_application_name=~\"$application\", gen_ai_environment=~\"$environment\"})",
"fullMetaSearch": false,
"includeNullMetadata": true,
+ "instant": false,
"legendFormat": "__auto",
"range": true,
- "rawQuery": true,
- "rawSql": "SELECT AVG(requestduration) FROM doku\nWHERE $__timeFilter(time)",
"refId": "A",
- "sql": {
- "columns": [
- {
- "name": "AVG",
- "parameters": [
- {
- "name": "requestduration",
- "type": "functionParameter"
- }
- ],
- "type": "function"
- }
- ],
- "groupBy": [
- {
- "property": {
- "type": "string"
- },
- "type": "groupBy"
- }
- ],
- "limit": 50
- },
- "table": "doku",
"useBackend": false
}
],
- "title": "Avg Request Duration",
- "transformations": [
- {
- "id": "groupBy",
- "options": {
- "fields": {
- "Value": {
- "aggregations": [
- "mean"
- ],
- "operation": "aggregate"
- },
- "job": {
- "aggregations": [],
- "operation": "groupby"
- }
- }
- }
- }
- ],
- "transparent": true,
+ "title": "Total Successful VectorDB Requests",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
- "uid": "grafanacloud-prom"
+ "uid": "${metricsDatasource}"
},
- "description": "This panel shows the average count of Total tokens used per LLM request",
+ "description": "This panel displays the total number of tokens consumed by GenAI requests, providing a direct measure of usage. Monitoring this helps in assessing the demand on GenAI services and guiding resource allocation or optimization strategies.",
"fieldConfig": {
"defaults": {
"color": {
- "mode": "thresholds"
+ "mode": "palette-classic-by-name"
},
"mappings": [],
"thresholds": {
@@ -420,22 +256,21 @@ curl --request POST \
},
{
"color": "red",
- "value": 1000
+ "value": 80
}
]
},
- "unit": "none",
- "unitScale": true
+ "unit": "none"
},
"overrides": []
},
"gridPos": {
- "h": 3,
- "w": 4,
- "x": 10,
- "y": 4
+ "h": 4,
+ "w": 8,
+ "x": 0,
+ "y": 5
},
- "id": 5,
+ "id": 3,
"options": {
"colorMode": "background",
"graphMode": "area",
@@ -452,84 +287,38 @@ curl --request POST \
"textMode": "auto",
"wideLayout": true
},
- "pluginVersion": "10.4.0-65875",
+ "pluginVersion": "11.1.0-69372",
"targets": [
{
"datasource": {
"type": "prometheus",
- "uid": "grafanacloud-prom"
+ "uid": "${metricsDatasource}"
},
"disableTextWrap": false,
"editorMode": "code",
- "expr": "doku_llm_totalTokens{job=\"doku\", applicationName=~\"$Application\", environment=~\"$Environment\"}",
- "format": "table",
+ "expr": "sum(gen_ai_usage_tokens_total{telemetry_sdk_name=\"openlit\", gen_ai_application_name=~\"$application\", gen_ai_environment=~\"$environment\"})",
"fullMetaSearch": false,
"includeNullMetadata": true,
+ "instant": false,
"legendFormat": "__auto",
"range": true,
- "rawQuery": true,
- "rawSql": "SELECT AVG(totaltokens) FROM doku\nWHERE $__timeFilter(time)",
"refId": "A",
- "sql": {
- "columns": [
- {
- "name": "AVG",
- "parameters": [
- {
- "name": "totaltokens",
- "type": "functionParameter"
- }
- ],
- "type": "function"
- }
- ],
- "groupBy": [
- {
- "property": {
- "type": "string"
- },
- "type": "groupBy"
- }
- ],
- "limit": 50
- },
- "table": "doku",
"useBackend": false
}
],
- "title": "Avg Tokens per Request",
- "transformations": [
- {
- "id": "groupBy",
- "options": {
- "fields": {
- "Value": {
- "aggregations": [
- "mean"
- ],
- "operation": "aggregate"
- },
- "job": {
- "aggregations": [],
- "operation": "groupby"
- }
- }
- }
- }
- ],
- "transparent": true,
+ "title": "Total Usage Tokens",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
- "uid": "grafanacloud-prom"
+ "uid": "${metricsDatasource}"
},
- "description": "This panel shows the average cost incurred per request to the LLM, helping monitor and optimize budget allocation for LLM usage.",
+ "description": "This panel displays the total cost incurred from using GenAI models. It reflects the financial impact of operational activities, offering insights into budgetary allocation and efficiency. Tracking this helps in effective cost management and financial planning for GenAI usage.",
"fieldConfig": {
"defaults": {
"color": {
- "mode": "thresholds"
+ "mode": "continuous-BlPu"
},
"mappings": [],
"thresholds": {
@@ -541,22 +330,21 @@ curl --request POST \
},
{
"color": "red",
- "value": 5
+ "value": 80
}
]
},
- "unit": "currencyUSD",
- "unitScale": true
+ "unit": "currencyUSD"
},
"overrides": []
},
"gridPos": {
- "h": 3,
- "w": 4,
- "x": 14,
- "y": 4
+ "h": 4,
+ "w": 8,
+ "x": 8,
+ "y": 5
},
- "id": 6,
+ "id": 2,
"options": {
"colorMode": "background",
"graphMode": "area",
@@ -573,84 +361,38 @@ curl --request POST \
"textMode": "auto",
"wideLayout": true
},
- "pluginVersion": "10.4.0-65875",
+ "pluginVersion": "11.1.0-69372",
"targets": [
{
"datasource": {
"type": "prometheus",
- "uid": "grafanacloud-prom"
+ "uid": "${metricsDatasource}"
},
"disableTextWrap": false,
"editorMode": "code",
- "expr": "doku_llm_usageCost{job=\"doku\", applicationName=~\"$Application\", environment=~\"$Environment\"}",
- "format": "table",
+ "expr": "sum(gen_ai_usage_cost_USD_sum{telemetry_sdk_name=\"openlit\", gen_ai_application_name=~\"$application\", gen_ai_environment=~\"$environment\"})",
"fullMetaSearch": false,
"includeNullMetadata": true,
+ "instant": false,
"legendFormat": "__auto",
"range": true,
- "rawQuery": true,
- "rawSql": "SELECT AVG(usagecost) FROM doku \nWHERE $__timeFilter(time)",
"refId": "A",
- "sql": {
- "columns": [
- {
- "name": "AVG",
- "parameters": [
- {
- "name": "usagecost",
- "type": "functionParameter"
- }
- ],
- "type": "function"
- }
- ],
- "groupBy": [
- {
- "property": {
- "type": "string"
- },
- "type": "groupBy"
- }
- ],
- "limit": 50
- },
- "table": "doku",
"useBackend": false
}
],
- "title": "Avg Cost / Request",
- "transformations": [
- {
- "id": "groupBy",
- "options": {
- "fields": {
- "Value": {
- "aggregations": [
- "mean"
- ],
- "operation": "aggregate"
- },
- "job": {
- "aggregations": [],
- "operation": "groupby"
- }
- }
- }
- }
- ],
- "transparent": true,
+ "title": "Total Usage Cost",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
- "uid": "grafanacloud-prom"
+ "uid": "${metricsDatasource}"
},
- "description": "This panel shows the total cost on LLM requests over the selected period.",
+ "description": "This panel displays the average cost per use of the GenAI models and related services. It provides insights into the cost-effectiveness of interactions with GenAI, helping to identify trends in expense per operation. Monitoring this assists in optimizing budget allocation and improving cost efficiency in GenAI utilization.",
"fieldConfig": {
"defaults": {
"color": {
- "mode": "continuous-BlPu"
+ "mode": "thresholds"
},
"mappings": [],
"thresholds": {
@@ -659,24 +401,31 @@ curl --request POST \
{
"color": "green",
"value": null
+ },
+ {
+ "color": "#EAB839",
+ "value": 0.5
+ },
+ {
+ "color": "red",
+ "value": 1
}
]
},
- "unit": "currencyUSD",
- "unitScale": true
+ "unit": "currencyUSD"
},
"overrides": []
},
"gridPos": {
- "h": 3,
- "w": 6,
- "x": 18,
- "y": 4
+ "h": 4,
+ "w": 8,
+ "x": 16,
+ "y": 5
},
- "id": 10,
+ "id": 5,
"options": {
"colorMode": "background",
- "graphMode": "area",
+ "graphMode": "none",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
@@ -690,126 +439,54 @@ curl --request POST \
"textMode": "auto",
"wideLayout": true
},
- "pluginVersion": "10.4.0-65875",
+ "pluginVersion": "11.1.0-69372",
"targets": [
{
"datasource": {
"type": "prometheus",
- "uid": "grafanacloud-prom"
+ "uid": "${metricsDatasource}"
},
"disableTextWrap": false,
"editorMode": "code",
- "expr": "doku_llm_usageCost{job=\"doku\", applicationName=~\"$Application\", environment=~\"$Environment\"}",
- "format": "table",
+ "expr": "avg by() (gen_ai_usage_cost_USD_sum{telemetry_sdk_name=\"openlit\", gen_ai_application_name=~\"$application\", gen_ai_environment=~\"$environment\"})",
"fullMetaSearch": false,
"includeNullMetadata": true,
+ "instant": false,
"legendFormat": "__auto",
"range": true,
- "rawQuery": true,
- "rawSql": "SELECT SUM(usagecost) FROM doku\nWHERE $__timeFilter(time)",
"refId": "A",
- "sql": {
- "columns": [
- {
- "name": "SUM",
- "parameters": [
- {
- "name": "usagecost",
- "type": "functionParameter"
- }
- ],
- "type": "function"
- }
- ],
- "groupBy": [
- {
- "property": {
- "type": "string"
- },
- "type": "groupBy"
- }
- ],
- "limit": 50
- },
- "table": "doku",
"useBackend": false
}
],
- "title": "Total Cost",
- "transformations": [
- {
- "id": "groupBy",
- "options": {
- "fields": {
- "Value": {
- "aggregations": [
- "sum"
- ],
- "operation": "aggregate"
- },
- "job": {
- "aggregations": [],
- "operation": "groupby"
- }
- }
- }
- }
- ],
- "transparent": true,
+ "title": "Avg Usage Cost",
"type": "stat"
},
{
"datasource": {
- "type": "prometheus",
- "uid": "grafanacloud-prom"
+ "type": "tempo",
+ "uid": "${traceDatasource}"
},
- "description": "This panel shows the count of API requests made to LLMs over the selected time.",
+ "description": "This panel displays the distribution of request durations for both GenAI and VectorDB services. It highlights how long requests take to complete, from the shortest to the longest durations, offering insights into system performance and efficiency. Understanding this distribution helps in identifying bottlenecks and optimizing response times for both services.",
"fieldConfig": {
"defaults": {
"color": {
- "mode": "continuous-BlPu"
+ "fixedColor": "blue",
+ "mode": "palette-classic"
},
"custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "series",
- "axisGridShow": true,
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 54,
- "gradientMode": "hue",
+ "fillOpacity": 80,
+ "gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
- "insertNulls": false,
- "lineInterpolation": "smooth",
- "lineStyle": {
- "dash": [
- 10,
- 10
- ],
- "fill": "dash"
- },
- "lineWidth": 2,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "always",
- "spanNulls": true,
+ "lineWidth": 1,
"stacking": {
"group": "A",
"mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
}
},
- "fieldMinMax": false,
"mappings": [],
"thresholds": {
"mode": "absolute",
@@ -817,261 +494,151 @@ curl --request POST \
{
"color": "green",
"value": null
+ },
+ {
+ "color": "red",
+ "value": 10
}
]
},
- "unit": "none",
- "unitScale": true
+ "unit": "s"
},
"overrides": []
},
"gridPos": {
- "h": 5,
+ "h": 7,
"w": 24,
"x": 0,
- "y": 7
+ "y": 9
},
- "id": 17,
+ "id": 4,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
- "showLegend": false
- },
- "tooltip": {
- "mode": "single",
- "sort": "none"
+ "showLegend": true
}
},
- "pluginVersion": "10.2.3",
+ "pluginVersion": "11.1.0-69372",
"targets": [
{
"datasource": {
- "type": "prometheus",
- "uid": "grafanacloud-prom"
- },
- "disableTextWrap": false,
- "editorMode": "code",
- "expr": "count(count_over_time(doku_llm_requestDuration{job=\"doku\", applicationName=~\"$Application\", environment=~\"$Environment\"}[$__interval]))",
- "format": "time_series",
- "fullMetaSearch": false,
- "includeNullMetadata": true,
- "legendFormat": "Requests",
- "range": true,
- "rawQuery": true,
- "rawSql": "SELECT\n DATE_TRUNC('day', \"time\") AS \"time\",\n COUNT(endpoint) AS total_requests\nFROM \n doku\nWHERE\n $__timeFilter(\"time\")\nGROUP BY \n DATE_TRUNC('day', \"time\")\nORDER BY \n \"time\";",
+ "type": "tempo",
+ "uid": "${traceDatasource}"
+ },
+ "filters": [
+ {
+ "id": "status",
+ "operator": "=",
+ "scope": "intrinsic",
+ "tag": "status",
+ "value": "ok",
+ "valueType": "keyword"
+ },
+ {
+ "id": "f755ab99",
+ "operator": "=",
+ "scope": "span",
+ "tag": "telemetry.sdk.name",
+ "value": [
+ "openlit"
+ ],
+ "valueType": "string"
+ }
+ ],
+ "limit": 20,
+ "query": "{status=ok && span.telemetry.sdk.name=\"openlit\" && span.gen_ai.application_name=~\"$application\" && span.gen_ai.environment=~\"$environment\"}",
+ "queryType": "traceql",
"refId": "A",
- "sql": {
- "columns": [
- {
- "alias": "\"Prompt Tokens\"",
- "parameters": [
- {
- "name": "prompttokens",
- "type": "functionParameter"
- }
- ],
- "type": "function"
- },
- {
- "alias": "\"time\"",
- "parameters": [
- {
- "name": "\"time\"",
- "type": "functionParameter"
- }
- ],
- "type": "function"
- },
- {
- "alias": "\"Completion Tokens\"",
- "parameters": [
- {
- "name": "completiontokens",
- "type": "functionParameter"
- }
- ],
- "type": "function"
- },
- {
- "alias": "\"Total Tokens\"",
- "parameters": [
- {
- "name": "totaltokens",
- "type": "functionParameter"
- }
- ],
- "type": "function"
- }
- ],
- "groupBy": [
- {
- "property": {
- "type": "string"
- },
- "type": "groupBy"
- }
- ],
- "limit": 50
- },
- "table": "doku",
- "useBackend": false
+ "tableType": "traces"
}
],
- "title": "Total Requests overtime",
- "transformations": [],
- "type": "timeseries"
+ "title": "Request Duration Distribution",
+ "type": "histogram"
},
{
"datasource": {
"type": "prometheus",
- "uid": "grafanacloud-prom"
+ "uid": "${metricsDatasource}"
},
- "description": "This panel illustrates the distribution of LLM requests across different usage categories: Chat Generations, Embeddings, Image Generations, Audio Generations, and Fine Tuning. It visualizes the relative volume of requests, helping to identify which types of operations are most frequently performed and providing insights into usage patterns by category.",
+ "description": "This panel displays the ranking of GenAI models based on their usage frequency. It identifies which models are most popular or in-demand, providing insights into user preferences and operational trends. Analyzing this helps in resource allocation, optimizing model availability, and understanding which GenAI features are driving usage.",
"fieldConfig": {
"defaults": {
"color": {
- "fixedColor": "blue",
- "mode": "palette-classic"
- },
- "custom": {
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- }
+ "mode": "continuous-BlYlRd"
},
"mappings": [],
- "unit": "none",
- "unitScale": true
- },
- "overrides": [
- {
- "__systemRef": "hideSeriesFrom",
- "matcher": {
- "id": "byNames",
- "options": {
- "mode": "exclude",
- "names": [
- "Value (sum)",
- "Chat"
- ],
- "prefix": "All except:",
- "readOnly": true
- }
- },
- "properties": [
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
{
- "id": "custom.hideFrom",
- "value": {
- "legend": false,
- "tooltip": false,
- "viz": true
- }
+ "color": "red",
+ "value": 80
}
]
- }
- ]
+ },
+ "unit": "none"
+ },
+ "overrides": []
},
"gridPos": {
- "h": 9,
- "w": 6,
+ "h": 14,
+ "w": 14,
"x": 0,
- "y": 12
+ "y": 16
},
- "id": 12,
+ "id": 17,
"options": {
- "displayLabels": [
- "name",
- "value"
- ],
- "legend": {
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true,
- "values": []
- },
- "pieType": "pie",
+ "displayMode": "gradient",
+ "maxVizHeight": 300,
+ "minVizHeight": 16,
+ "minVizWidth": 8,
+ "namePlacement": "auto",
+ "orientation": "horizontal",
"reduceOptions": {
"calcs": [
- "allValues"
+ "lastNotNull"
],
"fields": "",
- "values": true
+ "values": false
},
- "tooltip": {
- "mode": "single",
- "sort": "asc"
- }
+ "showUnfilled": true,
+ "sizing": "auto",
+ "valueMode": "text"
},
- "pluginVersion": "10.4.0-65875",
+ "pluginVersion": "11.1.0-69372",
"targets": [
{
"datasource": {
"type": "prometheus",
- "uid": "grafanacloud-prom"
+ "uid": "${metricsDatasource}"
},
"disableTextWrap": false,
"editorMode": "code",
- "expr": "count_over_time(doku_llm_requestDuration{job=\"doku\", applicationName=~\"$Application\", environment=~\"$Environment\"}[$__interval])",
- "format": "table",
+ "expr": "topk(5, sum by(gen_ai_request_model) (gen_ai_requests_total{telemetry_sdk_name=\"openlit\", gen_ai_application_name=~\"$application\", gen_ai_environment=~\"$environment\"}))",
"fullMetaSearch": false,
"includeNullMetadata": true,
+ "instant": false,
"legendFormat": "__auto",
"range": true,
- "rawQuery": true,
- "rawSql": "SELECT \n category,\n CAST(COUNT(*) AS INTEGER) AS Requests,\n ROUND(100.0 * COUNT(*) / SUM(COUNT(*)) OVER(), 2) AS percentage\n FROM \n (\n SELECT \n CASE\n WHEN endpoint LIKE 'openai.chat%' THEN 'Chat'\n WHEN endpoint LIKE 'openai.completions%' THEN 'Chat'\n WHEN endpoint LIKE 'anthropic.messages%' THEN 'Chat'\n WHEN endpoint LIKE 'cohere.summarize%' THEN 'Chat'\n WHEN endpoint LIKE 'cohere.generate%' THEN 'Chat'\n WHEN endpoint LIKE 'cohere.chat%' THEN 'Chat'\n WHEN endpoint LIKE '%embed%' THEN 'Embeddings'\n WHEN endpoint LIKE 'openai.images%' THEN 'Image'\n WHEN endpoint LIKE 'openai.audio%' THEN 'Audio'\n WHEN endpoint LIKE 'openai.fine_tuning%' THEN 'Fine Tuning'\n END AS category,\n endpoint\n FROM doku\n WHERE $__timeFilter(time)\n ) AS subquery\n GROUP BY category",
"refId": "A",
- "sql": {
- "columns": [
- {
- "parameters": [],
- "type": "function"
- }
- ],
- "groupBy": [
- {
- "property": {
- "type": "string"
- },
- "type": "groupBy"
- }
- ],
- "limit": 50
- },
"useBackend": false
}
],
- "title": "Usage by Category",
- "transformations": [
- {
- "id": "groupBy",
- "options": {
- "fields": {
- "Value": {
- "aggregations": [
- "sum"
- ],
- "operation": "aggregate"
- },
- "generation": {
- "aggregations": [],
- "operation": "groupby"
- }
- }
- }
- }
- ],
- "transparent": true,
- "type": "piechart"
+ "title": "Top GenAI Models by Usage",
+ "type": "bargauge"
},
{
"datasource": {
"type": "prometheus",
- "uid": "grafanacloud-prom"
+ "uid": "${metricsDatasource}"
},
- "description": "Displays the cost breakdown by provider (OpenAI, Cohere, Anthropic) for LLM requests, offering insights into financial deployment across different services.",
+ "description": "This panel displays the distribution of GenAI requests across different platforms such as OpenAI, Cohere, Anthropic, etc. It shows where requests are being sent, giving insights into platform popularity and usage patterns. Monitoring this helps in understanding platform preferences, and it can guide strategic decisions for integration or diversification of GenAI services.",
"fieldConfig": {
"defaults": {
"color": {
@@ -1085,112 +652,65 @@ curl --request POST \
}
},
"mappings": [],
- "unit": "currencyUSD",
- "unitScale": true
+ "min": 0,
+ "unit": "none"
},
"overrides": []
},
"gridPos": {
- "h": 9,
- "w": 6,
- "x": 6,
- "y": 12
+ "h": 7,
+ "w": 5,
+ "x": 14,
+ "y": 16
},
"id": 16,
"options": {
- "displayLabels": [
- "name",
- "value"
- ],
"legend": {
"displayMode": "list",
"placement": "bottom",
- "showLegend": true,
- "values": []
+ "showLegend": true
},
"pieType": "pie",
"reduceOptions": {
"calcs": [
- "allValues"
+ "lastNotNull"
],
"fields": "",
- "values": true
+ "values": false
},
"tooltip": {
"mode": "single",
- "sort": "asc"
+ "sort": "none"
}
},
- "pluginVersion": "10.4.0-65875",
+ "pluginVersion": "11.1.0-69372",
"targets": [
{
"datasource": {
"type": "prometheus",
- "uid": "grafanacloud-prom"
+ "uid": "${metricsDatasource}"
},
"disableTextWrap": false,
"editorMode": "code",
- "expr": "doku_llm_usageCost{job=\"doku\", applicationName=~\"$Application\", environment=~\"$Environment\"}",
- "format": "table",
+ "expr": "sum by(gen_ai_system) (gen_ai_requests_total{telemetry_sdk_name=\"openlit\", gen_ai_application_name=~\"$application\", gen_ai_environment=~\"$environment\"})",
"fullMetaSearch": false,
"includeNullMetadata": true,
+ "instant": false,
"legendFormat": "__auto",
"range": true,
- "rawQuery": true,
- "rawSql": "SELECT\n CASE\n WHEN endpoint LIKE 'openai.%' THEN 'OpenAI'\n WHEN endpoint LIKE 'cohere.%' THEN 'Cohere'\n WHEN endpoint LIKE 'anthropic.%' THEN 'Anthropic'\n END AS provider,\n SUM(usageCost) AS total_cost\nFROM\n doku\nWHERE $__timeFilter(time)\nGROUP BY\n provider\nORDER BY\n total_cost DESC;",
"refId": "A",
- "sql": {
- "columns": [
- {
- "parameters": [],
- "type": "function"
- }
- ],
- "groupBy": [
- {
- "property": {
- "type": "string"
- },
- "type": "groupBy"
- }
- ],
- "limit": 50
- },
"useBackend": false
}
],
- "title": "Usage cost by Provider",
- "transformations": [
- {
- "id": "groupBy",
- "options": {
- "fields": {
- "Value": {
- "aggregations": [
- "sum"
- ],
- "operation": "aggregate"
- },
- "generation": {
- "aggregations": []
- },
- "platform": {
- "aggregations": [],
- "operation": "groupby"
- }
- }
- }
- }
- ],
- "transparent": true,
+ "title": "GenAI Requests by Platform",
"type": "piechart"
},
{
"datasource": {
"type": "prometheus",
- "uid": "grafanacloud-prom"
+ "uid": "${metricsDatasource}"
},
- "description": "This panel presents a pie chart detailing the distribution of expenses for LLM requests by various applications. It shows how budget allocations are consumed by different applications, enabling a deeper understanding of cost drivers and facilitating cost optimization efforts.",
+ "description": "This panel displays the breakdown of GenAI requests by type, such as Chat, Embedding, Image, Audio, and Fine Tuning. It reveals the diversity in usage, highlighting which types of requests are most common. Understanding this distribution assists in optimizing resources for the most demanded services and planning for future needs based on usage trends.",
"fieldConfig": {
"defaults": {
"color": {
@@ -1204,109 +724,65 @@ curl --request POST \
}
},
"mappings": [],
- "unit": "currencyUSD",
- "unitScale": true
+ "min": 0,
+ "unit": "none"
},
"overrides": []
},
"gridPos": {
- "h": 9,
- "w": 6,
- "x": 12,
- "y": 12
+ "h": 7,
+ "w": 5,
+ "x": 19,
+ "y": 16
},
- "id": 24,
+ "id": 13,
"options": {
- "displayLabels": [
- "name",
- "value"
- ],
"legend": {
"displayMode": "list",
"placement": "bottom",
- "showLegend": true,
- "values": []
+ "showLegend": true
},
"pieType": "pie",
"reduceOptions": {
"calcs": [
- "allValues"
+ "lastNotNull"
],
- "fields": "/.*/",
- "values": true
+ "fields": "",
+ "values": false
},
"tooltip": {
"mode": "single",
- "sort": "asc"
+ "sort": "none"
}
},
- "pluginVersion": "10.4.0-65875",
+ "pluginVersion": "11.1.0-69372",
"targets": [
{
"datasource": {
"type": "prometheus",
- "uid": "grafanacloud-prom"
+ "uid": "${metricsDatasource}"
},
"disableTextWrap": false,
"editorMode": "code",
- "expr": "doku_llm_usageCost{job=\"doku\", applicationName=~\"$Application\", environment=~\"$Environment\"}",
- "format": "table",
+ "expr": "sum by(gen_ai_type) (gen_ai_requests_total{telemetry_sdk_name=\"openlit\", gen_ai_application_name=~\"$application\", gen_ai_environment=~\"$environment\"})",
"fullMetaSearch": false,
"includeNullMetadata": true,
+ "instant": false,
"legendFormat": "__auto",
"range": true,
- "rawQuery": true,
- "rawSql": "SELECT\n CASE\n WHEN endpoint LIKE 'openai.%' THEN 'OpenAI'\n WHEN endpoint LIKE 'cohere.%' THEN 'Cohere'\n WHEN endpoint LIKE 'anthropic.%' THEN 'Anthropic'\n END AS provider,\n SUM(usageCost) AS total_cost\nFROM\n doku\nWHERE $__timeFilter(time)\nGROUP BY\n provider\nORDER BY\n total_cost DESC;",
"refId": "A",
- "sql": {
- "columns": [
- {
- "parameters": [],
- "type": "function"
- }
- ],
- "groupBy": [
- {
- "property": {
- "type": "string"
- },
- "type": "groupBy"
- }
- ],
- "limit": 50
- },
"useBackend": false
}
],
- "title": "Usage cost by Application",
- "transformations": [
- {
- "id": "groupBy",
- "options": {
- "fields": {
- "Value": {
- "aggregations": [
- "sum"
- ],
- "operation": "aggregate"
- },
- "applicationName": {
- "aggregations": [],
- "operation": "groupby"
- }
- }
- }
- }
- ],
- "transparent": true,
+ "title": "GenAI Requests by Type",
"type": "piechart"
},
{
"datasource": {
"type": "prometheus",
- "uid": "grafanacloud-prom"
+ "uid": "${metricsDatasource}"
},
- "description": "Visualizes the allocation of LLM request expenses across different environments, such as development, staging, and production. This breakdown helps in tracking and managing costs with respect to operational environments.",
+ "description": "This panel displays the distribution of GenAI requests based on the environment, such as production, development, testing, etc. It shows where the requests are being utilized, providing a clear picture of operational focus and deployment strategies. Tracking this helps in aligning resources appropriately and optimizing the performance across different environments.",
"fieldConfig": {
"defaults": {
"color": {
@@ -1320,502 +796,175 @@ curl --request POST \
}
},
"mappings": [],
- "unit": "currencyUSD",
- "unitScale": true
+ "min": 0,
+ "unit": "none"
},
"overrides": []
},
"gridPos": {
- "h": 9,
- "w": 6,
- "x": 18,
- "y": 12
+ "h": 7,
+ "w": 5,
+ "x": 14,
+ "y": 23
},
- "id": 25,
+ "id": 15,
"options": {
- "displayLabels": [
- "name",
- "value"
- ],
"legend": {
"displayMode": "list",
"placement": "bottom",
- "showLegend": true,
- "values": []
+ "showLegend": true
},
"pieType": "pie",
"reduceOptions": {
"calcs": [
- "allValues"
+ "lastNotNull"
],
- "fields": "/.*/",
- "values": true
+ "fields": "",
+ "values": false
},
"tooltip": {
"mode": "single",
- "sort": "asc"
+ "sort": "none"
}
},
- "pluginVersion": "10.4.0-65875",
+ "pluginVersion": "11.1.0-69372",
"targets": [
{
"datasource": {
"type": "prometheus",
- "uid": "grafanacloud-prom"
+ "uid": "${metricsDatasource}"
},
"disableTextWrap": false,
"editorMode": "code",
- "expr": "doku_llm_usageCost{job=\"doku\", applicationName=~\"$Application\", environment=~\"$Environment\"}",
- "format": "table",
+ "expr": "sum by(gen_ai_environment) (gen_ai_requests_total{telemetry_sdk_name=\"openlit\", gen_ai_application_name=~\"$application\", gen_ai_environment=~\"$environment\"})",
"fullMetaSearch": false,
"includeNullMetadata": true,
+ "instant": false,
"legendFormat": "__auto",
"range": true,
- "rawQuery": true,
- "rawSql": "SELECT\n CASE\n WHEN endpoint LIKE 'openai.%' THEN 'OpenAI'\n WHEN endpoint LIKE 'cohere.%' THEN 'Cohere'\n WHEN endpoint LIKE 'anthropic.%' THEN 'Anthropic'\n END AS provider,\n SUM(usageCost) AS total_cost\nFROM\n doku\nWHERE $__timeFilter(time)\nGROUP BY\n provider\nORDER BY\n total_cost DESC;",
"refId": "A",
- "sql": {
- "columns": [
- {
- "parameters": [],
- "type": "function"
- }
- ],
- "groupBy": [
- {
- "property": {
- "type": "string"
- },
- "type": "groupBy"
- }
- ],
- "limit": 50
- },
"useBackend": false
}
],
- "title": "Usage cost by Environment",
- "transformations": [
- {
- "id": "groupBy",
- "options": {
- "fields": {
- "Value": {
- "aggregations": [
- "sum"
- ],
- "operation": "aggregate"
- },
- "applicationName": {
- "aggregations": []
- },
- "environment": {
- "aggregations": [],
- "operation": "groupby"
- }
- }
- }
- }
- ],
- "transparent": true,
+ "title": "GenAI Requests by Environment",
"type": "piechart"
},
{
"datasource": {
"type": "prometheus",
- "uid": "grafanacloud-prom"
- },
- "description": "This panel shows the average count of Prompt tokens used per LLM request",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "continuous-YlBl"
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 5
- }
- ]
- },
- "unit": "none",
- "unitScale": true
- },
- "overrides": []
- },
- "gridPos": {
- "h": 3,
- "w": 6,
- "x": 0,
- "y": 21
- },
- "id": 9,
- "options": {
- "colorMode": "background",
- "graphMode": "area",
- "justifyMode": "auto",
- "orientation": "auto",
- "reduceOptions": {
- "calcs": [
- "lastNotNull"
- ],
- "fields": "",
- "values": false
- },
- "showPercentChange": false,
- "textMode": "auto",
- "wideLayout": true
- },
- "pluginVersion": "10.4.0-65875",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "grafanacloud-prom"
- },
- "disableTextWrap": false,
- "editorMode": "code",
- "expr": "doku_llm_promptTokens{job=\"doku\", applicationName=~\"$Application\", environment=~\"$Environment\"}",
- "format": "table",
- "fullMetaSearch": false,
- "includeNullMetadata": true,
- "legendFormat": "__auto",
- "range": true,
- "rawQuery": true,
- "rawSql": "SELECT AVG(prompttokens) FROM doku WHERE \n $__timeFilter(\"time\")",
- "refId": "A",
- "sql": {
- "columns": [
- {
- "name": "AVG",
- "parameters": [
- {
- "name": "prompttokens",
- "type": "functionParameter"
- }
- ],
- "type": "function"
- }
- ],
- "groupBy": [
- {
- "property": {
- "type": "string"
- },
- "type": "groupBy"
- }
- ],
- "limit": 50
- },
- "table": "doku",
- "useBackend": false
- }
- ],
- "title": "Avg Prompt Tokens / Request",
- "transformations": [
- {
- "id": "groupBy",
- "options": {
- "fields": {
- "Value": {
- "aggregations": [
- "mean"
- ],
- "operation": "aggregate"
- },
- "job": {
- "aggregations": [],
- "operation": "groupby"
- }
- }
- }
- }
- ],
- "type": "stat"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "grafanacloud-prom"
+ "uid": "${metricsDatasource}"
},
- "description": "Provides a time-series analysis of token consumption for LLM requests, highlighting trends in usage intensity and complexity over time.",
+ "description": "This panel displays the breakdown of GenAI requests by application name, offering insights into which applications are leveraging GenAI services the most. This identification helps in understanding application-specific demand, guiding resource allocation, and supporting targeted optimization efforts for enhanced efficiency in application performance.\"",
"fieldConfig": {
"defaults": {
"color": {
- "mode": "continuous-BlYlRd"
+ "mode": "palette-classic"
},
"custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "series",
- "axisGridShow": true,
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 50,
- "gradientMode": "hue",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
- },
- "insertNulls": 86400000,
- "lineInterpolation": "smooth",
- "lineStyle": {
- "dash": [
- 10,
- 10
- ],
- "fill": "dash"
- },
- "lineWidth": 1,
- "pointSize": 6,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "always",
- "spanNulls": true,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
}
},
- "fieldMinMax": false,
"mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "none",
- "unitScale": true
+ "min": 0,
+ "unit": "none"
},
"overrides": []
},
"gridPos": {
- "h": 6,
- "w": 18,
- "x": 6,
- "y": 21
+ "h": 7,
+ "w": 5,
+ "x": 19,
+ "y": 23
},
- "id": 11,
+ "id": 19,
"options": {
"legend": {
- "calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
+ "pieType": "pie",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
"tooltip": {
"mode": "single",
"sort": "none"
}
},
- "pluginVersion": "10.2.3",
+ "pluginVersion": "11.1.0-69372",
"targets": [
{
"datasource": {
"type": "prometheus",
- "uid": "grafanacloud-prom"
+ "uid": "${metricsDatasource}"
},
"disableTextWrap": false,
"editorMode": "code",
- "expr": "avg(doku_llm_completionTokens{job=\"doku\", applicationName=~\"$Application\", environment=~\"$Environment\"})",
- "format": "time_series",
+ "expr": "sum by(gen_ai_application_name) (gen_ai_requests_total{telemetry_sdk_name=\"openlit\", gen_ai_application_name=~\"$application\", gen_ai_environment=~\"$environment\"})",
"fullMetaSearch": false,
"includeNullMetadata": true,
- "legendFormat": "Completion Tokens",
+ "instant": false,
+ "legendFormat": "__auto",
"range": true,
- "rawQuery": true,
- "rawSql": "SELECT prompttokens AS \"Prompt Tokens\", \"time\" AS \"time\", completiontokens AS \"Completion Tokens\", totaltokens AS \"Total Tokens\" FROM doku",
"refId": "A",
- "sql": {
- "columns": [
- {
- "alias": "\"Prompt Tokens\"",
- "parameters": [
- {
- "name": "prompttokens",
- "type": "functionParameter"
- }
- ],
- "type": "function"
- },
- {
- "alias": "\"time\"",
- "parameters": [
- {
- "name": "\"time\"",
- "type": "functionParameter"
- }
- ],
- "type": "function"
- },
- {
- "alias": "\"Completion Tokens\"",
- "parameters": [
- {
- "name": "completiontokens",
- "type": "functionParameter"
- }
- ],
- "type": "function"
- },
- {
- "alias": "\"Total Tokens\"",
- "parameters": [
- {
- "name": "totaltokens",
- "type": "functionParameter"
- }
- ],
- "type": "function"
- }
- ],
- "groupBy": [
- {
- "property": {
- "type": "string"
- },
- "type": "groupBy"
- }
- ],
- "limit": 50
- },
- "table": "doku",
- "useBackend": false
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "grafanacloud-prom"
- },
- "disableTextWrap": false,
- "editorMode": "builder",
- "expr": "avg(doku_llm_promptTokens)",
- "format": "time_series",
- "fullMetaSearch": false,
- "hide": false,
- "includeNullMetadata": true,
- "legendFormat": "Prompt Tokens",
- "range": true,
- "rawQuery": true,
- "rawSql": "SELECT prompttokens AS \"Prompt Tokens\", \"time\" AS \"time\", completiontokens AS \"Completion Tokens\", totaltokens AS \"Total Tokens\" FROM doku",
- "refId": "B",
- "sql": {
- "columns": [
- {
- "alias": "\"Prompt Tokens\"",
- "parameters": [
- {
- "name": "prompttokens",
- "type": "functionParameter"
- }
- ],
- "type": "function"
- },
- {
- "alias": "\"time\"",
- "parameters": [
- {
- "name": "\"time\"",
- "type": "functionParameter"
- }
- ],
- "type": "function"
- },
- {
- "alias": "\"Completion Tokens\"",
- "parameters": [
- {
- "name": "completiontokens",
- "type": "functionParameter"
- }
- ],
- "type": "function"
- },
- {
- "alias": "\"Total Tokens\"",
- "parameters": [
- {
- "name": "totaltokens",
- "type": "functionParameter"
- }
- ],
- "type": "function"
- }
- ],
- "groupBy": [
- {
- "property": {
- "type": "string"
- },
- "type": "groupBy"
- }
- ],
- "limit": 50
- },
- "table": "doku",
"useBackend": false
}
],
- "title": "Token usage overtime",
- "type": "timeseries"
+ "title": "GenAI Requests by Application",
+ "type": "piechart"
},
{
"datasource": {
"type": "prometheus",
- "uid": "grafanacloud-prom"
+ "uid": "${metricsDatasource}"
},
- "description": "This panel shows the average count of Completion tokens used per LLM request",
+ "description": "This panel displays the distribution of requests across different VectorDB systems, such as Chroma, Pinecone, etc. It highlights which VectorDB services are most frequently used, providing insights into system preference and usage patterns. Monitoring this helps in evaluating the performance and scalability of each VectorDB system, guiding strategic decisions for technology adoption and integration.",
"fieldConfig": {
"defaults": {
"color": {
- "mode": "continuous-BlPu"
+ "mode": "continuous-BlYlRd"
},
"mappings": [],
+ "min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
- "color": "green"
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
}
]
},
- "unit": "none",
- "unitScale": true
+ "unit": "none"
},
"overrides": []
},
"gridPos": {
- "h": 3,
- "w": 6,
+ "h": 9,
+ "w": 14,
"x": 0,
- "y": 24
+ "y": 30
},
- "id": 7,
+ "id": 18,
"options": {
- "colorMode": "background",
- "graphMode": "area",
- "justifyMode": "auto",
- "orientation": "auto",
+ "displayMode": "gradient",
+ "maxVizHeight": 300,
+ "minVizHeight": 16,
+ "minVizWidth": 8,
+ "namePlacement": "auto",
+ "orientation": "horizontal",
"reduceOptions": {
"calcs": [
"lastNotNull"
@@ -1823,216 +972,110 @@ curl --request POST \
"fields": "",
"values": false
},
- "showPercentChange": false,
- "textMode": "auto",
- "wideLayout": true
+ "showUnfilled": true,
+ "sizing": "auto",
+ "valueMode": "text"
},
- "pluginVersion": "10.4.0-65875",
+ "pluginVersion": "11.1.0-69372",
"targets": [
{
"datasource": {
"type": "prometheus",
- "uid": "grafanacloud-prom"
+ "uid": "${metricsDatasource}"
},
"disableTextWrap": false,
"editorMode": "code",
- "expr": "doku_llm_completionTokens{job=\"doku\", applicationName=~\"$Application\", environment=~\"$Environment\"}",
- "format": "table",
+ "expr": "sum by(db_system) (db_requests_total{telemetry_sdk_name=\"openlit\", gen_ai_application_name=~\"$application\", gen_ai_environment=~\"$environment\"})",
"fullMetaSearch": false,
"includeNullMetadata": true,
+ "instant": false,
"legendFormat": "__auto",
"range": true,
- "rawQuery": true,
- "rawSql": "SELECT AVG(completiontokens) FROM doku WHERE \n $__timeFilter(\"time\")",
"refId": "A",
- "sql": {
- "columns": [
- {
- "name": "AVG",
- "parameters": [
- {
- "name": "completiontokens",
- "type": "functionParameter"
- }
- ],
- "type": "function"
- }
- ],
- "groupBy": [
- {
- "property": {
- "type": "string"
- },
- "type": "groupBy"
- }
- ],
- "limit": 50
- },
- "table": "doku",
"useBackend": false
}
],
- "title": "Avg Completion Tokens / Request",
- "transformations": [
- {
- "id": "groupBy",
- "options": {
- "fields": {
- "Value": {
- "aggregations": [
- "mean"
- ],
- "operation": "aggregate"
- },
- "job": {
- "aggregations": [],
- "operation": "groupby"
- }
- }
- }
- }
- ],
- "type": "stat"
+ "title": "VectorDB Usage by System",
+ "type": "bargauge"
},
{
"datasource": {
"type": "prometheus",
- "uid": "grafanacloud-prom"
+ "uid": "${metricsDatasource}"
},
- "description": "Displays the frequency of model usage over time, identifying the most utilised models and showcasing usage trends across different periods",
+ "description": "This panel displays the breakdown of database requests based on the type of operation, such as add, updates, deletes, and queries. It provides insights into the operational dynamics of database interactions, highlighting the most common actions performed. Understanding this distribution helps in optimizing database performance and planning for capacity based on operational needs.",
"fieldConfig": {
"defaults": {
"color": {
- "fixedColor": "purple",
- "mode": "continuous-YlBl"
+ "mode": "palette-classic"
},
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
+ "custom": {
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ }
},
- "unit": "none",
- "unitScale": true
+ "mappings": [],
+ "min": 0,
+ "unit": "none"
},
"overrides": []
},
"gridPos": {
- "h": 6,
- "w": 12,
- "x": 0,
- "y": 27
+ "h": 9,
+ "w": 10,
+ "x": 14,
+ "y": 30
},
- "id": 13,
+ "id": 14,
"options": {
- "displayMode": "lcd",
- "maxVizHeight": 165,
- "minVizHeight": 0,
- "minVizWidth": 75,
- "namePlacement": "left",
- "orientation": "horizontal",
+ "legend": {
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "pieType": "pie",
"reduceOptions": {
"calcs": [
- "allValues"
+ "lastNotNull"
],
"fields": "",
- "values": true
+ "values": false
},
- "showUnfilled": true,
- "sizing": "auto",
- "valueMode": "color"
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
},
- "pluginVersion": "10.4.0-65875",
+ "pluginVersion": "11.1.0-69372",
"targets": [
{
"datasource": {
"type": "prometheus",
- "uid": "grafanacloud-prom"
+ "uid": "${metricsDatasource}"
},
"disableTextWrap": false,
"editorMode": "code",
- "expr": "topk(5,count by(model) (sum_over_time(doku_llm_requestDuration{job=\"doku\", applicationName=~\"$Application\", environment=~\"$Environment\"}[$__interval])))",
- "format": "table",
+ "expr": "sum by(db_operation) (db_requests_total{telemetry_sdk_name=\"openlit\", gen_ai_application_name=~\"$application\", gen_ai_environment=~\"$environment\"})",
"fullMetaSearch": false,
"includeNullMetadata": true,
+ "instant": false,
"legendFormat": "__auto",
"range": true,
- "rawQuery": true,
- "rawSql": "SELECT \n model,\n total_count,\n percentage_usage\nFROM \n (\n SELECT \n model,\n COUNT(*) AS total_count,\n ROUND(100.0 * COUNT(*) / SUM(COUNT(*)) OVER(), 2) AS percentage_usage\n FROM \n doku\n WHERE $__timeFilter(time)\n GROUP BY \n model\n ORDER BY \n percentage_usage DESC\n ) sub\nLIMIT 5;",
"refId": "A",
- "sql": {
- "columns": [
- {
- "parameters": [
- {
- "name": "model",
- "type": "functionParameter"
- }
- ],
- "type": "function"
- }
- ],
- "groupBy": [
- {
- "property": {
- "type": "string"
- },
- "type": "groupBy"
- }
- ],
- "limit": 50
- },
- "table": "doku",
"useBackend": false
}
],
- "title": "Top models by usage",
- "transformations": [
- {
- "id": "groupBy",
- "options": {
- "fields": {
- "Value": {
- "aggregations": [
- "sum"
- ],
- "operation": "aggregate"
- },
- "model": {
- "aggregations": [],
- "operation": "groupby"
- }
- }
- }
- },
- {
- "id": "sortBy",
- "options": {
- "fields": {},
- "sort": [
- {
- "desc": true,
- "field": "Value (sum)"
- }
- ]
- }
- }
- ],
- "type": "bargauge"
+ "title": "DB Requests by Operation",
+ "type": "piechart"
},
{
"datasource": {
"type": "prometheus",
- "uid": "grafanacloud-prom"
+ "uid": "${metricsDatasource}"
},
- "description": "Breaks down token usage across different LLM models, offering insights into which models are driving the highest token consumption.",
+ "description": "This panel displays a comparative graph showing the average number of tokens consumed for completions and prompts against the average usage cost. It provides a visual representation of the relationship between the volume of data processed (in tokens) and the financial implications of using GenAI services. Analyzing this comparison helps in assessing cost-effectiveness and guiding strategic decisions for efficient resource utilization.",
"fieldConfig": {
"defaults": {
"color": {
@@ -2044,26 +1087,22 @@ curl --request POST \
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
- "barAlignment": 0,
+ "barAlignment": 1,
"drawStyle": "line",
- "fillOpacity": 23,
- "gradientMode": "hue",
+ "fillOpacity": 30,
+ "gradientMode": "opacity",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
- "lineInterpolation": "linear",
+ "lineInterpolation": "smooth",
"lineStyle": {
- "dash": [
- 10,
- 10
- ],
- "fill": "dash"
+ "fill": "solid"
},
- "lineWidth": 1,
- "pointSize": 9,
+ "lineWidth": 2,
+ "pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
@@ -2082,7 +1121,8 @@ curl --request POST \
"mode": "absolute",
"steps": [
{
- "color": "green"
+ "color": "green",
+ "value": null
},
{
"color": "red",
@@ -2090,18 +1130,17 @@ curl --request POST \
}
]
},
- "unit": "none",
- "unitScale": true
+ "unit": "none"
},
"overrides": []
},
"gridPos": {
- "h": 6,
- "w": 12,
- "x": 12,
- "y": 27
+ "h": 8,
+ "w": 24,
+ "x": 0,
+ "y": 39
},
- "id": 14,
+ "id": 6,
"options": {
"legend": {
"calcs": [],
@@ -2114,165 +1153,310 @@ curl --request POST \
"sort": "none"
}
},
- "pluginVersion": "10.4.0-65875",
"targets": [
{
"datasource": {
"type": "prometheus",
- "uid": "grafanacloud-prom"
+ "uid": "${metricsDatasource}"
},
"disableTextWrap": false,
"editorMode": "code",
- "expr": "sum_over_time(doku_llm_totalTokens{job=\"doku\", applicationName=~\"$Application\", environment=~\"$Environment\"}[$__interval])",
- "format": "time_series",
+ "expr": "avg(gen_ai_usage_prompt_tokens_total{telemetry_sdk_name=\"openlit\", gen_ai_application_name=~\"$application\", gen_ai_environment=~\"$environment\"})",
"fullMetaSearch": false,
"includeNullMetadata": true,
- "legendFormat": "{{model}}",
+ "instant": false,
+ "legendFormat": "Prompt Tokens",
"range": true,
- "rawQuery": true,
- "rawSql": "SELECT \n DATE_TRUNC('day', \"time\") AS \"time\",\n model,\n COUNT(*) AS \"Model:\"\nFROM \n doku\nWHERE \n $__timeFilter(\"time\")\nGROUP BY \n 1, model\nORDER BY \n \"time\" ASC, model;",
"refId": "A",
- "sql": {
- "columns": [
- {
- "parameters": [
- {
- "name": "model",
- "type": "functionParameter"
- }
- ],
- "type": "function"
- }
- ],
- "groupBy": [
- {
- "property": {
- "type": "string"
- },
- "type": "groupBy"
- }
- ],
- "limit": 50
+ "useBackend": false
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metricsDatasource}"
},
- "table": "doku",
+ "disableTextWrap": false,
+ "editorMode": "builder",
+ "expr": "avg(gen_ai_usage_completion_tokens_total{telemetry_sdk_name=\"openlit\"})",
+ "fullMetaSearch": false,
+ "hide": false,
+ "includeNullMetadata": true,
+ "instant": false,
+ "legendFormat": "Completion Tokens",
+ "range": true,
+ "refId": "B",
+ "useBackend": false
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metricsDatasource}"
+ },
+ "disableTextWrap": false,
+ "editorMode": "builder",
+ "expr": "avg(gen_ai_usage_cost_USD_bucket{telemetry_sdk_name=\"openlit\"})",
+ "fullMetaSearch": false,
+ "hide": false,
+ "includeNullMetadata": true,
+ "instant": false,
+ "legendFormat": "Usage Cost",
+ "range": true,
+ "refId": "C",
"useBackend": false
}
],
- "title": "Token usage by model",
- "transformations": [],
+ "title": "Average Token Consumption vs. Average Usage Cost Comparison",
"type": "timeseries"
},
+ {
+ "collapsed": false,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 47
+ },
+ "id": 10,
+ "panels": [],
+ "title": "GenAI Requests",
+ "type": "row"
+ },
{
"datasource": {
- "type": "loki",
- "uid": "grafanacloud-logs"
+ "type": "tempo",
+ "uid": "${traceDatasource}"
+ },
+ "description": "This panel displays a detailed table of GenAI request traces, providing comprehensive insights into each request's timing, source, type, and outcome. It allows for the in-depth analysis of individual requests, facilitating troubleshooting, performance monitoring, and understanding user interactions with GenAI services. Tracking this helps in identifying patterns, potential issues, and opportunities for optimization.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "continuous-BlPu"
+ },
+ "custom": {
+ "align": "auto",
+ "cellOptions": {
+ "type": "color-background"
+ },
+ "inspect": false
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
},
- "description": "Displays a log of all prompt messages sent to the LLM, providing insights into the inputs and interactions leading up to LLM responses.",
"gridPos": {
- "h": 8,
- "w": 12,
+ "h": 16,
+ "w": 24,
"x": 0,
- "y": 33
+ "y": 48
},
- "id": 26,
+ "id": 8,
"options": {
- "dedupStrategy": "none",
- "enableLogDetails": true,
- "prettifyLogMessage": true,
- "showCommonLabels": false,
- "showLabels": false,
- "showTime": true,
- "sortOrder": "Descending",
- "wrapLogMessage": false
- },
- "pluginVersion": "10.4.0-65875",
+ "cellHeight": "sm",
+ "footer": {
+ "countRows": false,
+ "fields": "",
+ "reducer": [
+ "sum"
+ ],
+ "show": false
+ },
+ "showHeader": true
+ },
+ "pluginVersion": "11.1.0-69372",
"targets": [
{
"datasource": {
- "type": "loki",
- "uid": "grafanacloud-logs"
- },
- "editorMode": "code",
- "expr": "{job=\"doku\", type=\"prompt\", applicationName=~\"$Application\", environment=~\"$Environment\"}",
- "hide": false,
- "queryType": "range",
- "refId": "B"
- }
- ],
- "title": "Prompts",
- "transformations": [
- {
- "id": "extractFields",
- "options": {
- "format": "auto",
- "source": "labels"
- }
+ "type": "tempo",
+ "uid": "${traceDatasource}"
+ },
+ "filters": [
+ {
+ "id": "e7e29fde",
+ "operator": "!=",
+ "scope": "span",
+ "tag": "gen_ai.type",
+ "value": [
+ "vectordb"
+ ],
+ "valueType": "string"
+ },
+ {
+ "id": "status",
+ "operator": "=",
+ "scope": "intrinsic",
+ "tag": "status",
+ "value": "ok",
+ "valueType": "keyword"
+ }
+ ],
+ "limit": 20,
+ "query": "{span.gen_ai.type!=\"vectordb\" && status=ok && span.gen_ai.application_name=~\"$application\" && span.gen_ai.environment=~\"$environment\"} | select(span.gen_ai.content.prompt, span.gen_ai.content.completion, span.gen_ai.usage.cost)",
+ "queryType": "traceql",
+ "refId": "A",
+ "tableType": "traces"
}
],
"transparent": true,
- "type": "logs"
+ "type": "table"
+ },
+ {
+ "collapsed": false,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 64
+ },
+ "id": 12,
+ "panels": [],
+ "title": "VectorDB Requests",
+ "type": "row"
},
{
"datasource": {
- "type": "loki",
- "uid": "grafanacloud-logs"
+ "type": "tempo",
+ "uid": "${traceDatasource}"
+ },
+ "description": "This panel displays a table with detailed traces of VectorDB requests, offering insights into specifics like request timing, source, operation type, and results. It's designed for thorough examination of VectorDB interactions, enabling precise troubleshooting, performance evaluation, and user behavior understanding. By monitoring these traces, users can spot trends, diagnose problems, and improve operational efficiency in handling VectorDB requests.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic-by-name"
+ },
+ "custom": {
+ "align": "auto",
+ "cellOptions": {
+ "type": "color-background"
+ },
+ "inspect": false
+ },
+ "mappings": [],
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
},
- "description": "Displays a log of all responses from the LLM, offering a comprehensive view of how the model interprets prompts and the information it generates.",
"gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 33
+ "h": 15,
+ "w": 24,
+ "x": 0,
+ "y": 65
},
- "id": 27,
+ "id": 9,
"options": {
- "dedupStrategy": "none",
- "enableLogDetails": true,
- "prettifyLogMessage": true,
- "showCommonLabels": false,
- "showLabels": false,
- "showTime": true,
- "sortOrder": "Descending",
- "wrapLogMessage": true
- },
- "pluginVersion": "10.4.0-65875",
+ "cellHeight": "sm",
+ "footer": {
+ "countRows": false,
+ "fields": "",
+ "reducer": [
+ "sum"
+ ],
+ "show": false
+ },
+ "showHeader": true
+ },
+ "pluginVersion": "11.1.0-69372",
"targets": [
{
"datasource": {
- "type": "loki",
- "uid": "grafanacloud-logs"
- },
- "editorMode": "code",
- "expr": "{job=\"doku\", type=\"response\", applicationName=~\"$Application\", environment=~\"$Environment\"}",
- "hide": false,
- "queryType": "range",
- "refId": "B"
- }
- ],
- "title": "Responses",
- "transformations": [
- {
- "id": "extractFields",
- "options": {
- "format": "auto",
- "source": "labels"
- }
+ "type": "tempo",
+ "uid": "${traceDatasource}"
+ },
+ "filters": [
+ {
+ "id": "e7e29fde",
+ "operator": "=",
+ "scope": "span",
+ "tag": "gen_ai.type",
+ "value": [
+ "vectordb"
+ ],
+ "valueType": "string"
+ },
+ {
+ "id": "status",
+ "operator": "=",
+ "scope": "intrinsic",
+ "tag": "status",
+ "value": "ok",
+ "valueType": "keyword"
+ }
+ ],
+ "limit": 20,
+ "query": "{span.gen_ai.type=\"vectordb\" && status=ok && span.gen_ai.application_name=~\"$application\" && span.gen_ai.environment=~\"$environment\"}",
+ "queryType": "traceql",
+ "refId": "A",
+ "tableType": "traces"
}
],
"transparent": true,
- "type": "logs"
+ "type": "table"
}
],
"refresh": "",
"schemaVersion": 39,
- "tags": [
- "Doku",
- "OpenAI",
- "Cohere",
- "Anthropic"
- ],
+ "tags": [],
"templating": {
"list": [
+ {
+ "current": {
+ "selected": true,
+ "value": "grafanacloud-traces"
+ },
+ "hide": 0,
+ "includeAll": false,
+ "label": "Trace Datasource",
+ "multi": false,
+ "name": "traceDatasource",
+ "options": [],
+ "query": "tempo",
+ "queryValue": "",
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "type": "datasource"
+ },
+ {
+ "current": {
+ "selected": true,
+ "value": "grafanacloud-prom"
+ },
+ "hide": 0,
+ "includeAll": false,
+ "label": "Metrics Datasource",
+ "multi": false,
+ "name": "metricsDatasource",
+ "options": [],
+ "query": "prometheus",
+ "queryValue": "",
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "type": "datasource"
+ },
{
"current": {
"selected": true,
@@ -2287,16 +1471,16 @@ curl --request POST \
"type": "prometheus",
"uid": "grafanacloud-prom"
},
- "definition": "label_values(applicationName)",
+ "definition": "label_values(gen_ai_application_name)",
"hide": 0,
"includeAll": true,
"label": "Application",
"multi": true,
- "name": "Application",
+ "name": "application",
"options": [],
"query": {
"qryType": 1,
- "query": "label_values(applicationName)",
+ "query": "label_values(gen_ai_application_name)",
"refId": "PrometheusVariableQueryEditor-VariableQuery"
},
"refresh": 1,
@@ -2319,22 +1503,22 @@ curl --request POST \
"type": "prometheus",
"uid": "grafanacloud-prom"
},
- "definition": "label_values(environment)",
+ "definition": "label_values(gen_ai_environment)",
"hide": 0,
"includeAll": true,
"label": "Environment",
"multi": true,
- "name": "Environment",
+ "name": "environment",
"options": [],
"query": {
"qryType": 1,
- "query": "label_values(environment)",
+ "query": "label_values(gen_ai_environment)",
"refId": "PrometheusVariableQueryEditor-VariableQuery"
},
"refresh": 1,
"regex": "",
"skipUrlSync": false,
- "sort": 0,
+ "sort": 1,
"type": "query"
}
]
@@ -2343,11 +1527,12 @@ curl --request POST \
"from": "now-24h",
"to": "now"
},
+ "timeRangeUpdatedDuringEditOrView": false,
"timepicker": {},
- "timezone": "",
- "title": "Doku - LLM Monitoring",
- "uid": "doku-llm-monitoring",
- "version": 1,
+ "timezone": "browser",
+ "title": "GenAI Observability",
+ "uid": "cdiz9piuoa3gge",
+ "version": 21,
"weekStart": ""
}
```
diff --git a/sdk/python/.pylintrc b/sdk/python/.pylintrc
new file mode 100644
index 00000000..7b5909c5
--- /dev/null
+++ b/sdk/python/.pylintrc
@@ -0,0 +1,7 @@
+[DESIGN]
+
+# Maximum number of locals for function / method body
+max-locals=25
+
+# Maximum number of arguments for function / method
+max-args=25
\ No newline at end of file
diff --git a/sdk/python/LICENSE b/sdk/python/LICENSE
new file mode 100644
index 00000000..261eeb9e
--- /dev/null
+++ b/sdk/python/LICENSE
@@ -0,0 +1,201 @@
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
diff --git a/sdk/python/README.md b/sdk/python/README.md
new file mode 100644
index 00000000..d11d25ba
--- /dev/null
+++ b/sdk/python/README.md
@@ -0,0 +1,90 @@
+# OpenLIT: OpenTelemetry-native Observability for LLMs
+
+OpenLIT Python SDK is an **OpenTelemetry-native** Auto instrumentation library for monitoring LLM Applications, facilitating the integration of observability into your GenAI-driven projects. Designed with simplicity and efficiency, OpenLIT offers the ability to embed observability into your GenAI-driven projects effortlessly using just **a single line of code**.
+
+Whether you're directly using LLM Libraries like OpenAI, Anthropic or building complex RAG Agents using Langchain, OpenLIT seamlessly integrates observability into your applications, ensuring enhanced performance and reliability across diverse scenarios.
+
+This project adheres to the [Semantic Conventions](https://github.com/open-telemetry/semantic-conventions/tree/main/docs/gen-ai) proposed by the OpenTelemetry community. You can check out the current definitions [here](src/openlit/semcov/__init__.py).
+
+## What can be Auto Instrumented?
+
+### LLMs
+- ✅ OpenAI
+- ✅ Anthropic
+- ✅ Cohere
+- ✅ Mistral
+- ✅ Azure OpenAI
+- ✅ HuggingFace Transformers
+
+### Vector DBs
+- ✅ ChromaDB
+- ✅ Pinecone
+
+### Frameworks
+- ✅ Langchain
+- ✅ LiteLLM
+
+## Supported Destinations
+- ✅ OpenTelemetry Collector
+- ✅ Grafana Cloud
+- ✅ Grafana Tempo
+- ✅ DataDog
+- ✅ New Relic
+- ✅ SigNoz
+- ✅ Dynatrace
+- ✅ OpenObserve
+
+## 💿 Installation
+
+```bash
+pip install openlit
+```
+
+## ⚡ Quick Integration
+
+```python
+import openlit
+openlit.init()
+```
+
+By default, OpenLIT directs traces and metrics straight to your console. To forward telemetry data to an HTTP OTLP endpoint, such as the OpenTelemetry Collector, set the `otlp_endpoint` parameter with the desired endpoint. Alternatively, you can configure the endpoint by setting the `OTEL_EXPORTER_OTLP_ENDPOINT` environment variable as recommended in the OpenTelemetry documentation.
+
+To send telemetry to OpenTelemetry backends requiring authentication, set the `otlp_headers` parameter with its desired value. Alternatively, you can configure the endpoint by setting the `OTEL_EXPORTER_OTLP_HEADERS` environment variable as recommended in the OpenTelemetry documentation.
+
+
+
+### Configuration
+
+Below is a detailed overview of the configuration options available, allowing you to adjust OpenLIT's behavior and functionality to align with your specific observability needs:
+
+| Argument | Description | Default Value | Required |
+|-------------------------|-----------------------------------------------------------------------------------------------|----------------|----------|
+| `environment` | The deployment environment of the application. | `"default"` | Yes |
+| `application_name` | Identifies the name of your application. | `"default"` | Yes |
+| `tracer` | An instance of OpenTelemetry Tracer for tracing operations. | `None` | No |
+| `meter` | An OpenTelemetry Metrics instance for capturing metrics. | `None` | No |
+| `otlp_endpoint` | Specifies the OTLP endpoint for transmitting telemetry data. | `None` | No |
+| `otlp_headers` | Defines headers for the OTLP exporter, useful for backends requiring authentication. | `None` | No |
+| `disable_batch` | A flag to disable batch span processing, favoring immediate dispatch. | `False` | No |
+| `trace_content` | Enables tracing of content for deeper insights. | `True` | No |
+| `disabled_instrumentors`| List of instrumentors to disable. Choices: `["openai", "anthropic", "langchain", "cohere", "mistral", "transformers", "chroma", "pinecone"]`. | `None` | No |
+| `disable_metrics` | If set, disables the collection of metrics. | `False` | No |
+
+## 🌱 Contributing
+
+Whether it's big or small, we love contributions 💚. Check out our [Contribution guide](../../CONTRIBUTING.md) to get started
+
+Unsure where to start? Here are a few ways to get involved:
+
+- Join our [Slack channel](https://join.slack.com/t/openlit/shared_invite/zt-2etnfttwg-TjP_7BZXfYg84oAukY8QRQ) to discuss ideas, share feedback, and connect with both our team and the wider OpenLIT community.
+
+Your input helps us grow and improve, and we're here to support you every step of the way.
+
+## 💚 Community & Support
+
+Connect with the OpenLIT community and maintainers for support, discussions, and updates:
+
+- 🌟 If you like it, Leave a star on our [GitHub](https://github.com/open-lit/openlit/)
+- 🌍 Join our [Slack](https://join.slack.com/t/openlit/shared_invite/zt-2etnfttwg-TjP_7BZXfYg84oAukY8QRQ) Community for live interactions and questions.
+- 🐞 Report bugs on our [GitHub Issues](https://github.com/open-lit/openlit/issues) to help us improve OpenLIT.
+- 𝕏 Follow us on [X](https://twitter.com/openlit) for the latest updates and news.
\ No newline at end of file
diff --git a/sdk/python/pyproject.toml b/sdk/python/pyproject.toml
new file mode 100644
index 00000000..3c802ed8
--- /dev/null
+++ b/sdk/python/pyproject.toml
@@ -0,0 +1,22 @@
+[tool.poetry]
+name = "openlit"
+version = "0.0.1"
+description = "OpenTelemetry-native Auto instrumentation library for monitoring LLM Applications, facilitating the integration of observability into your GenAI-driven projects"
+authors = ["OpenLIT"]
+repository = "https://github.com/open-lit/openlit/tree/main/openlit/python"
+readme = "README.md"
+homepage = "https://github.com/open-lit/openlit/tree/main/openlit/python"
+keywords = ["OpenTelemetry", "otel", "otlp","llm", "tracing", "openai", "anthropic", "claude", "cohere", "llm monitoring", "observability", "monitoring", "gpt", "Generative AI", "chatGPT"]
+
+[tool.poetry.dependencies]
+python = "^3.7.1"
+requests = "^2.26.0"
+tiktoken = "^0.1.1"
+opentelemetry-api = "^1.24.0"
+opentelemetry-sdk = "^1.24.0"
+opentelemetry-exporter-otlp = "^1.24.0"
+opentelemetry-instrumentation = "^0.45b0"
+
+[build-system]
+requires = ["poetry-core>=1.1.0"]
+build-backend = "poetry.core.masonry.api"
diff --git a/sdk/python/pytest.ini b/sdk/python/pytest.ini
new file mode 100644
index 00000000..f154c0fa
--- /dev/null
+++ b/sdk/python/pytest.ini
@@ -0,0 +1,2 @@
+[pytest]
+pythonpath = src/
\ No newline at end of file
diff --git a/sdk/python/src/openlit/__helpers.py b/sdk/python/src/openlit/__helpers.py
new file mode 100644
index 00000000..09802b6f
--- /dev/null
+++ b/sdk/python/src/openlit/__helpers.py
@@ -0,0 +1,143 @@
+# pylint: disable=bare-except, broad-exception-caught
+"""
+This module has functions to calculate model costs based on tokens and to fetch pricing information.
+"""
+
+import logging
+import requests
+import tiktoken
+from opentelemetry.trace import Status, StatusCode
+
+# Set up logging
+logger = logging.getLogger(__name__)
+
+def openai_tokens(text, model):
+ """
+ Calculate the number of tokens a given text would take up for a specified model.
+
+ Args:
+ text (str): The input text to be encoded.
+ model (str): The model identifier used for encoding.
+
+ Returns:
+ int: The number of tokens the text is encoded into.
+ """
+ try:
+ encoding = tiktoken.encoding_for_model(model)
+ except:
+ encoding = tiktoken.get_encoding("cl100k_base")
+
+ num_tokens = len(encoding.encode(text))
+ return num_tokens
+
+def general_tokens(text):
+ """
+ Calculate the number of tokens a given text would take up.
+
+ Args:
+ text (str): The input text to be encoded.
+ model (str): The model identifier used for encoding.
+
+ Returns:
+ int: The number of tokens the text is encoded into.
+ """
+
+ encoding = tiktoken.get_encoding("gpt2")
+
+ num_tokens = len(encoding.encode(text))
+ return num_tokens
+
+def get_chat_model_cost(model, pricing_info, prompt_tokens, completion_tokens):
+ """
+ Retrieve the cost of processing for a given model based on prompt and tokens.
+
+ Args:
+ model (str): The model identifier.
+ pricing_info (dict): A dictionary containing pricing information for various models.
+ prompt_tokens (int): Number of tokens in the prompt.
+ completion_tokens (int): Number of tokens in the completion if applicable.
+
+ Returns:
+ float: The calculated cost for the operation.
+ """
+ try:
+ cost = ((prompt_tokens / 1000) * pricing_info["chat"][model]["promptPrice"]) + \
+ ((completion_tokens / 1000) * pricing_info["chat"][model]["completionPrice"])
+ except:
+ cost = 0
+ return cost
+
+def get_embed_model_cost(model, pricing_info, prompt_tokens):
+ """
+ Retrieve the cost of processing for a given model based on prompt tokens.
+
+ Args:
+ model (str): The model identifier.
+ pricing_info (dict): A dictionary containing pricing information for various models.
+ prompt_tokens (int): Number of tokens in the prompt.
+
+ Returns:
+ float: The calculated cost for the operation.
+ """
+ try:
+ cost = (prompt_tokens / 1000) * pricing_info["embeddings"][model]
+ except:
+ cost = 0
+ return cost
+
+def get_image_model_cost(model, pricing_info, size, quality):
+ """
+ Retrieve the cost of processing for a given model based on image size and quailty.
+
+ Args:
+ model (str): The model identifier.
+ pricing_info (dict): A dictionary containing pricing information for various models.
+ size (str): Size of the Image.
+ quality (int): Quality of the Image.
+
+ Returns:
+ float: The calculated cost for the operation.
+ """
+ try:
+ cost = pricing_info["images"][model][quality][size]
+ except:
+ cost = 0
+ return cost
+
+def get_audio_model_cost(model, pricing_info, prompt):
+ """
+ Retrieve the cost of processing for a given model based on prompt.
+
+ Args:
+ model (str): The model identifier.
+ pricing_info (dict): A dictionary containing pricing information for various models.
+ prompt (str): Prompt to the LLM Model
+
+ Returns:
+ float: The calculated cost for the operation.
+ """
+ try:
+ cost = (len(prompt) / 1000) * pricing_info["audio"][model]
+ except:
+ cost = 0
+ return cost
+
+def fetch_pricing_info():
+ """Fetches pricing information from a specified URL."""
+ pricing_url = "https://raw.githubusercontent.com/dokulabs/doku/main/assets/pricing.json"
+ try:
+ # Set a timeout of 10 seconds for both the connection and the read
+ response = requests.get(pricing_url, timeout=20)
+ response.raise_for_status()
+ return response.json()
+ except requests.HTTPError as http_err:
+ logger.error("HTTP error occured while fetching pricing info: %s", http_err)
+ except Exception as err:
+ logger.error("Unexpected error occurred while fetching pricing info: %s", err)
+ return {}
+
+def handle_exception(span,e):
+ """Handles Exception when LLM Function fails or trace creation fails."""
+ # Record the exception details within the span
+ span.record_exception(e)
+ span.set_status(Status(StatusCode.ERROR))
diff --git a/sdk/python/src/openlit/__init__.py b/sdk/python/src/openlit/__init__.py
new file mode 100644
index 00000000..56ab1176
--- /dev/null
+++ b/sdk/python/src/openlit/__init__.py
@@ -0,0 +1,184 @@
+"""
+The __init__.py module for the openLIT package.
+This module sets up the openLIT configuration and instrumentation for various
+large language models (LLMs).
+"""
+from typing import Optional, Dict, Any
+import logging
+
+# Import internal modules for setting up tracing and fetching pricing info.
+from openlit.otel.tracing import setup_tracing
+from openlit.otel.metrics import setup_meter
+from openlit.__helpers import fetch_pricing_info
+
+# Instrumentors for various large language models.
+from openlit.instrumentation.openai import OpenAIInstrumentor
+from openlit.instrumentation.anthropic import AnthropicInstrumentor
+from openlit.instrumentation.cohere import CohereInstrumentor
+from openlit.instrumentation.mistral import MistralInstrumentor
+from openlit.instrumentation.langchain import LangChainInstrumentor
+from openlit.instrumentation.chroma import ChromaInstrumentor
+from openlit.instrumentation.pinecone import PineconeInstrumentor
+from openlit.instrumentation.transformers import TransformersInstrumentor
+
+# Set up logging for error and information messages.
+logger = logging.getLogger(__name__)
+
+class OpenlitConfig:
+ """
+ A Singleton Configuration class for openLIT.
+
+ This class maintains a single instance of configuration settings including
+ environment details, application name, and tracing information throughout the openLIT package.
+
+ Attributes:
+ environment (str): Deployment environment of the application.
+ application_name (str): Name of the application using openLIT.
+ pricing_info (Dict[str, Any]): Pricing information.
+ tracer (Optional[Any]): Tracer instance for OpenTelemetry.
+ otlp_endpoint (Optional[str]): Endpoint for OTLP.
+ otlp_headers (Optional[Dict[str, str]]): Headers for OTLP.
+ disable_batch (bool): Flag to disable batch span processing in tracing.
+ trace_content (bool): Flag to enable or disable tracing of content.
+ """
+ _instance = None
+
+ def __new__(cls):
+ """Ensures that only one instance of the configuration exists."""
+ if cls._instance is None:
+ cls._instance = super(OpenlitConfig, cls).__new__(cls)
+ cls.reset_to_defaults()
+ return cls._instance
+
+ @classmethod
+ def reset_to_defaults(cls):
+ """Resets configuration to default values."""
+ cls.environment = "default"
+ cls.application_name = "default"
+ cls.pricing_info = fetch_pricing_info()
+ cls.tracer = None
+ cls.metrics_dict = {}
+ cls.otlp_endpoint = None
+ cls.otlp_headers = None
+ cls.disable_batch = False
+ cls.trace_content = True
+ cls.disable_metrics = False
+
+ @classmethod
+ def update_config(cls, environment, application_name, tracer, otlp_endpoint,
+ otlp_headers, disable_batch, trace_content, metrics_dict, disable_metrics):
+ """
+ Updates the configuration based on provided parameters.
+
+ Args:
+ environment (str): Deployment environment.
+ application_name (str): Application name.
+ tracer: Tracer instance.
+ meter: Metric Instance
+ otlp_endpoint (str): OTLP endpoint.
+ otlp_headers (Dict[str, str]): OTLP headers.
+ disable_batch (bool): Disable batch span processing flag.
+ trace_content (bool): Enable or disable content tracing.
+ """
+ cls.environment = environment
+ cls.application_name = application_name
+ cls.pricing_info = fetch_pricing_info()
+ cls.tracer = tracer
+ cls.metrics_dict = metrics_dict
+ cls.otlp_endpoint = otlp_endpoint
+ cls.otlp_headers = otlp_headers
+ cls.disable_batch = disable_batch
+ cls.trace_content = trace_content
+ cls.disable_metrics = disable_metrics
+
+def init(environment="default", application_name="default", tracer=None, otlp_endpoint=None,
+ otlp_headers=None, disable_batch=False, trace_content=True, disabled_instrumentors=None,
+ meter=None, disable_metrics=False):
+ """
+ Initializes the openLIT configuration and setups tracing.
+
+ This function sets up the openLIT environment with provided configurations
+ and initializes instrumentors for tracing.
+
+ Args:
+ environment (str): Deployment environment.
+ application_name (str): Application name.
+ tracer: Tracer instance (Optional).
+ meter: OpenTelemetry Metrics Instance (Optional).
+ otlp_endpoint (str): OTLP endpoint for exporter (Optional).
+ otlp_headers (Dict[str, str]): OTLP headers for exporter (Optional).
+ disable_batch (bool): Flag to disable batch span processing (Optional).
+ trace_content (bool): Flag to trace content (Optional).
+ disabled_instrumentors (List[str]): Optional. List of instrumentor names to disable.
+ disable_metrics (bool): Flag to disable metrics (Optional)
+ """
+ disabled_instrumentors = disabled_instrumentors if disabled_instrumentors else []
+
+ # Check for invalid instrumentor names
+ valid_instruments = {
+ "openai", "anthropic", "langchain",
+ "cohere", "mistral", "chroma",
+ "pinecone", "transformers"
+ }
+ invalid_instrumentors = set(disabled_instrumentors) - valid_instruments
+ for invalid_name in invalid_instrumentors:
+ logger.warning("Invalid instrumentor name detected and ignored: '%s'", invalid_name)
+
+ try:
+ # Retrieve or create the single configuration instance.
+ config = OpenlitConfig()
+
+ # Setup tracing based on the provided or default configuration.
+ tracer = setup_tracing(
+ application_name=application_name,
+ environment=environment, tracer=tracer,
+ otlp_endpoint=otlp_endpoint, otlp_headers=otlp_headers,
+ disable_batch=disable_batch
+ )
+
+ if not tracer:
+ logger.error("openLIT tracing setup failed. Tracing will not be available.")
+ return
+
+ # Setup meter and receive metrics_dict instead of meter
+ metrics_dict = setup_meter(application_name=application_name,
+ environment=environment, meter=meter,
+ otlp_endpoint=otlp_endpoint, otlp_headers=otlp_headers)
+
+ if not metrics_dict:
+ logger.error("openLIT metrics setup failed. Metrics will not be available.")
+ return
+
+ # Update global configuration with the provided settings.
+ config.update_config(environment, application_name, tracer, otlp_endpoint,
+ otlp_headers, disable_batch, trace_content,
+ metrics_dict, disable_metrics)
+
+ # Map instrumentor names to their instances
+ instrumentor_instances = {
+ "openai": OpenAIInstrumentor(),
+ "anthropic": AnthropicInstrumentor(),
+ "cohere": CohereInstrumentor(),
+ "mistral": MistralInstrumentor(),
+ "langchain": LangChainInstrumentor(),
+ "chroma": ChromaInstrumentor(),
+ "pinecone": PineconeInstrumentor(),
+ "transformers": TransformersInstrumentor()
+ }
+
+ # Initialize and instrument only the enabled instrumentors
+ for name, instrumentor in instrumentor_instances.items():
+ if name not in disabled_instrumentors:
+ instrumentor.instrument(
+ environment=config.environment,
+ application_name=config.application_name,
+ tracer=config.tracer,
+ pricing_info=config.pricing_info,
+ trace_content=config.trace_content,
+ metrics_dict=config.metrics_dict,
+ disable_metrics=config.disable_metrics
+ )
+
+ # pylint: disable=broad-exception-caught
+ except Exception as e:
+ logger.error("Error during openLIT initialization: %s", e)
diff --git a/sdk/python/src/openlit/instrumentation/anthropic/__init__.py b/sdk/python/src/openlit/instrumentation/anthropic/__init__.py
new file mode 100644
index 00000000..7a833165
--- /dev/null
+++ b/sdk/python/src/openlit/instrumentation/anthropic/__init__.py
@@ -0,0 +1,50 @@
+# pylint: disable=useless-return, bad-staticmethod-argument, disable=duplicate-code
+"""Initializer of Auto Instrumentation of Anthropic Functions"""
+
+from typing import Collection
+import importlib.metadata
+from opentelemetry.instrumentation.instrumentor import BaseInstrumentor
+from wrapt import wrap_function_wrapper
+
+from openlit.instrumentation.anthropic.anthropic import messages
+from openlit.instrumentation.anthropic.async_anthropic import async_messages
+
+_instruments = ("anthropic >= 0.21.0",)
+
+class AnthropicInstrumentor(BaseInstrumentor):
+ """
+ An instrumentor for Anthropic's client library.
+ """
+
+ def instrumentation_dependencies(self) -> Collection[str]:
+ return _instruments
+
+ def _instrument(self, **kwargs):
+ application_name = kwargs.get("application_name", "default_application")
+ environment = kwargs.get("environment", "default_environment")
+ tracer = kwargs.get("tracer")
+ metrics = kwargs.get("metrics_dict")
+ pricing_info = kwargs.get("pricing_info", {})
+ trace_content = kwargs.get("trace_content", False)
+ disable_metrics = kwargs.get("disable_metrics")
+ version = importlib.metadata.version("anthropic")
+
+ #sync
+ wrap_function_wrapper(
+ "anthropic.resources.messages",
+ "Messages.create",
+ messages("anthropic.messages", version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics),
+ )
+
+ #async
+ wrap_function_wrapper(
+ "anthropic.resources.messages",
+ "AsyncMessages.create",
+ async_messages("anthropic.messages", version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics),
+ )
+
+ def _uninstrument(self, **kwargs):
+ # Proper uninstrumentation logic to revert patched methods
+ pass
diff --git a/sdk/python/src/openlit/instrumentation/anthropic/anthropic.py b/sdk/python/src/openlit/instrumentation/anthropic/anthropic.py
new file mode 100644
index 00000000..f5495186
--- /dev/null
+++ b/sdk/python/src/openlit/instrumentation/anthropic/anthropic.py
@@ -0,0 +1,291 @@
+# pylint: disable=duplicate-code, broad-exception-caught, too-many-statements, unused-argument
+"""
+Module for monitoring Anthropic API calls.
+"""
+
+import logging
+from opentelemetry.trace import SpanKind, Status, StatusCode
+from opentelemetry.sdk.resources import TELEMETRY_SDK_NAME
+from openlit.__helpers import get_chat_model_cost, handle_exception
+from openlit.semcov import SemanticConvetion
+
+# Initialize logger for logging potential issues and operations
+logger = logging.getLogger(__name__)
+
+def messages(gen_ai_endpoint, version, environment, application_name, tracer,
+ pricing_info, trace_content, metrics, disable_metrics):
+ """
+ Generates a telemetry wrapper for messages to collect metrics.
+
+ Args:
+ gen_ai_endpoint: Endpoint identifier for logging and tracing.
+ version: Version of the monitoring package.
+ environment: Deployment environment (e.g., production, staging).
+ application_name: Name of the application using the OpenAI API.
+ tracer: OpenTelemetry tracer for creating spans.
+ pricing_info: Information used for calculating the cost of OpenAI usage.
+ trace_content: Flag indicating whether to trace the actual content.
+
+ Returns:
+ A function that wraps the chat method to add telemetry.
+ """
+
+ def wrapper(wrapped, instance, args, kwargs):
+ """
+ Wraps the 'messages' API call to add telemetry.
+
+ This collects metrics such as execution time, cost, and token usage, and handles errors
+ gracefully, adding details to the trace for observability.
+
+ Args:
+ wrapped: The original 'messages' method to be wrapped.
+ instance: The instance of the class where the original method is defined.
+ args: Positional arguments for the 'messages' method.
+ kwargs: Keyword arguments for the 'messages' method.
+
+ Returns:
+ The response from the original 'messages' method.
+ """
+
+ # Check if streaming is enabled for the API call
+ streaming = kwargs.get("stream", False)
+
+ # pylint: disable=no-else-return
+ if streaming:
+ # Special handling for streaming response to accommodate the nature of data flow
+ def stream_generator():
+ with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
+ # Placeholder for aggregating streaming response
+ llmresponse = ""
+
+ # Loop through streaming events capturing relevant details
+ for event in wrapped(*args, **kwargs):
+
+ # Collect message IDs and input token from events
+ if event.type == "message_start":
+ response_id = event.message.id
+ prompt_tokens = event.message.usage.input_tokens
+
+ # Aggregate response content
+ if event.type == "content_block_delta":
+ llmresponse += event.delta.text
+
+ # Collect output tokens and stop reason from events
+ if event.type == "message_delta":
+ completion_tokens = event.usage.output_tokens
+ finish_reason = event.delta.stop_reason
+ yield event
+
+ # Handling exception ensure observability without disrupting operation
+ try:
+ # Format 'messages' into a single string
+ message_prompt = kwargs.get("messages", "")
+ formatted_messages = []
+ for message in message_prompt:
+ role = message["role"]
+ content = message["content"]
+
+ if isinstance(content, list):
+ content_str = ", ".join(
+ # pylint: disable=line-too-long
+ f'{item["type"]}: {item["text"] if "text" in item else item["image_url"]}'
+ if "type" in item else f'text: {item["text"]}'
+ for item in content
+ )
+ formatted_messages.append(f"{role}: {content_str}")
+ else:
+ formatted_messages.append(f"{role}: {content}")
+ prompt = "\n".join(formatted_messages)
+
+ # Calculate cost of the operation
+ cost = get_chat_model_cost(
+ kwargs.get("model", "claude-3-sonnet-20240229"),
+ pricing_info, prompt_tokens, completion_tokens
+ )
+
+ # Set Span attributes
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_ANTHROPIC)
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
+ SemanticConvetion.GEN_AI_TYPE_CHAT)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
+ gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,
+ response_id)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,
+ environment)
+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
+ application_name)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ kwargs.get("model", "claude-3-sonnet-20240229"))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS,
+ kwargs.get("max_tokens", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,
+ True)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,
+ kwargs.get("temperature", 1.0))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,
+ kwargs.get("top_p", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_K,
+ kwargs.get("top_k", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,
+ finish_reason)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
+ prompt_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,
+ completion_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ prompt_tokens + completion_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,
+ prompt)
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_COMPLETION,
+ llmresponse)
+
+ span.set_status(Status(StatusCode.OK))
+
+ if disable_metrics is False:
+ attributes = {
+ TELEMETRY_SDK_NAME:
+ "openlit",
+ SemanticConvetion.GEN_AI_APPLICATION_NAME:
+ application_name,
+ SemanticConvetion.GEN_AI_SYSTEM:
+ SemanticConvetion.GEN_AI_SYSTEM_ANTHROPIC,
+ SemanticConvetion.GEN_AI_ENVIRONMENT:
+ environment,
+ SemanticConvetion.GEN_AI_TYPE:
+ SemanticConvetion.GEN_AI_TYPE_CHAT,
+ SemanticConvetion.GEN_AI_REQUEST_MODEL:
+ kwargs.get("model", "claude-3-sonnet-20240229")
+ }
+
+ metrics["genai_requests"].add(1, attributes)
+ metrics["genai_total_tokens"].add(
+ prompt_tokens + completion_tokens, attributes
+ )
+ metrics["genai_completion_tokens"].add(completion_tokens, attributes)
+ metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)
+ metrics["genai_cost"].record(cost, attributes)
+
+ except Exception as e:
+ handle_exception(span, e)
+ logger.error("Error in trace creation: %s", e)
+
+ return stream_generator()
+
+ # Handling for non-streaming responses
+ else:
+ with tracer.start_as_current_span(gen_ai_endpoint, kind=SpanKind.CLIENT) as span:
+ response = wrapped(*args, **kwargs)
+
+ try:
+ # Format 'messages' into a single string
+ message_prompt = kwargs.get("messages", "")
+ formatted_messages = []
+ for message in message_prompt:
+ role = message["role"]
+ content = message["content"]
+
+ if isinstance(content, list):
+ content_str = ", ".join(
+ # pylint: disable=line-too-long
+ f'{item["type"]}: {item["text"] if "text" in item else item["image_url"]}'
+ if "type" in item else f'text: {item["text"]}'
+ for item in content
+ )
+ formatted_messages.append(f"{role}: {content_str}")
+ else:
+ formatted_messages.append(f"{role}: {content}")
+ prompt = "\n".join(formatted_messages)
+
+ # Calculate cost of the operation
+ cost = get_chat_model_cost(kwargs.get("model", "claude-3-sonnet-20240229"),
+ pricing_info, response.usage.input_tokens,
+ response.usage.output_tokens)
+
+ # Set Span attribues
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_ANTHROPIC)
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
+ SemanticConvetion.GEN_AI_TYPE_CHAT)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
+ gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,
+ response.id)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,
+ environment)
+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
+ application_name)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ kwargs.get("model", "claude-3-sonnet-20240229"))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS,
+ kwargs.get("max_tokens", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,
+ False)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,
+ kwargs.get("temperature", 1.0))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,
+ kwargs.get("top_p", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_K,
+ kwargs.get("top_k", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,
+ response.stop_reason)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
+ response.usage.input_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,
+ response.usage.output_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ response.usage.input_tokens +
+ response.usage.output_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,
+ prompt)
+ # pylint: disable=line-too-long
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_COMPLETION, response.content[0].text if response.content else "")
+
+ span.set_status(Status(StatusCode.OK))
+
+ if disable_metrics is False:
+ attributes = {
+ TELEMETRY_SDK_NAME:
+ "openlit",
+ SemanticConvetion.GEN_AI_APPLICATION_NAME:
+ application_name,
+ SemanticConvetion.GEN_AI_SYSTEM:
+ SemanticConvetion.GEN_AI_SYSTEM_ANTHROPIC,
+ SemanticConvetion.GEN_AI_ENVIRONMENT:
+ environment,
+ SemanticConvetion.GEN_AI_TYPE:
+ SemanticConvetion.GEN_AI_TYPE_CHAT,
+ SemanticConvetion.GEN_AI_REQUEST_MODEL:
+ kwargs.get("model", "claude-3-sonnet-20240229")
+ }
+
+ metrics["genai_requests"].add(1, attributes)
+ metrics["genai_total_tokens"].add(
+ response.usage.input_tokens +
+ response.usage.output_tokens, attributes)
+ metrics["genai_completion_tokens"].add(
+ response.usage.output_tokens, attributes)
+ metrics["genai_prompt_tokens"].add(
+ response.usage.input_tokens, attributes)
+ metrics["genai_cost"].record(cost, attributes)
+
+ # Return original response
+ return response
+
+ except Exception as e:
+ handle_exception(span, e)
+ logger.error("Error in trace creation: %s", e)
+
+ # Return original response
+ return response
+
+ return wrapper
diff --git a/sdk/python/src/openlit/instrumentation/anthropic/async_anthropic.py b/sdk/python/src/openlit/instrumentation/anthropic/async_anthropic.py
new file mode 100644
index 00000000..2b0c3e43
--- /dev/null
+++ b/sdk/python/src/openlit/instrumentation/anthropic/async_anthropic.py
@@ -0,0 +1,291 @@
+# pylint: disable=duplicate-code, broad-exception-caught, too-many-statements, unused-argument
+"""
+Module for monitoring Anthropic API calls.
+"""
+
+import logging
+from opentelemetry.trace import SpanKind, Status, StatusCode
+from opentelemetry.sdk.resources import TELEMETRY_SDK_NAME
+from openlit.__helpers import get_chat_model_cost, handle_exception
+from openlit.semcov import SemanticConvetion
+
+# Initialize logger for logging potential issues and operations
+logger = logging.getLogger(__name__)
+
+def async_messages(gen_ai_endpoint, version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics):
+ """
+ Generates a telemetry wrapper for messages to collect metrics.
+
+ Args:
+ gen_ai_endpoint: Endpoint identifier for logging and tracing.
+ version: Version of the monitoring package.
+ environment: Deployment environment (e.g., production, staging).
+ application_name: Name of the application using the OpenAI API.
+ tracer: OpenTelemetry tracer for creating spans.
+ pricing_info: Information used for calculating the cost of OpenAI usage.
+ trace_content: Flag indicating whether to trace the actual content.
+
+ Returns:
+ A function that wraps the chat method to add telemetry.
+ """
+
+ async def wrapper(wrapped, instance, args, kwargs):
+ """
+ Wraps the 'messages' API call to add telemetry.
+
+ This collects metrics such as execution time, cost, and token usage, and handles errors
+ gracefully, adding details to the trace for observability.
+
+ Args:
+ wrapped: The original 'messages' method to be wrapped.
+ instance: The instance of the class where the original method is defined.
+ args: Positional arguments for the 'messages' method.
+ kwargs: Keyword arguments for the 'messages' method.
+
+ Returns:
+ The response from the original 'messages' method.
+ """
+
+ # Check if streaming is enabled for the API call
+ streaming = kwargs.get("stream", False)
+
+ # pylint: disable=no-else-return
+ if streaming:
+ # Special handling for streaming response to accommodate the nature of data flow
+ async def stream_generator():
+ with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
+ # Placeholder for aggregating streaming response
+ llmresponse = ""
+
+ # Loop through streaming events capturing relevant details
+ async for event in await wrapped(*args, **kwargs):
+
+ # Collect message IDs and input token from events
+ if event.type == "message_start":
+ response_id = event.message.id
+ prompt_tokens = event.message.usage.input_tokens
+
+ # Aggregate response content
+ if event.type == "content_block_delta":
+ llmresponse += event.delta.text
+
+ # Collect output tokens and stop reason from events
+ if event.type == "message_delta":
+ completion_tokens = event.usage.output_tokens
+ finish_reason = event.delta.stop_reason
+ yield event
+
+ # Handling exception ensure observability without disrupting operation
+ try:
+ # Format 'messages' into a single string
+ message_prompt = kwargs.get("messages", "")
+ formatted_messages = []
+ for message in message_prompt:
+ role = message["role"]
+ content = message["content"]
+
+ if isinstance(content, list):
+ content_str = ", ".join(
+ # pylint: disable=line-too-long
+ f'{item["type"]}: {item["text"] if "text" in item else item["image_url"]}'
+ if "type" in item else f'text: {item["text"]}'
+ for item in content
+ )
+ formatted_messages.append(f"{role}: {content_str}")
+ else:
+ formatted_messages.append(f"{role}: {content}")
+ prompt = "\n".join(formatted_messages)
+
+ # Calculate cost of the operation
+ cost = get_chat_model_cost(
+ kwargs.get("model", "claude-3-sonnet-20240229"),
+ pricing_info, prompt_tokens, completion_tokens
+ )
+
+ # Set Span attributes
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_ANTHROPIC)
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
+ SemanticConvetion.GEN_AI_TYPE_CHAT)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
+ gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,
+ response_id)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,
+ environment)
+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
+ application_name)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ kwargs.get("model", "claude-3-sonnet-20240229"))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS,
+ kwargs.get("max_tokens", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,
+ True)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,
+ kwargs.get("temperature", 1.0))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,
+ kwargs.get("top_p", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_K,
+ kwargs.get("top_k", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,
+ finish_reason)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
+ prompt_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,
+ completion_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ prompt_tokens + completion_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,
+ prompt)
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_COMPLETION,
+ llmresponse)
+
+ span.set_status(Status(StatusCode.OK))
+
+ if disable_metrics is False:
+ attributes = {
+ TELEMETRY_SDK_NAME:
+ "openlit",
+ SemanticConvetion.GEN_AI_APPLICATION_NAME:
+ application_name,
+ SemanticConvetion.GEN_AI_SYSTEM:
+ SemanticConvetion.GEN_AI_SYSTEM_ANTHROPIC,
+ SemanticConvetion.GEN_AI_ENVIRONMENT:
+ environment,
+ SemanticConvetion.GEN_AI_TYPE:
+ SemanticConvetion.GEN_AI_TYPE_CHAT,
+ SemanticConvetion.GEN_AI_REQUEST_MODEL:
+ kwargs.get("model", "claude-3-sonnet-20240229")
+ }
+
+ metrics["genai_requests"].add(1, attributes)
+ metrics["genai_total_tokens"].add(
+ prompt_tokens + completion_tokens, attributes
+ )
+ metrics["genai_completion_tokens"].add(completion_tokens, attributes)
+ metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)
+ metrics["genai_cost"].record(cost, attributes)
+
+ except Exception as e:
+ handle_exception(span, e)
+ logger.error("Error in trace creation: %s", e)
+
+ return stream_generator()
+
+ # Handling for non-streaming responses
+ else:
+ with tracer.start_as_current_span(gen_ai_endpoint, kind=SpanKind.CLIENT) as span:
+ response = await wrapped(*args, **kwargs)
+
+ try:
+ # Format 'messages' into a single string
+ message_prompt = kwargs.get("messages", "")
+ formatted_messages = []
+ for message in message_prompt:
+ role = message["role"]
+ content = message["content"]
+
+ if isinstance(content, list):
+ content_str = ", ".join(
+ # pylint: disable=line-too-long
+ f'{item["type"]}: {item["text"] if "text" in item else item["image_url"]}'
+ if "type" in item else f'text: {item["text"]}'
+ for item in content
+ )
+ formatted_messages.append(f"{role}: {content_str}")
+ else:
+ formatted_messages.append(f"{role}: {content}")
+ prompt = "\n".join(formatted_messages)
+
+ # Calculate cost of the operation
+ cost = get_chat_model_cost(kwargs.get("model", "claude-3-sonnet-20240229"),
+ pricing_info, response.usage.input_tokens,
+ response.usage.output_tokens)
+
+ # Set Span attribues
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_ANTHROPIC)
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
+ SemanticConvetion.GEN_AI_TYPE_CHAT)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
+ gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,
+ response.id)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,
+ environment)
+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
+ application_name)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ kwargs.get("model", "claude-3-sonnet-20240229"))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS,
+ kwargs.get("max_tokens", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,
+ False)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,
+ kwargs.get("temperature", 1.0))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,
+ kwargs.get("top_p", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_K,
+ kwargs.get("top_k", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,
+ response.stop_reason)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
+ response.usage.input_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,
+ response.usage.output_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ response.usage.input_tokens +
+ response.usage.output_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,
+ prompt)
+ # pylint: disable=line-too-long
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_COMPLETION, response.content[0].text if response.content else "")
+
+ span.set_status(Status(StatusCode.OK))
+
+ if disable_metrics is False:
+ attributes = {
+ TELEMETRY_SDK_NAME:
+ "openlit",
+ SemanticConvetion.GEN_AI_APPLICATION_NAME:
+ application_name,
+ SemanticConvetion.GEN_AI_SYSTEM:
+ SemanticConvetion.GEN_AI_SYSTEM_ANTHROPIC,
+ SemanticConvetion.GEN_AI_ENVIRONMENT:
+ environment,
+ SemanticConvetion.GEN_AI_TYPE:
+ SemanticConvetion.GEN_AI_TYPE_CHAT,
+ SemanticConvetion.GEN_AI_REQUEST_MODEL:
+ kwargs.get("model", "claude-3-sonnet-20240229")
+ }
+
+ metrics["genai_requests"].add(1, attributes)
+ metrics["genai_total_tokens"].add(
+ response.usage.input_tokens +
+ response.usage.output_tokens, attributes)
+ metrics["genai_completion_tokens"].add(
+ response.usage.output_tokens, attributes)
+ metrics["genai_prompt_tokens"].add(
+ response.usage.input_tokens, attributes)
+ metrics["genai_cost"].record(cost, attributes)
+
+ # Return original response
+ return response
+
+ except Exception as e:
+ handle_exception(span, e)
+ logger.error("Error in trace creation: %s", e)
+
+ # Return original response
+ return response
+
+ return wrapper
diff --git a/sdk/python/src/openlit/instrumentation/chroma/__init__.py b/sdk/python/src/openlit/instrumentation/chroma/__init__.py
new file mode 100644
index 00000000..320bc861
--- /dev/null
+++ b/sdk/python/src/openlit/instrumentation/chroma/__init__.py
@@ -0,0 +1,86 @@
+# pylint: disable=useless-return, bad-staticmethod-argument, disable=duplicate-code
+"""Initializer of Auto Instrumentation of Chromadb Functions"""
+from typing import Collection
+import importlib.metadata
+from opentelemetry.instrumentation.instrumentor import BaseInstrumentor
+from wrapt import wrap_function_wrapper
+
+from openlit.instrumentation.chroma.chroma import general_wrap
+
+_instruments = ("chromadb >= 0.4.0",)
+
+class ChromaInstrumentor(BaseInstrumentor):
+ """An instrumentor for Chromadb's client library."""
+
+ def instrumentation_dependencies(self) -> Collection[str]:
+ return _instruments
+
+ def _instrument(self, **kwargs):
+ application_name = kwargs.get("application_name")
+ environment = kwargs.get("environment")
+ tracer = kwargs.get("tracer")
+ metrics = kwargs.get("metrics_dict")
+ pricing_info = kwargs.get("pricing_info")
+ trace_content = kwargs.get("trace_content")
+ disable_metrics = kwargs.get("disable_metrics")
+ version = importlib.metadata.version("chromadb")
+
+ wrap_function_wrapper(
+ "chromadb.db",
+ "DB.create_collection",
+ general_wrap("chroma.create_collection", version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics),
+ )
+
+ wrap_function_wrapper(
+ "chromadb",
+ "Collection.add",
+ general_wrap("chroma.add", version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics),
+ )
+
+ wrap_function_wrapper(
+ "chromadb",
+ "Collection.get",
+ general_wrap("chroma.get", version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics),
+ )
+
+ wrap_function_wrapper(
+ "chromadb",
+ "Collection.peek",
+ general_wrap("chroma.peek", version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics),
+ )
+
+ wrap_function_wrapper(
+ "chromadb",
+ "Collection.query",
+ general_wrap("chroma.query", version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics),
+ )
+
+ wrap_function_wrapper(
+ "chromadb",
+ "Collection.update",
+ general_wrap("chroma.update", version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics),
+ )
+
+ wrap_function_wrapper(
+ "chromadb",
+ "Collection.upsert",
+ general_wrap("chroma.upsert", version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics),
+ )
+ wrap_function_wrapper(
+ "chromadb",
+ "Collection.delete",
+ general_wrap("chroma.delete", version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics),
+ )
+
+
+ @staticmethod
+ def _uninstrument(self, **kwargs):
+ pass
diff --git a/sdk/python/src/openlit/instrumentation/chroma/chroma.py b/sdk/python/src/openlit/instrumentation/chroma/chroma.py
new file mode 100644
index 00000000..206f8aca
--- /dev/null
+++ b/sdk/python/src/openlit/instrumentation/chroma/chroma.py
@@ -0,0 +1,197 @@
+# pylint: disable=duplicate-code, broad-exception-caught, too-many-statements, unused-argument
+"""
+Module for monitoring ChromaDB.
+"""
+
+import logging
+from opentelemetry.trace import SpanKind, Status, StatusCode
+from opentelemetry.sdk.resources import TELEMETRY_SDK_NAME
+from openlit.__helpers import handle_exception
+from openlit.semcov import SemanticConvetion
+
+# Initialize logger for logging potential issues and operations
+logger = logging.getLogger(__name__)
+
+def object_count(obj):
+ """
+ Counts Length of object if it exists, Else returns None
+ """
+
+ if obj:
+ return len(obj)
+
+ return None
+
+def general_wrap(gen_ai_endpoint, version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics):
+ """
+ Creates a wrapper around a function call to trace and log its execution metrics.
+
+ This function wraps any given function to measure its execution time,
+ log its operation, and trace its execution using OpenTelemetry.
+
+ Parameters:
+ - gen_ai_endpoint (str): A descriptor or name for the endpoint being traced.
+ - version (str): The version of the Langchain application.
+ - environment (str): The deployment environment (e.g., 'production', 'development').
+ - application_name (str): Name of the Langchain application.
+ - tracer (opentelemetry.trace.Tracer): The tracer object used for OpenTelemetry tracing.
+ - pricing_info (dict): Information about the pricing for internal metrics (currently not used).
+ - trace_content (bool): Flag indicating whether to trace the content of the response.
+
+ Returns:
+ - function: A higher-order function that takes a function 'wrapped' and returns
+ a new function that wraps 'wrapped' with additional tracing and logging.
+ """
+
+ def wrapper(wrapped, instance, args, kwargs):
+ """
+ An inner wrapper function that executes the wrapped function, measures execution
+ time, and records trace data using OpenTelemetry.
+
+ Parameters:
+ - wrapped (Callable): The original function that this wrapper will execute.
+ - instance (object): The instance to which the wrapped function belongs. This
+ is used for instance methods. For static and classmethods,
+ this may be None.
+ - args (tuple): Positional arguments passed to the wrapped function.
+ - kwargs (dict): Keyword arguments passed to the wrapped function.
+
+ Returns:
+ - The result of the wrapped function call.
+
+ The wrapper initiates a span with the provided tracer, sets various attributes
+ on the span based on the function's execution and response, and ensures
+ errors are handled and logged appropriately.
+ """
+ with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
+ response = wrapped(*args, **kwargs)
+
+ try:
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
+ gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,
+ environment)
+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
+ application_name)
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
+ SemanticConvetion.GEN_AI_TYPE_VECTORDB)
+ span.set_attribute(SemanticConvetion.DB_SYSTEM,
+ SemanticConvetion.DB_SYSTEM_CHROMA)
+ span.set_attribute(SemanticConvetion.DB_COLLECTION_NAME,
+ instance.name)
+
+ if gen_ai_endpoint == "chroma.add":
+ db_operation = SemanticConvetion.DB_OPERATION_ADD
+ span.set_attribute(SemanticConvetion.DB_OPERATION,
+ SemanticConvetion.DB_OPERATION_GET)
+ span.set_attribute(SemanticConvetion.DB_ID_COUNT,
+ object_count(kwargs.get("ids")))
+ span.set_attribute(SemanticConvetion.DB_VECTOR_COUNT,
+ object_count(kwargs.get("embeddings")))
+ span.set_attribute(SemanticConvetion.DB_VECTOR_COUNT,
+ object_count(kwargs.get("metadatas")))
+ span.set_attribute(SemanticConvetion.DB_DOCUMENTS_COUNT,
+ object_count(kwargs.get("documents")))
+
+ elif gen_ai_endpoint == "chroma.get":
+ db_operation = SemanticConvetion.DB_OPERATION_GET
+ span.set_attribute(SemanticConvetion.DB_OPERATION,
+ SemanticConvetion.DB_OPERATION_GET)
+ span.set_attribute(SemanticConvetion.DB_ID_COUNT,
+ object_count(kwargs.get("ids")))
+ span.set_attribute(SemanticConvetion.DB_QUERY_LIMIT,
+ kwargs.get("limit"))
+ span.set_attribute(SemanticConvetion.DB_OFFSET,
+ kwargs.get("offset"))
+ span.set_attribute(SemanticConvetion.DB_WHERE_DOCUMENT,
+ str(kwargs.get("where_document", "")))
+
+ elif gen_ai_endpoint == "chroma.query":
+ db_operation = SemanticConvetion.DB_OPERATION_QUERY
+ span.set_attribute(SemanticConvetion.DB_OPERATION,
+ SemanticConvetion.DB_OPERATION_QUERY)
+ span.set_attribute(SemanticConvetion.DB_STATEMENT,
+ str(kwargs.get("query_texts")))
+ span.set_attribute(SemanticConvetion.DB_N_RESULTS,
+ kwargs.get("n_results", ""))
+ span.set_attribute(SemanticConvetion.DB_FILTER,
+ str(kwargs.get("where", "")))
+ span.set_attribute(SemanticConvetion.DB_WHERE_DOCUMENT,
+ str(kwargs.get("where_document", "")))
+
+ elif gen_ai_endpoint == "chroma.update":
+ db_operation = SemanticConvetion.DB_OPERATION_UPDATE
+ span.set_attribute(SemanticConvetion.DB_OPERATION,
+ SemanticConvetion.DB_OPERATION_UPDATE)
+ span.set_attribute(SemanticConvetion.DB_VECTOR_COUNT,
+ object_count(kwargs.get("embeddings")))
+ span.set_attribute(SemanticConvetion.DB_VECTOR_COUNT,
+ object_count(kwargs.get("metadatas")))
+ span.set_attribute(SemanticConvetion.DB_ID_COUNT,
+ object_count(kwargs.get("ids")))
+ span.set_attribute(SemanticConvetion.DB_DOCUMENTS_COUNT,
+ object_count(kwargs.get("documents")))
+
+ elif gen_ai_endpoint == "chroma.upsert":
+ db_operation = SemanticConvetion.DB_OPERATION_UPSERT
+ span.set_attribute(SemanticConvetion.DB_OPERATION,
+ SemanticConvetion.DB_OPERATION_UPSERT)
+ span.set_attribute(SemanticConvetion.DB_VECTOR_COUNT,
+ object_count(kwargs.get("embeddings")))
+ span.set_attribute(SemanticConvetion.DB_VECTOR_COUNT,
+ object_count(kwargs.get("metadatas")))
+ span.set_attribute(SemanticConvetion.DB_ID_COUNT,
+ object_count(kwargs.get("ids")))
+ span.set_attribute(SemanticConvetion.DB_DOCUMENTS_COUNT,
+ object_count(kwargs.get("documents")))
+
+ elif gen_ai_endpoint == "chroma.delete":
+ db_operation = SemanticConvetion.DB_OPERATION_DELETE
+ span.set_attribute(SemanticConvetion.DB_OPERATION,
+ SemanticConvetion.DB_OPERATION_DELETE)
+ span.set_attribute(SemanticConvetion.DB_ID_COUNT,
+ object_count(kwargs.get("ids")))
+ span.set_attribute(SemanticConvetion.DB_FILTER,
+ str(kwargs.get("where", "")))
+ span.set_attribute(SemanticConvetion.DB_DELETE_ALL,
+ kwargs.get("delete_all", False))
+ span.set_attribute(SemanticConvetion.DB_WHERE_DOCUMENT,
+ str(kwargs.get("where_document", "")))
+
+ elif gen_ai_endpoint == "chroma.peek":
+ db_operation = SemanticConvetion.DB_OPERATION_PEEK
+ span.set_attribute(SemanticConvetion.DB_OPERATION,
+ SemanticConvetion.DB_OPERATION_PEEK)
+
+ span.set_status(Status(StatusCode.OK))
+
+ if disable_metrics is False:
+ attributes = {
+ TELEMETRY_SDK_NAME:
+ "openlit",
+ SemanticConvetion.GEN_AI_APPLICATION_NAME:
+ application_name,
+ SemanticConvetion.DB_SYSTEM:
+ SemanticConvetion.DB_SYSTEM_CHROMA,
+ SemanticConvetion.GEN_AI_ENVIRONMENT:
+ environment,
+ SemanticConvetion.GEN_AI_TYPE:
+ SemanticConvetion.GEN_AI_TYPE_VECTORDB,
+ SemanticConvetion.DB_OPERATION:
+ db_operation
+ }
+
+ metrics["db_requests"].add(1, attributes)
+
+ return response
+
+ except Exception as e:
+ handle_exception(span, e)
+ logger.error("Error in trace creation: %s", e)
+
+ # Return original response
+ return response
+
+ return wrapper
diff --git a/sdk/python/src/openlit/instrumentation/cohere/__init__.py b/sdk/python/src/openlit/instrumentation/cohere/__init__.py
new file mode 100644
index 00000000..a9a5600a
--- /dev/null
+++ b/sdk/python/src/openlit/instrumentation/cohere/__init__.py
@@ -0,0 +1,51 @@
+# pylint: disable=useless-return, bad-staticmethod-argument, disable=duplicate-code
+"""Initializer of Auto Instrumentation of Cohere Functions"""
+from typing import Collection
+import importlib.metadata
+from opentelemetry.instrumentation.instrumentor import BaseInstrumentor
+from wrapt import wrap_function_wrapper
+
+from openlit.instrumentation.cohere.cohere import chat, chat_stream, embed
+
+_instruments = ("cohere >= 5.0.0",)
+
+class CohereInstrumentor(BaseInstrumentor):
+ """An instrumentor for Cohere's client library."""
+
+ def instrumentation_dependencies(self) -> Collection[str]:
+ return _instruments
+
+ def _instrument(self, **kwargs):
+ application_name = kwargs.get("application_name")
+ environment = kwargs.get("environment")
+ tracer = kwargs.get("tracer")
+ metrics = kwargs.get("metrics_dict")
+ pricing_info = kwargs.get("pricing_info")
+ trace_content = kwargs.get("trace_content")
+ disable_metrics = kwargs.get("disable_metrics")
+ version = importlib.metadata.version("cohere")
+
+ wrap_function_wrapper(
+ "cohere.client",
+ "Client.chat",
+ chat("cohere.chat", version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics),
+ )
+
+ wrap_function_wrapper(
+ "cohere.client",
+ "Client.chat_stream",
+ chat_stream("cohere.chat", version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics),
+ )
+
+ wrap_function_wrapper(
+ "cohere.client",
+ "Client.embed",
+ embed("cohere.embed", version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics),
+ )
+
+ @staticmethod
+ def _uninstrument(self, **kwargs):
+ pass
diff --git a/sdk/python/src/openlit/instrumentation/cohere/cohere.py b/sdk/python/src/openlit/instrumentation/cohere/cohere.py
new file mode 100644
index 00000000..1326ea5c
--- /dev/null
+++ b/sdk/python/src/openlit/instrumentation/cohere/cohere.py
@@ -0,0 +1,397 @@
+# pylint: disable=duplicate-code, broad-exception-caught, too-many-statements, unused-argument
+"""
+Module for monitoring Cohere API calls.
+"""
+
+import logging
+from opentelemetry.trace import SpanKind, Status, StatusCode
+from opentelemetry.sdk.resources import TELEMETRY_SDK_NAME
+from openlit.__helpers import get_chat_model_cost, get_embed_model_cost, handle_exception
+from openlit.semcov import SemanticConvetion
+
+# Initialize logger for logging potential issues and operations
+logger = logging.getLogger(__name__)
+
+def embed(gen_ai_endpoint, version, environment, application_name, tracer,
+ pricing_info, trace_content, metrics, disable_metrics):
+ """
+ Generates a telemetry wrapper for embeddings to collect metrics.
+
+ Args:
+ gen_ai_endpoint: Endpoint identifier for logging and tracing.
+ version: Version of the monitoring package.
+ environment: Deployment environment (e.g., production, staging).
+ application_name: Name of the application using the OpenAI API.
+ tracer: OpenTelemetry tracer for creating spans.
+ pricing_info: Information used for calculating the cost of OpenAI usage.
+ trace_content: Flag indicating whether to trace the actual content.
+
+ Returns:
+ A function that wraps the embeddings method to add telemetry.
+ """
+
+ def wrapper(wrapped, instance, args, kwargs):
+ """
+ Wraps the 'embed' API call to add telemetry.
+
+ This collects metrics such as execution time, cost, and token usage, and handles errors
+ gracefully, adding details to the trace for observability.
+
+ Args:
+ wrapped: The original 'embed' method to be wrapped.
+ instance: The instance of the class where the original method is defined.
+ args: Positional arguments for the 'embed' method.
+ kwargs: Keyword arguments for the 'embed' method.
+
+ Returns:
+ The response from the original 'embed' method.
+ """
+
+ with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
+ response = wrapped(*args, **kwargs)
+
+ try:
+ # Get prompt from kwargs and store as a single string
+ prompt = " ".join(kwargs.get("texts", []))
+
+
+ # Calculate cost of the operation
+ cost = get_embed_model_cost(kwargs.get("model", "embed-english-v2.0"),
+ pricing_info,
+ response.meta.billed_units.input_tokens)
+
+ # Set Span attributes
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_COHERE)
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
+ SemanticConvetion.GEN_AI_TYPE_EMBEDDING)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
+ gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,
+ environment)
+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
+ application_name)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ kwargs.get("model", "embed-english-v2.0"))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_EMBEDDING_FORMAT,
+ kwargs.get("embedding_types", "float"))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_EMBEDDING_DIMENSION,
+ kwargs.get("input_type", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_USER,
+ kwargs.get("user", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,
+ response.id)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
+ response.meta.billed_units.input_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ response.meta.billed_units.input_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,
+ prompt)
+
+ span.set_status(Status(StatusCode.OK))
+
+ if disable_metrics is False:
+ attributes = {
+ TELEMETRY_SDK_NAME:
+ "openlit",
+ SemanticConvetion.GEN_AI_APPLICATION_NAME:
+ application_name,
+ SemanticConvetion.GEN_AI_SYSTEM:
+ SemanticConvetion.GEN_AI_SYSTEM_COHERE,
+ SemanticConvetion.GEN_AI_ENVIRONMENT:
+ environment,
+ SemanticConvetion.GEN_AI_TYPE:
+ SemanticConvetion.GEN_AI_TYPE_EMBEDDING,
+ SemanticConvetion.GEN_AI_REQUEST_MODEL:
+ kwargs.get("model", "embed-english-v2.0")
+ }
+
+ metrics["genai_requests"].add(1, attributes)
+ metrics["genai_total_tokens"].add(
+ response.meta.billed_units.input_tokens, attributes
+ )
+ metrics["genai_prompt_tokens"].add(
+ response.meta.billed_units.input_tokens, attributes
+ )
+ metrics["genai_cost"].record(cost, attributes)
+
+ # Return original response
+ return response
+
+ except Exception as e:
+ handle_exception(span, e)
+ logger.error("Error in trace creation: %s", e)
+
+ # Return original response
+ return response
+
+ return wrapper
+
+def chat(gen_ai_endpoint, version, environment, application_name, tracer,
+ pricing_info, trace_content, metrics, disable_metrics):
+ """
+ Generates a telemetry wrapper for chat to collect metrics.
+
+ Args:
+ gen_ai_endpoint: Endpoint identifier for logging and tracing.
+ version: Version of the monitoring package.
+ environment: Deployment environment (e.g., production, staging).
+ application_name: Name of the application using the OpenAI API.
+ tracer: OpenTelemetry tracer for creating spans.
+ pricing_info: Information used for calculating the cost of OpenAI usage.
+ trace_content: Flag indicating whether to trace the actual content.
+
+ Returns:
+ A function that wraps the chat method to add telemetry.
+ """
+
+ def wrapper(wrapped, instance, args, kwargs):
+ """
+ Wraps the 'chat' API call to add telemetry.
+
+ This collects metrics such as execution time, cost, and token usage, and handles errors
+ gracefully, adding details to the trace for observability.
+
+ Args:
+ wrapped: The original 'chat' method to be wrapped.
+ instance: The instance of the class where the original method is defined.
+ args: Positional arguments for the 'chat' method.
+ kwargs: Keyword arguments for the 'chat' method.
+
+ Returns:
+ The response from the original 'chat' method.
+ """
+
+ with tracer.start_as_current_span(gen_ai_endpoint, kind=SpanKind.CLIENT) as span:
+ response = wrapped(*args, **kwargs)
+
+ try:
+ # Calculate cost of the operation
+ cost = get_chat_model_cost(kwargs.get("model", "command"),
+ pricing_info,
+ response.meta["billed_units"]["input_tokens"],
+ response.meta["billed_units"]["output_tokens"])
+
+ # Set Span attributes
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_COHERE)
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
+ SemanticConvetion.GEN_AI_TYPE_CHAT)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
+ gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,
+ environment)
+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
+ application_name)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ kwargs.get("model", "command"))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,
+ kwargs.get("temperature", 0.3))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS,
+ kwargs.get("max_tokens", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_SEED,
+ kwargs.get("seed", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_FREQUENCY_PENALTY,
+ kwargs.get("frequency_penalty", 0.0))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_PRESENCE_PENALTY,
+ kwargs.get("presence_penalty", 0.0))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,
+ False)
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,
+ response.response_id)
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,
+ response.response_id)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
+ response.meta["billed_units"]["input_tokens"])
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,
+ response.meta["billed_units"]["output_tokens"])
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ response.meta["billed_units"]["input_tokens"] +
+ response.meta["billed_units"]["output_tokens"])
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,
+ kwargs.get("message", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_COMPLETION,
+ response.text)
+
+ span.set_status(Status(StatusCode.OK))
+
+ if disable_metrics is False:
+ attributes = {
+ TELEMETRY_SDK_NAME:
+ "openlit",
+ SemanticConvetion.GEN_AI_APPLICATION_NAME:
+ application_name,
+ SemanticConvetion.GEN_AI_SYSTEM:
+ SemanticConvetion.GEN_AI_SYSTEM_COHERE,
+ SemanticConvetion.GEN_AI_ENVIRONMENT:
+ environment,
+ SemanticConvetion.GEN_AI_TYPE:
+ SemanticConvetion.GEN_AI_TYPE_CHAT,
+ SemanticConvetion.GEN_AI_REQUEST_MODEL:
+ kwargs.get("model", "command")
+ }
+
+ metrics["genai_requests"].add(1, attributes)
+ metrics["genai_total_tokens"].add(
+ response.meta["billed_units"]["input_tokens"] +
+ response.meta["billed_units"]["output_tokens"], attributes)
+ metrics["genai_completion_tokens"].add(
+ response.meta["billed_units"]["output_tokens"], attributes)
+ metrics["genai_prompt_tokens"].add(
+ response.meta["billed_units"]["input_tokens"], attributes)
+ metrics["genai_cost"].record(cost, attributes)
+
+ # Return original response
+ return response
+
+ except Exception as e:
+ handle_exception(span, e)
+ logger.error("Error in trace creation: %s", e)
+
+ # Return original response
+ return response
+
+ return wrapper
+
+def chat_stream(gen_ai_endpoint, version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics):
+ """
+ Generates a telemetry wrapper for chat_stream to collect metrics.
+
+ Args:
+ gen_ai_endpoint: Endpoint identifier for logging and tracing.
+ version: Version of the monitoring package.
+ environment: Deployment environment (e.g., production, staging).
+ application_name: Name of the application using the OpenAI API.
+ tracer: OpenTelemetry tracer for creating spans.
+ pricing_info: Information used for calculating the cost of OpenAI usage.
+ trace_content: Flag indicating whether to trace the actual content.
+
+ Returns:
+ A function that wraps the chat method to add telemetry.
+ """
+
+ def wrapper(wrapped, instance, args, kwargs):
+ """
+ Wraps the 'chat_stream' API call to add telemetry.
+
+ This collects metrics such as execution time, cost, and token usage, and handles errors
+ gracefully, adding details to the trace for observability.
+
+ Args:
+ wrapped: The original 'chat_stream' method to be wrapped.
+ instance: The instance of the class where the original method is defined.
+ args: Positional arguments for the 'chat_stream' method.
+ kwargs: Keyword arguments for the 'chat_stream' method.
+
+ Returns:
+ The response from the original 'chat_stream' method.
+ """
+
+ def stream_generator():
+ with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
+ # Placeholder for aggregating streaming response
+ llmresponse = ""
+
+ # Loop through streaming events capturing relevant details
+ for event in wrapped(*args, **kwargs):
+ # Collect message IDs and aggregated response from events
+ if event.event_type == "stream-end":
+ llmresponse = event.response.text
+ response_id = event.response.response_id
+ prompt_tokens = event.response.meta["billed_units"]["input_tokens"]
+ completion_tokens = event.response.meta["billed_units"]["output_tokens"]
+ finish_reason = event.finish_reason
+ yield event
+
+ # Handling exception ensure observability without disrupting operation
+ try:
+ # Calculate cost of the operation
+ cost = get_chat_model_cost(kwargs.get("model", "command"),
+ pricing_info, prompt_tokens, completion_tokens)
+
+ # Set Span attributes
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_COHERE)
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
+ SemanticConvetion.GEN_AI_TYPE_CHAT)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
+ gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,
+ environment)
+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
+ application_name)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ kwargs.get("model", "command"))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,
+ kwargs.get("temperature", 0.3))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS,
+ kwargs.get("max_tokens", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_SEED,
+ kwargs.get("seed", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_FREQUENCY_PENALTY,
+ kwargs.get("frequency_penalty", 0.0))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_PRESENCE_PENALTY,
+ kwargs.get("presence_penalty", 0.0))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,
+ True)
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,
+ response_id)
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,
+ finish_reason)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
+ prompt_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,
+ completion_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ prompt_tokens + completion_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,
+ kwargs.get("message", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_COMPLETION,
+ llmresponse)
+
+ span.set_status(Status(StatusCode.OK))
+
+ if disable_metrics is False:
+ attributes = {
+ TELEMETRY_SDK_NAME:
+ "openlit",
+ SemanticConvetion.GEN_AI_APPLICATION_NAME:
+ application_name,
+ SemanticConvetion.GEN_AI_SYSTEM:
+ SemanticConvetion.GEN_AI_SYSTEM_COHERE,
+ SemanticConvetion.GEN_AI_ENVIRONMENT:
+ environment,
+ SemanticConvetion.GEN_AI_TYPE:
+ SemanticConvetion.GEN_AI_TYPE_CHAT,
+ SemanticConvetion.GEN_AI_REQUEST_MODEL:
+ kwargs.get("model", "command")
+ }
+
+ metrics["genai_requests"].add(1, attributes)
+ metrics["genai_total_tokens"].add(
+ prompt_tokens + completion_tokens, attributes
+ )
+ metrics["genai_completion_tokens"].add(completion_tokens, attributes)
+ metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)
+ metrics["genai_cost"].record(cost, attributes)
+
+ except Exception as e:
+ handle_exception(span, e)
+ logger.error("Error in trace creation: %s", e)
+
+ return stream_generator()
+
+ return wrapper
diff --git a/sdk/python/src/openlit/instrumentation/langchain/__init__.py b/sdk/python/src/openlit/instrumentation/langchain/__init__.py
new file mode 100644
index 00000000..780b8b8e
--- /dev/null
+++ b/sdk/python/src/openlit/instrumentation/langchain/__init__.py
@@ -0,0 +1,74 @@
+# pylint: disable=useless-return, bad-staticmethod-argument, disable=duplicate-code
+"""Initializer of Auto Instrumentation of LangChain Functions"""
+from typing import Collection
+import importlib.metadata
+from opentelemetry.instrumentation.instrumentor import BaseInstrumentor
+from wrapt import wrap_function_wrapper
+
+from openlit.instrumentation.langchain.langchain import general_wrap, hub
+
+_instruments = ("langchain >= 0.1.1", "langchain-openai >= 0.1.1",
+ "langchain-core > 0.1.1", "langchain-community >= 0.0.31")
+
+WRAPPED_METHODS = [
+ {
+ "package": "langchain_community.document_loaders.base",
+ "object": "BaseLoader.load",
+ "endpoint": "langchain.retrieve.load",
+ "wrapper": general_wrap,
+ },
+ {
+ "package": "langchain_community.document_loaders.base",
+ "object": "BaseLoader.aload",
+ "endpoint": "langchain.retrieve.load",
+ "wrapper": general_wrap,
+ },
+ {
+ "package": "langchain_text_splitters.base",
+ "object": "TextSplitter.split_documents",
+ "endpoint": "langchain.retrieve.split_documents",
+ "wrapper": general_wrap,
+ },
+ {
+ "package": "langchain_text_splitters.base",
+ "object": "TextSplitter.create_documents",
+ "endpoint": "langchain.retrieve.create_documents",
+ "wrapper": general_wrap,
+ },
+ {
+ "package": "langchain.hub",
+ "object": "pull",
+ "endpoint": "langchain.retrieve.prompt",
+ "wrapper": hub,
+ },
+]
+
+class LangChainInstrumentor(BaseInstrumentor):
+ """An instrumentor for Cohere's client library."""
+
+ def instrumentation_dependencies(self) -> Collection[str]:
+ return _instruments
+
+ def _instrument(self, **kwargs):
+ application_name = kwargs.get("application_name")
+ environment = kwargs.get("environment")
+ tracer = kwargs.get("tracer")
+ pricing_info = kwargs.get("pricing_info")
+ trace_content = kwargs.get("trace_content")
+ version = importlib.metadata.version("langchain")
+
+ for wrapped_method in WRAPPED_METHODS:
+ wrap_package = wrapped_method.get("package")
+ wrap_object = wrapped_method.get("object")
+ gen_ai_endpoint = wrapped_method.get("endpoint")
+ wrapper = wrapped_method.get("wrapper")
+ wrap_function_wrapper(
+ wrap_package,
+ wrap_object,
+ wrapper(gen_ai_endpoint, version, environment, application_name,
+ tracer, pricing_info, trace_content),
+ )
+
+ @staticmethod
+ def _uninstrument(self, **kwargs):
+ pass
diff --git a/sdk/python/src/openlit/instrumentation/langchain/langchain.py b/sdk/python/src/openlit/instrumentation/langchain/langchain.py
new file mode 100644
index 00000000..64148b96
--- /dev/null
+++ b/sdk/python/src/openlit/instrumentation/langchain/langchain.py
@@ -0,0 +1,161 @@
+# pylint: disable=duplicate-code, broad-exception-caught, too-many-statements, unused-argument
+"""
+Module for monitoring Langchain applications.
+"""
+
+import logging
+from opentelemetry.trace import SpanKind, Status, StatusCode
+from opentelemetry.sdk.resources import TELEMETRY_SDK_NAME
+from openlit.__helpers import handle_exception
+from openlit.semcov import SemanticConvetion
+
+# Initialize logger for logging potential issues and operations
+logger = logging.getLogger(__name__)
+
+def general_wrap(gen_ai_endpoint, version, environment, application_name,
+ tracer, pricing_info, trace_content):
+ """
+ Creates a wrapper around a function call to trace and log its execution metrics.
+
+ This function wraps any given function to measure its execution time,
+ log its operation, and trace its execution using OpenTelemetry.
+
+ Parameters:
+ - gen_ai_endpoint (str): A descriptor or name for the endpoint being traced.
+ - version (str): The version of the Langchain application.
+ - environment (str): The deployment environment (e.g., 'production', 'development').
+ - application_name (str): Name of the Langchain application.
+ - tracer (opentelemetry.trace.Tracer): The tracer object used for OpenTelemetry tracing.
+ - pricing_info (dict): Information about the pricing for internal metrics (currently not used).
+ - trace_content (bool): Flag indicating whether to trace the content of the response.
+
+ Returns:
+ - function: A higher-order function that takes a function 'wrapped' and returns
+ a new function that wraps 'wrapped' with additional tracing and logging.
+ """
+
+ def wrapper(wrapped, instance, args, kwargs):
+ """
+ An inner wrapper function that executes the wrapped function, measures execution
+ time, and records trace data using OpenTelemetry.
+
+ Parameters:
+ - wrapped (Callable): The original function that this wrapper will execute.
+ - instance (object): The instance to which the wrapped function belongs. This
+ is used for instance methods. For static and classmethods,
+ this may be None.
+ - args (tuple): Positional arguments passed to the wrapped function.
+ - kwargs (dict): Keyword arguments passed to the wrapped function.
+
+ Returns:
+ - The result of the wrapped function call.
+
+ The wrapper initiates a span with the provided tracer, sets various attributes
+ on the span based on the function's execution and response, and ensures
+ errors are handled and logged appropriately.
+ """
+ with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
+ response = wrapped(*args, **kwargs)
+
+ try:
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ "langchain")
+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
+ gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,
+ environment)
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
+ SemanticConvetion.GEN_AI_TYPE_FRAMEWORK)
+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
+ application_name)
+ span.set_attribute(SemanticConvetion.GEN_AI_RETRIEVAL_SOURCE,
+ response[0].metadata["source"])
+ span.set_status(Status(StatusCode.OK))
+
+ # Return original response
+ return response
+
+ except Exception as e:
+ handle_exception(span, e)
+ logger.error("Error in trace creation: %s", e)
+
+ # Return original response
+ return response
+
+ return wrapper
+
+def hub(gen_ai_endpoint, version, environment, application_name, tracer,
+ pricing_info, trace_content):
+ """
+ Creates a wrapper around Langchain hub operations for tracing and logging.
+
+ Similar to `general_wrap`, this function focuses on wrapping functions involved
+ in interacting with the Langchain hub, adding specific metadata relevant to
+ hub operations to the span attributes.
+
+ Parameters:
+ - gen_ai_endpoint (str): A descriptor or name for the Langchain hub endpoint.
+ - version (str): The version of the Langchain application.
+ - environment (str): The deployment environment, such as 'production' or 'development'.
+ - application_name (str): Name of the Langchain application.
+ - tracer (opentelemetry.trace.Tracer): The tracer for OpenTelemetry tracing.
+ - pricing_info (dict): Pricing information for the operation (not currently used).
+ - trace_content (bool): Indicates if the content of the response should be traced.
+
+ Returns:
+ - function: A new function that wraps the original hub operation call with added
+ logging, tracing, and metric calculation functionalities.
+ """
+
+ def wrapper(wrapped, instance, args, kwargs):
+ """
+ An inner wrapper specifically designed for Langchain hub operations,
+ providing tracing, logging, and execution metrics.
+
+ Parameters:
+ - wrapped (Callable): The original hub operation function to be executed.
+ - instance (object): The instance of the class where the hub operation
+ method is defined. May be None for static or class methods.
+ - args (tuple): Positional arguments to pass to the hub operation function.
+ - kwargs (dict): Keyword arguments to pass to the hub operation function.
+
+ Returns:
+ - The result of executing the hub operation function.
+
+ This wrapper captures additional metadata relevant to Langchain hub operations,
+ creating spans with specific attributes and metrics that reflect the nature of
+ each hub call.
+ """
+
+ with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
+ response = wrapped(*args, **kwargs)
+
+ try:
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_LANGCHAIN)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
+ gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,
+ environment)
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
+ SemanticConvetion.GEN_AI_TYPE_FRAMEWORK)
+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
+ application_name)
+ span.set_attribute(SemanticConvetion.GEN_AI_HUB_OWNER,
+ response.metadata["lc_hub_owner"])
+ span.set_attribute(SemanticConvetion.GEN_AI_HUB_REPO,
+ response.metadata["lc_hub_repo"])
+ span.set_status(Status(StatusCode.OK))
+
+ return response
+
+ except Exception as e:
+ handle_exception(span, e)
+ logger.error("Error in trace creation: %s", e)
+
+ # Return original response
+ return response
+
+ return wrapper
diff --git a/sdk/python/src/openlit/instrumentation/mistral/__init__.py b/sdk/python/src/openlit/instrumentation/mistral/__init__.py
new file mode 100644
index 00000000..9027d21d
--- /dev/null
+++ b/sdk/python/src/openlit/instrumentation/mistral/__init__.py
@@ -0,0 +1,80 @@
+# pylint: disable=useless-return, bad-staticmethod-argument, disable=duplicate-code
+"""Initializer of Auto Instrumentation of Mistral Functions"""
+from typing import Collection
+import importlib.metadata
+from opentelemetry.instrumentation.instrumentor import BaseInstrumentor
+from wrapt import wrap_function_wrapper
+
+from openlit.instrumentation.mistral.mistral import chat, chat_stream, embeddings
+from openlit.instrumentation.mistral.async_mistral import async_chat, async_chat_stream
+from openlit.instrumentation.mistral.async_mistral import async_embeddings
+
+_instruments = ("mistralai >= 0.1.0",)
+
+class MistralInstrumentor(BaseInstrumentor):
+ """An instrumentor for Azure Mistral's client library."""
+
+ def instrumentation_dependencies(self) -> Collection[str]:
+ return _instruments
+
+ def _instrument(self, **kwargs):
+ application_name = kwargs.get("application_name")
+ environment = kwargs.get("environment")
+ tracer = kwargs.get("tracer")
+ metrics = kwargs.get("metrics_dict")
+ pricing_info = kwargs.get("pricing_info")
+ trace_content = kwargs.get("trace_content")
+ disable_metrics = kwargs.get("disable_metrics")
+ version = importlib.metadata.version("mistralai")
+
+ #sync
+ wrap_function_wrapper(
+ "mistralai.client",
+ "MistralClient.chat",
+ chat("mistral.chat", version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics),
+ )
+
+ #sync
+ wrap_function_wrapper(
+ "mistralai.client",
+ "MistralClient.chat_stream",
+ chat_stream("mistral.chat", version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics),
+ )
+
+ #sync
+ wrap_function_wrapper(
+ "mistralai.client",
+ "MistralClient.embeddings",
+ embeddings("mistral.embeddings", version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics),
+ )
+
+ # Async
+ wrap_function_wrapper(
+ "mistralai.async_client",
+ "MistralAsyncClient.chat",
+ async_chat("mistral.chat", version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics),
+ )
+
+ #sync
+ wrap_function_wrapper(
+ "mistralai.async_client",
+ "MistralAsyncClient.chat_stream",
+ async_chat_stream("mistral.chat", version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics),
+ )
+
+ #sync
+ wrap_function_wrapper(
+ "mistralai.async_client",
+ "MistralAsyncClient.embeddings",
+ async_embeddings("mistral.embeddings", version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics),
+ )
+
+ @staticmethod
+ def _uninstrument(self, **kwargs):
+ pass
diff --git a/sdk/python/src/openlit/instrumentation/mistral/async_mistral.py b/sdk/python/src/openlit/instrumentation/mistral/async_mistral.py
new file mode 100644
index 00000000..f0b09054
--- /dev/null
+++ b/sdk/python/src/openlit/instrumentation/mistral/async_mistral.py
@@ -0,0 +1,417 @@
+# pylint: disable=duplicate-code, broad-exception-caught, too-many-statements, unused-argument
+"""
+Module for monitoring Mistral API calls.
+"""
+
+import logging
+from opentelemetry.trace import SpanKind, Status, StatusCode
+from opentelemetry.sdk.resources import TELEMETRY_SDK_NAME
+from openlit.__helpers import get_chat_model_cost, get_embed_model_cost, handle_exception
+from openlit.semcov import SemanticConvetion
+
+# Initialize logger for logging potential issues and operations
+logger = logging.getLogger(__name__)
+
+def async_chat(gen_ai_endpoint, version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics):
+ """
+ Generates a telemetry wrapper for chat to collect metrics.
+
+ Args:
+ gen_ai_endpoint: Endpoint identifier for logging and tracing.
+ version: Version of the monitoring package.
+ environment: Deployment environment (e.g., production, staging).
+ application_name: Name of the application using the OpenAI API.
+ tracer: OpenTelemetry tracer for creating spans.
+ pricing_info: Information used for calculating the cost of OpenAI usage.
+ trace_content: Flag indicating whether to trace the actual content.
+
+ Returns:
+ A function that wraps the chat method to add telemetry.
+ """
+
+ async def wrapper(wrapped, instance, args, kwargs):
+ """
+ Wraps the 'chat' API call to add telemetry.
+
+ This collects metrics such as execution time, cost, and token usage, and handles errors
+ gracefully, adding details to the trace for observability.
+
+ Args:
+ wrapped: The original 'chat' method to be wrapped.
+ instance: The instance of the class where the original method is defined.
+ args: Positional arguments for the 'chat' method.
+ kwargs: Keyword arguments for the 'chat' method.
+
+ Returns:
+ The response from the original 'chat' method.
+ """
+
+ with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
+ # Handling exception ensure observability without disrupting operation
+ response = await wrapped(*args, **kwargs)
+
+ try:
+ # Format 'messages' into a single string
+ message_prompt = kwargs.get('messages', "")
+ formatted_messages = []
+ for message in message_prompt:
+ role = message.role
+ content = message.content
+
+ if isinstance(content, list):
+ content_str = ", ".join(
+ # pylint: disable=line-too-long
+ f"{item['type']}: {item['text'] if 'text' in item else item['image_url']}"
+ if 'type' in item else f"text: {item['text']}"
+ for item in content
+ )
+ formatted_messages.append(f"{role}: {content_str}")
+ else:
+ formatted_messages.append(f"{role}: {content}")
+ prompt = " ".join(formatted_messages)
+
+ # Calculate cost of the operation
+ cost = get_chat_model_cost(kwargs.get("model", "mistral-small-latest"),
+ pricing_info, response.usage.prompt_tokens,
+ response.usage.completion_tokens)
+
+ # Set Span attributes
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_MISTRAL)
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
+ SemanticConvetion.GEN_AI_TYPE_CHAT)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
+ gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,
+ response.id)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,
+ environment)
+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
+ application_name)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ kwargs.get("model", "mistral-small-latest"))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,
+ kwargs.get("temperature", 0.7))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,
+ kwargs.get("top_p", 1))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS,
+ kwargs.get("max_tokens", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_SEED,
+ kwargs.get("random_seed", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,
+ False)
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,
+ response.choices[0].finish_reason)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
+ response.usage.prompt_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,
+ response.usage.completion_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ response.usage.total_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,
+ prompt)
+ # pylint: disable=line-too-long
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_COMPLETION, response.choices[0].message.content if response.choices[0].message.content else "")
+
+ span.set_status(Status(StatusCode.OK))
+
+ if disable_metrics is False:
+ attributes = {
+ TELEMETRY_SDK_NAME:
+ "openlit",
+ SemanticConvetion.GEN_AI_APPLICATION_NAME:
+ application_name,
+ SemanticConvetion.GEN_AI_SYSTEM:
+ SemanticConvetion.GEN_AI_SYSTEM_MISTRAL,
+ SemanticConvetion.GEN_AI_ENVIRONMENT:
+ environment,
+ SemanticConvetion.GEN_AI_TYPE:
+ SemanticConvetion.GEN_AI_TYPE_CHAT,
+ SemanticConvetion.GEN_AI_REQUEST_MODEL:
+ kwargs.get("model", "mistral-small-latest")
+ }
+
+ metrics["genai_requests"].add(1, attributes)
+ metrics["genai_total_tokens"].add(response.usage.total_tokens, attributes)
+ metrics["genai_completion_tokens"].add(
+ response.usage.completion_tokens, attributes
+ )
+ metrics["genai_prompt_tokens"].add(response.usage.prompt_tokens, attributes)
+ metrics["genai_cost"].record(cost)
+
+ # Return original response
+ return response
+
+ except Exception as e:
+ handle_exception(span, e)
+ logger.error("Error in trace creation: %s", e)
+
+ # Return original response
+ return response
+
+ return wrapper
+
+def async_chat_stream(gen_ai_endpoint, version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics):
+ """
+ Generates a telemetry wrapper for chat_stream to collect metrics.
+
+ Args:
+ gen_ai_endpoint: Endpoint identifier for logging and tracing.
+ version: Version of the monitoring package.
+ environment: Deployment environment (e.g., production, staging).
+ application_name: Name of the application using the OpenAI API.
+ tracer: OpenTelemetry tracer for creating spans.
+ pricing_info: Information used for calculating the cost of OpenAI usage.
+ trace_content: Flag indicating whether to trace the actual content.
+
+ Returns:
+ A function that wraps the chat method to add telemetry.
+ """
+
+ async def wrapper(wrapped, instance, args, kwargs):
+ """
+ Wraps the 'chat_stream' API call to add telemetry.
+
+ This collects metrics such as execution time, cost, and token usage, and handles errors
+ gracefully, adding details to the trace for observability.
+
+ Args:
+ wrapped: The original 'chat_stream' method to be wrapped.
+ instance: The instance of the class where the original method is defined.
+ args: Positional arguments for the 'chat_stream' method.
+ kwargs: Keyword arguments for the 'chat_stream' method.
+
+ Returns:
+ The response from the original 'chat_stream' method.
+ """
+
+ async def stream_generator():
+ # pylint: disable=line-too-long
+ with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
+ # Placeholder for aggregating streaming response
+ llmresponse = ""
+
+ # Loop through streaming events capturing relevant details
+ async for event in wrapped(*args, **kwargs):
+ response_id = event.id
+ llmresponse += event.choices[0].delta.content
+ if event.usage is not None:
+ prompt_tokens = event.usage.prompt_tokens
+ completion_tokens = event.usage.completion_tokens
+ total_tokens = event.usage.total_tokens
+ finish_reason = event.choices[0].finish_reason
+ yield event
+
+ # Handling exception ensure observability without disrupting operation
+ try:
+ # Format 'messages' into a single string
+ message_prompt = kwargs.get('messages', "")
+ formatted_messages = []
+ for message in message_prompt:
+ role = message.role
+ content = message.content
+
+ if isinstance(content, list):
+ content_str = ", ".join(
+ # pylint: disable=line-too-long
+ f"{item['type']}: {item['text'] if 'text' in item else item['image_url']}"
+ if 'type' in item else f"text: {item['text']}"
+ for item in content
+ )
+ formatted_messages.append(f"{role}: {content_str}")
+ else:
+ formatted_messages.append(f"{role}: {content}")
+ prompt = " ".join(formatted_messages)
+
+ # Calculate cost of the operation
+ cost = get_chat_model_cost(kwargs.get("model", "mistral-small-latest"),
+ pricing_info, prompt_tokens, completion_tokens)
+
+ # Set Span attributes
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_MISTRAL)
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
+ SemanticConvetion.GEN_AI_TYPE_CHAT)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
+ gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,
+ response_id)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,
+ environment)
+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
+ application_name)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ kwargs.get("model", "mistral-small-latest"))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,
+ kwargs.get("temperature", 0.7))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,
+ kwargs.get("top_p", 1))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS,
+ kwargs.get("max_tokens", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_SEED,
+ kwargs.get("random_seed", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,
+ True)
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,
+ finish_reason)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
+ prompt_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,
+ completion_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ total_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,
+ prompt)
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_COMPLETION,
+ llmresponse)
+
+ span.set_status(Status(StatusCode.OK))
+
+ if disable_metrics is False:
+ attributes = {
+ TELEMETRY_SDK_NAME:
+ "openlit",
+ SemanticConvetion.GEN_AI_APPLICATION_NAME:
+ application_name,
+ SemanticConvetion.GEN_AI_SYSTEM:
+ SemanticConvetion.GEN_AI_SYSTEM_MISTRAL,
+ SemanticConvetion.GEN_AI_ENVIRONMENT:
+ environment,
+ SemanticConvetion.GEN_AI_TYPE:
+ SemanticConvetion.GEN_AI_TYPE_CHAT,
+ SemanticConvetion.GEN_AI_REQUEST_MODEL:
+ kwargs.get("model", "mistral-small-latest")
+ }
+
+ metrics["genai_requests"].add(1, attributes)
+ metrics["genai_total_tokens"].add(prompt_tokens + completion_tokens, attributes)
+ metrics["genai_completion_tokens"].add(completion_tokens, attributes)
+ metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)
+ metrics["genai_cost"].record(cost)
+
+ except Exception as e:
+ handle_exception(span, e)
+ logger.error("Error in trace creation: %s", e)
+
+ return stream_generator()
+
+ return wrapper
+
+def async_embeddings(gen_ai_endpoint, version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics):
+ """
+ Generates a telemetry wrapper for embeddings to collect metrics.
+
+ Args:
+ gen_ai_endpoint: Endpoint identifier for logging and tracing.
+ version: Version of the monitoring package.
+ environment: Deployment environment (e.g., production, staging).
+ application_name: Name of the application using the OpenAI API.
+ tracer: OpenTelemetry tracer for creating spans.
+ pricing_info: Information used for calculating the cost of OpenAI usage.
+ trace_content: Flag indicating whether to trace the actual content.
+
+ Returns:
+ A function that wraps the embeddings method to add telemetry.
+ """
+
+ async def wrapper(wrapped, instance, args, kwargs):
+ """
+ Wraps the 'embeddings' API call to add telemetry.
+
+ This collects metrics such as execution time, cost, and token usage, and handles errors
+ gracefully, adding details to the trace for observability.
+
+ Args:
+ wrapped: The original 'embeddings' method to be wrapped.
+ instance: The instance of the class where the original method is defined.
+ args: Positional arguments for the 'embeddings' method.
+ kwargs: Keyword arguments for the 'embeddings' method.
+
+ Returns:
+ The response from the original 'embeddings' method.
+ """
+
+ with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
+ response = await wrapped(*args, **kwargs)
+
+ try:
+ # Get prompt from kwargs and store as a single string
+ prompt = ', '.join(kwargs.get('input', []))
+
+ # Calculate cost of the operation
+ cost = get_embed_model_cost(kwargs.get('model', "mistral-embed"),
+ pricing_info, response.usage.prompt_tokens)
+
+ # Set Span attributes
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_MISTRAL)
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
+ SemanticConvetion.GEN_AI_TYPE_EMBEDDING)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
+ gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,
+ environment)
+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
+ application_name)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ kwargs.get('model', "mistral-embed"))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_EMBEDDING_FORMAT,
+ kwargs.get("encoding_format", "float"))
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,
+ response.id)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
+ response.usage.prompt_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ response.usage.total_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,
+ prompt)
+
+ span.set_status(Status(StatusCode.OK))
+
+ if disable_metrics is False:
+ attributes = {
+ TELEMETRY_SDK_NAME:
+ "openlit",
+ SemanticConvetion.GEN_AI_APPLICATION_NAME:
+ application_name,
+ SemanticConvetion.GEN_AI_SYSTEM:
+ SemanticConvetion.GEN_AI_SYSTEM_MISTRAL,
+ SemanticConvetion.GEN_AI_ENVIRONMENT:
+ environment,
+ SemanticConvetion.GEN_AI_TYPE:
+ SemanticConvetion.GEN_AI_TYPE_EMBEDDING,
+ SemanticConvetion.GEN_AI_REQUEST_MODEL:
+ kwargs.get('model', "mistral-embed")
+ }
+
+ metrics["genai_requests"].add(1, attributes)
+ metrics["genai_total_tokens"].add(response.usage.total_tokens, attributes)
+ metrics["genai_prompt_tokens"].add(response.usage.prompt_tokens, attributes)
+ metrics["genai_cost"].record(cost, attributes)
+
+ # Return original response
+ return response
+
+ except Exception as e:
+ handle_exception(span, e)
+ logger.error("Error in trace creation: %s", e)
+
+ # Return original response
+ return response
+
+ return wrapper
diff --git a/sdk/python/src/openlit/instrumentation/mistral/mistral.py b/sdk/python/src/openlit/instrumentation/mistral/mistral.py
new file mode 100644
index 00000000..86f09931
--- /dev/null
+++ b/sdk/python/src/openlit/instrumentation/mistral/mistral.py
@@ -0,0 +1,416 @@
+# pylint: disable=duplicate-code, broad-exception-caught, too-many-statements, unused-argument
+"""
+Module for monitoring Mistral API calls.
+"""
+
+import logging
+from opentelemetry.trace import SpanKind, Status, StatusCode
+from opentelemetry.sdk.resources import TELEMETRY_SDK_NAME
+from openlit.__helpers import get_chat_model_cost, get_embed_model_cost, handle_exception
+from openlit.semcov import SemanticConvetion
+
+# Initialize logger for logging potential issues and operations
+logger = logging.getLogger(__name__)
+
+def chat(gen_ai_endpoint, version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics):
+ """
+ Generates a telemetry wrapper for chat to collect metrics.
+
+ Args:
+ gen_ai_endpoint: Endpoint identifier for logging and tracing.
+ version: Version of the monitoring package.
+ environment: Deployment environment (e.g., production, staging).
+ application_name: Name of the application using the OpenAI API.
+ tracer: OpenTelemetry tracer for creating spans.
+ pricing_info: Information used for calculating the cost of OpenAI usage.
+ trace_content: Flag indicating whether to trace the actual content.
+
+ Returns:
+ A function that wraps the chat method to add telemetry.
+ """
+
+ def wrapper(wrapped, instance, args, kwargs):
+ """
+ Wraps the 'chat' API call to add telemetry.
+
+ This collects metrics such as execution time, cost, and token usage, and handles errors
+ gracefully, adding details to the trace for observability.
+
+ Args:
+ wrapped: The original 'chat' method to be wrapped.
+ instance: The instance of the class where the original method is defined.
+ args: Positional arguments for the 'chat' method.
+ kwargs: Keyword arguments for the 'chat' method.
+
+ Returns:
+ The response from the original 'chat' method.
+ """
+
+ with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
+ response = wrapped(*args, **kwargs)
+
+ try:
+ # Format 'messages' into a single string
+ message_prompt = kwargs.get('messages', "")
+ formatted_messages = []
+ for message in message_prompt:
+ role = message.role
+ content = message.content
+
+ if isinstance(content, list):
+ content_str = ", ".join(
+ # pylint: disable=line-too-long
+ f"{item['type']}: {item['text'] if 'text' in item else item['image_url']}"
+ if 'type' in item else f"text: {item['text']}"
+ for item in content
+ )
+ formatted_messages.append(f"{role}: {content_str}")
+ else:
+ formatted_messages.append(f"{role}: {content}")
+ prompt = " ".join(formatted_messages)
+
+ # Calculate cost of the operation
+ cost = get_chat_model_cost(kwargs.get("model", "mistral-small-latest"),
+ pricing_info, response.usage.prompt_tokens,
+ response.usage.completion_tokens)
+
+ # Set Span attributes
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_MISTRAL)
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
+ SemanticConvetion.GEN_AI_TYPE_CHAT)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
+ gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,
+ response.id)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,
+ environment)
+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
+ application_name)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ kwargs.get("model", "mistral-small-latest"))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,
+ kwargs.get("temperature", 0.7))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,
+ kwargs.get("top_p", 1))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS,
+ kwargs.get("max_tokens", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_SEED,
+ kwargs.get("random_seed", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,
+ False)
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,
+ response.choices[0].finish_reason)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
+ response.usage.prompt_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,
+ response.usage.completion_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ response.usage.total_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,
+ prompt)
+ # pylint: disable=line-too-long
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_COMPLETION, response.choices[0].message.content if response.choices[0].message.content else "")
+
+ span.set_status(Status(StatusCode.OK))
+
+ if disable_metrics is False:
+ attributes = {
+ TELEMETRY_SDK_NAME:
+ "openlit",
+ SemanticConvetion.GEN_AI_APPLICATION_NAME:
+ application_name,
+ SemanticConvetion.GEN_AI_SYSTEM:
+ SemanticConvetion.GEN_AI_SYSTEM_MISTRAL,
+ SemanticConvetion.GEN_AI_ENVIRONMENT:
+ environment,
+ SemanticConvetion.GEN_AI_TYPE:
+ SemanticConvetion.GEN_AI_TYPE_CHAT,
+ SemanticConvetion.GEN_AI_REQUEST_MODEL:
+ kwargs.get("model", "mistral-small-latest")
+ }
+
+ metrics["genai_requests"].add(1, attributes)
+ metrics["genai_total_tokens"].add(response.usage.total_tokens, attributes)
+ metrics["genai_completion_tokens"].add(
+ response.usage.completion_tokens, attributes
+ )
+ metrics["genai_prompt_tokens"].add(response.usage.prompt_tokens, attributes)
+ metrics["genai_cost"].record(cost, attributes)
+
+ # Return original response
+ return response
+
+ except Exception as e:
+ handle_exception(span, e)
+ logger.error("Error in trace creation: %s", e)
+
+ # Return original response
+ return response
+
+ return wrapper
+
+def chat_stream(gen_ai_endpoint, version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics):
+ """
+ Generates a telemetry wrapper for chat_stream to collect metrics.
+
+ Args:
+ gen_ai_endpoint: Endpoint identifier for logging and tracing.
+ version: Version of the monitoring package.
+ environment: Deployment environment (e.g., production, staging).
+ application_name: Name of the application using the OpenAI API.
+ tracer: OpenTelemetry tracer for creating spans.
+ pricing_info: Information used for calculating the cost of OpenAI usage.
+ trace_content: Flag indicating whether to trace the actual content.
+
+ Returns:
+ A function that wraps the chat method to add telemetry.
+ """
+
+ def wrapper(wrapped, instance, args, kwargs):
+ """
+ Wraps the 'chat_stream' API call to add telemetry.
+
+ This collects metrics such as execution time, cost, and token usage, and handles errors
+ gracefully, adding details to the trace for observability.
+
+ Args:
+ wrapped: The original 'chat_stream' method to be wrapped.
+ instance: The instance of the class where the original method is defined.
+ args: Positional arguments for the 'chat_stream' method.
+ kwargs: Keyword arguments for the 'chat_stream' method.
+
+ Returns:
+ The response from the original 'chat_stream' method.
+ """
+
+ def stream_generator():
+ # pylint: disable=line-too-long
+ with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
+ # Placeholder for aggregating streaming response
+ llmresponse = ""
+
+ # Loop through streaming events capturing relevant details
+ for event in wrapped(*args, **kwargs):
+ response_id = event.id
+ llmresponse += event.choices[0].delta.content
+ if event.usage is not None:
+ prompt_tokens = event.usage.prompt_tokens
+ completion_tokens = event.usage.completion_tokens
+ total_tokens = event.usage.total_tokens
+ finish_reason = event.choices[0].finish_reason
+ yield event
+
+ # Handling exception ensure observability without disrupting operation
+ try:
+ # Format 'messages' into a single string
+ message_prompt = kwargs.get('messages', "")
+ formatted_messages = []
+ for message in message_prompt:
+ role = message.role
+ content = message.content
+
+ if isinstance(content, list):
+ content_str = ", ".join(
+ # pylint: disable=line-too-long
+ f"{item['type']}: {item['text'] if 'text' in item else item['image_url']}"
+ if 'type' in item else f"text: {item['text']}"
+ for item in content
+ )
+ formatted_messages.append(f"{role}: {content_str}")
+ else:
+ formatted_messages.append(f"{role}: {content}")
+ prompt = " ".join(formatted_messages)
+
+ # Calculate cost of the operation
+ cost = get_chat_model_cost(kwargs.get("model", "mistral-small-latest"),
+ pricing_info, prompt_tokens, completion_tokens)
+
+ # Set Span attributes
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_MISTRAL)
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
+ SemanticConvetion.GEN_AI_TYPE_CHAT)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
+ gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,
+ response_id)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,
+ environment)
+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
+ application_name)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ kwargs.get("model", "mistral-small-latest"))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,
+ kwargs.get("temperature", 0.7))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,
+ kwargs.get("top_p", 1))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS,
+ kwargs.get("max_tokens", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_SEED,
+ kwargs.get("random_seed", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,
+ True)
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,
+ finish_reason)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
+ prompt_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,
+ completion_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ total_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,
+ prompt)
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_COMPLETION,
+ llmresponse)
+
+ span.set_status(Status(StatusCode.OK))
+
+ if disable_metrics is False:
+ attributes = {
+ TELEMETRY_SDK_NAME:
+ "openlit",
+ SemanticConvetion.GEN_AI_APPLICATION_NAME:
+ application_name,
+ SemanticConvetion.GEN_AI_SYSTEM:
+ SemanticConvetion.GEN_AI_SYSTEM_MISTRAL,
+ SemanticConvetion.GEN_AI_ENVIRONMENT:
+ environment,
+ SemanticConvetion.GEN_AI_TYPE:
+ SemanticConvetion.GEN_AI_TYPE_CHAT,
+ SemanticConvetion.GEN_AI_REQUEST_MODEL:
+ kwargs.get("model", "mistral-small-latest")
+ }
+
+ metrics["genai_requests"].add(1, attributes)
+ metrics["genai_total_tokens"].add(prompt_tokens + completion_tokens, attributes)
+ metrics["genai_completion_tokens"].add(completion_tokens, attributes)
+ metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)
+ metrics["genai_cost"].record(cost)
+
+ except Exception as e:
+ handle_exception(span, e)
+ logger.error("Error in trace creation: %s", e)
+
+ return stream_generator()
+
+ return wrapper
+
+def embeddings(gen_ai_endpoint, version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics):
+ """
+ Generates a telemetry wrapper for embeddings to collect metrics.
+
+ Args:
+ gen_ai_endpoint: Endpoint identifier for logging and tracing.
+ version: Version of the monitoring package.
+ environment: Deployment environment (e.g., production, staging).
+ application_name: Name of the application using the OpenAI API.
+ tracer: OpenTelemetry tracer for creating spans.
+ pricing_info: Information used for calculating the cost of OpenAI usage.
+ trace_content: Flag indicating whether to trace the actual content.
+
+ Returns:
+ A function that wraps the embeddings method to add telemetry.
+ """
+
+ def wrapper(wrapped, instance, args, kwargs):
+ """
+ Wraps the 'embeddings' API call to add telemetry.
+
+ This collects metrics such as execution time, cost, and token usage, and handles errors
+ gracefully, adding details to the trace for observability.
+
+ Args:
+ wrapped: The original 'embeddings' method to be wrapped.
+ instance: The instance of the class where the original method is defined.
+ args: Positional arguments for the 'embeddings' method.
+ kwargs: Keyword arguments for the 'embeddings' method.
+
+ Returns:
+ The response from the original 'embeddings' method.
+ """
+
+ with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
+ response = wrapped(*args, **kwargs)
+
+ try:
+ # Get prompt from kwargs and store as a single string
+ prompt = ', '.join(kwargs.get('input', []))
+
+ # Calculate cost of the operation
+ cost = get_embed_model_cost(kwargs.get('model', "mistral-embed"),
+ pricing_info, response.usage.prompt_tokens)
+
+ # Set Span attributes
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_MISTRAL)
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
+ SemanticConvetion.GEN_AI_TYPE_EMBEDDING)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
+ gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,
+ environment)
+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
+ application_name)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ kwargs.get('model', "mistral-embed"))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_EMBEDDING_FORMAT,
+ kwargs.get("encoding_format", "float"))
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,
+ response.id)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
+ response.usage.prompt_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ response.usage.total_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,
+ prompt)
+
+ span.set_status(Status(StatusCode.OK))
+
+ if disable_metrics is False:
+ attributes = {
+ TELEMETRY_SDK_NAME:
+ "openlit",
+ SemanticConvetion.GEN_AI_APPLICATION_NAME:
+ application_name,
+ SemanticConvetion.GEN_AI_SYSTEM:
+ SemanticConvetion.GEN_AI_SYSTEM_MISTRAL,
+ SemanticConvetion.GEN_AI_ENVIRONMENT:
+ environment,
+ SemanticConvetion.GEN_AI_TYPE:
+ SemanticConvetion.GEN_AI_TYPE_EMBEDDING,
+ SemanticConvetion.GEN_AI_REQUEST_MODEL:
+ kwargs.get('model', "mistral-embed")
+ }
+
+ metrics["genai_requests"].add(1, attributes)
+ metrics["genai_total_tokens"].add(response.usage.total_tokens, attributes)
+ metrics["genai_prompt_tokens"].add(response.usage.prompt_tokens, attributes)
+ metrics["genai_cost"].record(cost, attributes)
+
+ # Return original response
+ return response
+
+ except Exception as e:
+ handle_exception(span, e)
+ logger.error("Error in trace creation: %s", e)
+
+ # Return original response
+ return response
+
+ return wrapper
diff --git a/sdk/python/src/openlit/instrumentation/openai/__init__.py b/sdk/python/src/openlit/instrumentation/openai/__init__.py
new file mode 100644
index 00000000..cc4155f2
--- /dev/null
+++ b/sdk/python/src/openlit/instrumentation/openai/__init__.py
@@ -0,0 +1,335 @@
+# pylint: disable=useless-return, bad-staticmethod-argument, disable=duplicate-code
+"""Initializer of Auto Instrumentation of OpenAI Functions"""
+from typing import Collection
+import importlib.metadata
+from opentelemetry.instrumentation.instrumentor import BaseInstrumentor
+from wrapt import wrap_function_wrapper
+
+from openlit.instrumentation.openai.openai import chat_completions, embedding, finetune
+from openlit.instrumentation.openai.openai import image_generate, image_variatons, audio_create
+from openlit.instrumentation.openai.async_openai import async_chat_completions, async_embedding
+from openlit.instrumentation.openai.async_openai import async_finetune
+from openlit.instrumentation.openai.async_openai import async_image_generate, async_image_variatons
+from openlit.instrumentation.openai.async_openai import async_audio_create
+
+from openlit.instrumentation.openai.azure_openai import azure_chat_completions, azure_completions
+from openlit.instrumentation.openai.azure_openai import azure_image_generate, azure_embedding
+from openlit.instrumentation.openai.async_azure_openai import azure_async_chat_completions
+from openlit.instrumentation.openai.async_azure_openai import azure_async_completions
+from openlit.instrumentation.openai.async_azure_openai import azure_async_image_generate
+from openlit.instrumentation.openai.async_azure_openai import azure_async_embedding
+
+_instruments = ("openai >= 1.1.1",)
+
+class OpenAIInstrumentor(BaseInstrumentor):
+ """An instrumentor for OpenAI's client library."""
+
+ def instrumentation_dependencies(self) -> Collection[str]:
+ return _instruments
+
+ def _instrument(self, **kwargs):
+ application_name = kwargs.get("application_name")
+ environment = kwargs.get("environment")
+ tracer = kwargs.get("tracer")
+ metrics = kwargs.get("metrics_dict")
+ pricing_info = kwargs.get("pricing_info")
+ trace_content = kwargs.get("trace_content")
+ disable_metrics = kwargs.get("disable_metrics")
+ version = importlib.metadata.version("openai")
+
+ wrap_function_wrapper(
+ "openai.resources.chat.completions",
+ "Completions.create",
+ chat_wrapper(version, environment, application_name,
+ tracer, pricing_info, trace_content,
+ metrics, disable_metrics),
+ )
+
+ wrap_function_wrapper(
+ "openai.resources.chat.completions",
+ "AsyncCompletions.create",
+ async_chat_wrapper(version, environment, application_name,
+ tracer, pricing_info, trace_content,
+ metrics, disable_metrics),
+ )
+
+ wrap_function_wrapper(
+ "openai.resources.images",
+ "Images.generate",
+ image_generate_wrapper(version, environment, application_name,
+ tracer, pricing_info, trace_content,
+ metrics, disable_metrics),
+ )
+
+ wrap_function_wrapper(
+ "openai.resources.images",
+ "AsyncImages.generate",
+ async_image_generate_wrapper(version, environment, application_name,
+ tracer, pricing_info, trace_content,
+ metrics, disable_metrics),
+ )
+
+ wrap_function_wrapper(
+ "openai.resources.embeddings",
+ "Embeddings.create",
+ embedding_wrapper(version, environment, application_name,
+ tracer, pricing_info, trace_content,
+ metrics, disable_metrics),
+ )
+
+ wrap_function_wrapper(
+ "openai.resources.embeddings",
+ "AsyncEmbeddings.create",
+ async_embedding_wrapper(version, environment, application_name,
+ tracer, pricing_info, trace_content,
+ metrics, disable_metrics),
+ )
+
+ wrap_function_wrapper(
+ "openai.resources.completions",
+ "Completions.create",
+ azure_completions("azure_openai.completions", version, environment, application_name,
+ tracer, pricing_info, trace_content,
+ metrics, disable_metrics),
+ )
+
+ wrap_function_wrapper(
+ "openai.resources.completions",
+ "AsyncCompletions.create",
+ azure_async_completions("azure_openai.completions", version,
+ environment, application_name,
+ tracer, pricing_info, trace_content,
+ metrics, disable_metrics),
+ )
+
+ wrap_function_wrapper(
+ "openai.resources.images",
+ "Images.create_variation",
+ image_variatons("openai.images.variations", version,
+ environment, application_name,
+ tracer, pricing_info, trace_content,
+ metrics, disable_metrics),
+ )
+
+ wrap_function_wrapper(
+ "openai.resources.images",
+ "AsyncImages.create_variation",
+ async_image_variatons("openai.images.variations", version,
+ environment, application_name,
+ tracer, pricing_info, trace_content,
+ metrics, disable_metrics),
+ )
+
+ wrap_function_wrapper(
+ "openai.resources.audio.speech",
+ "Speech.create",
+ audio_create("openai.audio.speech", version, environment, application_name,
+ tracer, pricing_info, trace_content,
+ metrics, disable_metrics),
+ )
+
+ wrap_function_wrapper(
+ "openai.resources.audio.speech",
+ "AsyncSpeech.create",
+ async_audio_create("openai.audio.speech", version, environment, application_name,
+ tracer, pricing_info, trace_content,
+ metrics, disable_metrics),
+ )
+
+ wrap_function_wrapper(
+ "openai.resources.fine_tuning.jobs",
+ "Jobs.create",
+ finetune("openai.audio.speech", version, environment, application_name,
+ tracer, pricing_info, trace_content,
+ metrics, disable_metrics),
+ )
+
+ wrap_function_wrapper(
+ "openai.resources.fine_tuning.jobs",
+ "AsyncJobs.create",
+ async_finetune("openai.fine_tuning.jo", version, environment, application_name,
+ tracer, pricing_info, trace_content,
+ metrics, disable_metrics),
+ )
+
+ @staticmethod
+ def _uninstrument(self, **kwargs):
+ pass
+
+def chat_wrapper(version, environment, application_name, tracer, pricing_info, trace_content,
+ metrics, disable_metrics):
+ """
+ Decorator for making a custom wrapper execute conditionally,
+ based on whether the instance is for Azure OpenAI or not.
+ """
+ def wrapper(wrapped, instance, args, kwargs):
+ # Default to using the standard OpenAI chat completions
+ completion_func = chat_completions("openai.chat.completions", version, environment,
+ application_name, tracer, pricing_info, trace_content,
+ metrics, disable_metrics)
+
+ # Check if it's an Azure instance by inspecting `base_url`
+ try:
+ base_url = getattr(instance, 'base_url', '')
+ if 'azure.com' in base_url:
+ # Switch to the Azure-specific chat completions logic
+ completion_func = azure_chat_completions("azure_openai.chat.completions",
+ version, environment, application_name,
+ tracer, pricing_info, trace_content,
+ metrics, disable_metrics)
+ except AttributeError:
+ pass # base_url attribute not found, proceed with the default
+
+ # Execute the selected completion function
+ return completion_func(wrapped, instance, args, kwargs)
+
+ return wrapper
+
+def async_chat_wrapper(version, environment, application_name, tracer, pricing_info,
+ trace_content, metrics, disable_metrics):
+ """
+ Decorator for making a custom wrapper execute conditionally,
+ based on whether the instance is for Azure OpenAI or not.
+ """
+ def wrapper(wrapped, instance, args, kwargs):
+ # Default to using the standard OpenAI chat completions
+ completion_func = async_chat_completions("openai.chat.completions", version, environment,
+ application_name, tracer,
+ pricing_info, trace_content,
+ metrics, disable_metrics)
+
+ # Check if it's an Azure instance by inspecting `base_url`
+ try:
+ base_url = getattr(instance, 'base_url', '')
+ if 'azure.com' in base_url:
+ # Switch to the Azure-specific chat completions logic
+ completion_func = azure_async_chat_completions("azure_openai.chat.completions",
+ version, environment,
+ application_name, tracer,
+ pricing_info, trace_content,
+ metrics, disable_metrics)
+ except AttributeError:
+ pass # base_url attribute not found, proceed with the default
+
+ # Execute the selected completion function
+ return completion_func(wrapped, instance, args, kwargs)
+
+ return wrapper
+
+def image_generate_wrapper(version, environment, application_name, tracer, pricing_info,
+ trace_content, metrics, disable_metrics):
+ """
+ Decorator for making a custom wrapper execute conditionally,
+ based on whether the instance is for Azure OpenAI or not.
+ """
+ def wrapper(wrapped, instance, args, kwargs):
+ # Default to using the standard OpenAI chat completions
+ completion_func = image_generate("openai.images.generate", version, environment,
+ application_name, tracer, pricing_info, trace_content,
+ metrics, disable_metrics)
+
+ # Check if it's an Azure instance by inspecting `base_url`
+ try:
+ base_url = getattr(instance, 'base_url', '')
+ if 'azure.com' in base_url:
+ # Switch to the Azure-specific chat completions logic
+ completion_func = azure_image_generate("azure_openai.images.generate",
+ version, environment, application_name,
+ tracer, pricing_info, trace_content,
+ metrics, disable_metrics)
+ except AttributeError:
+ pass # base_url attribute not found, proceed with the default
+
+ # Execute the selected completion function
+ return completion_func(wrapped, instance, args, kwargs)
+
+ return wrapper
+
+def async_image_generate_wrapper(version, environment, application_name, tracer,
+ pricing_info, trace_content, metrics, disable_metrics):
+ """
+ Decorator for making a custom wrapper execute conditionally,
+ based on whether the instance is for Azure OpenAI or not.
+ """
+ def wrapper(wrapped, instance, args, kwargs):
+ # Default to using the standard OpenAI chat completions
+ completion_func = async_image_generate("openai.images.generate", version,
+ environment, application_name, tracer,
+ pricing_info, trace_content,
+ metrics, disable_metrics)
+
+ # Check if it's an Azure instance by inspecting `base_url`
+ try:
+ base_url = getattr(instance, 'base_url', '')
+ if 'azure.com' in base_url:
+ # Switch to the Azure-specific chat completions logic
+ completion_func = azure_async_image_generate("azure_openai.images.generate",
+ version, environment,
+ application_name, tracer,
+ pricing_info, trace_content,
+ metrics, disable_metrics)
+ except AttributeError:
+ pass # base_url attribute not found, proceed with the default
+
+ # Execute the selected completion function
+ return completion_func(wrapped, instance, args, kwargs)
+
+ return wrapper
+
+def embedding_wrapper(version, environment, application_name, tracer, pricing_info,
+ trace_content, metrics, disable_metrics):
+ """
+ Decorator for making a custom wrapper execute conditionally,
+ based on whether the instance is for Azure OpenAI or not.
+ """
+ def wrapper(wrapped, instance, args, kwargs):
+ # Default to using the standard OpenAI chat completions
+ completion_func = embedding("openai.embeddings", version, environment,
+ application_name, tracer, pricing_info, trace_content,
+ metrics, disable_metrics)
+
+ # Check if it's an Azure instance by inspecting `base_url`
+ try:
+ base_url = getattr(instance, 'base_url', '')
+ if 'azure.com' in base_url:
+ # Switch to the Azure-specific chat completions logic
+ completion_func = azure_embedding("azure_openai.embeddings",
+ version, environment, application_name,
+ tracer, pricing_info, trace_content,
+ metrics, disable_metrics)
+ except AttributeError:
+ pass # base_url attribute not found, proceed with the default
+
+ # Execute the selected completion function
+ return completion_func(wrapped, instance, args, kwargs)
+
+ return wrapper
+
+def async_embedding_wrapper(version, environment, application_name, tracer,
+ pricing_info, trace_content, metrics, disable_metrics):
+ """
+ Decorator for making a custom wrapper execute conditionally,
+ based on whether the instance is for Azure OpenAI or not.
+ """
+ def wrapper(wrapped, instance, args, kwargs):
+ # Default to using the standard OpenAI chat completions
+ completion_func = async_embedding("openai.embeddings", version, environment,
+ application_name, tracer, pricing_info, trace_content,
+ metrics, disable_metrics)
+
+ # Check if it's an Azure instance by inspecting `base_url`
+ try:
+ base_url = getattr(instance, 'base_url', '')
+ if 'azure.com' in base_url:
+ # Switch to the Azure-specific chat completions logic
+ completion_func = azure_async_embedding("azure_openai.embeddings", version,
+ environment, application_name, tracer,
+ pricing_info, trace_content,
+ metrics, disable_metrics)
+ except AttributeError:
+ pass # base_url attribute not found, proceed with the default
+
+ # Execute the selected completion function
+ return completion_func(wrapped, instance, args, kwargs)
+
+ return wrapper
diff --git a/sdk/python/src/openlit/instrumentation/openai/async_azure_openai.py b/sdk/python/src/openlit/instrumentation/openai/async_azure_openai.py
new file mode 100644
index 00000000..1c4c286a
--- /dev/null
+++ b/sdk/python/src/openlit/instrumentation/openai/async_azure_openai.py
@@ -0,0 +1,841 @@
+# pylint: disable=duplicate-code, broad-exception-caught, too-many-statements, unused-argument, too-many-branches
+"""
+Module for monitoring Azure OpenAI API calls.
+"""
+
+import logging
+from opentelemetry.trace import SpanKind, Status, StatusCode
+from opentelemetry.sdk.resources import TELEMETRY_SDK_NAME
+from openlit.__helpers import get_chat_model_cost, get_embed_model_cost
+from openlit.__helpers import get_image_model_cost, openai_tokens, handle_exception
+from openlit.semcov import SemanticConvetion
+
+# Initialize logger for logging potential issues and operations
+logger = logging.getLogger(__name__)
+
+def azure_async_chat_completions(gen_ai_endpoint, version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics):
+ """
+ Generates a telemetry wrapper for chat completions to collect metrics.
+
+ Args:
+ gen_ai_endpoint: Endpoint identifier for logging and tracing.
+ version: Version of the monitoring package.
+ environment: Deployment environment (e.g., production, staging).
+ application_name: Name of the application using the OpenAI API.
+ tracer: OpenTelemetry tracer for creating spans.
+ pricing_info: Information used for calculating the cost of OpenAI usage.
+ trace_content: Flag indicating whether to trace the actual content.
+
+ Returns:
+ A function that wraps the chat completions method to add telemetry.
+ """
+
+ async def wrapper(wrapped, instance, args, kwargs):
+ """
+ Wraps the 'chat.completions' API call to add telemetry.
+
+ This collects metrics such as execution time, cost, and token usage, and handles errors
+ gracefully, adding details to the trace for observability.
+
+ Args:
+ wrapped: The original 'chat.completions' method to be wrapped.
+ instance: The instance of the class where the original method is defined.
+ args: Positional arguments for the 'chat.completions' method.
+ kwargs: Keyword arguments for the 'chat.completions' method.
+
+ Returns:
+ The response from the original 'chat.completions' method.
+ """
+
+ # Check if streaming is enabled for the API call
+ streaming = kwargs.get("stream", False)
+
+ # pylint: disable=no-else-return
+ if streaming:
+ # Special handling for streaming response to accommodate the nature of data flow
+ async def stream_generator():
+ # pylint: disable=line-too-long
+ with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
+ # Placeholder for aggregating streaming response
+ llmresponse = ""
+
+ # Loop through streaming events capturing relevant details
+ async for chunk in await wrapped(*args, **kwargs):
+ # Collect message IDs and aggregated response from events
+ if len(chunk.choices) > 0:
+ # pylint: disable=line-too-long
+ if hasattr(chunk.choices[0], "delta") and hasattr(chunk.choices[0].delta, "content"):
+ content = chunk.choices[0].delta.content
+ if content:
+ llmresponse += content
+ yield chunk
+ response_id = chunk.id
+ model = "azure_" + chunk.model
+
+ # Handling exception ensure observability without disrupting operation
+ try:
+ # Format 'messages' into a single string
+ message_prompt = kwargs.get("messages", "")
+ formatted_messages = []
+ for message in message_prompt:
+ role = message["role"]
+ content = message["content"]
+
+ if isinstance(content, list):
+ content_str = ", ".join(
+ # pylint: disable=line-too-long
+ f'{item["type"]}: {item["text"] if "text" in item else item["image_url"]}'
+ if "type" in item else f'text: {item["text"]}'
+ for item in content
+ )
+ formatted_messages.append(f"{role}: {content_str}")
+ else:
+ formatted_messages.append(f"{role}: {content}")
+ prompt = "\n".join(formatted_messages)
+
+ # Calculate tokens using input prompt and aggregated response
+ prompt_tokens = openai_tokens(prompt,
+ kwargs.get("model", "gpt-3.5-turbo"))
+ completion_tokens = openai_tokens(llmresponse,
+ kwargs.get("model", "gpt-3.5-turbo"))
+
+ # Calculate cost of the operation
+ cost = get_chat_model_cost(model, pricing_info,
+ prompt_tokens, completion_tokens)
+
+ # Set Span attributes
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_AZURE_OPENAI)
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
+ SemanticConvetion.GEN_AI_TYPE_CHAT)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
+ gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,
+ response_id)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,
+ environment)
+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
+ application_name)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ model)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_USER,
+ kwargs.get("user", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOOL_CHOICE,
+ kwargs.get("tool_choice", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,
+ kwargs.get("temperature", 1))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_PRESENCE_PENALTY,
+ kwargs.get("presence_penalty", 0))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_FREQUENCY_PENALTY,
+ kwargs.get("frequency_penalty", 0))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_SEED,
+ kwargs.get("seed", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,
+ True)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
+ prompt_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,
+ completion_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ prompt_tokens + completion_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,
+ prompt)
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_COMPLETION,
+ llmresponse)
+
+ span.set_status(Status(StatusCode.OK))
+
+ if disable_metrics is False:
+ attributes = {
+ TELEMETRY_SDK_NAME:
+ "openlit",
+ SemanticConvetion.GEN_AI_APPLICATION_NAME:
+ application_name,
+ SemanticConvetion.GEN_AI_SYSTEM:
+ SemanticConvetion.GEN_AI_SYSTEM_AZURE_OPENAI,
+ SemanticConvetion.GEN_AI_ENVIRONMENT:
+ environment,
+ SemanticConvetion.GEN_AI_TYPE:
+ SemanticConvetion.GEN_AI_TYPE_CHAT,
+ SemanticConvetion.GEN_AI_REQUEST_MODEL:
+ model
+ }
+
+ metrics["genai_requests"].add(1, attributes)
+ metrics["genai_total_tokens"].add(prompt_tokens + completion_tokens, attributes)
+ metrics["genai_completion_tokens"].add(completion_tokens, attributes)
+ metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)
+ metrics["genai_cost"].record(cost, attributes)
+
+ except Exception as e:
+ handle_exception(span, e)
+ logger.error("Error in trace creation: %s", e)
+
+ return stream_generator()
+
+ # Handling for non-streaming responses
+ else:
+ # pylint: disable=line-too-long
+ with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
+ response = await wrapped(*args, **kwargs)
+
+ try:
+ # Find base model from response
+ model = "azure_" + response.model
+
+ # Format 'messages' into a single string
+ message_prompt = kwargs.get("messages", "")
+ formatted_messages = []
+ for message in message_prompt:
+ role = message["role"]
+ content = message["content"]
+
+ if isinstance(content, list):
+ content_str = ", ".join(
+ # pylint: disable=line-too-long
+ f'{item["type"]}: {item["text"] if "text" in item else item["image_url"]}'
+ if "type" in item else f'text: {item["text"]}'
+ for item in content
+ )
+ formatted_messages.append(f"{role}: {content_str}")
+ else:
+ formatted_messages.append(f"{role}: {content}")
+ prompt = "\n".join(formatted_messages)
+
+ # Set base span attribues
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_AZURE_OPENAI)
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
+ SemanticConvetion.GEN_AI_TYPE_CHAT)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
+ gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,
+ response.id)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,
+ environment)
+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
+ application_name)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ model)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_USER,
+ kwargs.get("user", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOOL_CHOICE,
+ kwargs.get("tool_choice", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,
+ kwargs.get("temperature", 1))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_PRESENCE_PENALTY,
+ kwargs.get("presence_penalty", 0))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_FREQUENCY_PENALTY,
+ kwargs.get("frequency_penalty", 0))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_SEED,
+ kwargs.get("seed", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,
+ False)
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,
+ prompt)
+
+ # Set span attributes when tools is not passed to the function call
+ if "tools" not in kwargs:
+ # Calculate cost of the operation
+ cost = get_chat_model_cost(model, pricing_info,
+ response.usage.prompt_tokens,
+ response.usage.completion_tokens)
+
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
+ response.usage.prompt_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,
+ response.usage.completion_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ response.usage.total_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,
+ response.choices[0].finish_reason)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+
+ # Set span attributes for when n = 1 (default)
+ if "n" not in kwargs or kwargs["n"] == 1:
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_COMPLETION,
+ response.choices[0].message.content)
+
+ # Set span attributes for when n > 0
+ else:
+ i = 0
+ while i < kwargs["n"] and trace_content is True:
+ attribute_name = f"gen_ai.content.completion.{i}"
+ span.set_attribute(attribute_name,
+ response.choices[i].message.content)
+ i += 1
+
+ # Return original response
+ return response
+
+ # Set span attributes when tools is passed to the function call
+ elif "tools" in kwargs:
+ # Calculate cost of the operation
+ cost = get_chat_model_cost(model, pricing_info,
+ response.usage.prompt_tokens,
+ response.usage.completion_tokens)
+
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_COMPLETION,
+ "Function called with tools")
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
+ response.usage.prompt_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,
+ response.usage.completion_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ response.usage.total_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+
+ span.set_status(Status(StatusCode.OK))
+
+ if disable_metrics is False:
+ attributes = {
+ TELEMETRY_SDK_NAME:
+ "openlit",
+ SemanticConvetion.GEN_AI_APPLICATION_NAME:
+ application_name,
+ SemanticConvetion.GEN_AI_SYSTEM:
+ SemanticConvetion.GEN_AI_SYSTEM_AZURE_OPENAI,
+ SemanticConvetion.GEN_AI_ENVIRONMENT:
+ environment,
+ SemanticConvetion.GEN_AI_TYPE:
+ SemanticConvetion.GEN_AI_TYPE_CHAT,
+ SemanticConvetion.GEN_AI_REQUEST_MODEL:
+ model
+ }
+
+ metrics["genai_requests"].add(1, attributes)
+ metrics["genai_total_tokens"].add(response.usage.total_tokens, attributes)
+ metrics["genai_completion_tokens"].add(response.usage.completion_tokens, attributes)
+ metrics["genai_prompt_tokens"].add(response.usage.prompt_tokens, attributes)
+ metrics["genai_cost"].record(cost, attributes)
+
+ # Return original response
+ return response
+
+ except Exception as e:
+ handle_exception(span, e)
+ logger.error("Error in trace creation: %s", e)
+
+ # Return original response
+ return response
+
+ return wrapper
+
+def azure_async_completions(gen_ai_endpoint, version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics):
+ """
+ Generates a telemetry wrapper for completions to collect metrics.
+
+ Args:
+ gen_ai_endpoint: Endpoint identifier for logging and tracing.
+ version: Version of the monitoring package.
+ environment: Deployment environment (e.g., production, staging).
+ application_name: Name of the application using the OpenAI API.
+ tracer: OpenTelemetry tracer for creating spans.
+ pricing_info: Information used for calculating the cost of OpenAI usage.
+ trace_content: Flag indicating whether to trace the actual content.
+
+ Returns:
+ A function that wraps the chat completions method to add telemetry.
+ """
+
+ async def wrapper(wrapped, instance, args, kwargs):
+ """
+ Wraps the 'completions' API call to add telemetry.
+
+ This collects metrics such as execution time, cost, and token usage, and handles errors
+ gracefully, adding details to the trace for observability.
+
+ Args:
+ wrapped: The original 'completions' method to be wrapped.
+ instance: The instance of the class where the original method is defined.
+ args: Positional arguments for the 'completions' method.
+ kwargs: Keyword arguments for the 'completions' method.
+
+ Returns:
+ The response from the original 'chat.completions' method.
+ """
+
+ # Check if streaming is enabled for the API call
+ streaming = kwargs.get("stream", False)
+
+ # pylint: disable=no-else-return
+ if streaming:
+ # Special handling for streaming response to accommodate the nature of data flow
+ async def stream_generator():
+ # pylint: disable=line-too-long
+ with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
+ # Placeholder for aggregating streaming response
+ llmresponse = ""
+
+ # Loop through streaming events capturing relevant details
+ async for chunk in await wrapped(*args, **kwargs):
+ # Collect message IDs and aggregated response from events
+ if len(chunk.choices) > 0:
+ if hasattr(chunk.choices[0], "text"):
+ content = chunk.choices[0].text
+ if content:
+ llmresponse += content
+ yield chunk
+ response_id = chunk.id
+ model = "azure_" + chunk.model
+
+ # Handling exception ensure observability without disrupting operation
+ try:
+ prompt = kwargs.get("prompt", "")
+
+ # Calculate tokens using input prompt and aggregated response
+ prompt_tokens = openai_tokens(prompt,
+ "gpt-3.5-turbo")
+ completion_tokens = openai_tokens(llmresponse,
+ "gpt-3.5-turbo")
+
+ # Calculate cost of the operation
+ cost = get_chat_model_cost(model, pricing_info,
+ prompt_tokens, completion_tokens)
+
+ # Set Span attributes
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_AZURE_OPENAI)
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
+ SemanticConvetion.GEN_AI_TYPE_CHAT)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
+ gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,
+ response_id)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,
+ environment)
+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
+ application_name)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ model)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_USER,
+ kwargs.get("user", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOOL_CHOICE,
+ kwargs.get("tool_choice", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,
+ kwargs.get("temperature", 1))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_PRESENCE_PENALTY,
+ kwargs.get("presence_penalty", 0))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_FREQUENCY_PENALTY,
+ kwargs.get("frequency_penalty", 0))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_SEED,
+ kwargs.get("seed", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,
+ True)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
+ prompt_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,
+ completion_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ prompt_tokens + completion_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,
+ prompt)
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_COMPLETION,
+ llmresponse)
+
+ span.set_status(Status(StatusCode.OK))
+
+ if disable_metrics is False:
+ attributes = {
+ TELEMETRY_SDK_NAME:
+ "openlit",
+ SemanticConvetion.GEN_AI_APPLICATION_NAME:
+ application_name,
+ SemanticConvetion.GEN_AI_SYSTEM:
+ SemanticConvetion.GEN_AI_SYSTEM_AZURE_OPENAI,
+ SemanticConvetion.GEN_AI_ENVIRONMENT:
+ environment,
+ SemanticConvetion.GEN_AI_TYPE:
+ SemanticConvetion.GEN_AI_TYPE_CHAT,
+ SemanticConvetion.GEN_AI_REQUEST_MODEL:
+ model
+ }
+
+ metrics["genai_requests"].add(1, attributes)
+ metrics["genai_total_tokens"].add(prompt_tokens + completion_tokens, attributes)
+ metrics["genai_completion_tokens"].add(completion_tokens, attributes)
+ metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)
+ metrics["genai_cost"].record(cost, attributes)
+
+ except Exception as e:
+ handle_exception(span, e)
+ logger.error("Error in trace creation: %s", e)
+
+ return stream_generator()
+
+ # Handling for non-streaming responses
+ else:
+ # pylint: disable=line-too-long
+ with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
+ response = wrapped(*args, **kwargs)
+
+ try:
+ # Find base model from response
+ model = "azure_" + response.model
+
+ # Set base span attribues
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_AZURE_OPENAI)
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
+ SemanticConvetion.GEN_AI_TYPE_CHAT)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
+ gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,
+ response.id)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,
+ environment)
+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
+ application_name)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ model)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_USER,
+ kwargs.get("user", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOOL_CHOICE,
+ kwargs.get("tool_choice", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,
+ kwargs.get("temperature", 1))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_PRESENCE_PENALTY,
+ kwargs.get("presence_penalty", 0))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_FREQUENCY_PENALTY,
+ kwargs.get("frequency_penalty", 0))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_SEED,
+ kwargs.get("seed", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,
+ False)
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,
+ kwargs.get("prompt", ""))
+
+ # Set span attributes when tools is not passed to the function call
+ if "tools" not in kwargs:
+ # Calculate cost of the operation
+ cost = get_chat_model_cost(model, pricing_info,
+ response.usage.prompt_tokens,
+ response.usage.completion_tokens)
+
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
+ response.usage.prompt_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,
+ response.usage.completion_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ response.usage.total_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,
+ response.choices[0].finish_reason)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+
+ # Set span attributes for when n = 1 (default)
+ if "n" not in kwargs or kwargs["n"] == 1:
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_COMPLETION,
+ response.choices[0].text)
+
+ # Set span attributes for when n > 0
+ else:
+ i = 0
+ while i < kwargs["n"] and trace_content is True:
+ attribute_name = f"gen_ai.content.completion.{i}"
+ span.set_attribute(attribute_name,
+ response.choices[i].text)
+ i += 1
+ return response
+
+ # Set span attributes when tools is passed to the function call
+ elif "tools" in kwargs:
+ # Calculate cost of the operation
+ cost = get_chat_model_cost(model, pricing_info,
+ response.usage.prompt_tokens,
+ response.usage.completion_tokens)
+
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_COMPLETION,
+ "Function called with tools")
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
+ response.usage.prompt_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,
+ response.usage.completion_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ response.usage.total_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+
+ span.set_status(Status(StatusCode.OK))
+
+ if disable_metrics is False:
+ attributes = {
+ TELEMETRY_SDK_NAME:
+ "openlit",
+ SemanticConvetion.GEN_AI_APPLICATION_NAME:
+ application_name,
+ SemanticConvetion.GEN_AI_SYSTEM:
+ SemanticConvetion.GEN_AI_SYSTEM_AZURE_OPENAI,
+ SemanticConvetion.GEN_AI_ENVIRONMENT:
+ environment,
+ SemanticConvetion.GEN_AI_TYPE:
+ SemanticConvetion.GEN_AI_TYPE_CHAT,
+ SemanticConvetion.GEN_AI_REQUEST_MODEL:
+ model
+ }
+
+ metrics["genai_requests"].add(1, attributes)
+ metrics["genai_total_tokens"].add(response.usage.total_tokens, attributes)
+ metrics["genai_completion_tokens"].add(response.usage.completion_tokens, attributes)
+ metrics["genai_prompt_tokens"].add(response.usage.prompt_tokens, attributes)
+ metrics["genai_cost"].record(cost, attributes)
+
+ # Return original response
+ return response
+
+ except Exception as e:
+ handle_exception(span, e)
+ logger.error("Error in trace creation: %s", e)
+
+ # Return original response
+ return response
+
+ return wrapper
+
+def azure_async_embedding(gen_ai_endpoint, version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics):
+ """
+ Generates a telemetry wrapper for embeddings to collect metrics.
+
+ Args:
+ gen_ai_endpoint: Endpoint identifier for logging and tracing.
+ version: Version of the monitoring package.
+ environment: Deployment environment (e.g., production, staging).
+ application_name: Name of the application using the OpenAI API.
+ tracer: OpenTelemetry tracer for creating spans.
+ pricing_info: Information used for calculating the cost of OpenAI usage.
+ trace_content: Flag indicating whether to trace the actual content.
+
+ Returns:
+ A function that wraps the embeddings method to add telemetry.
+ """
+
+ async def wrapper(wrapped, instance, args, kwargs):
+ """
+ Wraps the 'embeddings' API call to add telemetry.
+
+ This collects metrics such as execution time, cost, and token usage, and handles errors
+ gracefully, adding details to the trace for observability.
+
+ Args:
+ wrapped: The original 'embeddings' method to be wrapped.
+ instance: The instance of the class where the original method is defined.
+ args: Positional arguments for the 'embeddings' method.
+ kwargs: Keyword arguments for the 'embeddings' method.
+
+ Returns:
+ The response from the original 'embeddings' method.
+ """
+
+ with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
+ response = await wrapped(*args, **kwargs)
+
+ try:
+ # Calculate cost of the operation
+ cost = get_embed_model_cost("azure_" + response.model,
+ pricing_info, response.usage.prompt_tokens)
+
+ # Set Span attributes
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_AZURE_OPENAI)
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
+ SemanticConvetion.GEN_AI_TYPE_EMBEDDING)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
+ gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,
+ environment)
+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
+ application_name)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ "azure_" + response.model)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_EMBEDDING_FORMAT,
+ kwargs.get("encoding_format", "float"))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_EMBEDDING_DIMENSION,
+ kwargs.get("dimensions", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_USER,
+ kwargs.get("user", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
+ response.usage.prompt_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ response.usage.total_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST, cost)
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,
+ kwargs.get("input", ""))
+
+ span.set_status(Status(StatusCode.OK))
+
+ if disable_metrics is False:
+ attributes = {
+ TELEMETRY_SDK_NAME:
+ "openlit",
+ SemanticConvetion.GEN_AI_APPLICATION_NAME:
+ application_name,
+ SemanticConvetion.GEN_AI_SYSTEM:
+ SemanticConvetion.GEN_AI_SYSTEM_AZURE_OPENAI,
+ SemanticConvetion.GEN_AI_ENVIRONMENT:
+ environment,
+ SemanticConvetion.GEN_AI_TYPE:
+ SemanticConvetion.GEN_AI_TYPE_EMBEDDING,
+ SemanticConvetion.GEN_AI_REQUEST_MODEL:
+ "azure_" + response.model
+ }
+
+ metrics["genai_requests"].add(1, attributes)
+ metrics["genai_total_tokens"].add(response.usage.total_tokens, attributes)
+ metrics["genai_prompt_tokens"].add(response.usage.prompt_tokens, attributes)
+ metrics["genai_cost"].record(cost, attributes)
+
+ # Return original response
+ return response
+
+ except Exception as e:
+ handle_exception(span, e)
+ logger.error("Error in trace creation: %s", e)
+
+ # Return original response
+ return response
+
+ return wrapper
+
+def azure_async_image_generate(gen_ai_endpoint, version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics):
+ """
+ Generates a telemetry wrapper for image generation to collect metrics.
+
+ Args:
+ gen_ai_endpoint: Endpoint identifier for logging and tracing.
+ version: Version of the monitoring package.
+ environment: Deployment environment (e.g., production, staging).
+ application_name: Name of the application using the OpenAI API.
+ tracer: OpenTelemetry tracer for creating spans.
+ pricing_info: Information used for calculating the cost of OpenAI image generation.
+ trace_content: Flag indicating whether to trace the input prompt and generated images.
+
+ Returns:
+ A function that wraps the image generation method to add telemetry.
+ """
+
+ async def wrapper(wrapped, instance, args, kwargs):
+ """
+ Wraps the 'images.generate' API call to add telemetry.
+
+ This collects metrics such as execution time, cost, and handles errors
+ gracefully, adding details to the trace for observability.
+
+ Args:
+ wrapped: The original 'images.generate' method to be wrapped.
+ instance: The instance of the class where the original method is defined.
+ args: Positional arguments for the 'images.generate' method.
+ kwargs: Keyword arguments for the 'images.generate' method.
+
+ Returns:
+ The response from the original 'images.generate' method.
+ """
+
+ with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
+ response = await wrapped(*args, **kwargs)
+ images_count = 0
+
+ try:
+ # Find Image format
+ if "response_format" in kwargs and kwargs["response_format"] == "b64_json":
+ image = "b64_json"
+ else:
+ image = "url"
+
+ # Calculate cost of the operation
+ cost = get_image_model_cost("azure_" + kwargs.get("model", "dall-e-3"),
+ pricing_info, kwargs.get("size", "1024x1024"),
+ kwargs.get("quality", "standard"))
+
+ for items in response.data:
+ # Set Span attributes
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_AZURE_OPENAI)
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
+ SemanticConvetion.GEN_AI_TYPE_IMAGE)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
+ gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,
+ response.created)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,
+ environment)
+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
+ application_name)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ "azure_" + kwargs.get("model", "dall-e-3"))
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_IMAGE_SIZE,
+ kwargs.get("size", "1024x1024"))
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_IMAGE_QUALITY,
+ kwargs.get("quality", "standard"))
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_IMAGE_STYLE,
+ kwargs.get("style", "vivid"))
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_REVISED_PROMPT,
+ items.revised_prompt if response.revised_prompt else "")
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_USER,
+ kwargs.get("user", ""))
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,
+ kwargs.get("prompt", ""))
+
+ attribute_name = f"gen_ai.response.image.{images_count}"
+ span.set_attribute(attribute_name,
+ getattr(items, image))
+
+ images_count+=1
+
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ len(response.data) * cost)
+ span.set_status(Status(StatusCode.OK))
+
+ if disable_metrics is False:
+ attributes = {
+ TELEMETRY_SDK_NAME:
+ "openlit",
+ SemanticConvetion.GEN_AI_APPLICATION_NAME:
+ application_name,
+ SemanticConvetion.GEN_AI_SYSTEM:
+ SemanticConvetion.GEN_AI_SYSTEM_AZURE_OPENAI,
+ SemanticConvetion.GEN_AI_ENVIRONMENT:
+ environment,
+ SemanticConvetion.GEN_AI_TYPE:
+ SemanticConvetion.GEN_AI_TYPE_IMAGE,
+ SemanticConvetion.GEN_AI_REQUEST_MODEL:
+ "azure_" + kwargs.get("model", "dall-e-3")
+ }
+
+ metrics["genai_requests"].add(1, attributes)
+ metrics["genai_cost"].record(cost, attributes)
+
+ # Return original response
+ return response
+
+ except Exception as e:
+ handle_exception(span, e)
+ logger.error("Error in trace creation: %s", e)
+
+ # Return original response
+ return response
+
+ return wrapper
diff --git a/sdk/python/src/openlit/instrumentation/openai/async_openai.py b/sdk/python/src/openlit/instrumentation/openai/async_openai.py
new file mode 100644
index 00000000..480390f4
--- /dev/null
+++ b/sdk/python/src/openlit/instrumentation/openai/async_openai.py
@@ -0,0 +1,875 @@
+# pylint: disable=duplicate-code, broad-exception-caught, too-many-statements, unused-argument, too-many-branches
+"""
+Module for monitoring OpenAI API calls.
+"""
+
+import logging
+from opentelemetry.trace import SpanKind, Status, StatusCode
+from opentelemetry.sdk.resources import TELEMETRY_SDK_NAME
+from openlit.__helpers import get_chat_model_cost, get_embed_model_cost, get_audio_model_cost
+from openlit.__helpers import get_image_model_cost, openai_tokens, handle_exception
+from openlit.semcov import SemanticConvetion
+
+# Initialize logger for logging potential issues and operations
+logger = logging.getLogger(__name__)
+
+def async_chat_completions(gen_ai_endpoint, version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics):
+ """
+ Generates a telemetry wrapper for chat completions to collect metrics.
+
+ Args:
+ gen_ai_endpoint: Endpoint identifier for logging and tracing.
+ version: Version of the monitoring package.
+ environment: Deployment environment (e.g., production, staging).
+ application_name: Name of the application using the OpenAI API.
+ tracer: OpenTelemetry tracer for creating spans.
+ pricing_info: Information used for calculating the cost of OpenAI usage.
+ trace_content: Flag indicating whether to trace the actual content.
+
+ Returns:
+ A function that wraps the chat completions method to add telemetry.
+ """
+
+ async def wrapper(wrapped, instance, args, kwargs):
+ """
+ Wraps the 'chat.completions' API call to add telemetry.
+
+ This collects metrics such as execution time, cost, and token usage, and handles errors
+ gracefully, adding details to the trace for observability.
+
+ Args:
+ wrapped: The original 'chat.completions' method to be wrapped.
+ instance: The instance of the class where the original method is defined.
+ args: Positional arguments for the 'chat.completions' method.
+ kwargs: Keyword arguments for the 'chat.completions' method.
+
+ Returns:
+ The response from the original 'chat.completions' method.
+ """
+
+ # Check if streaming is enabled for the API call
+ streaming = kwargs.get("stream", False)
+
+ # pylint: disable=no-else-return
+ if streaming:
+ # Special handling for streaming response to accommodate the nature of data flow
+ async def stream_generator():
+ with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
+ # Placeholder for aggregating streaming response
+ llmresponse = ""
+
+ # Loop through streaming events capturing relevant details
+ async for chunk in await wrapped(*args, **kwargs):
+ # Collect message IDs and aggregated response from events
+ if len(chunk.choices) > 0:
+ # pylint: disable=line-too-long
+ if hasattr(chunk.choices[0], "delta") and hasattr(chunk.choices[0].delta, "content"):
+ content = chunk.choices[0].delta.content
+ if content:
+ llmresponse += content
+ yield chunk
+ response_id = chunk.id
+
+ # Handling exception ensure observability without disrupting operation
+ try:
+ # Format 'messages' into a single string
+ message_prompt = kwargs.get("messages", "")
+ formatted_messages = []
+ for message in message_prompt:
+ role = message["role"]
+ content = message["content"]
+
+ if isinstance(content, list):
+ content_str = ", ".join(
+ # pylint: disable=line-too-long
+ f'{item["type"]}: {item["text"] if "text" in item else item["image_url"]}'
+ if "type" in item else f'text: {item["text"]}'
+ for item in content
+ )
+ formatted_messages.append(f"{role}: {content_str}")
+ else:
+ formatted_messages.append(f"{role}: {content}")
+ prompt = "\n".join(formatted_messages)
+
+ # Calculate tokens using input prompt and aggregated response
+ prompt_tokens = openai_tokens(prompt,
+ kwargs.get("model", "gpt-3.5-turbo"))
+ completion_tokens = openai_tokens(llmresponse,
+ kwargs.get("model", "gpt-3.5-turbo"))
+
+ # Calculate cost of the operation
+ cost = get_chat_model_cost(kwargs.get("model", "gpt-3.5-turbo"),
+ pricing_info, prompt_tokens,
+ completion_tokens)
+
+ # Set Span attributes
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_OPENAI)
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
+ SemanticConvetion.GEN_AI_TYPE_CHAT)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
+ gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,
+ response_id)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,
+ environment)
+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
+ application_name)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ kwargs.get("model", "gpt-3.5-turbo"))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_USER,
+ kwargs.get("user", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,
+ kwargs.get("top_p", 1))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS,
+ kwargs.get("max_tokens", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,
+ kwargs.get("temperature", 1))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_PRESENCE_PENALTY,
+ kwargs.get("presence_penalty", 0))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_FREQUENCY_PENALTY,
+ kwargs.get("frequency_penalty", 0))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_SEED,
+ kwargs.get("seed", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,
+ True)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
+ prompt_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,
+ completion_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ prompt_tokens + completion_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,
+ prompt)
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_COMPLETION,
+ llmresponse)
+
+ span.set_status(Status(StatusCode.OK))
+
+ if disable_metrics is False:
+ attributes = {
+ TELEMETRY_SDK_NAME:
+ "openlit",
+ SemanticConvetion.GEN_AI_APPLICATION_NAME:
+ application_name,
+ SemanticConvetion.GEN_AI_SYSTEM:
+ SemanticConvetion.GEN_AI_SYSTEM_OPENAI,
+ SemanticConvetion.GEN_AI_ENVIRONMENT:
+ environment,
+ SemanticConvetion.GEN_AI_TYPE:
+ SemanticConvetion.GEN_AI_TYPE_CHAT,
+ SemanticConvetion.GEN_AI_REQUEST_MODEL:
+ kwargs.get("model", "gpt-3.5-turbo")
+ }
+
+ metrics["genai_requests"].add(1, attributes)
+ metrics["genai_total_tokens"].add(
+ prompt_tokens + completion_tokens, attributes
+ )
+ metrics["genai_completion_tokens"].add(completion_tokens, attributes)
+ metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)
+ metrics["genai_cost"].record(cost, attributes)
+
+ except Exception as e:
+ handle_exception(span, e)
+ logger.error("Error in trace creation: %s", e)
+
+ return stream_generator()
+
+ # Handling for non-streaming responses
+ else:
+ # pylint: disable=line-too-long
+ with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
+ response = await wrapped(*args, **kwargs)
+
+ try:
+ # Format 'messages' into a single string
+ message_prompt = kwargs.get("messages", "")
+ formatted_messages = []
+ for message in message_prompt:
+ role = message["role"]
+ content = message["content"]
+
+ if isinstance(content, list):
+ content_str = ", ".join(
+ # pylint: disable=line-too-long
+ f'{item["type"]}: {item["text"] if "text" in item else item["image_url"]}'
+ if "type" in item else f'text: {item["text"]}'
+ for item in content
+ )
+ formatted_messages.append(f"{role}: {content_str}")
+ else:
+ formatted_messages.append(f"{role}: {content}")
+ prompt = "\n".join(formatted_messages)
+
+ # Set base span attribues
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_OPENAI)
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
+ SemanticConvetion.GEN_AI_TYPE_CHAT)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
+ gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,
+ response.id)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,
+ environment)
+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
+ application_name)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ kwargs.get("model", "gpt-3.5-turbo"))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,
+ kwargs.get("top_p", 1))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS,
+ kwargs.get("max_tokens", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_USER,
+ kwargs.get("user", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,
+ kwargs.get("temperature", 1))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_PRESENCE_PENALTY,
+ kwargs.get("presence_penalty", 0))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_FREQUENCY_PENALTY,
+ kwargs.get("frequency_penalty", 0))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_SEED,
+ kwargs.get("seed", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,
+ False)
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,
+ prompt)
+
+ span.set_status(Status(StatusCode.OK))
+
+ # Set span attributes when tools is not passed to the function call
+ if "tools" not in kwargs:
+ # Calculate cost of the operation
+ cost = get_chat_model_cost(kwargs.get("model", "gpt-3.5-turbo"),
+ pricing_info, response.usage.prompt_tokens,
+ response.usage.completion_tokens)
+
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
+ response.usage.prompt_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,
+ response.usage.completion_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ response.usage.total_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,
+ response.choices[0].finish_reason)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+
+ # Set span attributes for when n = 1 (default)
+ if "n" not in kwargs or kwargs["n"] == 1:
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_COMPLETION,
+ response.choices[0].message.content)
+
+ # Set span attributes for when n > 0
+ else:
+ i = 0
+ while i < kwargs["n"] and trace_content is True:
+ attribute_name = f"gen_ai.content.completion.{i}"
+ span.set_attribute(attribute_name,
+ response.choices[i].message.content)
+ i += 1
+
+ # Return original response
+ return response
+
+ # Set span attributes when tools is passed to the function call
+ elif "tools" in kwargs:
+ # Calculate cost of the operation
+ cost = get_chat_model_cost(kwargs.get("model", "gpt-3.5-turbo"),
+ pricing_info, response.usage.prompt_tokens,
+ response.usage.completion_tokens)
+
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_COMPLETION,
+ "Function called with tools")
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
+ response.usage.prompt_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,
+ response.usage.completion_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ response.usage.total_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+
+ span.set_status(Status(StatusCode.OK))
+
+ if disable_metrics is False:
+ attributes = {
+ TELEMETRY_SDK_NAME:
+ "openlit",
+ SemanticConvetion.GEN_AI_APPLICATION_NAME:
+ application_name,
+ SemanticConvetion.GEN_AI_SYSTEM:
+ SemanticConvetion.GEN_AI_SYSTEM_OPENAI,
+ SemanticConvetion.GEN_AI_ENVIRONMENT:
+ environment,
+ SemanticConvetion.GEN_AI_TYPE:
+ SemanticConvetion.GEN_AI_TYPE_CHAT,
+ SemanticConvetion.GEN_AI_REQUEST_MODEL:
+ kwargs.get("model", "gpt-3.5-turbo")
+ }
+
+ metrics["genai_requests"].add(1, attributes)
+ metrics["genai_total_tokens"].add(response.usage.total_tokens, attributes)
+ metrics["genai_completion_tokens"].add(response.usage.completion_tokens, attributes)
+ metrics["genai_prompt_tokens"].add(response.usage.prompt_tokens, attributes)
+ metrics["genai_cost"].record(cost, attributes)
+
+ # Return original response
+ return response
+
+ except Exception as e:
+ handle_exception(span, e)
+ logger.error("Error in trace creation: %s", e)
+
+ # Return original response
+ return response
+
+ return wrapper
+
+def async_embedding(gen_ai_endpoint, version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics):
+ """
+ Generates a telemetry wrapper for embeddings to collect metrics.
+
+ Args:
+ gen_ai_endpoint: Endpoint identifier for logging and tracing.
+ version: Version of the monitoring package.
+ environment: Deployment environment (e.g., production, staging).
+ application_name: Name of the application using the OpenAI API.
+ tracer: OpenTelemetry tracer for creating spans.
+ pricing_info: Information used for calculating the cost of OpenAI usage.
+ trace_content: Flag indicating whether to trace the actual content.
+
+ Returns:
+ A function that wraps the embeddings method to add telemetry.
+ """
+
+ async def wrapper(wrapped, instance, args, kwargs):
+ """
+ Wraps the 'embeddings' API call to add telemetry.
+
+ This collects metrics such as execution time, cost, and token usage, and handles errors
+ gracefully, adding details to the trace for observability.
+
+ Args:
+ wrapped: The original 'embeddings' method to be wrapped.
+ instance: The instance of the class where the original method is defined.
+ args: Positional arguments for the 'embeddings' method.
+ kwargs: Keyword arguments for the 'embeddings' method.
+
+ Returns:
+ The response from the original 'embeddings' method.
+ """
+
+ with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
+ response = await wrapped(*args, **kwargs)
+
+ try:
+ # Calculate cost of the operation
+ cost = get_embed_model_cost(kwargs.get("model", "text-embedding-ada-002"),
+ pricing_info, response.usage.prompt_tokens)
+
+ # Set Span attributes
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_OPENAI)
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
+ SemanticConvetion.GEN_AI_TYPE_EMBEDDING)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
+ gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,
+ environment)
+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
+ application_name)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ kwargs.get("model", "text-embedding-ada-002"))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_EMBEDDING_FORMAT,
+ kwargs.get("encoding_format", "float"))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_EMBEDDING_DIMENSION,
+ kwargs.get("dimensions", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_USER,
+ kwargs.get("user", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
+ response.usage.prompt_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ response.usage.total_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,
+ kwargs.get("input", ""))
+
+ span.set_status(Status(StatusCode.OK))
+
+ if disable_metrics is False:
+ attributes = {
+ TELEMETRY_SDK_NAME:
+ "openlit",
+ SemanticConvetion.GEN_AI_APPLICATION_NAME:
+ application_name,
+ SemanticConvetion.GEN_AI_SYSTEM:
+ SemanticConvetion.GEN_AI_SYSTEM_OPENAI,
+ SemanticConvetion.GEN_AI_ENVIRONMENT:
+ environment,
+ SemanticConvetion.GEN_AI_TYPE:
+ SemanticConvetion.GEN_AI_TYPE_EMBEDDING,
+ SemanticConvetion.GEN_AI_REQUEST_MODEL:
+ kwargs.get("model", "text-embedding-ada-002")
+ }
+
+ metrics["genai_requests"].add(1, attributes)
+ metrics["genai_total_tokens"].add(response.usage.total_tokens, attributes)
+ metrics["genai_prompt_tokens"].add(response.usage.prompt_tokens, attributes)
+ metrics["genai_cost"].record(cost, attributes)
+
+ # Return original response
+ return response
+
+ except Exception as e:
+ handle_exception(span, e)
+ logger.error("Error in trace creation: %s", e)
+
+ # Return original response
+ return response
+
+ return wrapper
+
+def async_finetune(gen_ai_endpoint, version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics):
+ """
+ Generates a telemetry wrapper for fine-tuning jobs to collect metrics.
+
+ Args:
+ gen_ai_endpoint: Endpoint identifier for logging and tracing.
+ version: Version of the monitoring package.
+ environment: Deployment environment (e.g., production, staging).
+ application_name: Name of the application using the OpenAI API.
+ tracer: OpenTelemetry tracer for creating spans.
+ pricing_info: Information used for calculating the cost of OpenAI usage.
+ trace_content: Flag indicating whether to trace the actual content.
+
+ Returns:
+ A function that wraps the fine tuning creation method to add telemetry.
+ """
+
+ async def wrapper(wrapped, instance, args, kwargs):
+ """
+ Wraps the 'fine_tuning.jobs.create' API call to add telemetry.
+
+ This collects metrics such as execution time, usage stats, and handles errors
+ gracefully, adding details to the trace for observability.
+
+ Args:
+ wrapped: The original 'fine_tuning.jobs.create' method to be wrapped.
+ instance: The instance of the class where the original method is defined.
+ args: Positional arguments for the method.
+ kwargs: Keyword arguments for the method.
+
+ Returns:
+ The response from the original 'fine_tuning.jobs.create' method.
+ """
+
+ with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
+ response = await wrapped(*args, **kwargs)
+
+ try:
+ # Set Span attributes
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_OPENAI)
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
+ "fine_tuning")
+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
+ gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,
+ environment)
+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
+ application_name)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ kwargs.get("model", "gpt-3.5-turbo"))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TRAINING_FILE,
+ kwargs.get("training_file", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_VALIDATION_FILE,
+ kwargs.get("validation_file", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_FINETUNE_BATCH_SIZE,
+ kwargs.get("hyperparameters.batch_size", "auto"))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_FINETUNE_MODEL_LRM,
+ kwargs.get("hyperparameters.learning_rate_multiplier",
+ "auto"))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_FINETUNE_MODEL_EPOCHS,
+ kwargs.get("hyperparameters.n_epochs", "auto"))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_FINETUNE_MODEL_SUFFIX,
+ kwargs.get("suffix", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,
+ response.id)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
+ response.usage.prompt_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_FINETUNE_STATUS,
+ response.status)
+ span.set_status(Status(StatusCode.OK))
+
+ if disable_metrics is False:
+ metrics["genai_requests"].add(1)
+
+ # Return original response
+ return response
+
+ except Exception as e:
+ handle_exception(span, e)
+ logger.error("Error in trace creation: %s", e)
+
+ # Return original response
+ return response
+
+ return wrapper
+
+def async_image_generate(gen_ai_endpoint, version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics):
+ """
+ Generates a telemetry wrapper for image generation to collect metrics.
+
+ Args:
+ gen_ai_endpoint: Endpoint identifier for logging and tracing.
+ version: Version of the monitoring package.
+ environment: Deployment environment (e.g., production, staging).
+ application_name: Name of the application using the OpenAI API.
+ tracer: OpenTelemetry tracer for creating spans.
+ pricing_info: Information used for calculating the cost of OpenAI image generation.
+ trace_content: Flag indicating whether to trace the input prompt and generated images.
+
+ Returns:
+ A function that wraps the image generation method to add telemetry.
+ """
+
+ async def wrapper(wrapped, instance, args, kwargs):
+ """
+ Wraps the 'images.generate' API call to add telemetry.
+
+ This collects metrics such as execution time, cost, and handles errors
+ gracefully, adding details to the trace for observability.
+
+ Args:
+ wrapped: The original 'images.generate' method to be wrapped.
+ instance: The instance of the class where the original method is defined.
+ args: Positional arguments for the 'images.generate' method.
+ kwargs: Keyword arguments for the 'images.generate' method.
+
+ Returns:
+ The response from the original 'images.generate' method.
+ """
+
+ with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
+ response = await wrapped(*args, **kwargs)
+ images_count = 0
+
+ try:
+ # Find Image format
+ if "response_format" in kwargs and kwargs["response_format"] == "b64_json":
+ image = "b64_json"
+ else:
+ image = "url"
+
+ # Calculate cost of the operation
+ cost = get_image_model_cost(kwargs.get("model", "dall-e-2"),
+ pricing_info, kwargs.get("size", "1024x1024"),
+ kwargs.get("quality", "standard"))
+
+ for items in response.data:
+ # Set Span attributes
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_OPENAI)
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
+ SemanticConvetion.GEN_AI_TYPE_IMAGE)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
+ gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,
+ response.created)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,
+ environment)
+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
+ application_name)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ kwargs.get("model", "dall-e-2"))
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_IMAGE_SIZE,
+ kwargs.get("size", "1024x1024"))
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_IMAGE_QUALITY,
+ kwargs.get("quality", "standard"))
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_IMAGE_STYLE,
+ kwargs.get("style", "vivid"))
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_REVISED_PROMPT,
+ items.revised_prompt if items.revised_prompt else "")
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_USER,
+ kwargs.get("user", ""))
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,
+ kwargs.get("prompt", ""))
+
+ attribute_name = f"gen_ai.response.image.{images_count}"
+ span.set_attribute(attribute_name,
+ getattr(items, image))
+
+ images_count+=1
+
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ len(response.data) * cost)
+ span.set_status(Status(StatusCode.OK))
+
+ if disable_metrics is False:
+ attributes = {
+ TELEMETRY_SDK_NAME:
+ "openlit",
+ SemanticConvetion.GEN_AI_APPLICATION_NAME:
+ application_name,
+ SemanticConvetion.GEN_AI_SYSTEM:
+ SemanticConvetion.GEN_AI_SYSTEM_OPENAI,
+ SemanticConvetion.GEN_AI_ENVIRONMENT:
+ environment,
+ SemanticConvetion.GEN_AI_TYPE:
+ SemanticConvetion.GEN_AI_TYPE_IMAGE,
+ SemanticConvetion.GEN_AI_REQUEST_MODEL:
+ kwargs.get("model", "dall-e-2")
+ }
+
+ metrics["genai_requests"].add(1, attributes)
+ metrics["genai_cost"].record(cost, attributes)
+
+ # Return original response
+ return response
+
+ except Exception as e:
+ handle_exception(span, e)
+ logger.error("Error in trace creation: %s", e)
+
+ # Return original response
+ return response
+
+ return wrapper
+
+def async_image_variatons(gen_ai_endpoint, version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics):
+ """
+ Generates a telemetry wrapper for creating image variations to collect metrics.
+
+ Args:
+ gen_ai_endpoint: Endpoint identifier for logging and tracing.
+ version: Version of the monitoring package.
+ environment: Deployment environment (e.g., production, staging).
+ application_name: Name of the application using the OpenAI API.
+ tracer: OpenTelemetry tracer for creating spans.
+ pricing_info: Information used for calculating the cost of generating image variations.
+ trace_content: Flag indicating whether to trace the input image and generated variations.
+
+ Returns:
+ A function that wraps the image variations creation method to add telemetry.
+ """
+
+ async def wrapper(wrapped, instance, args, kwargs):
+ """
+ Wraps the 'images.create.variations' API call to add telemetry.
+
+ This collects metrics such as execution time, cost, and handles errors
+ gracefully, adding details to the trace for observability.
+
+ Args:
+ wrapped: The original 'images.create.variations' method to be wrapped.
+ instance: The instance of the class where the original method is defined.
+ args: Positional arguments for the method.
+ kwargs: Keyword arguments for the method.
+
+ Returns:
+ The response from the original 'images.create.variations' method.
+ """
+
+ with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
+ response = await wrapped(*args, **kwargs)
+ images_count = 0
+
+ try:
+ # Find Image format
+ if "response_format" in kwargs and kwargs["response_format"] == "b64_json":
+ image = "b64_json"
+ else:
+ image = "url"
+
+ # Calculate cost of the operation
+ cost = get_image_model_cost(kwargs.get("model", "dall-e-2"), pricing_info,
+ kwargs.get("size", "1024x1024"), "standard")
+
+ for items in response.data:
+ # Set Span attributes
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_OPENAI)
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
+ SemanticConvetion.GEN_AI_TYPE_IMAGE)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
+ gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,
+ response.created)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,
+ environment)
+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
+ application_name)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ kwargs.get("model", "dall-e-2"))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_USER,
+ kwargs.get("user", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_IMAGE_SIZE,
+ kwargs.get("size", "1024x1024"))
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_IMAGE_QUALITY,
+ "standard")
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,
+ kwargs.get(SemanticConvetion.GEN_AI_TYPE_IMAGE, ""))
+
+ attribute_name = f"gen_ai.response.image.{images_count}"
+ span.set_attribute(attribute_name, getattr(items, image))
+
+ images_count+=1
+
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ len(response.data) * cost)
+ span.set_status(Status(StatusCode.OK))
+
+ if disable_metrics is False:
+ attributes = {
+ TELEMETRY_SDK_NAME:
+ "openlit",
+ SemanticConvetion.GEN_AI_APPLICATION_NAME:
+ application_name,
+ SemanticConvetion.GEN_AI_SYSTEM:
+ SemanticConvetion.GEN_AI_SYSTEM_OPENAI,
+ SemanticConvetion.GEN_AI_ENVIRONMENT:
+ environment,
+ SemanticConvetion.GEN_AI_TYPE:
+ SemanticConvetion.GEN_AI_TYPE_IMAGE,
+ SemanticConvetion.GEN_AI_REQUEST_MODEL:
+ kwargs.get("model", "dall-e-2")
+ }
+
+ metrics["genai_requests"].add(1, attributes)
+ metrics["genai_cost"].record(cost, attributes)
+
+ # Return original response
+ return response
+
+ except Exception as e:
+ handle_exception(span, e)
+ logger.error("Error in trace creation: %s", e)
+
+ # Return original response
+ return response
+
+ return wrapper
+
+def async_audio_create(gen_ai_endpoint, version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics):
+ """
+ Generates a telemetry wrapper for creating speech audio to collect metrics.
+
+ Args:
+ gen_ai_endpoint: Endpoint identifier for logging and tracing.
+ version: Version of the monitoring package.
+ environment: Deployment environment (e.g., production, staging).
+ application_name: Name of the application using the OpenAI API.
+ tracer: OpenTelemetry tracer for creating spans.
+ pricing_info: Information used for calculating the cost of generating speech audio.
+ trace_content: Flag indicating whether to trace the input text and generated audio.
+
+ Returns:
+ A function that wraps the speech audio creation method to add telemetry.
+ """
+
+ async def wrapper(wrapped, instance, args, kwargs):
+ """
+ Wraps the 'audio.speech.create' API call to add telemetry.
+
+ This collects metrics such as execution time, cost, and handles errors
+ gracefully, adding details to the trace for observability.
+
+ Args:
+ wrapped: The original 'audio.speech.create' method to be wrapped.
+ instance: The instance of the class where the original method is defined.
+ args: Positional arguments for the 'audio.speech.create' method.
+ kwargs: Keyword arguments for the 'audio.speech.create' method.
+
+ Returns:
+ The response from the original 'audio.speech.create' method.
+ """
+
+ with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
+ response = await wrapped(*args, **kwargs)
+
+ try:
+ # Calculate cost of the operation
+ cost = get_audio_model_cost(kwargs.get("model", "tts-1"),
+ pricing_info, kwargs.get("input", ""))
+
+ # Set Span attributes
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_OPENAI)
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
+ SemanticConvetion.GEN_AI_TYPE_AUDIO)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
+ gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,
+ environment)
+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
+ application_name)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ kwargs.get("model", "tts-1"))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_AUDIO_VOICE,
+ kwargs.get("voice", "alloy"))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_AUDIO_RESPONSE_FORMAT,
+ kwargs.get("response_format", "mp3"))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_AUDIO_SPEED,
+ kwargs.get("speed", 1))
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,
+ kwargs.get("input", ""))
+
+ span.set_status(Status(StatusCode.OK))
+
+ if disable_metrics is False:
+ attributes = {
+ TELEMETRY_SDK_NAME:
+ "openlit",
+ SemanticConvetion.GEN_AI_APPLICATION_NAME:
+ application_name,
+ SemanticConvetion.GEN_AI_SYSTEM:
+ SemanticConvetion.GEN_AI_SYSTEM_OPENAI,
+ SemanticConvetion.GEN_AI_ENVIRONMENT:
+ environment,
+ SemanticConvetion.GEN_AI_TYPE:
+ SemanticConvetion.GEN_AI_TYPE_AUDIO,
+ SemanticConvetion.GEN_AI_REQUEST_MODEL:
+ kwargs.get("model", "tts-1")
+ }
+
+ metrics["genai_requests"].add(1, attributes)
+ metrics["genai_cost"].record(cost, attributes)
+
+ # Return original response
+ return response
+
+ except Exception as e:
+ handle_exception(span, e)
+ logger.error("Error in trace creation: %s", e)
+
+ # Return original response
+ return response
+
+ return wrapper
diff --git a/sdk/python/src/openlit/instrumentation/openai/azure_openai.py b/sdk/python/src/openlit/instrumentation/openai/azure_openai.py
new file mode 100644
index 00000000..40dd6a3f
--- /dev/null
+++ b/sdk/python/src/openlit/instrumentation/openai/azure_openai.py
@@ -0,0 +1,840 @@
+# pylint: disable=duplicate-code, broad-exception-caught, too-many-statements, unused-argument, too-many-branches
+"""
+Module for monitoring Azure OpenAI API calls.
+"""
+
+import logging
+from opentelemetry.trace import SpanKind, Status, StatusCode
+from opentelemetry.sdk.resources import TELEMETRY_SDK_NAME
+from openlit.__helpers import get_chat_model_cost, get_embed_model_cost
+from openlit.__helpers import get_image_model_cost, openai_tokens, handle_exception
+from openlit.semcov import SemanticConvetion
+
+# Initialize logger for logging potential issues and operations
+logger = logging.getLogger(__name__)
+
+def azure_chat_completions(gen_ai_endpoint, version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics):
+ """
+ Generates a telemetry wrapper for chat completions to collect metrics.
+
+ Args:
+ gen_ai_endpoint: Endpoint identifier for logging and tracing.
+ version: Version of the monitoring package.
+ environment: Deployment environment (e.g., production, staging).
+ application_name: Name of the application using the OpenAI API.
+ tracer: OpenTelemetry tracer for creating spans.
+ pricing_info: Information used for calculating the cost of OpenAI usage.
+ trace_content: Flag indicating whether to trace the actual content.
+
+ Returns:
+ A function that wraps the chat completions method to add telemetry.
+ """
+
+ def wrapper(wrapped, instance, args, kwargs):
+ """
+ Wraps the 'chat.completions' API call to add telemetry.
+
+ This collects metrics such as execution time, cost, and token usage, and handles errors
+ gracefully, adding details to the trace for observability.
+
+ Args:
+ wrapped: The original 'chat.completions' method to be wrapped.
+ instance: The instance of the class where the original method is defined.
+ args: Positional arguments for the 'chat.completions' method.
+ kwargs: Keyword arguments for the 'chat.completions' method.
+
+ Returns:
+ The response from the original 'chat.completions' method.
+ """
+
+ # Check if streaming is enabled for the API call
+ streaming = kwargs.get("stream", False)
+
+ # pylint: disable=no-else-return
+ if streaming:
+ # Special handling for streaming response to accommodate the nature of data flow
+ def stream_generator():
+ # pylint: disable=line-too-long
+ with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
+ # Placeholder for aggregating streaming response
+ llmresponse = ""
+
+ # Loop through streaming events capturing relevant details
+ for chunk in wrapped(*args, **kwargs):
+ # Collect message IDs and aggregated response from events
+ if len(chunk.choices) > 0:
+ # pylint: disable=line-too-long
+ if hasattr(chunk.choices[0], "delta") and hasattr(chunk.choices[0].delta, "content"):
+ content = chunk.choices[0].delta.content
+ if content:
+ llmresponse += content
+ yield chunk
+ response_id = chunk.id
+ model = "azure_" + chunk.model
+
+ # Handling exception ensure observability without disrupting operation
+ try:
+ # Format 'messages' into a single string
+ message_prompt = kwargs.get("messages", "")
+ formatted_messages = []
+ for message in message_prompt:
+ role = message["role"]
+ content = message["content"]
+
+ if isinstance(content, list):
+ content_str = ", ".join(
+ # pylint: disable=line-too-long
+ f'{item["type"]}: {item["text"] if "text" in item else item["image_url"]}'
+ if "type" in item else f'text: {item["text"]}'
+ for item in content
+ )
+ formatted_messages.append(f"{role}: {content_str}")
+ else:
+ formatted_messages.append(f"{role}: {content}")
+ prompt = "\n".join(formatted_messages)
+
+ # Calculate tokens using input prompt and aggregated response
+ prompt_tokens = openai_tokens(prompt,
+ kwargs.get("model", "gpt-3.5-turbo"))
+ completion_tokens = openai_tokens(llmresponse,
+ kwargs.get("model", "gpt-3.5-turbo"))
+
+ # Calculate cost of the operation
+ cost = get_chat_model_cost(model, pricing_info,
+ prompt_tokens, completion_tokens)
+
+ # Set Span attributes
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_AZURE_OPENAI)
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
+ SemanticConvetion.GEN_AI_TYPE_CHAT)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
+ gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,
+ response_id)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,
+ environment)
+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
+ application_name)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ model)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_USER,
+ kwargs.get("user", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOOL_CHOICE,
+ kwargs.get("tool_choice", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,
+ kwargs.get("temperature", 1))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_PRESENCE_PENALTY,
+ kwargs.get("presence_penalty", 0))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_FREQUENCY_PENALTY,
+ kwargs.get("frequency_penalty", 0))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_SEED,
+ kwargs.get("seed", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,
+ True)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
+ prompt_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,
+ completion_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ prompt_tokens + completion_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,
+ prompt)
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_COMPLETION,
+ llmresponse)
+
+ span.set_status(Status(StatusCode.OK))
+
+ if disable_metrics is False:
+ attributes = {
+ TELEMETRY_SDK_NAME:
+ "openlit",
+ SemanticConvetion.GEN_AI_APPLICATION_NAME:
+ application_name,
+ SemanticConvetion.GEN_AI_SYSTEM:
+ SemanticConvetion.GEN_AI_SYSTEM_AZURE_OPENAI,
+ SemanticConvetion.GEN_AI_ENVIRONMENT:
+ environment,
+ SemanticConvetion.GEN_AI_TYPE:
+ SemanticConvetion.GEN_AI_TYPE_CHAT,
+ SemanticConvetion.GEN_AI_REQUEST_MODEL:
+ model
+ }
+
+ metrics["genai_requests"].add(1, attributes)
+ metrics["genai_total_tokens"].add(prompt_tokens + completion_tokens, attributes)
+ metrics["genai_completion_tokens"].add(completion_tokens, attributes)
+ metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)
+ metrics["genai_cost"].record(cost, attributes)
+
+ except Exception as e:
+ handle_exception(span, e)
+ logger.error("Error in trace creation: %s", e)
+
+ return stream_generator()
+
+ # Handling for non-streaming responses
+ else:
+ # pylint: disable=line-too-long
+ with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
+ response = wrapped(*args, **kwargs)
+
+ try:
+ # Find base model from response
+ model = "azure_" + response.model
+
+ # Format 'messages' into a single string
+ message_prompt = kwargs.get("messages", "")
+ formatted_messages = []
+ for message in message_prompt:
+ role = message["role"]
+ content = message["content"]
+
+ if isinstance(content, list):
+ content_str = ", ".join(
+ # pylint: disable=line-too-long
+ f'{item["type"]}: {item["text"] if "text" in item else item["image_url"]}'
+ if "type" in item else f'text: {item["text"]}'
+ for item in content
+ )
+ formatted_messages.append(f"{role}: {content_str}")
+ else:
+ formatted_messages.append(f"{role}: {content}")
+ prompt = "\n".join(formatted_messages)
+
+ # Set base span attribues
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_AZURE_OPENAI)
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
+ SemanticConvetion.GEN_AI_TYPE_CHAT)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
+ gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,
+ response.id)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,
+ environment)
+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
+ application_name)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ model)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_USER,
+ kwargs.get("user", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOOL_CHOICE,
+ kwargs.get("tool_choice", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,
+ kwargs.get("temperature", 1))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_PRESENCE_PENALTY,
+ kwargs.get("presence_penalty", 0))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_FREQUENCY_PENALTY,
+ kwargs.get("frequency_penalty", 0))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_SEED,
+ kwargs.get("seed", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,
+ False)
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,
+ prompt)
+
+ # Set span attributes when tools is not passed to the function call
+ if "tools" not in kwargs:
+ # Calculate cost of the operation
+ cost = get_chat_model_cost(model, pricing_info,
+ response.usage.prompt_tokens,
+ response.usage.completion_tokens)
+
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
+ response.usage.prompt_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,
+ response.usage.completion_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ response.usage.total_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,
+ response.choices[0].finish_reason)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+
+ # Set span attributes for when n = 1 (default)
+ if "n" not in kwargs or kwargs["n"] == 1:
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_COMPLETION,
+ response.choices[0].message.content)
+
+ # Set span attributes for when n > 0
+ else:
+ i = 0
+ while i < kwargs["n"] and trace_content is True:
+ attribute_name = f"gen_ai.content.completion.{i}"
+ span.set_attribute(attribute_name,
+ response.choices[i].message.content)
+ i += 1
+
+ # Return original response
+ return response
+
+ # Set span attributes when tools is passed to the function call
+ elif "tools" in kwargs:
+ # Calculate cost of the operation
+ cost = get_chat_model_cost(model, pricing_info,
+ response.usage.prompt_tokens,
+ response.usage.completion_tokens)
+
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_COMPLETION,
+ "Function called with tools")
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
+ response.usage.prompt_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,
+ response.usage.completion_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ response.usage.total_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+
+ span.set_status(Status(StatusCode.OK))
+
+ if disable_metrics is False:
+ attributes = {
+ TELEMETRY_SDK_NAME:
+ "openlit",
+ SemanticConvetion.GEN_AI_APPLICATION_NAME:
+ application_name,
+ SemanticConvetion.GEN_AI_SYSTEM:
+ SemanticConvetion.GEN_AI_SYSTEM_AZURE_OPENAI,
+ SemanticConvetion.GEN_AI_ENVIRONMENT:
+ environment,
+ SemanticConvetion.GEN_AI_TYPE:
+ SemanticConvetion.GEN_AI_TYPE_CHAT,
+ SemanticConvetion.GEN_AI_REQUEST_MODEL:
+ model
+ }
+
+ metrics["genai_requests"].add(1, attributes)
+ metrics["genai_total_tokens"].add(response.usage.total_tokens, attributes)
+ metrics["genai_completion_tokens"].add(response.usage.completion_tokens, attributes)
+ metrics["genai_prompt_tokens"].add(response.usage.prompt_tokens, attributes)
+ metrics["genai_cost"].record(cost, attributes)
+
+ # Return original response
+ return response
+
+ except Exception as e:
+ handle_exception(span, e)
+ logger.error("Error in trace creation: %s", e)
+
+ # Return original response
+ return response
+
+ return wrapper
+
+def azure_completions(gen_ai_endpoint, version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics):
+ """
+ Generates a telemetry wrapper for completions to collect metrics.
+
+ Args:
+ gen_ai_endpoint: Endpoint identifier for logging and tracing.
+ version: Version of the monitoring package.
+ environment: Deployment environment (e.g., production, staging).
+ application_name: Name of the application using the OpenAI API.
+ tracer: OpenTelemetry tracer for creating spans.
+ pricing_info: Information used for calculating the cost of OpenAI usage.
+ trace_content: Flag indicating whether to trace the actual content.
+
+ Returns:
+ A function that wraps the chat completions method to add telemetry.
+ """
+
+ def wrapper(wrapped, instance, args, kwargs):
+ """
+ Wraps the 'completions' API call to add telemetry.
+
+ This collects metrics such as execution time, cost, and token usage, and handles errors
+ gracefully, adding details to the trace for observability.
+
+ Args:
+ wrapped: The original 'completions' method to be wrapped.
+ instance: The instance of the class where the original method is defined.
+ args: Positional arguments for the 'completions' method.
+ kwargs: Keyword arguments for the 'completions' method.
+
+ Returns:
+ The response from the original 'chat.completions' method.
+ """
+
+ # Check if streaming is enabled for the API call
+ streaming = kwargs.get("stream", False)
+
+ # pylint: disable=no-else-return
+ if streaming:
+ # Special handling for streaming response to accommodate the nature of data flow
+ def stream_generator():
+ # pylint: disable=line-too-long
+ with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
+ # Placeholder for aggregating streaming response
+ llmresponse = ""
+
+ # Loop through streaming events capturing relevant details
+ for chunk in wrapped(*args, **kwargs):
+ # Collect message IDs and aggregated response from events
+ if len(chunk.choices) > 0:
+ if hasattr(chunk.choices[0], "text"):
+ content = chunk.choices[0].text
+ if content:
+ llmresponse += content
+ yield chunk
+ response_id = chunk.id
+ model = "azure_" + chunk.model
+
+ # Handling exception ensure observability without disrupting operation
+ try:
+ prompt = kwargs.get("prompt", "")
+
+ # Calculate tokens using input prompt and aggregated response
+ prompt_tokens = openai_tokens(prompt, "gpt-3.5-turbo")
+ completion_tokens = openai_tokens(llmresponse, "gpt-3.5-turbo")
+
+ # Calculate cost of the operation
+ cost = get_chat_model_cost(model, pricing_info,
+ prompt_tokens, completion_tokens)
+
+ # Set Span attributes
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_AZURE_OPENAI)
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
+ SemanticConvetion.GEN_AI_TYPE_CHAT)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
+ gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,
+ response_id)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,
+ environment)
+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
+ application_name)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ model)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_USER,
+ kwargs.get("user", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOOL_CHOICE,
+ kwargs.get("tool_choice", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,
+ kwargs.get("temperature", 1))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_PRESENCE_PENALTY,
+ kwargs.get("presence_penalty", 0))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_FREQUENCY_PENALTY,
+ kwargs.get("frequency_penalty", 0))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_SEED,
+ kwargs.get("seed", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,
+ True)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
+ prompt_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,
+ completion_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ prompt_tokens + completion_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,
+ prompt)
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_COMPLETION,
+ llmresponse)
+
+ span.set_status(Status(StatusCode.OK))
+
+ if disable_metrics is False:
+ attributes = {
+ TELEMETRY_SDK_NAME:
+ "openlit",
+ SemanticConvetion.GEN_AI_APPLICATION_NAME:
+ application_name,
+ SemanticConvetion.GEN_AI_SYSTEM:
+ SemanticConvetion.GEN_AI_SYSTEM_AZURE_OPENAI,
+ SemanticConvetion.GEN_AI_ENVIRONMENT:
+ environment,
+ SemanticConvetion.GEN_AI_TYPE:
+ SemanticConvetion.GEN_AI_TYPE_CHAT,
+ SemanticConvetion.GEN_AI_REQUEST_MODEL:
+ model
+ }
+
+ metrics["genai_requests"].add(1, attributes)
+ metrics["genai_total_tokens"].add(prompt_tokens + completion_tokens, attributes)
+ metrics["genai_completion_tokens"].add(completion_tokens, attributes)
+ metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)
+ metrics["genai_cost"].record(cost, attributes)
+
+ except Exception as e:
+ handle_exception(span, e)
+ logger.error("Error in trace creation: %s", e)
+
+ return stream_generator()
+
+ # Handling for non-streaming responses
+ else:
+ # pylint: disable=line-too-long
+ with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
+ response = wrapped(*args, **kwargs)
+
+ try:
+ # Find base model from response
+ model = "azure_" + response.model
+
+ # Set base span attribues
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_AZURE_OPENAI)
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
+ SemanticConvetion.GEN_AI_TYPE_CHAT)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
+ gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,
+ response.id)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,
+ environment)
+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
+ application_name)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ model)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_USER,
+ kwargs.get("user", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOOL_CHOICE,
+ kwargs.get("tool_choice", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,
+ kwargs.get("temperature", 1))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_PRESENCE_PENALTY,
+ kwargs.get("presence_penalty", 0))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_FREQUENCY_PENALTY,
+ kwargs.get("frequency_penalty", 0))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_SEED,
+ kwargs.get("seed", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,
+ False)
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,
+ kwargs.get("prompt", ""))
+
+ # Set span attributes when tools is not passed to the function call
+ if "tools" not in kwargs:
+ # Calculate cost of the operation
+ cost = get_chat_model_cost(model, pricing_info,
+ response.usage.prompt_tokens,
+ response.usage.completion_tokens)
+
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
+ response.usage.prompt_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,
+ response.usage.completion_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ response.usage.total_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,
+ response.choices[0].finish_reason)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+
+ # Set span attributes for when n = 1 (default)
+ if "n" not in kwargs or kwargs["n"] == 1:
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_COMPLETION,
+ response.choices[0].text)
+
+ # Set span attributes for when n > 0
+ else:
+ i = 0
+ while i < kwargs["n"] and trace_content is True:
+ attribute_name = f"gen_ai.content.completion.{i}"
+ span.set_attribute(attribute_name,
+ response.choices[i].text)
+ i += 1
+ return response
+
+ # Set span attributes when tools is passed to the function call
+ elif "tools" in kwargs:
+ # Calculate cost of the operation
+ cost = get_chat_model_cost(model, pricing_info,
+ response.usage.prompt_tokens,
+ response.usage.completion_tokens)
+
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_COMPLETION,
+ "Function called with tools")
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
+ response.usage.prompt_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,
+ response.usage.completion_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ response.usage.total_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+
+ span.set_status(Status(StatusCode.OK))
+
+ if disable_metrics is False:
+ attributes = {
+ TELEMETRY_SDK_NAME:
+ "openlit",
+ SemanticConvetion.GEN_AI_APPLICATION_NAME:
+ application_name,
+ SemanticConvetion.GEN_AI_SYSTEM:
+ SemanticConvetion.GEN_AI_SYSTEM_AZURE_OPENAI,
+ SemanticConvetion.GEN_AI_ENVIRONMENT:
+ environment,
+ SemanticConvetion.GEN_AI_TYPE:
+ SemanticConvetion.GEN_AI_TYPE_CHAT,
+ SemanticConvetion.GEN_AI_REQUEST_MODEL:
+ model
+ }
+
+ metrics["genai_requests"].add(1, attributes)
+ metrics["genai_total_tokens"].add(response.usage.total_tokens, attributes)
+ metrics["genai_completion_tokens"].add(response.usage.completion_tokens, attributes)
+ metrics["genai_prompt_tokens"].add(response.usage.prompt_tokens, attributes)
+ metrics["genai_cost"].record(cost, attributes)
+
+ # Return original response
+ return response
+
+ except Exception as e:
+ handle_exception(span, e)
+ logger.error("Error in trace creation: %s", e)
+
+ # Return original response
+ return response
+
+ return wrapper
+
+def azure_embedding(gen_ai_endpoint, version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics):
+ """
+ Generates a telemetry wrapper for embeddings to collect metrics.
+
+ Args:
+ gen_ai_endpoint: Endpoint identifier for logging and tracing.
+ version: Version of the monitoring package.
+ environment: Deployment environment (e.g., production, staging).
+ application_name: Name of the application using the OpenAI API.
+ tracer: OpenTelemetry tracer for creating spans.
+ pricing_info: Information used for calculating the cost of OpenAI usage.
+ trace_content: Flag indicating whether to trace the actual content.
+
+ Returns:
+ A function that wraps the embeddings method to add telemetry.
+ """
+
+ def wrapper(wrapped, instance, args, kwargs):
+ """
+ Wraps the 'embeddings' API call to add telemetry.
+
+ This collects metrics such as execution time, cost, and token usage, and handles errors
+ gracefully, adding details to the trace for observability.
+
+ Args:
+ wrapped: The original 'embeddings' method to be wrapped.
+ instance: The instance of the class where the original method is defined.
+ args: Positional arguments for the 'embeddings' method.
+ kwargs: Keyword arguments for the 'embeddings' method.
+
+ Returns:
+ The response from the original 'embeddings' method.
+ """
+
+ with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
+ response = wrapped(*args, **kwargs)
+
+ try:
+ # Calculate cost of the operation
+ cost = get_embed_model_cost("azure_" + response.model,
+ pricing_info, response.usage.prompt_tokens)
+
+ # Set Span attributes
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_AZURE_OPENAI)
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
+ SemanticConvetion.GEN_AI_TYPE_EMBEDDING)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
+ gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,
+ environment)
+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
+ application_name)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ "azure_" + response.model)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_EMBEDDING_FORMAT,
+ kwargs.get("encoding_format", "float"))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_EMBEDDING_DIMENSION,
+ kwargs.get("dimensions", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_USER,
+ kwargs.get("user", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
+ response.usage.prompt_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ response.usage.total_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,
+ kwargs.get("input", ""))
+
+ span.set_status(Status(StatusCode.OK))
+
+ if disable_metrics is False:
+ attributes = {
+ TELEMETRY_SDK_NAME:
+ "openlit",
+ SemanticConvetion.GEN_AI_APPLICATION_NAME:
+ application_name,
+ SemanticConvetion.GEN_AI_SYSTEM:
+ SemanticConvetion.GEN_AI_SYSTEM_AZURE_OPENAI,
+ SemanticConvetion.GEN_AI_ENVIRONMENT:
+ environment,
+ SemanticConvetion.GEN_AI_TYPE:
+ SemanticConvetion.GEN_AI_TYPE_EMBEDDING,
+ SemanticConvetion.GEN_AI_REQUEST_MODEL:
+ "azure_" + response.model
+ }
+
+ metrics["genai_requests"].add(1, attributes)
+ metrics["genai_total_tokens"].add(response.usage.total_tokens, attributes)
+ metrics["genai_prompt_tokens"].add(response.usage.prompt_tokens, attributes)
+ metrics["genai_cost"].record(cost, attributes)
+
+ # Return original response
+ return response
+
+ except Exception as e:
+ handle_exception(span, e)
+ logger.error("Error in trace creation: %s", e)
+
+ # Return original response
+ return response
+
+ return wrapper
+
+def azure_image_generate(gen_ai_endpoint, version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics):
+ """
+ Generates a telemetry wrapper for image generation to collect metrics.
+
+ Args:
+ gen_ai_endpoint: Endpoint identifier for logging and tracing.
+ version: Version of the monitoring package.
+ environment: Deployment environment (e.g., production, staging).
+ application_name: Name of the application using the OpenAI API.
+ tracer: OpenTelemetry tracer for creating spans.
+ pricing_info: Information used for calculating the cost of OpenAI image generation.
+ trace_content: Flag indicating whether to trace the input prompt and generated images.
+
+ Returns:
+ A function that wraps the image generation method to add telemetry.
+ """
+
+ def wrapper(wrapped, instance, args, kwargs):
+ """
+ Wraps the 'images.generate' API call to add telemetry.
+
+ This collects metrics such as execution time, cost, and handles errors
+ gracefully, adding details to the trace for observability.
+
+ Args:
+ wrapped: The original 'images.generate' method to be wrapped.
+ instance: The instance of the class where the original method is defined.
+ args: Positional arguments for the 'images.generate' method.
+ kwargs: Keyword arguments for the 'images.generate' method.
+
+ Returns:
+ The response from the original 'images.generate' method.
+ """
+
+ with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
+ response = wrapped(*args, **kwargs)
+ images_count = 0
+
+ try:
+ # Find Image format
+ if "response_format" in kwargs and kwargs["response_format"] == "b64_json":
+ image = "b64_json"
+ else:
+ image = "url"
+
+ # Calculate cost of the operation
+ cost = get_image_model_cost("azure_" + kwargs.get("model", "dall-e-3"),
+ pricing_info, kwargs.get("size", "1024x1024"),
+ kwargs.get("quality", "standard"))
+
+ for items in response.data:
+ # Set Span attributes
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_AZURE_OPENAI)
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
+ SemanticConvetion.GEN_AI_TYPE_IMAGE)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
+ gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,
+ response.created)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,
+ environment)
+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
+ application_name)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ "azure_" + kwargs.get("model", "dall-e-3"))
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_IMAGE_SIZE,
+ kwargs.get("size", "1024x1024"))
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_IMAGE_QUALITY,
+ kwargs.get("quality", "standard"))
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_IMAGE_STYLE,
+ kwargs.get("style", "vivid"))
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_REVISED_PROMPT,
+ items.revised_prompt if response.revised_prompt else "")
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_USER,
+ kwargs.get("user", ""))
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,
+ kwargs.get("prompt", ""))
+
+ attribute_name = f"gen_ai.response.image.{images_count}"
+ span.set_attribute(attribute_name,
+ getattr(items, image))
+
+ images_count+=1
+
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ len(response.data) * cost)
+ span.set_status(Status(StatusCode.OK))
+
+ if disable_metrics is False:
+ attributes = {
+ TELEMETRY_SDK_NAME:
+ "openlit",
+ SemanticConvetion.GEN_AI_APPLICATION_NAME:
+ application_name,
+ SemanticConvetion.GEN_AI_SYSTEM:
+ SemanticConvetion.GEN_AI_SYSTEM_AZURE_OPENAI,
+ SemanticConvetion.GEN_AI_ENVIRONMENT:
+ environment,
+ SemanticConvetion.GEN_AI_TYPE:
+ SemanticConvetion.GEN_AI_TYPE_IMAGE,
+ SemanticConvetion.GEN_AI_REQUEST_MODEL:
+ "azure_" + kwargs.get("model", "dall-e-3")
+ }
+
+ metrics["genai_requests"].add(1, attributes)
+ metrics["genai_cost"].record(cost, attributes)
+
+ # Return original response
+ return response
+
+ except Exception as e:
+ handle_exception(span, e)
+ logger.error("Error in trace creation: %s", e)
+
+ # Return original response
+ return response
+
+ return wrapper
diff --git a/sdk/python/src/openlit/instrumentation/openai/openai.py b/sdk/python/src/openlit/instrumentation/openai/openai.py
new file mode 100644
index 00000000..5334e23e
--- /dev/null
+++ b/sdk/python/src/openlit/instrumentation/openai/openai.py
@@ -0,0 +1,891 @@
+# pylint: disable=duplicate-code, broad-exception-caught, too-many-statements, unused-argument, too-many-branches
+"""
+Module for monitoring OpenAI API calls.
+"""
+
+import logging
+from opentelemetry.trace import SpanKind, Status, StatusCode
+from opentelemetry.sdk.resources import TELEMETRY_SDK_NAME
+from openlit.__helpers import get_chat_model_cost, get_embed_model_cost, get_audio_model_cost
+from openlit.__helpers import get_image_model_cost, openai_tokens, handle_exception
+from openlit.semcov import SemanticConvetion
+
+# Initialize logger for logging potential issues and operations
+logger = logging.getLogger(__name__)
+
+def chat_completions(gen_ai_endpoint, version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics):
+ """
+ Generates a telemetry wrapper for chat completions to collect metrics.
+
+ Args:
+ gen_ai_endpoint: Endpoint identifier for logging and tracing.
+ version: Version of the monitoring package.
+ environment: Deployment environment (e.g., production, staging).
+ application_name: Name of the application using the OpenAI API.
+ tracer: OpenTelemetry tracer for creating spans.
+ pricing_info: Information used for calculating the cost of OpenAI usage.
+ trace_content: Flag indicating whether to trace the actual content.
+
+ Returns:
+ A function that wraps the chat completions method to add telemetry.
+ """
+
+ def wrapper(wrapped, instance, args, kwargs):
+ """
+ Wraps the 'chat.completions' API call to add telemetry.
+
+ This collects metrics such as execution time, cost, and token usage, and handles errors
+ gracefully, adding details to the trace for observability.
+
+ Args:
+ wrapped: The original 'chat.completions' method to be wrapped.
+ instance: The instance of the class where the original method is defined.
+ args: Positional arguments for the 'chat.completions' method.
+ kwargs: Keyword arguments for the 'chat.completions' method.
+
+ Returns:
+ The response from the original 'chat.completions' method.
+ """
+
+ # Check if streaming is enabled for the API call
+ streaming = kwargs.get("stream", False)
+
+ # pylint: disable=no-else-return
+ if streaming:
+ # Special handling for streaming response to accommodate the nature of data flow
+ def stream_generator():
+ with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
+ # Placeholder for aggregating streaming response
+ llmresponse = ""
+
+
+ # Loop through streaming events capturing relevant details
+ for chunk in wrapped(*args, **kwargs):
+ # Collect message IDs and aggregated response from events
+ if len(chunk.choices) > 0:
+ # pylint: disable=line-too-long
+ if hasattr(chunk.choices[0], "delta") and hasattr(chunk.choices[0].delta, "content"):
+ content = chunk.choices[0].delta.content
+ if content:
+ llmresponse += content
+ yield chunk
+ response_id = chunk.id
+
+ # Handling exception ensure observability without disrupting operation
+ try:
+ # Format 'messages' into a single string
+ message_prompt = kwargs.get("messages", "")
+ formatted_messages = []
+ for message in message_prompt:
+ role = message["role"]
+ content = message["content"]
+
+ if isinstance(content, list):
+ content_str = ", ".join(
+ # pylint: disable=line-too-long
+ f'{item["type"]}: {item["text"] if "text" in item else item["image_url"]}'
+ if "type" in item else f'text: {item["text"]}'
+ for item in content
+ )
+ formatted_messages.append(f"{role}: {content_str}")
+ else:
+ formatted_messages.append(f"{role}: {content}")
+ prompt = "\n".join(formatted_messages)
+
+ # Calculate tokens using input prompt and aggregated response
+ prompt_tokens = openai_tokens(prompt,
+ kwargs.get("model", "gpt-3.5-turbo"))
+ completion_tokens = openai_tokens(llmresponse,
+ kwargs.get("model", "gpt-3.5-turbo"))
+
+ # Calculate cost of the operation
+ cost = get_chat_model_cost(kwargs.get("model", "gpt-3.5-turbo"),
+ pricing_info, prompt_tokens,
+ completion_tokens)
+
+ # Set Span attributes
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_OPENAI)
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
+ SemanticConvetion.GEN_AI_TYPE_CHAT)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
+ gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,
+ response_id)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,
+ environment)
+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
+ application_name)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ kwargs.get("model", "gpt-3.5-turbo"))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_USER,
+ kwargs.get("user", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,
+ kwargs.get("top_p", 1))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS,
+ kwargs.get("max_tokens", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,
+ kwargs.get("temperature", 1))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_PRESENCE_PENALTY,
+ kwargs.get("presence_penalty", 0))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_FREQUENCY_PENALTY,
+ kwargs.get("frequency_penalty", 0))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_SEED,
+ kwargs.get("seed", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,
+ True)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
+ prompt_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,
+ completion_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ prompt_tokens + completion_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,
+ prompt)
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_COMPLETION,
+ llmresponse)
+
+ span.set_status(Status(StatusCode.OK))
+
+ if disable_metrics is False:
+ attributes = {
+ TELEMETRY_SDK_NAME:
+ "openlit",
+ SemanticConvetion.GEN_AI_APPLICATION_NAME:
+ application_name,
+ SemanticConvetion.GEN_AI_SYSTEM:
+ SemanticConvetion.GEN_AI_SYSTEM_OPENAI,
+ SemanticConvetion.GEN_AI_ENVIRONMENT:
+ environment,
+ SemanticConvetion.GEN_AI_TYPE:
+ SemanticConvetion.GEN_AI_TYPE_CHAT,
+ SemanticConvetion.GEN_AI_REQUEST_MODEL:
+ kwargs.get("model", "gpt-3.5-turbo")
+ }
+
+ metrics["genai_requests"].add(1, attributes)
+ metrics["genai_total_tokens"].add(
+ prompt_tokens + completion_tokens, attributes
+ )
+ metrics["genai_completion_tokens"].add(completion_tokens, attributes)
+ metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)
+ metrics["genai_cost"].record(cost, attributes)
+
+ except Exception as e:
+ handle_exception(span, e)
+ logger.error("Error in trace creation: %s", e)
+
+ return stream_generator()
+
+ # Handling for non-streaming responses
+ else:
+ # pylint: disable=line-too-long
+ with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
+ response = wrapped(*args, **kwargs)
+
+ try:
+ # Format 'messages' into a single string
+ message_prompt = kwargs.get("messages", "")
+ formatted_messages = []
+ for message in message_prompt:
+ role = message["role"]
+ content = message["content"]
+
+ if isinstance(content, list):
+ content_str = ", ".join(
+ # pylint: disable=line-too-long
+ f'{item["type"]}: {item["text"] if "text" in item else item["image_url"]}'
+ if "type" in item else f'text: {item["text"]}'
+ for item in content
+ )
+ formatted_messages.append(f"{role}: {content_str}")
+ else:
+ formatted_messages.append(f"{role}: {content}")
+ prompt = "\n".join(formatted_messages)
+
+ # Set base span attribues
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_OPENAI)
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
+ SemanticConvetion.GEN_AI_TYPE_CHAT)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
+ gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,
+ response.id)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,
+ environment)
+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
+ application_name)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ kwargs.get("model", "gpt-3.5-turbo"))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,
+ kwargs.get("top_p", 1))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS,
+ kwargs.get("max_tokens", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_USER,
+ kwargs.get("user", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,
+ kwargs.get("temperature", 1))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_PRESENCE_PENALTY,
+ kwargs.get("presence_penalty", 0))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_FREQUENCY_PENALTY,
+ kwargs.get("frequency_penalty", 0))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_SEED,
+ kwargs.get("seed", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,
+ False)
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,
+ prompt)
+
+ # Set span attributes when tools is not passed to the function call
+ if "tools" not in kwargs:
+ # Calculate cost of the operation
+ cost = get_chat_model_cost(kwargs.get("model", "gpt-3.5-turbo"),
+ pricing_info, response.usage.prompt_tokens,
+ response.usage.completion_tokens)
+
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
+ response.usage.prompt_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,
+ response.usage.completion_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ response.usage.total_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,
+ response.choices[0].finish_reason)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+
+ # Set span attributes for when n = 1 (default)
+ if "n" not in kwargs or kwargs["n"] == 1:
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_COMPLETION,
+ response.choices[0].message.content)
+
+ # Set span attributes for when n > 0
+ else:
+ i = 0
+ while i < kwargs["n"] and trace_content is True:
+ attribute_name = f"gen_ai.content.completion.{i}"
+ span.set_attribute(attribute_name,
+ response.choices[i].message.content)
+ i += 1
+
+ # Return original response
+ return response
+
+ # Set span attributes when tools is passed to the function call
+ elif "tools" in kwargs:
+ # Calculate cost of the operation
+ cost = get_chat_model_cost(kwargs.get("model", "gpt-3.5-turbo"),
+ pricing_info, response.usage.prompt_tokens,
+ response.usage.completion_tokens)
+
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_COMPLETION,
+ "Function called with tools")
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
+ response.usage.prompt_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,
+ response.usage.completion_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ response.usage.total_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+
+ span.set_status(Status(StatusCode.OK))
+
+ if disable_metrics is False:
+ attributes = {
+ TELEMETRY_SDK_NAME:
+ "openlit",
+ SemanticConvetion.GEN_AI_APPLICATION_NAME:
+ application_name,
+ SemanticConvetion.GEN_AI_SYSTEM:
+ SemanticConvetion.GEN_AI_SYSTEM_OPENAI,
+ SemanticConvetion.GEN_AI_ENVIRONMENT:
+ environment,
+ SemanticConvetion.GEN_AI_TYPE:
+ SemanticConvetion.GEN_AI_TYPE_CHAT,
+ SemanticConvetion.GEN_AI_REQUEST_MODEL:
+ kwargs.get("model", "gpt-3.5-turbo")
+ }
+
+ metrics["genai_requests"].add(1, attributes)
+ metrics["genai_total_tokens"].add(response.usage.total_tokens, attributes)
+ metrics["genai_completion_tokens"].add(response.usage.completion_tokens, attributes)
+ metrics["genai_prompt_tokens"].add(response.usage.prompt_tokens, attributes)
+ metrics["genai_cost"].record(cost, attributes)
+
+ # Return original response
+ return response
+
+ except Exception as e:
+ handle_exception(span, e)
+ logger.error("Error in trace creation: %s", e)
+
+ # Return original response
+ return response
+
+ return wrapper
+
+def embedding(gen_ai_endpoint, version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics):
+ """
+ Generates a telemetry wrapper for embeddings to collect metrics.
+
+ Args:
+ gen_ai_endpoint: Endpoint identifier for logging and tracing.
+ version: Version of the monitoring package.
+ environment: Deployment environment (e.g., production, staging).
+ application_name: Name of the application using the OpenAI API.
+ tracer: OpenTelemetry tracer for creating spans.
+ pricing_info: Information used for calculating the cost of OpenAI usage.
+ trace_content: Flag indicating whether to trace the actual content.
+
+ Returns:
+ A function that wraps the embeddings method to add telemetry.
+ """
+
+ def wrapper(wrapped, instance, args, kwargs):
+ """
+ Wraps the 'embeddings' API call to add telemetry.
+
+ This collects metrics such as execution time, cost, and token usage, and handles errors
+ gracefully, adding details to the trace for observability.
+
+ Args:
+ wrapped: The original 'embeddings' method to be wrapped.
+ instance: The instance of the class where the original method is defined.
+ args: Positional arguments for the 'embeddings' method.
+ kwargs: Keyword arguments for the 'embeddings' method.
+
+ Returns:
+ The response from the original 'embeddings' method.
+ """
+
+ with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
+ response = wrapped(*args, **kwargs)
+
+ try:
+ # Calculate cost of the operation
+ cost = get_embed_model_cost(kwargs.get("model", "text-embedding-ada-002"),
+ pricing_info, response.usage.prompt_tokens)
+
+ # Set Span attributes
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_OPENAI)
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
+ SemanticConvetion.GEN_AI_TYPE_EMBEDDING)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
+ gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,
+ environment)
+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
+ application_name)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ kwargs.get("model", "text-embedding-ada-002"))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_EMBEDDING_FORMAT,
+ kwargs.get("encoding_format", "float"))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_EMBEDDING_DIMENSION,
+ kwargs.get("dimensions", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_USER,
+ kwargs.get("user", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
+ response.usage.prompt_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ response.usage.total_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,
+ kwargs.get("input", ""))
+
+ span.set_status(Status(StatusCode.OK))
+
+ if disable_metrics is False:
+ attributes = {
+ TELEMETRY_SDK_NAME:
+ "openlit",
+ SemanticConvetion.GEN_AI_APPLICATION_NAME:
+ application_name,
+ SemanticConvetion.GEN_AI_SYSTEM:
+ SemanticConvetion.GEN_AI_SYSTEM_OPENAI,
+ SemanticConvetion.GEN_AI_ENVIRONMENT:
+ environment,
+ SemanticConvetion.GEN_AI_TYPE:
+ SemanticConvetion.GEN_AI_TYPE_EMBEDDING,
+ SemanticConvetion.GEN_AI_REQUEST_MODEL:
+ kwargs.get("model", "text-embedding-ada-002")
+ }
+
+ metrics["genai_requests"].add(1, attributes)
+ metrics["genai_total_tokens"].add(response.usage.total_tokens, attributes)
+ metrics["genai_prompt_tokens"].add(response.usage.prompt_tokens, attributes)
+ metrics["genai_cost"].record(cost, attributes)
+
+ # Return original response
+ return response
+
+ except Exception as e:
+ handle_exception(span, e)
+ logger.error("Error in trace creation: %s", e)
+
+ # Return original response
+ return response
+
+ return wrapper
+
+def finetune(gen_ai_endpoint, version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics):
+ """
+ Generates a telemetry wrapper for fine-tuning jobs to collect metrics.
+
+ Args:
+ gen_ai_endpoint: Endpoint identifier for logging and tracing.
+ version: Version of the monitoring package.
+ environment: Deployment environment (e.g., production, staging).
+ application_name: Name of the application using the OpenAI API.
+ tracer: OpenTelemetry tracer for creating spans.
+ pricing_info: Information used for calculating the cost of OpenAI usage.
+ trace_content: Flag indicating whether to trace the actual content.
+
+ Returns:
+ A function that wraps the fine tuning creation method to add telemetry.
+ """
+
+ def wrapper(wrapped, instance, args, kwargs):
+ """
+ Wraps the 'fine_tuning.jobs.create' API call to add telemetry.
+
+ This collects metrics such as execution time, usage stats, and handles errors
+ gracefully, adding details to the trace for observability.
+
+ Args:
+ wrapped: The original 'fine_tuning.jobs.create' method to be wrapped.
+ instance: The instance of the class where the original method is defined.
+ args: Positional arguments for the method.
+ kwargs: Keyword arguments for the method.
+
+ Returns:
+ The response from the original 'fine_tuning.jobs.create' method.
+ """
+
+ with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
+ response = wrapped(*args, **kwargs)
+
+ # Handling exception ensure observability without disrupting operation
+ try:
+ # Set Span attributes
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_OPENAI)
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
+ SemanticConvetion.GEN_AI_TYPE_FINETUNING)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
+ gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,
+ environment)
+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
+ application_name)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ kwargs.get("model", "gpt-3.5-turbo"))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TRAINING_FILE,
+ kwargs.get("training_file", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_VALIDATION_FILE,
+ kwargs.get("validation_file", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_FINETUNE_BATCH_SIZE,
+ kwargs.get("hyperparameters.batch_size", "auto"))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_FINETUNE_MODEL_LRM,
+ kwargs.get("hyperparameters.learning_rate_multiplier",
+ "auto"))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_FINETUNE_MODEL_EPOCHS,
+ kwargs.get("hyperparameters.n_epochs", "auto"))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_FINETUNE_MODEL_SUFFIX,
+ kwargs.get("suffix", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,
+ response.id)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
+ response.usage.prompt_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_FINETUNE_STATUS,
+ response.status)
+ span.set_status(Status(StatusCode.OK))
+
+ if disable_metrics is False:
+ attributes = {
+ TELEMETRY_SDK_NAME:
+ "openlit",
+ SemanticConvetion.GEN_AI_APPLICATION_NAME:
+ application_name,
+ SemanticConvetion.GEN_AI_SYSTEM:
+ SemanticConvetion.GEN_AI_SYSTEM_OPENAI,
+ SemanticConvetion.GEN_AI_ENVIRONMENT:
+ environment,
+ SemanticConvetion.GEN_AI_TYPE:
+ SemanticConvetion.GEN_AI_TYPE_FINETUNING,
+ SemanticConvetion.GEN_AI_REQUEST_MODEL:
+ kwargs.get("model", "gpt-3.5-turbo")
+ }
+
+ metrics["genai_requests"].add(1, attributes)
+
+ # Return original response
+ return response
+
+ except Exception as e:
+ handle_exception(span, e)
+ logger.error("Error in trace creation: %s", e)
+
+ # Return original response
+ return response
+
+ return wrapper
+
+def image_generate(gen_ai_endpoint, version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics):
+ """
+ Generates a telemetry wrapper for image generation to collect metrics.
+
+ Args:
+ gen_ai_endpoint: Endpoint identifier for logging and tracing.
+ version: Version of the monitoring package.
+ environment: Deployment environment (e.g., production, staging).
+ application_name: Name of the application using the OpenAI API.
+ tracer: OpenTelemetry tracer for creating spans.
+ pricing_info: Information used for calculating the cost of OpenAI image generation.
+ trace_content: Flag indicating whether to trace the input prompt and generated images.
+
+ Returns:
+ A function that wraps the image generation method to add telemetry.
+ """
+
+ def wrapper(wrapped, instance, args, kwargs):
+ """
+ Wraps the 'images.generate' API call to add telemetry.
+
+ This collects metrics such as execution time, cost, and handles errors
+ gracefully, adding details to the trace for observability.
+
+ Args:
+ wrapped: The original 'images.generate' method to be wrapped.
+ instance: The instance of the class where the original method is defined.
+ args: Positional arguments for the 'images.generate' method.
+ kwargs: Keyword arguments for the 'images.generate' method.
+
+ Returns:
+ The response from the original 'images.generate' method.
+ """
+
+ with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
+ response = wrapped(*args, **kwargs)
+ images_count = 0
+
+ try:
+ # Find Image format
+ if "response_format" in kwargs and kwargs["response_format"] == "b64_json":
+ image = "b64_json"
+ else:
+ image = "url"
+
+ # Calculate cost of the operation
+ cost = get_image_model_cost(kwargs.get("model", "dall-e-2"),
+ pricing_info, kwargs.get("size", "1024x1024"),
+ kwargs.get("quality", "standard"))
+
+ for items in response.data:
+ # Set Span attributes
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_OPENAI)
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
+ SemanticConvetion.GEN_AI_TYPE_IMAGE)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
+ gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,
+ response.created)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,
+ environment)
+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
+ application_name)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ kwargs.get("model", "dall-e-2"))
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_IMAGE_SIZE,
+ kwargs.get("size", "1024x1024"))
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_IMAGE_QUALITY,
+ kwargs.get("quality", "standard"))
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_IMAGE_STYLE,
+ kwargs.get("style", "vivid"))
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_REVISED_PROMPT,
+ items.revised_prompt if items.revised_prompt else "")
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_USER,
+ kwargs.get("user", ""))
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,
+ kwargs.get("prompt", ""))
+
+ attribute_name = f"gen_ai.response.image.{images_count}"
+ span.set_attribute(attribute_name,
+ getattr(items, image))
+
+ images_count+=1
+
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ len(response.data) * cost)
+ span.set_status(Status(StatusCode.OK))
+
+ if disable_metrics is False:
+ attributes = {
+ TELEMETRY_SDK_NAME:
+ "openlit",
+ SemanticConvetion.GEN_AI_APPLICATION_NAME:
+ application_name,
+ SemanticConvetion.GEN_AI_SYSTEM:
+ SemanticConvetion.GEN_AI_SYSTEM_OPENAI,
+ SemanticConvetion.GEN_AI_ENVIRONMENT:
+ environment,
+ SemanticConvetion.GEN_AI_TYPE:
+ SemanticConvetion.GEN_AI_TYPE_IMAGE,
+ SemanticConvetion.GEN_AI_REQUEST_MODEL:
+ kwargs.get("model", "dall-e-2")
+ }
+
+ metrics["genai_requests"].add(1, attributes)
+ metrics["genai_cost"].record(cost, attributes)
+
+ # Return original response
+ return response
+
+ except Exception as e:
+ handle_exception(span, e)
+ logger.error("Error in trace creation: %s", e)
+
+ # Return original response
+ return response
+
+ return wrapper
+
+def image_variatons(gen_ai_endpoint, version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics):
+ """
+ Generates a telemetry wrapper for creating image variations to collect metrics.
+
+ Args:
+ gen_ai_endpoint: Endpoint identifier for logging and tracing.
+ version: Version of the monitoring package.
+ environment: Deployment environment (e.g., production, staging).
+ application_name: Name of the application using the OpenAI API.
+ tracer: OpenTelemetry tracer for creating spans.
+ pricing_info: Information used for calculating the cost of generating image variations.
+ trace_content: Flag indicating whether to trace the input image and generated variations.
+
+ Returns:
+ A function that wraps the image variations creation method to add telemetry.
+ """
+
+ def wrapper(wrapped, instance, args, kwargs):
+ """
+ Wraps the 'images.create.variations' API call to add telemetry.
+
+ This collects metrics such as execution time, cost, and handles errors
+ gracefully, adding details to the trace for observability.
+
+ Args:
+ wrapped: The original 'images.create.variations' method to be wrapped.
+ instance: The instance of the class where the original method is defined.
+ args: Positional arguments for the method.
+ kwargs: Keyword arguments for the method.
+
+ Returns:
+ The response from the original 'images.create.variations' method.
+ """
+
+ with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
+ response = wrapped(*args, **kwargs)
+ images_count = 0
+
+ try:
+ # Find Image format
+ if "response_format" in kwargs and kwargs["response_format"] == "b64_json":
+ image = "b64_json"
+ else:
+ image = "url"
+
+ # Calculate cost of the operation
+ cost = get_image_model_cost(kwargs.get("model", "dall-e-2"), pricing_info,
+ kwargs.get("size", "1024x1024"), "standard")
+
+ for items in response.data:
+ # Set Span attributes
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_OPENAI)
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
+ SemanticConvetion.GEN_AI_TYPE_IMAGE)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
+ gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,
+ response.created)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,
+ environment)
+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
+ application_name)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ kwargs.get("model", "dall-e-2"))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_USER,
+ kwargs.get("user", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_IMAGE_SIZE,
+ kwargs.get("size", "1024x1024"))
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_IMAGE_QUALITY,
+ "standard")
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,
+ kwargs.get(SemanticConvetion.GEN_AI_TYPE_IMAGE, ""))
+
+ attribute_name = f"gen_ai.response.image.{images_count}"
+ span.set_attribute(attribute_name,
+ getattr(items, image))
+
+ images_count+=1
+
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ len(response.data) * cost)
+ span.set_status(Status(StatusCode.OK))
+
+ if disable_metrics is False:
+ attributes = {
+ TELEMETRY_SDK_NAME:
+ "openlit",
+ SemanticConvetion.GEN_AI_APPLICATION_NAME:
+ application_name,
+ SemanticConvetion.GEN_AI_SYSTEM:
+ SemanticConvetion.GEN_AI_SYSTEM_OPENAI,
+ SemanticConvetion.GEN_AI_ENVIRONMENT:
+ environment,
+ SemanticConvetion.GEN_AI_TYPE:
+ SemanticConvetion.GEN_AI_TYPE_IMAGE,
+ SemanticConvetion.GEN_AI_REQUEST_MODEL:
+ kwargs.get("model", "dall-e-2")
+ }
+
+ metrics["genai_requests"].add(1, attributes)
+ metrics["genai_cost"].record(cost, attributes)
+
+ # Return original response
+ return response
+
+ except Exception as e:
+ handle_exception(span, e)
+ logger.error("Error in trace creation: %s", e)
+
+ # Return original response
+ return response
+
+ return wrapper
+
+def audio_create(gen_ai_endpoint, version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics):
+ """
+ Generates a telemetry wrapper for creating speech audio to collect metrics.
+
+ Args:
+ gen_ai_endpoint: Endpoint identifier for logging and tracing.
+ version: Version of the monitoring package.
+ environment: Deployment environment (e.g., production, staging).
+ application_name: Name of the application using the OpenAI API.
+ tracer: OpenTelemetry tracer for creating spans.
+ pricing_info: Information used for calculating the cost of generating speech audio.
+ trace_content: Flag indicating whether to trace the input text and generated audio.
+
+ Returns:
+ A function that wraps the speech audio creation method to add telemetry.
+ """
+
+ def wrapper(wrapped, instance, args, kwargs):
+ """
+ Wraps the 'audio.speech.create' API call to add telemetry.
+
+ This collects metrics such as execution time, cost, and handles errors
+ gracefully, adding details to the trace for observability.
+
+ Args:
+ wrapped: The original 'audio.speech.create' method to be wrapped.
+ instance: The instance of the class where the original method is defined.
+ args: Positional arguments for the 'audio.speech.create' method.
+ kwargs: Keyword arguments for the 'audio.speech.create' method.
+
+ Returns:
+ The response from the original 'audio.speech.create' method.
+ """
+
+ with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
+ response = wrapped(*args, **kwargs)
+
+ try:
+ # Calculate cost of the operation
+ cost = get_audio_model_cost(kwargs.get("model", "tts-1"),
+ pricing_info, kwargs.get("input", ""))
+
+ # Set Span attributes
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_OPENAI)
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
+ SemanticConvetion.GEN_AI_TYPE_AUDIO)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
+ gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,
+ environment)
+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
+ application_name)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ kwargs.get("model", "tts-1"))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_AUDIO_VOICE,
+ kwargs.get("voice", "alloy"))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_AUDIO_RESPONSE_FORMAT,
+ kwargs.get("response_format", "mp3"))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_AUDIO_SPEED,
+ kwargs.get("speed", 1))
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,
+ kwargs.get("input", ""))
+
+ span.set_status(Status(StatusCode.OK))
+
+ if disable_metrics is False:
+ attributes = {
+ TELEMETRY_SDK_NAME:
+ "openlit",
+ SemanticConvetion.GEN_AI_APPLICATION_NAME:
+ application_name,
+ SemanticConvetion.GEN_AI_SYSTEM:
+ SemanticConvetion.GEN_AI_SYSTEM_OPENAI,
+ SemanticConvetion.GEN_AI_ENVIRONMENT:
+ environment,
+ SemanticConvetion.GEN_AI_TYPE:
+ SemanticConvetion.GEN_AI_TYPE_AUDIO,
+ SemanticConvetion.GEN_AI_REQUEST_MODEL:
+ kwargs.get("model", "tts-1")
+ }
+
+ metrics["genai_requests"].add(1, attributes)
+ metrics["genai_cost"].record(cost, attributes)
+
+ # Return original response
+ return response
+
+ except Exception as e:
+ handle_exception(span, e)
+ logger.error("Error in trace creation: %s", e)
+
+ # Return original response
+ return response
+
+ return wrapper
diff --git a/sdk/python/src/openlit/instrumentation/pinecone/__init__.py b/sdk/python/src/openlit/instrumentation/pinecone/__init__.py
new file mode 100644
index 00000000..5b9741d1
--- /dev/null
+++ b/sdk/python/src/openlit/instrumentation/pinecone/__init__.py
@@ -0,0 +1,66 @@
+# pylint: disable=useless-return, bad-staticmethod-argument, disable=duplicate-code
+"""Initializer of Auto Instrumentation of Pinecone Functions"""
+from typing import Collection
+import importlib.metadata
+from opentelemetry.instrumentation.instrumentor import BaseInstrumentor
+from wrapt import wrap_function_wrapper
+
+from openlit.instrumentation.pinecone.pinecone import general_wrap
+
+_instruments = ("pinecone-client >= 2.2.0",)
+
+class PineconeInstrumentor(BaseInstrumentor):
+ """An instrumentor for Pinecone's client library."""
+
+ def instrumentation_dependencies(self) -> Collection[str]:
+ return _instruments
+
+ def _instrument(self, **kwargs):
+ application_name = kwargs.get("application_name")
+ environment = kwargs.get("environment")
+ tracer = kwargs.get("tracer")
+ metrics = kwargs.get("metrics_dict")
+ pricing_info = kwargs.get("pricing_info")
+ trace_content = kwargs.get("trace_content")
+ disable_metrics = kwargs.get("disable_metrics")
+ version = importlib.metadata.version("pinecone-client")
+
+ wrap_function_wrapper(
+ "pinecone.control.pinecone",
+ "Pinecone.create_index",
+ general_wrap("pinecone.create_index", version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics),
+ )
+
+ wrap_function_wrapper(
+ "pinecone.data.index",
+ "Index.upsert",
+ general_wrap("pinecone.upsert", version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics),
+ )
+
+ wrap_function_wrapper(
+ "pinecone.data.index",
+ "Index.query",
+ general_wrap("pinecone.query", version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics),
+ )
+
+ wrap_function_wrapper(
+ "pinecone.data.index",
+ "Index.update",
+ general_wrap("pinecone.update", version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics),
+ )
+
+ wrap_function_wrapper(
+ "pinecone.data.index",
+ "Index.delete",
+ general_wrap("pinecone.delete", version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics),
+ )
+
+
+ @staticmethod
+ def _uninstrument(self, **kwargs):
+ pass
diff --git a/sdk/python/src/openlit/instrumentation/pinecone/pinecone.py b/sdk/python/src/openlit/instrumentation/pinecone/pinecone.py
new file mode 100644
index 00000000..bbd78248
--- /dev/null
+++ b/sdk/python/src/openlit/instrumentation/pinecone/pinecone.py
@@ -0,0 +1,173 @@
+# pylint: disable=duplicate-code, broad-exception-caught, too-many-statements, unused-argument
+"""
+Module for monitoring Pinecone.
+"""
+
+import logging
+from opentelemetry.trace import SpanKind, Status, StatusCode
+from opentelemetry.sdk.resources import TELEMETRY_SDK_NAME
+from openlit.__helpers import handle_exception
+from openlit.semcov import SemanticConvetion
+
+# Initialize logger for logging potential issues and operations
+logger = logging.getLogger(__name__)
+
+def object_count(obj):
+ """
+ Counts Length of object if it exists, Else returns None
+ """
+
+ if obj:
+ return len(obj)
+
+ return None
+
+def general_wrap(gen_ai_endpoint, version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics):
+ """
+ Wraps a Pinecone operation to trace and log its execution metrics.
+
+ This function is intended to wrap around Pinecone operations in order to
+ measure their execution time, log relevant information, and trace the execution
+ using OpenTelemetry. This helps in monitoring and debugging operations within
+ the Pinecone space.
+
+ Parameters:
+ - pinecone_operation (str): The specific Pinecone operation being monitored.
+ Examples include 'create_index', 'query', 'upsert', etc.
+ - version (str): The version of the application interfacing with Pinecone.
+ - environment (str): The deployment environment, such as 'production' or 'development'.
+ - application_name (str): The name of the application performing the Pinecone operation.
+ - tracer (opentelemetry.trace.Tracer): An object used for OpenTelemetry tracing.
+ - pricing_info (dict): Information about pricing, not used in current implementation.
+ - trace_content (bool): A flag indicating whether the content of responses should be traced.
+
+ Returns:
+ - function: A decorator function that, when applied, wraps the target function with
+ additional functionality for tracing and logging Pinecone operations.
+ """
+
+ def wrapper(wrapped, instance, args, kwargs):
+ """
+ Executes the wrapped Pinecone operation, adding tracing and logging.
+
+ This inner wrapper function captures the execution of Pinecone operations,
+ annotating the operation with relevant metrics and tracing information, and
+ ensuring any exceptions are caught and logged appropriately.
+
+ Parameters:
+ - wrapped (Callable): The Pinecone operation to be wrapped and executed.
+ - instance (object): The instance on which the operation is called (for class methods).
+ - args (tuple): Positional arguments for the Pinecone operation.
+ - kwargs (dict): Keyword arguments for the Pinecone operation.
+
+ Returns:
+ - Any: The result of executing the wrapped Pinecone operation.
+ """
+
+ with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
+ response = wrapped(*args, **kwargs)
+
+ try:
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
+ gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,
+ environment)
+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
+ application_name)
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
+ SemanticConvetion.GEN_AI_TYPE_VECTORDB)
+ span.set_attribute(SemanticConvetion.DB_SYSTEM,
+ SemanticConvetion.DB_SYSTEM_PINECONE)
+
+ if gen_ai_endpoint == "pinecone.create_index":
+ db_operation = SemanticConvetion.DB_OPERATION_CREATE_INDEX
+ span.set_attribute(SemanticConvetion.DB_OPERATION,
+ SemanticConvetion.DB_OPERATION_CREATE_INDEX)
+ span.set_attribute(SemanticConvetion.DB_INDEX_NAME,
+ kwargs.get("name", ""))
+ span.set_attribute(SemanticConvetion.DB_INDEX_DIMENSION,
+ kwargs.get("dimensions", ""))
+ span.set_attribute(SemanticConvetion.DB_INDEX_METRIC,
+ kwargs.get("metric", ""))
+ span.set_attribute(SemanticConvetion.DB_INDEX_SPEC,
+ str(kwargs.get("spec", "")))
+
+ elif gen_ai_endpoint == "pinecone.query":
+ db_operation = SemanticConvetion.DB_OPERATION_QUERY
+ span.set_attribute(SemanticConvetion.DB_OPERATION,
+ SemanticConvetion.DB_OPERATION_QUERY)
+ span.set_attribute(SemanticConvetion.DB_STATEMENT,
+ str(kwargs.get("vector")))
+ span.set_attribute(SemanticConvetion.DB_N_RESULTS,
+ kwargs.get("top_k", ""))
+ span.set_attribute(SemanticConvetion.DB_FILTER,
+ str(kwargs.get("filter", "")))
+ span.set_attribute(SemanticConvetion.DB_NAMESPACE,
+ str(kwargs.get("namespace", "")))
+
+ elif gen_ai_endpoint == "pinecone.update":
+ db_operation = SemanticConvetion.DB_OPERATION_UPDATE
+ span.set_attribute(SemanticConvetion.DB_OPERATION,
+ SemanticConvetion.DB_OPERATION_UPDATE)
+ span.set_attribute(SemanticConvetion.DB_UPDATE_ID,
+ kwargs.get("id",""))
+ span.set_attribute(SemanticConvetion.DB_UPDATE_VALUES,
+ str(kwargs.get("values",[])))
+ span.set_attribute(SemanticConvetion.DB_NAMESPACE,
+ str(kwargs.get("namespace", "")))
+ span.set_attribute(SemanticConvetion.DB_UPDATE_METADATA,
+ str(kwargs.get("set_metadata", "")))
+
+ elif gen_ai_endpoint == "pinecone.upsert":
+ db_operation = SemanticConvetion.DB_OPERATION_UPSERT
+ span.set_attribute(SemanticConvetion.DB_OPERATION,
+ SemanticConvetion.DB_OPERATION_UPSERT)
+ span.set_attribute(SemanticConvetion.DB_VECTOR_COUNT,
+ object_count(kwargs.get("vectors")))
+
+ elif gen_ai_endpoint == "pinecone.delete":
+ db_operation = SemanticConvetion.DB_OPERATION_DELETE
+ span.set_attribute(SemanticConvetion.DB_OPERATION,
+ SemanticConvetion.DB_OPERATION_DELETE)
+ span.set_attribute(SemanticConvetion.DB_ID_COUNT,
+ object_count(kwargs.get("ids")))
+ span.set_attribute(SemanticConvetion.DB_FILTER,
+ str(kwargs.get("filter", "")))
+ span.set_attribute(SemanticConvetion.DB_DELETE_ALL,
+ kwargs.get("delete_all", False))
+ span.set_attribute(SemanticConvetion.DB_NAMESPACE,
+ kwargs.get("namespace", ""))
+
+ span.set_status(Status(StatusCode.OK))
+
+ if disable_metrics is False:
+ attributes = {
+ TELEMETRY_SDK_NAME:
+ "openlit",
+ SemanticConvetion.GEN_AI_APPLICATION_NAME:
+ application_name,
+ SemanticConvetion.DB_SYSTEM:
+ SemanticConvetion.DB_SYSTEM_PINECONE,
+ SemanticConvetion.GEN_AI_ENVIRONMENT:
+ environment,
+ SemanticConvetion.GEN_AI_TYPE:
+ SemanticConvetion.GEN_AI_TYPE_VECTORDB,
+ SemanticConvetion.DB_OPERATION:
+ db_operation
+ }
+
+ metrics["db_requests"].add(1, attributes)
+
+ # Return original response
+ return response
+
+ except Exception as e:
+ handle_exception(span, e)
+ logger.error("Error in trace creation: %s", e)
+
+ # Return original response
+ return response
+
+ return wrapper
diff --git a/sdk/python/src/openlit/instrumentation/transformers/__init__.py b/sdk/python/src/openlit/instrumentation/transformers/__init__.py
new file mode 100644
index 00000000..fe0d9e7f
--- /dev/null
+++ b/sdk/python/src/openlit/instrumentation/transformers/__init__.py
@@ -0,0 +1,37 @@
+# pylint: disable=useless-return, bad-staticmethod-argument, disable=duplicate-code
+"""Initializer of Auto Instrumentation of HuggingFace Transformer Functions"""
+from typing import Collection
+import importlib.metadata
+from opentelemetry.instrumentation.instrumentor import BaseInstrumentor
+from wrapt import wrap_function_wrapper
+
+from openlit.instrumentation.transformers.transformers import text_wrap
+
+_instruments = ("transformers >= 4.39.3",)
+
+class TransformersInstrumentor(BaseInstrumentor):
+ """An instrumentor for HuggingFace Transformer Functions."""
+
+ def instrumentation_dependencies(self) -> Collection[str]:
+ return _instruments
+
+ def _instrument(self, **kwargs):
+ application_name = kwargs.get("application_name")
+ environment = kwargs.get("environment")
+ tracer = kwargs.get("tracer")
+ metrics = kwargs.get("metrics_dict")
+ pricing_info = kwargs.get("pricing_info")
+ trace_content = kwargs.get("trace_content")
+ disable_metrics = kwargs.get("disable_metrics")
+ version = importlib.metadata.version("transformers")
+
+ wrap_function_wrapper(
+ "transformers.pipelines",
+ "TextGenerationPipeline.__call__",
+ text_wrap("huggingface.text_generation", version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics),
+ )
+
+ @staticmethod
+ def _uninstrument(self, **kwargs):
+ pass
diff --git a/sdk/python/src/openlit/instrumentation/transformers/transformers.py b/sdk/python/src/openlit/instrumentation/transformers/transformers.py
new file mode 100644
index 00000000..df05b9c2
--- /dev/null
+++ b/sdk/python/src/openlit/instrumentation/transformers/transformers.py
@@ -0,0 +1,156 @@
+# pylint: disable=duplicate-code, broad-exception-caught, too-many-statements, unused-argument
+"""
+Module for monitoring ChromaDB.
+"""
+
+import logging
+from opentelemetry.trace import SpanKind, Status, StatusCode
+from opentelemetry.sdk.resources import TELEMETRY_SDK_NAME
+from openlit.__helpers import handle_exception, general_tokens
+from openlit.semcov import SemanticConvetion
+
+# Initialize logger for logging potential issues and operations
+logger = logging.getLogger(__name__)
+
+def text_wrap(gen_ai_endpoint, version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics):
+ """
+ Creates a wrapper around a function call to trace and log its execution metrics.
+
+ This function wraps any given function to measure its execution time,
+ log its operation, and trace its execution using OpenTelemetry.
+
+ Parameters:
+ - gen_ai_endpoint (str): A descriptor or name for the endpoint being traced.
+ - version (str): The version of the Langchain application.
+ - environment (str): The deployment environment (e.g., 'production', 'development').
+ - application_name (str): Name of the Langchain application.
+ - tracer (opentelemetry.trace.Tracer): The tracer object used for OpenTelemetry tracing.
+ - pricing_info (dict): Information about the pricing for internal metrics (currently not used).
+ - trace_content (bool): Flag indicating whether to trace the content of the response.
+
+ Returns:
+ - function: A higher-order function that takes a function 'wrapped' and returns
+ a new function that wraps 'wrapped' with additional tracing and logging.
+ """
+
+ def wrapper(wrapped, instance, args, kwargs):
+ """
+ An inner wrapper function that executes the wrapped function, measures execution
+ time, and records trace data using OpenTelemetry.
+
+ Parameters:
+ - wrapped (Callable): The original function that this wrapper will execute.
+ - instance (object): The instance to which the wrapped function belongs. This
+ is used for instance methods. For static and classmethods,
+ this may be None.
+ - args (tuple): Positional arguments passed to the wrapped function.
+ - kwargs (dict): Keyword arguments passed to the wrapped function.
+
+ Returns:
+ - The result of the wrapped function call.
+
+ The wrapper initiates a span with the provided tracer, sets various attributes
+ on the span based on the function's execution and response, and ensures
+ errors are handled and logged appropriately.
+ """
+
+ with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
+ response = wrapped(*args, **kwargs)
+
+ # pylint: disable=protected-access
+ forward_params = instance._forward_params
+
+ try:
+ if args and len(args) > 0:
+ prompt = args[0]
+ else:
+ prompt = kwargs.get("args", "")
+
+ prompt_tokens = general_tokens(prompt)
+
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
+ gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_HUGGING_FACE)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,
+ environment)
+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
+ application_name)
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
+ SemanticConvetion.GEN_AI_TYPE_CHAT)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ instance.model.config.name_or_path)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,
+ forward_params.get("temperature"))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,
+ forward_params.get("top_p"))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS,
+ forward_params.get("max_length"))
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,
+ prompt)
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
+ prompt_tokens)
+
+ i = 0
+ completion_tokens = 0
+ for completion in response:
+ if len(response) > 1:
+ attribute_name = f"gen_ai.content.completion.{i}"
+ else:
+ attribute_name = SemanticConvetion.GEN_AI_CONTENT_COMPLETION
+ if i == 0:
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_COMPLETION,
+ completion["generated_text"])
+ completion_tokens += general_tokens(completion["generated_text"])
+ else:
+ if trace_content:
+ span.set_attribute(attribute_name,
+ completion["generated_text"])
+ completion_tokens += general_tokens(completion["generated_text"])
+ i=i+1
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,
+ completion_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ prompt_tokens + completion_tokens)
+ span.set_status(Status(StatusCode.OK))
+
+ if disable_metrics is False:
+ attributes = {
+ TELEMETRY_SDK_NAME:
+ "openlit",
+ SemanticConvetion.GEN_AI_APPLICATION_NAME:
+ application_name,
+ SemanticConvetion.GEN_AI_SYSTEM:
+ SemanticConvetion.GEN_AI_SYSTEM_HUGGING_FACE,
+ SemanticConvetion.GEN_AI_ENVIRONMENT:
+ environment,
+ SemanticConvetion.GEN_AI_TYPE:
+ SemanticConvetion.GEN_AI_TYPE_CHAT,
+ SemanticConvetion.GEN_AI_REQUEST_MODEL:
+ instance.model.config.name_or_path
+ }
+
+ metrics["genai_requests"].add(1, attributes)
+ metrics["genai_total_tokens"].add(
+ prompt_tokens +
+ completion_tokens, attributes)
+ metrics["genai_completion_tokens"].add(
+ completion_tokens, attributes)
+ metrics["genai_prompt_tokens"].add(
+ prompt_tokens, attributes)
+
+ # Return original response
+ return response
+
+ except Exception as e:
+ handle_exception(span, e)
+ logger.error("Error in trace creation: %s", e)
+
+ # Return original response
+ return response
+
+ return wrapper
diff --git a/sdk/python/src/openlit/otel/metrics.py b/sdk/python/src/openlit/otel/metrics.py
new file mode 100644
index 00000000..7dca7864
--- /dev/null
+++ b/sdk/python/src/openlit/otel/metrics.py
@@ -0,0 +1,109 @@
+# pylint: disable=duplicate-code, line-too-long
+"""
+Setups up OpenTelemetry Meter
+"""
+import os
+from opentelemetry import metrics
+from opentelemetry.sdk.metrics import MeterProvider
+from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader, ConsoleMetricExporter
+from opentelemetry.sdk.resources import SERVICE_NAME, TELEMETRY_SDK_NAME, DEPLOYMENT_ENVIRONMENT
+from opentelemetry.sdk.resources import Resource
+from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
+
+from openlit.semcov import SemanticConvetion
+
+# Global flag to check if the meter provider initialization is complete.
+METER_SET = False
+
+def setup_meter(application_name, environment, meter, otlp_endpoint, otlp_headers):
+ """
+ Sets up OpenTelemetry metrics with a counter for total requests.
+
+ Params:
+ application_name (str): The name of the application for which metrics are being collected.
+ otlp_endpoint (str): The OTLP exporter endpoint for metrics.
+ otlp_headers (dict): Headers for the OTLP request.
+
+ Returns:
+ A dictionary containing the meter and created metrics for easy access.
+ """
+
+ # pylint: disable=global-statement
+ global METER_SET
+
+ try:
+ if meter is None and not METER_SET:
+ # Create a resource with the service name attribute.
+ resource = Resource(attributes={
+ SERVICE_NAME: application_name,
+ DEPLOYMENT_ENVIRONMENT: environment,
+ TELEMETRY_SDK_NAME: "openlit"}
+ )
+
+ # Only set environment variables if you have a non-None value.
+ if otlp_endpoint is not None:
+ os.environ["OTEL_EXPORTER_OTLP_ENDPOINT"] = otlp_endpoint
+
+ if otlp_headers is not None:
+ if isinstance(otlp_headers, dict):
+ headers_str = ','.join(f"{key}={value}" for key, value in otlp_headers.items())
+ else:
+ headers_str = otlp_headers
+ # Now, we have either converted the dict to a string or used the provided string.
+ os.environ["OTEL_EXPORTER_OTLP_HEADERS"] = headers_str
+
+ # Configure the span exporter and processor based on whether the endpoint is effectively set.
+ if os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT"):
+ metric_exporter = OTLPMetricExporter()
+ else:
+ metric_exporter = ConsoleMetricExporter()
+
+ metric_reader = PeriodicExportingMetricReader(metric_exporter)
+
+ meter_provider = MeterProvider(resource=resource, metric_readers=[metric_reader])
+
+ metrics.set_meter_provider(meter_provider)
+
+ meter = metrics.get_meter(__name__, version="0.1.0")
+
+ METER_SET = True
+
+ # Define and create the metrics
+ metrics_dict = {
+ "genai_requests": meter.create_counter(
+ name=SemanticConvetion.GEN_AI_REQUESTS,
+ description="Number of requests to OpenAI",
+ unit="1",
+ ),
+ "genai_prompt_tokens": meter.create_counter(
+ name=SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
+ description="Number of prompt tokens processed.",
+ unit="1",
+ ),
+ "genai_completion_tokens": meter.create_counter(
+ name=SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,
+ description="Number of completion tokens processed.",
+ unit="1",
+ ),
+ "genai_total_tokens": meter.create_counter(
+ name=SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ description="Number of total tokens processed.",
+ unit="1",
+ ),
+ "genai_cost": meter.create_histogram(
+ name=SemanticConvetion.GEN_AI_USAGE_COST,
+ description="The distribution of OpenAI request costs.",
+ unit="USD",
+ ),
+ "db_requests": meter.create_counter(
+ name=SemanticConvetion.DB_REQUESTS,
+ description="Number of requests to VectorDBs",
+ unit="1",
+ ),
+ }
+
+ return metrics_dict
+
+ # pylint: disable=bare-except
+ except:
+ return None
diff --git a/sdk/python/src/openlit/otel/tracing.py b/sdk/python/src/openlit/otel/tracing.py
new file mode 100644
index 00000000..04a40036
--- /dev/null
+++ b/sdk/python/src/openlit/otel/tracing.py
@@ -0,0 +1,83 @@
+# pylint: disable=duplicate-code, line-too-long
+"""
+Setups up OpenTelemetry tracer
+"""
+
+import os
+from opentelemetry import trace
+from opentelemetry.sdk.resources import SERVICE_NAME, TELEMETRY_SDK_NAME, DEPLOYMENT_ENVIRONMENT
+from opentelemetry.sdk.resources import Resource
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import BatchSpanProcessor, SimpleSpanProcessor
+from opentelemetry.sdk.trace.export import ConsoleSpanExporter
+from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
+
+
+# Global flag to check if the tracer provider initialization is complete.
+TRACER_SET = False
+
+def setup_tracing(application_name, environment, tracer, otlp_endpoint, otlp_headers, disable_batch):
+ """
+ Sets up tracing with OpenTelemetry. Initializes the tracer provider and configures the span processor and exporter.
+
+ Params:
+ application_name (str): The name of the application to be used in traces.
+ tracer (Tracer): Optional custom tracer. If provided, it is immediately returned and no setup is performed.
+ otlp_endpoint (str): The OTLP exporter endpoint. Falls back to the OTEL_EXPORTER_OTLP_ENDPOINT environment variable if not specified.
+ otlp_headers (dict): Headers for the OTLP request. Falls back to the OTEL_EXPORTER_OTLP_HEADERS environment variable if not specified.
+ disable_batch (bool): Flag to disable the batch span processor in favor of a simpler processor for exporting.
+
+ Returns:
+ The provided custom tracer if not None; otherwise, a tracer instance configured according to the given parameters or environment variables.
+ """
+
+ # If an external tracer is provided, return it immediately.
+ if tracer is not None:
+ return tracer
+
+ # Proceed with setting up a new tracer or configuration only if TRACER_SET is False.
+ # pylint: disable=global-statement
+ global TRACER_SET
+
+ try:
+ if not TRACER_SET:
+ # Create a resource with the service name attribute.
+ resource = Resource(attributes={
+ SERVICE_NAME: application_name,
+ DEPLOYMENT_ENVIRONMENT: environment,
+ TELEMETRY_SDK_NAME: "openlit"}
+ )
+
+ # Initialize the TracerProvider with the created resource.
+ trace.set_tracer_provider(TracerProvider(resource=resource))
+
+ # Only set environment variables if you have a non-None value.
+ if otlp_endpoint is not None:
+ os.environ["OTEL_EXPORTER_OTLP_ENDPOINT"] = otlp_endpoint
+
+ if otlp_headers is not None:
+ if isinstance(otlp_headers, dict):
+ headers_str = ','.join(f"{key}={value}" for key, value in otlp_headers.items())
+ else:
+ headers_str = otlp_headers
+
+ os.environ["OTEL_EXPORTER_OTLP_HEADERS"] = headers_str
+
+ # Configure the span exporter and processor based on whether the endpoint is effectively set.
+ if os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT"):
+ span_exporter = OTLPSpanExporter()
+
+ span_processor = BatchSpanProcessor(span_exporter) if not disable_batch else SimpleSpanProcessor(span_exporter)
+ else:
+ span_exporter = ConsoleSpanExporter()
+ span_processor = SimpleSpanProcessor(span_exporter)
+
+ trace.get_tracer_provider().add_span_processor(span_processor)
+
+ TRACER_SET = True
+
+ return trace.get_tracer(__name__)
+
+ # pylint: disable=bare-except
+ except:
+ return None
diff --git a/sdk/python/src/openlit/semcov/__init__.py b/sdk/python/src/openlit/semcov/__init__.py
new file mode 100644
index 00000000..03556a08
--- /dev/null
+++ b/sdk/python/src/openlit/semcov/__init__.py
@@ -0,0 +1,123 @@
+# pylint: disable=too-few-public-methods
+"""
+This module defines the `SemanticConvetion` class which encapsulates various constants used
+for semantic tagging within a generalized AI application context. These constants are
+intended for use across different components of AI applications, including request handling,
+response processing, usage metrics, and interaction with vector databases and AI systems.
+The purpose is to standardize the semantics for easier integration, analytics, and maintenance.
+"""
+class SemanticConvetion:
+ """
+ The SemanticConvetion class provides a centralized repository of constant values that
+ represent the keys for various semantic conventions within AI applications. These
+ conventions cover a broad range of areas including general AI configurations, request
+ parameters, usage metrics, response attributes, and integrations with external AI and
+ database systems. It is designed to facilitate consistency and understandability across
+ the application's data logging and processing functionalities.
+ """
+
+ # GenAI General
+ GEN_AI_ENDPOINT = "gen_ai.endpoint"
+ GEN_AI_SYSTEM = "gen_ai.system"
+ GEN_AI_ENVIRONMENT = "gen_ai.environment"
+ GEN_AI_APPLICATION_NAME = "gen_ai.application_name"
+ GEN_AI_TYPE = "gen_ai.type"
+ GEN_AI_HUB_OWNER = "gen_ai.hub.owner"
+ GEN_AI_HUB_REPO = "gen_ai.hub.repo"
+ GEN_AI_RETRIEVAL_SOURCE = "gen_ai.retrieval.source"
+ GEN_AI_REQUESTS = "gen_ai.total.requests"
+
+ # GenAI Request
+ GEN_AI_REQUEST_MODEL = "gen_ai.request.model"
+ GEN_AI_REQUEST_TEMPERATURE = "gen_ai.request.temperature"
+ GEN_AI_REQUEST_TOP_P = "gen_ai.request.top_p"
+ GEN_AI_REQUEST_TOP_K = "gen_ai.request.top_k"
+ GEN_AI_REQUEST_MAX_TOKENS = "gen_ai.request.max_tokens"
+ GEN_AI_REQUEST_IS_STREAM = "gen_ai.request.is_stream"
+ GEN_AI_REQUEST_USER = "gen_ai.request.user"
+ GEN_AI_REQUEST_SEED = "gen_ai.request.seed"
+ GEN_AI_REQUEST_FREQUENCY_PENALTY = "gen_ai.request.frequency_penalty"
+ GEN_AI_REQUEST_PRESENCE_PENALTY = "gen_ai.request.presence_penalty"
+ GEN_AI_REQUEST_EMBEDDING_FORMAT = "gen_ai.request.embedding_format"
+ GEN_AI_REQUEST_EMBEDDING_DIMENSION = "gen_ai.request.embedding_dimension"
+ GEN_AI_REQUEST_TOOL_CHOICE = "gen_ai.request.tool_choice"
+ GEN_AI_REQUEST_AUDIO_VOICE = "gen_ai.request.audio_voice"
+ GEN_AI_REQUEST_AUDIO_RESPONSE_FORMAT = "gen_ai.request.audio_response_format"
+ GEN_AI_REQUEST_AUDIO_SPEED = "gen_ai.request.audio_speed"
+ GEN_AI_REQUEST_FINETUNE_STATUS = "gen_ai.request.fine_tune_status"
+ GEN_AI_REQUEST_FINETUNE_MODEL_SUFFIX = "gen_ai.request.fine_tune_model_suffix"
+ GEN_AI_REQUEST_FINETUNE_MODEL_EPOCHS = "gen_ai.request.fine_tune_n_epochs"
+ GEN_AI_REQUEST_FINETUNE_MODEL_LRM = "gen_ai.request.learning_rate_multiplier"
+ GEN_AI_REQUEST_FINETUNE_BATCH_SIZE = "gen_ai.request.fine_tune_batch_size"
+ GEN_AI_REQUEST_VALIDATION_FILE = "gen_ai.request.validation_file"
+ GEN_AI_REQUEST_TRAINING_FILE = "gen_ai.request.training_file"
+
+ # GenAI Usage
+ GEN_AI_USAGE_PROMPT_TOKENS = "gen_ai.usage.prompt_tokens"
+ GEN_AI_USAGE_COMPLETION_TOKENS = "gen_ai.usage.completion_tokens"
+ GEN_AI_USAGE_TOTAL_TOKENS = "gen_ai.usage.total_tokens"
+ GEN_AI_USAGE_COST = "gen_ai.usage.cost"
+
+ # GenAI Response
+ GEN_AI_RESPONSE_ID = "gen_ai.response.id"
+ GEN_AI_RESPONSE_FINISH_REASON = "gen_ai.response.finish_reason"
+ GEN_AI_RESPONSE_IMAGE = "gen_ai.response.image" # Not used directly in code yet
+ GEN_AI_RESPONSE_IMAGE_SIZE = "gen_ai.request.image_size"
+ GEN_AI_RESPONSE_IMAGE_QUALITY = "gen_ai.request.image_quality"
+ GEN_AI_RESPONSE_IMAGE_STYLE = "gen_ai.request.image_style"
+
+ # GenAI Content
+ GEN_AI_CONTENT_PROMPT = "gen_ai.content.prompt"
+ GEN_AI_CONTENT_COMPLETION = "gen_ai.content.completion"
+ GEN_AI_CONTENT_REVISED_PROMPT = "gen_ai.content.revised_prompt"
+
+ GEN_AI_TYPE_CHAT = "chat"
+ GEN_AI_TYPE_EMBEDDING = "embedding"
+ GEN_AI_TYPE_IMAGE = "image"
+ GEN_AI_TYPE_AUDIO = "audio"
+ GEN_AI_TYPE_FINETUNING = "fine_tuning"
+ GEN_AI_TYPE_VECTORDB = "vectordb"
+ GEN_AI_TYPE_FRAMEWORK = "framework"
+
+ GEN_AI_SYSTEM_HUGGING_FACE = "huggingface"
+ GEN_AI_SYSTEM_OPENAI = "openai"
+ GEN_AI_SYSTEM_AZURE_OPENAI = "azure_openai"
+ GEN_AI_SYSTEM_ANTHROPIC = "anthropic"
+ GEN_AI_SYSTEM_COHERE = "cohere"
+ GEN_AI_SYSTEM_MISTRAL = "mistral"
+ GEN_AI_SYSTEM_LANGCHAIN = "langchain"
+
+ # Vector DB
+ DB_REQUESTS = "db.total.requests"
+ DB_SYSTEM = "db.system"
+ DB_SYSTEM_CHROMA = "chroma"
+ DB_SYSTEM_PINECONE = "pinecone"
+ DB_COLLECTION_NAME = "db.collection.name"
+ DB_OPERATION = "db.operation"
+ DB_OPERATION_CREATE_INDEX = "create_index"
+ DB_OPERATION_QUERY = "query"
+ DB_OPERATION_DELETE = "delete"
+ DB_OPERATION_UPDATE = "update"
+ DB_OPERATION_UPSERT = "upsert"
+ DB_OPERATION_GET = "get"
+ DB_OPERATION_ADD = "add"
+ DB_OPERATION_PEEK = "peek"
+ DB_ID_COUNT = "db.ids_count"
+ DB_VECTOR_COUNT = "db.vector_count"
+ DB_METADATA_COUNT = "db.metadatas_count"
+ DB_DOCUMENTS_COUNT = "db.documents_count"
+ DB_QUERY_LIMIT = "db.limit"
+ DB_OFFSET = "db.offset"
+ DB_WHERE_DOCUMENT = "db.where_document"
+ DB_FILTER = "db.filter"
+ DB_STATEMENT = "db.statement"
+ DB_N_RESULTS = "db.n_results"
+ DB_DELETE_ALL = "db.delete_all"
+ DB_INDEX_NAME = "db.create_index.name"
+ DB_INDEX_DIMENSION = "db.create_index.dimensions"
+ DB_INDEX_METRIC = "db.create_index.metric"
+ DB_INDEX_SPEC = "db.create_index.spec"
+ DB_NAMESPACE = "db.query.namespace"
+ DB_UPDATE_METADATA = "db.update.metadata"
+ DB_UPDATE_VALUES = "db.update.values"
+ DB_UPDATE_ID = "db.update.id"
diff --git a/sdk/python/tests/requirements.txt b/sdk/python/tests/requirements.txt
new file mode 100644
index 00000000..11a339eb
--- /dev/null
+++ b/sdk/python/tests/requirements.txt
@@ -0,0 +1,33 @@
+# Development
+pytest>=7.4.0
+pytest-asyncio>=0.23.6
+requests>=2.31.0
+pylint>=3.0.2
+
+# Gen AI
+cohere>=5.0.0
+openai>=1.1.1
+anthropic>=0.21.0
+mistralai>=0.1.0
+transformers>=4.39.3
+
+# Vector DBs
+chromadb>=0.4.0
+pinecone-client>=2.2.0
+
+# Frameworks
+langchain>=0.1.16
+langchainhub>=0.1.15
+langchain_community>=0.0.33
+
+# Utilities
+bs4
+tiktoken>=0.1.1
+tensorflow>=2.16.1
+tf_keras>=2.16.0
+
+# OpenTelemetry
+opentelemetry-api>=1.24.0
+opentelemetry-sdk>=1.24.0
+opentelemetry-exporter-otlp>=1.24.0
+opentelemetry-instrumentation>=0.45b0
\ No newline at end of file
diff --git a/sdk/python/tests/test-image-for-openai.png b/sdk/python/tests/test-image-for-openai.png
new file mode 100644
index 00000000..6ca1e59b
Binary files /dev/null and b/sdk/python/tests/test-image-for-openai.png differ
diff --git a/sdk/python/tests/test_anthropic.py.hold b/sdk/python/tests/test_anthropic.py.hold
new file mode 100644
index 00000000..78cff604
--- /dev/null
+++ b/sdk/python/tests/test_anthropic.py.hold
@@ -0,0 +1,84 @@
+# pylint: disable=duplicate-code, no-name-in-module
+"""
+This module contains tests for Anthropic functionality using the Anthropic Python library.
+
+Tests cover various API endpoints, including chat.
+These tests validate integration with OpenLIT.
+
+Environment Variables:
+ - ANTHROPIC_API_TOKEN: Anthropic API api_key for authentication.
+
+Note: Ensure the environment is properly configured for Anthropic access and OpenLIT monitoring
+prior to running these tests.
+"""
+
+import os
+import pytest
+from anthropic import Anthropic, AsyncAnthropic
+import openlit
+
+# Initialize synchronous Anthropic client
+sync_client = Anthropic(
+ api_key=os.getenv("ANTHROPIC_API_TOKEN")
+)
+
+# Initialize asynchronous Anthropic client
+async_client = AsyncAnthropic(
+ api_key=os.getenv("ANTHROPIC_API_TOKEN")
+)
+
+# Initialize environment and application name for OpenLIT monitoring
+openlit.init(environment="openlit-testing", application_name="openlit-python-test")
+
+def test_sync_anthropic_messages():
+ """
+ Tests synchronous messages with the 'claude-3-haiku-20240307' model.
+
+ Raises:
+ AssertionError: If the messages response object is not as expected.
+ """
+
+ try:
+ message = sync_client.messages.create(
+ max_tokens=1,
+ messages=[
+ {
+ "role": "user",
+ "content": "Hello, Claude",
+ }
+ ],
+ model="claude-3-haiku-20240307",
+ )
+ assert message.type == 'message'
+
+ # pylint: disable=broad-exception-caught
+ except Exception as e:
+ if "rate limit" in str(e).lower():
+ print("Rate Limited:", e)
+
+async def test_async_anthropic_messages():
+ """
+ Tests synchronous messages with the 'claude-3-haiku-20240307' model.
+
+ Raises:
+ AssertionError: If the messages response object is not as expected.
+ """
+
+ try:
+ message = await async_client.messages.create(
+ max_tokens=1,
+ messages=[
+ {
+ "role": "user",
+ "content": "Hello, Claude",
+ }
+ ],
+ model="claude-3-haiku-20240307",
+ )
+ assert message.type == 'message'
+
+ # pylint: disable=broad-exception-caught
+ except Exception as e:
+ if "rate limit" in str(e).lower():
+ print("Rate Limited:", e)
diff --git a/sdk/python/tests/test_azure.py.hold b/sdk/python/tests/test_azure.py.hold
new file mode 100644
index 00000000..56a3e071
--- /dev/null
+++ b/sdk/python/tests/test_azure.py.hold
@@ -0,0 +1,99 @@
+# pylint: disable=duplicate-code
+"""
+Azure OpenAI Test Suite
+
+This module contains a suite of tests for OpenAI functionality using the OpenAI Python library.
+It includes tests for various OpenAI API endpoints such as completions, chat completions,
+embeddings creation, fine-tuning job creation, image generation, image variation creation,
+and audio speech generation.
+
+The tests are designed to cover different aspects of OpenAI's capabilities and serve as a
+validation mechanism for the integration with the OpenLIT monitoring system.
+
+Global client and initialization are set up for the OpenAI client and OpenLIT monitoring.
+
+Environment Variables:
+ - AZURE_OPENAI_API_TOKEN: OpenAI API key for authentication.
+
+Note: Ensure the environment is properly configured for Azure OpenAI access and OpenLIT monitoring
+prior to running these tests.
+"""
+
+import os
+from openai import AzureOpenAI
+import openlit
+
+# Global client
+client = AzureOpenAI(
+ api_key=os.getenv("AZURE_OPENAI_API_TOKEN"),
+ api_version = "2024-02-01",
+ azure_endpoint = os.getenv("AZURE_OPENAI_ENDPOINT")
+)
+
+azure_chat_model = os.getenv("AZURE_OPENAI_CHAT_MODEL")
+azure_embedding_model = os.getenv("AZURE_OPENAI_EMBEDDING_MODEL")
+azure_image_model = os.getenv("AZURE_OPENAI_IMAGE_MODEL")
+
+# Global initialization
+# pylint: disable=line-too-long
+openlit.init(environment="openlit-testing", application_name="openlit-python-test")
+
+def test_completion():
+ """
+ Test the completions.
+
+ Raises:
+ AssertionError: If the completion response object is not as expected.
+ """
+
+ completions_resp = client.completions.create(
+ model=azure_chat_model,
+ prompt="Hello world",
+ max_tokens=100
+ )
+ assert completions_resp.object == 'text_completion'
+
+def test_chat_completion():
+ """
+ Test chat completions.
+
+ Raises:
+ AssertionError: If the chat completion response object is not as expected.
+ """
+
+ chat_completions_resp = client.chat.completions.create(
+ model=azure_chat_model,
+ max_tokens=100,
+ messages=[{"role": "user", "content": "What is Grafana?"}]
+ )
+ assert chat_completions_resp.object == 'chat.completion'
+
+def test_embedding_creation():
+ """
+ Test embedding creation.
+
+ Raises:
+ AssertionError: If the embedding response object is not as expected.
+ """
+
+ embeddings_resp = client.embeddings.create(
+ model=azure_embedding_model,
+ input="The quick brown fox jumped over the lazy dog",
+ encoding_format="float"
+ )
+ assert embeddings_resp.data[0].object == 'embedding'
+
+def test_image_generation():
+ """
+ Test image generation.
+
+ Raises:
+ AssertionError: If the image generation response created timestamp is not present.
+ """
+
+ image_generation_resp = client.images.generate(
+ model=azure_chat_model,
+ prompt='Generate an image for LLM Observability Dashboard',
+ n=1
+ )
+ assert image_generation_resp.created is not None
diff --git a/sdk/python/tests/test_chroma.py b/sdk/python/tests/test_chroma.py
new file mode 100644
index 00000000..861c92f2
--- /dev/null
+++ b/sdk/python/tests/test_chroma.py
@@ -0,0 +1,68 @@
+# pylint: disable=duplicate-code, no-name-in-module
+"""
+This module contains tests for ChromaDB functionality using the ChromaDB Python library.
+
+Tests cover various API endpoints, including create_collection, add, query,
+upsert, update, get, peek and delete.
+These tests validate integration with OpenLIT.
+
+Note: Ensure the environment is properly configured for ChromaDB access and OpenLIT monitoring
+prior to running these tests.
+"""
+
+import chromadb
+import openlit
+
+# Initialize ChromaDB client
+chroma_client = chromadb.Client()
+
+# Initialize environment and application name for OpenLIT monitoring
+openlit.init(environment="openlit-testing", application_name="openlit-python-test")
+
+def test_db_chroma():
+ """
+ Tests basic operations within a ChromaDB collection.
+
+ This includes creating a new collection, adding documents to the collection,
+ querying the collection, and deleting documents from the collection. The test
+ verifies correct behavior of these operations by asserting expected outcomes at
+ each step.
+
+ - A new collection named "openlit" is created and verified for correct naming.
+ - Documents are added to the collection with specific metadata and ids,
+ and the absence of an error response is verified.
+ - A query operation is performed, and its results are verified to match the
+ expected document ids.
+ - A delete operation targets a specific document by id, and successful deletion
+ is implicitly verified by the absence of an error response.
+
+ The test ensures the basic CRUD operations perform as expected in ChromaDB.
+ Raises:
+ AssertionError: If the responses from ChromaDB operations do not meet the expected outcomes.
+ """
+
+ # Create a new collection named "openlit"
+ collection = chroma_client.create_collection(name="openlit")
+ assert collection.name == 'openlit'
+
+ # Add documents to the collection
+ db_add = collection.add(
+ documents=["This is a document", "This is another document"],
+ metadatas=[{"source": "my_source"}, {"source": "my_source"}],
+ ids=["id1", "id2"]
+ )
+ assert db_add is None
+
+ # Query the documents in the collection
+ db_query = collection.query(
+ query_texts=["This is a query document"],
+ n_results=2
+ )
+ assert db_query["ids"] == [['id1', 'id2']]
+
+ # Delete a document from the collection
+ db_delete = collection.delete(
+ ids=["id2"],
+ where={"source": "my_source"}
+ )
+ assert db_delete is None
diff --git a/sdk/python/tests/test_cohere.py.hold b/sdk/python/tests/test_cohere.py.hold
new file mode 100644
index 00000000..185b7e36
--- /dev/null
+++ b/sdk/python/tests/test_cohere.py.hold
@@ -0,0 +1,59 @@
+# pylint: disable=duplicate-code, no-member
+"""
+This module contains tests for Cohere functionality using the Cohere Python library.
+
+Tests cover various API endpoints, including chat and embeddings.
+These tests validate integration with OpenLIT.
+
+Environment Variables:
+ - COHERE_API_TOKEN: Cohere API api_key for authentication.
+
+Note: Ensure the environment is properly configured for Cohere access and OpenLIT monitoring
+prior to running these tests.
+"""
+
+import os
+import cohere
+import openlit
+
+# Initialize synchronous Cohere client
+sync_client = cohere.Client(os.getenv("COHERE_API_TOKEN"))
+
+# Initialize environment and application name for OpenLIT monitoring
+openlit.init(environment="openlit-testing", application_name="openlit-python-test")
+
+def test_embed():
+ """
+ Tests synchronous embedding creation with the 'embed-english-v3.0' model.
+
+ Raises:
+ AssertionError: If the embedding response object is not as expected.
+ """
+
+ try:
+ embeddings_resp = sync_client.embed(
+ texts=['This is a test'],
+ model='embed-english-v3.0'
+ )
+ assert embeddings_resp.meta is not None
+
+ except cohere.core.api_error.ApiError as e:
+ print("Rate Limited:", e)
+
+def test_chat():
+ """
+ Tests synchronous chat with the 'command' model.
+
+ Raises:
+ AssertionError: If the chat response object is not as expected.
+ """
+
+ try:
+ chat_resp = sync_client.chat(
+ message='Say this is a test',
+ model='command'
+ )
+ assert chat_resp.response_id is not None
+
+ except cohere.core.api_error.ApiError as e:
+ print("Rate Limited:", e)
diff --git a/sdk/python/tests/test_langchain.py b/sdk/python/tests/test_langchain.py
new file mode 100644
index 00000000..b68be789
--- /dev/null
+++ b/sdk/python/tests/test_langchain.py
@@ -0,0 +1,59 @@
+# pylint: disable=duplicate-code, no-member
+"""
+This module contains tests for Langchain functionality using the langchain Python library.
+
+Tests cover various functions, including WebBaseLoader, TextSplitter,
+and Prompt pull from Hub.
+These tests validate integration with OpenLIT.
+
+Note: Ensure the environment is properly configured for Langchain and OpenLIT monitoring
+prior to running these tests.
+"""
+
+import bs4
+from langchain import hub
+from langchain_community.document_loaders import WebBaseLoader
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+import openlit
+
+# Initialize environment and application name for OpenLIT monitoring
+openlit.init(environment="openlit-testing", application_name="openlit-python-test")
+
+def test_langchain():
+ """
+ Verifies the functionality of LangChain operations including document loading, text splitting,
+ and prompt pulling from the Hub. Each operation is tested to ensure it functions as intended,
+ confirming the efficient use of the LangChain library for processing and interacting
+ with text data.
+
+ Steps involved in the test:
+ - Documents are loaded using the WebBaseLoader, focusing on specific content from
+ a provided URL.
+ - The loaded documents are then split into smaller chunks using
+ the RecursiveCharacterTextSplitter.
+ - A prompt is retrieved from the LangChain Hub to verify the functionality of
+ pulling resources from the Hub.
+
+ This test checks for the expected outcomes and metadata from each operation to
+ confirm that the LangChain library is working as expected within the intended use cases.
+
+ Raises:
+ AssertionError: If the outcomes from the LangChain operations deviate from what is expected.
+ """
+
+ docs = WebBaseLoader(
+ web_paths=("https://lilianweng.github.io/posts/2023-06-23-agent/",),
+ bs_kwargs={
+ "parse_only": bs4.SoupStrainer(
+ class_=("post-content", "post-title", "post-header")
+ )
+ },
+ ).load()
+ assert docs[0].metadata["source"] == "https://lilianweng.github.io/posts/2023-06-23-agent/"
+
+ texts = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)
+ splits = texts.split_documents(docs)
+ assert splits[0].metadata["source"] == "https://lilianweng.github.io/posts/2023-06-23-agent/"
+
+ prompt = hub.pull("rlm/rag-prompt")
+ assert prompt.metadata["lc_hub_repo"] == "rag-prompt"
diff --git a/sdk/python/tests/test_mistral.py.hold b/sdk/python/tests/test_mistral.py.hold
new file mode 100644
index 00000000..0dfd7976
--- /dev/null
+++ b/sdk/python/tests/test_mistral.py.hold
@@ -0,0 +1,94 @@
+# pylint: disable=duplicate-code
+"""
+This module contains tests for Mistral functionality using the Mistral Python library.
+
+Tests cover various API endpoints, including chat and embeddings.
+These tests validate integration with OpenLIT.
+
+Environment Variables:
+ - Mistral_API_TOKEN: Mistral API api_key for authentication.
+
+Note: Ensure the environment is properly configured for Mistral access and OpenLIT monitoring
+prior to running these tests.
+"""
+
+import os
+import pytest
+from mistralai.client import MistralClient
+from mistralai.async_client import MistralAsyncClient
+from mistralai.models.chat_completion import ChatMessage
+import openlit
+
+# Initialize synchronous Mistral client
+sync_client = MistralClient(
+ api_key=os.getenv("MISTRAL_API_TOKEN")
+)
+
+# Initialize asynchronous Mistral client
+async_client = MistralAsyncClient(
+ api_key=os.getenv("MISTRAL_API_TOKEN")
+)
+
+# Initialize environment and application name for OpenLIT monitoring
+openlit.init(environment="openlit-testing", application_name="openlit-python-test")
+
+def test_sync_mistral_chat():
+ """
+ Tests synchronous chat with the 'open-mistral-7b' model.
+
+ Raises:
+ AssertionError: If the chat response object is not as expected.
+ """
+
+ messages = [
+ ChatMessage(role="user", content="What is the best French cheese?")
+ ]
+
+ message = sync_client.chat(
+ model="open-mistral-7b",
+ messages=messages,
+ max_tokens=1,
+ )
+ assert message.object == 'chat.completion'
+
+def test_sync_mistral_embeddings():
+ """
+ Tests synchronous embedding creation with the 'mistral-embed' model.
+
+ Raises:
+ AssertionError: If the embedding response object is not as expected.
+ """
+
+ response = sync_client.embeddings(
+ model="mistral-embed",
+ input=["Embed this sentence.", "As well as this one."],
+ )
+ assert response.object == 'list'
+
+async def test_async_mistral():
+ """
+ Tests asynchronous Mistral.
+
+ Raises:
+ AssertionError: If the chat response object is not as expected.
+ """
+
+ # Tests synchronous chat with the 'open-mistral-7b' model.
+ messages = [
+ ChatMessage(role="user", content="What is the best French cheese?")
+ ]
+
+ message = await async_client.chat(
+ model="open-mistral-7b",
+ messages=messages,
+ max_tokens=1,
+ )
+ assert message.object == 'chat.completion'
+
+ # Tests asynchronous embedding creation with the 'mistral-embed' model.
+ response = await async_client.embeddings(
+ model="mistral-embed",
+ input=["Embed this sentence.", "As well as this one."],
+ )
+ assert response.object == 'list'
diff --git a/sdk/python/tests/test_openai.py.hold b/sdk/python/tests/test_openai.py.hold
new file mode 100644
index 00000000..37194c46
--- /dev/null
+++ b/sdk/python/tests/test_openai.py.hold
@@ -0,0 +1,193 @@
+# pylint: disable=duplicate-code, consider-using-with, no-name-in-module
+"""
+This module contains tests for OpenAI functionality using the OpenAI Python library.
+
+Tests cover various API endpoints, including completions, chat completions,
+embeddings, fine-tuning job creation, image generation, image variation creation,
+and audio speech generation. These tests validate integration with OpenLIT.
+
+Environment Variables:
+ - OPENAI_API_TOKEN: OpenAI API key for authentication.
+
+Note: Ensure the environment is properly configured for OpenAI access and OpenLIT monitoring
+prior to running these tests.
+"""
+
+import os
+import pytest
+from openai import OpenAI, AsyncOpenAI
+import openlit
+
+# Initialize synchronous OpenAI client
+sync_client = OpenAI(
+ api_key=os.getenv("OPENAI_API_TOKEN"),
+)
+
+# Initialize asynchronous OpenAI client
+async_client = AsyncOpenAI(
+ api_key=os.getenv("OPENAI_API_TOKEN"),
+)
+
+# Initialize environment and application name for OpenLIT monitoring
+openlit.init(environment="openlit-testing", application_name="openlit-python-test")
+
+def test_sync_openai_chat_completions():
+ """
+ Tests synchronous chat completions with the 'gpt-3.5-turbo' model.
+
+ Raises:
+ AssertionError: If the chat completion response object is not as expected.
+ """
+
+ chat_completions_resp = sync_client.chat.completions.create(
+ model="gpt-3.5-turbo",
+ max_tokens=1,
+ messages=[{"role": "user", "content": "What is LLM Observability?"}]
+ )
+ assert chat_completions_resp.object == 'chat.completion'
+
+def test_sync_openai_embeddings():
+ """
+ Tests synchronous embedding creation with the 'text-embedding-ada-002' model.
+
+ Raises:
+ AssertionError: If the embedding response object is not as expected.
+ """
+
+ embeddings_resp = sync_client.embeddings.create(
+ model="text-embedding-ada-002",
+ input="The quick brown fox jumped over the lazy dog",
+ encoding_format="float"
+ )
+ assert embeddings_resp.data[0].object == 'embedding'
+
+# def test_sync_openai_fine_tuning_job_creation():
+# """
+# Test fine-tuning job creation.
+
+# Raises:
+# AssertionError: If the fine-tuning job response object is not as expected.
+# """
+# try:
+# fine_tuning_job_resp = sync_client.fine_tuning.jobs.create(
+# training_file="",
+# model="gpt-3.5-turbo-1106"
+# )
+# assert fine_tuning_job_resp.object == 'fine_tuning.job'
+
+# # pylint: disable=broad-exception-caught
+# except Exception as e:
+# if 'rate_limit_exceeded' in str(e):
+# error_json = e.response.json()
+# rate_limit_code = error_json['error']['code']
+# print(rate_limit_code)
+
+def test_sync_openai_image_generation():
+ """
+ Tests synchronous image generation with the 'dall-e-2' model.
+
+ Raises:
+ AssertionError: If the image generation response created timestamp is not present.
+ """
+
+ image_generation_resp = sync_client.images.generate(
+ model='dall-e-2',
+ prompt='Generate an image of a cat.',
+ size='256x256',
+ n=1
+ )
+ assert image_generation_resp.created is not None
+
+def test_sync_openai_image_variations():
+ """
+ Tests synchronous image variation creation with the 'dall-e-2' model
+
+ Raises:
+ AssertionError: If the image variation response created timestamp is not present.
+ """
+
+ image_variation_resp = sync_client.images.create_variation(
+ image=open("tests/test-image-for-openai.png", "rb"),
+ model='dall-e-2',
+ n=1,
+ size="256x256"
+ )
+ assert image_variation_resp.created is not None
+
+def test_sync_openai_audio_speech_create():
+ """
+ Tests synchronous audio speech generation with the 'tts-1' model.
+
+ Raises:
+ AssertionError: If the audio speech response is not present or not an instance of an object.
+ """
+
+ audio_speech_resp = sync_client.audio.speech.create(
+ model='tts-1',
+ voice='alloy',
+ input='LLM Observability!')
+ assert audio_speech_resp is not None and isinstance(audio_speech_resp, object)
+
+async def test_async_openai_chat_completions():
+ """
+ Tests asynchronous chat completions with the 'gpt-3.5-turbo' model.
+
+ Raises:
+ AssertionError: If the chat completion response object is not as expected.
+ """
+
+ chat_completions_resp = await async_client.chat.completions.create(
+ model="gpt-3.5-turbo",
+ max_tokens=1,
+ messages=[{"role": "user", "content": "What is LLM Observability?"}]
+ )
+ assert chat_completions_resp.object == 'chat.completion'
+
+async def test_async_openai_embeddings():
+ """
+ Tests asynchronous embedding creation with the 'text-embedding-ada-002' model.
+
+ Raises:
+ AssertionError: If the embedding response object is not as expected.
+ """
+
+ embeddings_resp = await async_client.embeddings.create(
+ model="text-embedding-ada-002",
+ input="The quick brown fox jumped over the lazy dog",
+ encoding_format="float"
+ )
+ assert embeddings_resp.data[0].object == 'embedding'
+
+async def test_async_openai_image_variations():
+ """
+ Tests asynchronous image variation creation with the 'dall-e-2' model.
+
+ Raises:
+ AssertionError: If the image variation response created timestamp is not present.
+ """
+
+ image_variation_resp = await async_client.images.create_variation(
+ image=open("tests/test-image-for-openai.png", "rb"),
+ model='dall-e-2',
+ n=1,
+ size="256x256"
+ )
+ assert image_variation_resp.created is not None
+
+async def test_async_openai_audio_speech_create():
+ """
+ Tests asynchronous audio speech generation with the 'tts-1' model.
+
+ Raises:
+ AssertionError: If the audio speech response is not present or not an instance of an object.
+ """
+
+ audio_speech_resp = await async_client.audio.speech.create(
+ model='tts-1',
+ voice='alloy',
+ input='LLM Observability!')
+ assert audio_speech_resp is not None and isinstance(audio_speech_resp, object)
diff --git a/sdk/python/tests/test_pinecone.py b/sdk/python/tests/test_pinecone.py
new file mode 100644
index 00000000..cab460c3
--- /dev/null
+++ b/sdk/python/tests/test_pinecone.py
@@ -0,0 +1,72 @@
+# pylint: disable=duplicate-code, no-name-in-module
+"""
+This module contains tests for ChromaDB functionality using the ChromaDB Python library.
+
+Tests cover various API endpoints, including upsert, query and delete.
+These tests validate integration with OpenLIT.
+
+Environment Variables:
+ - PINECONE_API_TOKEN: Cohere API api_key for authentication.
+
+Note: Ensure the environment is properly configured for Pinecone access and OpenLIT monitoring
+prior to running these tests.
+"""
+
+import os
+from pinecone import Pinecone
+import openlit
+
+# Initialize the Chroma client
+pc = Pinecone(api_key=os.getenv("PINECONE_API_TOKEN"))
+index = pc.Index("openlit-tests")
+
+# Initialize environment and application name for OpenLIT monitoring
+openlit.init(environment="openlit-testing", application_name="openlit-python-test")
+
+def test_db_pinecone():
+ """
+ Tests basic operations within a Pinecone index.
+
+ This includes adding vectors to the index,
+ querying the index, and deleting vectors from the index. The test
+ verifies correct behavior of these operations by asserting expected outcomes at
+ each step.
+
+ - Vectors are upserted into the index with unique IDs and vector values. The test
+ verifies that the upsert operation reports the correct number of vectors inserted.
+ - A query operation is performed using a vector similar to one of the upserted vectors.
+ The test verifies that the query successfully executes within the specified namespace
+ and that the results match expected vectors vectors.
+ - A delete operation targets specific vectors by ID within a namespace, and the test
+ verifies that the delete operation completes successfully by asserting the absence
+ of an error response.
+
+ The test ensures the basic CRUD operations perform as expected in Pinecone.
+ Raises:
+ AssertionError: If the responses from Pinecone operations do not meet the expected outcomes.
+ """
+
+ # Upsert vectors to the index
+ db_upsert = index.upsert(
+ vectors=[
+ {"id": "vec1", "values": [0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1]},
+ {"id": "vec2", "values": [0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2]},
+ {"id": "vec3", "values": [0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3]},
+ {"id": "vec4", "values": [0.4, 0.4, 0.4, 0.4, 0.4, 0.4, 0.4, 0.4]}
+ ],
+ namespace="python-tests"
+ )
+ assert db_upsert.upserted_count == 4
+
+ # Query the vectors from the index
+ db_query = index.query(
+ namespace="python-tests",
+ vector=[0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3],
+ top_k=3,
+ include_values=True
+ )
+ assert db_query["namespace"] == "python-tests"
+
+ # Delete the vectors from the index
+ db_delete = index.delete(ids=["vec1", "vec2", "vec3", "vec4"], namespace='python-tests')
+ assert db_delete == {}
diff --git a/sdk/python/tests/test_transformers.py b/sdk/python/tests/test_transformers.py
new file mode 100644
index 00000000..0b2417d3
--- /dev/null
+++ b/sdk/python/tests/test_transformers.py
@@ -0,0 +1,33 @@
+# pylint: disable=duplicate-code, no-name-in-module
+"""
+This module contains tests for Text Generation functionality in HuggingFace Transformers library.
+
+Tests cover the usage of the Transformers' pipeline for generating text.
+These tests validate integration with OpenLIT.
+
+Note: Ensure the environment is properly configured for Transformers and OpenLIT monitoring
+prior to running these tests.
+"""
+
+from transformers import pipeline
+import openlit
+
+# Initialize environment and application name for OpenLIT monitoring
+openlit.init(environment="openlit-testing", application_name="openlit-python-test")
+
+def test_text_trasnformers():
+ """
+ Test text generation capabilities using the GPT-2 model from HuggingFace Transformers library.
+
+ This test sends a prompt to a pre-specified model and verifies that the response contains
+ generated text matching expected criteria.
+ In this case, simply the presence of the 'generated_text' field in the response object.
+
+ Raises:
+ AssertionError: If the 'generated_text' field is missing from any part of the response.
+ """
+
+ generator = pipeline(model="openai-community/gpt2")
+ response = generator("My tart needs some", num_return_sequences=1, return_full_text=False)
+ for output in response:
+ assert 'generated_text' in output