Skip to content

Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.

[Feat]: Add OpenTelemetry instrumentation for Monitoring LLM Applications using Ollama (#234)

patcher96dbc20c587ed
diff --git a/docs/images/docs-ollama-banner.gif b/docs/images/docs-ollama-banner.gif
new file mode 100644
index 00000000..203f03d9
Binary files /dev/null and b/docs/images/docs-ollama-banner.gif differ
diff --git a/docs/latest/integrations/introduction.mdx b/docs/latest/integrations/introduction.mdx
index 35f1734a..c329caa5 100644
--- a/docs/latest/integrations/introduction.mdx
+++ b/docs/latest/integrations/introduction.mdx
@@ -20,6 +20,21 @@ Start integrating and monitoring your applications with OpenLIT effortlessly. Ch
iconType="duotone"
>
</Card>
+ <Card
+ title="Ollama"
+ href="/latest/integrations/ollama"
+ icon={
+ <svg role="img" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 204 204" >
+ <path d="M0 0 C9.42981215 6.49609281 13.26228354 17.00573537 15.55078125 27.90234375 C15.85445328 30.26373375 15.85799649 32.55927413 15.8125 34.9375 C16.64394531 34.58816406 17.47539063 34.23882812 18.33203125 33.87890625 C19.41871094 33.42386719 20.50539063 32.96882813 21.625 32.5 C22.70394531 32.04753906 23.78289062 31.59507813 24.89453125 31.12890625 C35.73230506 26.70384465 49.73257469 27.64159908 60.5 31.8671875 C62.62642222 32.85137616 64.72849057 33.86655071 66.8125 34.9375 C66.83828125 34.35484375 66.8640625 33.7721875 66.890625 33.171875 C67.72257469 21.14642046 71.51980469 9.79117005 80.5 1.4375 C85.05280972 -1.51567387 89.54457924 -1.68704731 94.8125 -1.0625 C102.04053595 2.33892868 106.21055073 8.87893629 109.0625 16.125 C112.04324874 25.17902431 112.96893113 34.20631584 113 43.6875 C113.00241699 44.41928467 113.00483398 45.15106934 113.00732422 45.9050293 C112.9733355 49.73486101 112.59977712 53.16682322 111.6875 56.875 C111.27118897 58.90802182 111.27118897 58.90802182 111.8125 60.9375 C114.07530185 63.76154797 116.55791309 66.10948173 119.234375 68.54296875 C127.92729796 76.61685022 131.22080846 88.81065807 131.96875 100.34765625 C132.27896879 109.33107549 130.93180174 118.42529822 126.5625 126.375 C124.19783912 131.18877393 125.19029248 133.94643593 126.75 138.9375 C127.100625 139.9275 127.45125 140.9175 127.8125 141.9375 C132.50870737 155.19737962 132.33304687 170.61927346 127.8125 183.9375 C126.85613894 185.8656473 125.87742423 187.78289397 124.875 189.6875 C123.04760883 193.33897414 123.04760883 193.33897414 123.3125 197.26953125 C124.03103471 198.77792581 124.80013109 200.26309018 125.61328125 201.72265625 C129.09036432 208.14450022 128.99832746 215.55518093 129.0625 222.6875 C129.07651855 223.7693457 129.07651855 223.7693457 129.09082031 224.87304688 C129.06274646 229.54194446 128.34698249 232.33405254 126.8125 236.9375 C121.8625 236.9375 116.9125 236.9375 111.8125 236.9375 C112.1425 234.6275 112.4725 232.3175 112.8125 229.9375 C113.39751255 218.90101316 112.9527192 209.57592042 107.5625 199.75 C105.50093265 195.77809998 105.4845874 193.31180034 106.8125 188.9375 C107.56843585 187.64509354 108.36334563 186.37500922 109.1875 185.125 C115.38871571 175.67039287 116.00140556 164.95398858 114.8125 153.9375 C113.53116994 148.08280725 111.37736611 142.89282087 108.5625 137.625 C106.62257102 133.86153777 106.04854769 131.21563295 106.8125 126.9375 C108.18168159 124.85153682 109.70860216 122.95913374 111.27734375 121.01953125 C116.46726428 113.98078406 116.7081271 105.37048329 115.8125 96.9375 C113.47952029 86.94654736 108.43578836 77.96377973 99.85546875 72.15234375 C93.65970651 68.46805683 89.38680789 67.90309767 82.25 67.875 C75.52263087 67.57437278 75.52263087 67.57437278 72.46484375 64.83203125 C70.74119973 62.26740047 69.2628723 59.66385966 67.8125 56.9375 C60.68945029 48.71859648 51.63496171 44.70777794 40.9375 43.5 C31.98889418 43.91663549 23.19136687 47.70133708 16.8125 53.9375 C13.92903469 57.28074338 11.40073885 60.608233 9.609375 64.65625 C7.8125 66.9375 7.8125 66.9375 4.44140625 67.43359375 C3.08580742 67.42822504 1.73021092 67.40786892 0.375 67.375 C-9.73004214 67.59427759 -17.45400989 70.71791051 -24.625 78.0625 C-31.02314856 86.0068678 -34.68766235 95.69131702 -34.1875 105.9375 C-32.95591354 113.99788294 -30.79256758 119.81742276 -25.4375 125.9375 C-23.6692606 128.76668304 -23.92704325 130.6265422 -24.1875 133.9375 C-25.4296875 137.12109375 -25.4296875 137.12109375 -27.0625 140.375 C-32.88635652 152.0649149 -34.45598408 164.91132193 -30.5 177.5 C-29.142422 181.07462983 -27.52800379 184.09867601 -25.375 187.25 C-23.55179697 190.00456061 -23.19276513 190.79797413 -23.0625 194.25 C-24.3522059 198.47736934 -26.07937523 202.32857241 -28.03125 206.2890625 C-30.26522742 211.40607834 -30.34343741 216.29532767 -30.3125 221.8125 C-30.31636719 222.49441406 -30.32023437 223.17632812 -30.32421875 223.87890625 C-30.32228516 224.89404297 -30.32228516 224.89404297 -30.3203125 225.9296875 C-30.31918457 226.53151855 -30.31805664 227.13334961 -30.31689453 227.75341797 C-30.13447983 230.83244052 -29.62370305 233.88407866 -29.1875 236.9375 C-34.1375 236.9375 -39.0875 236.9375 -44.1875 236.9375 C-45.86057528 231.91827416 -46.57318397 228.20208426 -46.5625 223.0625 C-46.56233887 222.32942627 -46.56217773 221.59635254 -46.56201172 220.84106445 C-46.39919176 212.82946572 -45.23710148 202.76755744 -40.8828125 195.87109375 C-39.89763327 193.77838652 -39.89763327 193.77838652 -41.234375 191.0546875 C-42.07742187 189.44980469 -42.07742187 189.44980469 -42.9375 187.8125 C-50.7448363 171.73437821 -49.60851776 154.61603275 -44.1875 137.9375 C-43.52717276 136.60101575 -42.86274687 135.26650861 -42.1875 133.9375 C-42.41233588 129.89045409 -44.20815517 126.75998184 -46.15234375 123.29296875 C-51.36155903 111.43954681 -50.86036483 95.85172197 -46.5078125 83.859375 C-43.17208753 76.47735643 -39.21291676 70.72881553 -33.1875 65.3125 C-30.20093382 62.5975436 -30.20093382 62.5975436 -28.83984375 58.9765625 C-29.10194997 56.89183651 -29.10194997 56.89183651 -29.63671875 54.69140625 C-32.45782164 40.58589178 -30.27631884 21.49996377 -23.1875 8.9375 C-16.95200134 0.31269812 -10.62565263 -3.54188421 0 0 Z M-8.1875 13.9375 C-8.99259102 15.40940305 -9.78074574 16.89057067 -10.5625 18.375 C-11.00335938 19.19871094 -11.44421875 20.02242187 -11.8984375 20.87109375 C-15.09803557 28.48225885 -15.46039753 35.82581742 -15.3125 44 C-15.30347656 44.96099609 -15.29445312 45.92199219 -15.28515625 46.91210938 C-15.26185208 49.25417805 -15.22914058 51.59569792 -15.1875 53.9375 C-12.54004679 53.65693562 -9.89512036 53.36364369 -7.25 53.0625 C-6.49525391 52.98386719 -5.74050781 52.90523437 -4.96289062 52.82421875 C-3.02830701 52.59866192 -1.10573981 52.27491026 0.8125 51.9375 C4.84629197 43.86991607 1.37861498 31.09252641 -1.1875 22.9375 C-2.32115234 19.99540227 -3.52204825 17.64385909 -5.1875 14.9375 C-6.1775 14.6075 -7.1675 14.2775 -8.1875 13.9375 Z M88.8125 13.9375 C83.08718073 23.21646571 80.66423018 32.25068631 80.6875 43.1875 C80.68492187 44.40953125 80.68234375 45.6315625 80.6796875 46.890625 C80.60141322 49.85519183 80.60141322 49.85519183 81.8125 51.9375 C84.51849842 52.41347366 87.14845404 52.77843629 89.875 53.0625 C90.63361328 53.14886719 91.39222656 53.23523438 92.17382812 53.32421875 C94.05246986 53.53689517 95.93238636 53.73825459 97.8125 53.9375 C98.56616639 33.00082307 98.56616639 33.00082307 90.8125 13.9375 C90.1525 13.9375 89.4925 13.9375 88.8125 13.9375 Z " transform="translate(49.1875,19.0625)"/>
+ <path d="M0 0 C8.85196305 4.48296841 15.69526326 10.78350142 19.3125 20.0625 C21.41443442 27.33288959 20.70031048 33.81961716 17.625 40.70703125 C13.27893441 48.50666682 6.54180384 52.12519686 -1.6875 55.0625 C-6.65051511 56.45531774 -11.7585299 56.2836545 -16.875 56.3125 C-17.77025391 56.333125 -18.66550781 56.35375 -19.58789062 56.375 C-30.05929223 56.43548463 -39.62226194 53.68763008 -47.41015625 46.46484375 C-52.24230857 41.25276791 -54.76495918 35.75358207 -55.0625 28.625 C-54.65052783 19.71419456 -51.4972564 13.8253097 -45.37109375 7.3671875 C-33.43145496 -2.88665076 -14.7276191 -5.60960361 0 0 Z M-41.6875 19.0625 C-44.44761959 23.03635296 -45.31582302 27.18795618 -44.453125 32.0078125 C-42.68762065 36.74584969 -39.12402119 40.56548486 -34.84375 43.1953125 C-25.16081958 47.56961879 -11.87575235 47.69893072 -1.91015625 44.171875 C3.25440444 41.59413293 7.2039602 38.19089641 9.875 33.0625 C10.60425973 28.06186183 10.06160841 24.47428124 7.6875 20 C3.00959812 14.21417399 -2.15253725 9.48930828 -9.6875 8.0625 C-22.98089201 6.87006949 -32.59735891 8.97180473 -41.6875 19.0625 Z " transform="translate(107.6875,119.9375)"/>
+ <path d="M0 0 C3.03006112 2.19418219 4.52287359 4.55729417 5.13671875 8.27734375 C5.33042441 11.8642382 5.14666101 13.87809068 2.9375 16.75 C-0.89089524 19.5649965 -3.41297965 19.94910271 -8.1875 19.3125 C-11.48738887 16.95440948 -13.73310444 14.18211227 -14.69921875 10.20703125 C-14.89087449 6.68453095 -14.46365794 4.67150532 -12.3125 1.875 C-8.31023517 -1.06784179 -4.81461227 -1.51316385 0 0 Z " transform="translate(141.1875,119.6875)"/>
+ <path d="M0 0 C1.75 1.1875 1.75 1.1875 3.75 4.1875 C4.42344388 9.23832908 4.31869254 12.07937538 1.1875 16.1875 C-1.35470987 18.82586911 -2.07382046 19.1601618 -5.875 19.75 C-8.86438088 19.25176985 -10.79574203 18.91457042 -13.25 17.1875 C-15.72361688 13.47707469 -15.79607278 10.48887325 -15.25 6.1875 C-13.98515069 3.2783466 -12.99549933 1.72734434 -10.4375 -0.125 C-6.78468101 -1.27302883 -3.60841326 -1.30303812 0 0 Z " transform="translate(50.25,119.8125)"/>
+ <path d="M0 0 C1.26714844 -0.01417969 2.53429687 -0.02835938 3.83984375 -0.04296875 C7.0625 0.1875 7.0625 0.1875 9.0625 2.1875 C8.71979828 7.55649365 8.71979828 7.55649365 6.5 9.6875 C4.71568402 11.21784687 4.71568402 11.21784687 4.9375 14.3125 C4.999375 15.735625 4.999375 15.735625 5.0625 17.1875 C4.0625 18.1875 4.0625 18.1875 0.5625 18.3125 C-2.9375 18.1875 -2.9375 18.1875 -3.9375 17.1875 C-4.07865161 14.51691149 -3.97997107 11.86317762 -3.9375 9.1875 C-5.2575 8.5275 -6.5775 7.8675 -7.9375 7.1875 C-7.98004356 5.18795254 -7.97830783 3.18708364 -7.9375 1.1875 C-6.00408053 -0.74591947 -2.56270879 0.01300867 0 0 Z " transform="translate(89.9375,138.8125)"/>
+ </svg>
+ }
+ iconType="duotone"
+ >
+ </Card>
<Card
title="Cohere"
href="/latest/integrations/cohere"
diff --git a/docs/latest/integrations/ollama.mdx b/docs/latest/integrations/ollama.mdx
new file mode 100644
index 00000000..a9b251dc
--- /dev/null
+++ b/docs/latest/integrations/ollama.mdx
@@ -0,0 +1,52 @@
+---
+title: 'Monitor Ollama using OpenTelemetry'
+sidebarTitle: 'Ollama'
+---
+
+<Frame>
+ <img src="/images/docs-ollama-banner.gif" />
+</Frame>
+
+OpenLIT SDKs provide OpenTelemetry Auto-Instrumentation for monitoring LLM Application using Ollama which includes performance, token usage and user interactions.
+
+The integration is compatible with
+- Ollama Python SDK client `>=0.2.0`
+
+## Get Started
+
+<Steps>
+ <Step title="Install the OpenLIT Python SDK using pip">
+ ```shell
+ pip install openlit
+ ```
+ </Step>
+ <Step title="Add the following two lines to your application code:">
+ Integrating the OpenLIT into LLM applications is straightforward. Start monitoring for your LLM Application with just **one line of code**:
+
+ ```python
+ import openlit
+
+ openlit.init(otlp_endpoint="otlp_backend_endpoint", otlp_headers="otlp_backend_token")
+ ```
+
+ Alternatively, you can configure the OTLP endpoint by setting the `OTEL_EXPORTER_OTLP_ENDPOINT` and the OTLP headers by setting the Appli`OTEL_EXPORTER_OTLP_HEADERS` environmentcation variable as recommended in the OpenTelemetry documentation.
+
+ If your aim is primarily to test or develop without sending telemetry data to an external backend, you can initialize OpenLIT without any parameters. This approach directs all telemetry output to your console, enabling quick diagnostics and local debugging:
+
+ ```python
+ import openlit
+
+ openlit.init()
+ ```
+
+ Refer to the OpenLIT [Python SDK repository](https://github.com/openlit/openlit/tree/main/sdk/python) for more advanced configurations and use cases.
+ </Step>
+</Steps>
+
+---
+
+<CardGroup cols={2}>
+<Card title="Connections" href="/latest/connections/intro" icon='link'>
+Connect to your existing Observablity Stack
+</Card>
+</CardGroup>
\ No newline at end of file
diff --git a/docs/mint.json b/docs/mint.json
index 4da53942..1a87469b 100644
--- a/docs/mint.json
+++ b/docs/mint.json
@@ -60,6 +60,7 @@
"version": "latest",
"pages": [
"latest/integrations/openai",
+ "latest/integrations/ollama",
"latest/integrations/anthropic",
"latest/integrations/cohere",
"latest/integrations/mistral",
diff --git a/sdk/python/README.md b/sdk/python/README.md
index f4206c86..d05f91a3 100644
--- a/sdk/python/README.md
+++ b/sdk/python/README.md
@@ -25,6 +25,7 @@ This project adheres to the [Semantic Conventions](https://github.com/open-telem
### LLMs
- [✅ OpenAI](https://docs.openlit.io/latest/integrations/openai)
+- [✅ Ollama](https://docs.openlit.io/latest/integrations/ollama)
- [✅ Anthropic](https://docs.openlit.io/latest/integrations/anthropic)
- [✅ Cohere](https://docs.openlit.io/latest/integrations/cohere)
- [✅ Mistral](https://docs.openlit.io/latest/integrations/mistral)
diff --git a/sdk/python/pyproject.toml b/sdk/python/pyproject.toml
index cc61adce..9e94f549 100644
--- a/sdk/python/pyproject.toml
+++ b/sdk/python/pyproject.toml
@@ -1,6 +1,6 @@
[tool.poetry]
name = "openlit"
-version = "1.5.0"
+version = "1.6.0"
description = "OpenTelemetry-native Auto instrumentation library for monitoring LLM Applications, facilitating the integration of observability into your GenAI-driven projects"
authors = ["OpenLIT"]
repository = "https://github.com/openlit/openlit/tree/main/openlit/python"
diff --git a/sdk/python/src/openlit/__init__.py b/sdk/python/src/openlit/__init__.py
index c4d381fb..856cd981 100644
--- a/sdk/python/src/openlit/__init__.py
+++ b/sdk/python/src/openlit/__init__.py
@@ -20,6 +20,7 @@ from openlit.instrumentation.mistral import MistralInstrumentor
from openlit.instrumentation.bedrock import BedrockInstrumentor
from openlit.instrumentation.vertexai import VertexAIInstrumentor
from openlit.instrumentation.groq import GroqInstrumentor
+from openlit.instrumentation.ollama import OllamaInstrumentor
from openlit.instrumentation.langchain import LangChainInstrumentor
from openlit.instrumentation.llamaindex import LlamaIndexInstrumentor
from openlit.instrumentation.haystack import HaystackInstrumentor
@@ -154,6 +155,7 @@ def init(environment="default", application_name="default", tracer=None, otlp_en
"bedrock": "boto3",
"vertexai": "vertexai",
"groq": "groq",
+ "ollama": "ollama",
"langchain": "langchain",
"llama_index": "llama_index",
"haystack": "haystack",
@@ -206,6 +208,7 @@ def init(environment="default", application_name="default", tracer=None, otlp_en
"bedrock": BedrockInstrumentor(),
"vertexai": VertexAIInstrumentor(),
"groq": GroqInstrumentor(),
+ "ollama": OllamaInstrumentor(),
"langchain": LangChainInstrumentor(),
"llama_index": LlamaIndexInstrumentor(),
"haystack": HaystackInstrumentor(),
diff --git a/sdk/python/src/openlit/instrumentation/ollama/__init__.py b/sdk/python/src/openlit/instrumentation/ollama/__init__.py
new file mode 100644
index 00000000..b32bd094
--- /dev/null
+++ b/sdk/python/src/openlit/instrumentation/ollama/__init__.py
@@ -0,0 +1,104 @@
+# pylint: disable=useless-return, bad-staticmethod-argument, disable=duplicate-code
+"""Initializer of Auto Instrumentation of Ollama Functions"""
+
+from typing import Collection
+import importlib.metadata
+from opentelemetry.instrumentation.instrumentor import BaseInstrumentor
+from wrapt import wrap_function_wrapper
+
+from openlit.instrumentation.ollama.ollama import (
+ chat, embeddings, generate
+)
+from openlit.instrumentation.ollama.async_ollama import (
+ async_chat, async_embeddings, async_generate
+)
+
+_instruments = ("ollama >= 0.2.0",)
+
+class OllamaInstrumentor(BaseInstrumentor):
+ """
+ An instrumentor for Ollama's client library.
+ """
+
+ def instrumentation_dependencies(self) -> Collection[str]:
+ return _instruments
+
+ def _instrument(self, **kwargs):
+ application_name = kwargs.get("application_name", "default_application")
+ environment = kwargs.get("environment", "default_environment")
+ tracer = kwargs.get("tracer")
+ metrics = kwargs.get("metrics_dict")
+ pricing_info = kwargs.get("pricing_info", {})
+ trace_content = kwargs.get("trace_content", False)
+ disable_metrics = kwargs.get("disable_metrics")
+ version = importlib.metadata.version("ollama")
+
+ # sync chat
+ wrap_function_wrapper(
+ "ollama",
+ "chat",
+ chat("ollama.chat", version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics),
+ )
+ wrap_function_wrapper(
+ "ollama",
+ "Client.chat",
+ chat("ollama.chat", version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics),
+ )
+
+ # sync embeddings
+ wrap_function_wrapper(
+ "ollama",
+ "embeddings",
+ embeddings("ollama.embeddings", version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics),
+ )
+ wrap_function_wrapper(
+ "ollama",
+ "Client.embeddings",
+ embeddings("ollama.embeddings", version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics),
+ )
+
+ # sync generate
+ wrap_function_wrapper(
+ "ollama",
+ "generate",
+ generate("ollama.generate", version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics),
+ )
+ wrap_function_wrapper(
+ "ollama",
+ "Client.generate",
+ generate("ollama.generate", version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics),
+ )
+
+ # async chat
+ wrap_function_wrapper(
+ "ollama",
+ "AsyncClient.chat",
+ async_chat("ollama.chat", version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics),
+ )
+
+ # async embeddings
+ wrap_function_wrapper(
+ "ollama",
+ "AsyncClient.embeddings",
+ async_embeddings("ollama.embeddings", version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics),
+ )
+
+ # aync generate
+ wrap_function_wrapper(
+ "ollama",
+ "AsyncClient.generate",
+ async_generate("ollama.generate", version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics),
+ )
+
+ def _uninstrument(self, **kwargs):
+ # Proper uninstrumentation logic to revert patched methods
+ pass
diff --git a/sdk/python/src/openlit/instrumentation/ollama/async_ollama.py b/sdk/python/src/openlit/instrumentation/ollama/async_ollama.py
new file mode 100644
index 00000000..dfd74fdf
--- /dev/null
+++ b/sdk/python/src/openlit/instrumentation/ollama/async_ollama.py
@@ -0,0 +1,564 @@
+# pylint: disable=duplicate-code, broad-exception-caught, too-many-statements, unused-argument, possibly-used-before-assignment
+"""
+Module for monitoring Ollama API calls.
+"""
+
+import logging
+from opentelemetry.trace import SpanKind, Status, StatusCode
+from opentelemetry.sdk.resources import TELEMETRY_SDK_NAME
+from openlit.__helpers import handle_exception, general_tokens
+from openlit.semcov import SemanticConvetion
+
+# Initialize logger for logging potential issues and operations
+logger = logging.getLogger(__name__)
+
+def async_chat(gen_ai_endpoint, version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics):
+ """
+ Generates a telemetry wrapper for chat to collect metrics.
+
+ Args:
+ gen_ai_endpoint: Endpoint identifier for logging and tracing.
+ version: Version of the monitoring package.
+ environment: Deployment environment (e.g., production, staging).
+ application_name: Name of the application using the Ollama API.
+ tracer: OpenTelemetry tracer for creating spans.
+ pricing_info: Information used for calculating the cost of Ollama usage.
+ trace_content: Flag indicating whether to trace the actual content.
+
+ Returns:
+ A function that wraps the chat method to add telemetry.
+ """
+
+ async def wrapper(wrapped, instance, args, kwargs):
+ """
+ Wraps the 'chat' API call to add telemetry.
+
+ This collects metrics such as execution time, cost, and token usage, and handles errors
+ gracefully, adding details to the trace for observability.
+
+ Args:
+ wrapped: The original 'chat' method to be wrapped.
+ instance: The instance of the class where the original method is defined.
+ args: Positional arguments for the 'chat' method.
+ kwargs: Keyword arguments for the 'chat' method.
+
+ Returns:
+ The response from the original 'chat' method.
+ """
+
+ # Check if streaming is enabled for the API call
+ streaming = kwargs.get("stream", False)
+
+ # pylint: disable=no-else-return
+ if streaming:
+ # Special handling for streaming response to accommodate the nature of data flow
+ async def stream_generator():
+ with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
+ # Placeholder for aggregating streaming response
+ llmresponse = ""
+
+ # Loop through streaming events capturing relevant details
+ async for chunk in await wrapped(*args, **kwargs):
+ # Collect aggregated response from events
+ content = chunk['message']['content']
+ llmresponse += content
+
+ if chunk['done'] is True:
+ completion_tokens = chunk["eval_count"]
+
+ yield chunk
+
+ # Handling exception ensure observability without disrupting operation
+ try:
+ # Format 'messages' into a single string
+ message_prompt = kwargs.get("messages", "")
+ formatted_messages = []
+ for message in message_prompt:
+ role = message["role"]
+ content = message["content"]
+
+ if isinstance(content, list):
+ content_str = ", ".join(
+ # pylint: disable=line-too-long
+ f'{item["type"]}: {item["text"] if "text" in item else item["image_url"]}'
+ if "type" in item else f'text: {item["text"]}'
+ for item in content
+ )
+ formatted_messages.append(f"{role}: {content_str}")
+ else:
+ formatted_messages.append(f"{role}: {content}")
+ prompt = "\n".join(formatted_messages)
+
+ # Calculate cost of the operation
+ cost = 0
+ prompt_tokens = general_tokens(prompt)
+ total_tokens = prompt_tokens + completion_tokens
+
+ # Set Span attributes
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_OLLAMA)
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
+ SemanticConvetion.GEN_AI_TYPE_CHAT)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
+ gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,
+ environment)
+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
+ application_name)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ kwargs.get("model", "llama3"))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,
+ True)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
+ prompt_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,
+ completion_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ total_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,
+ prompt)
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_COMPLETION,
+ llmresponse)
+
+ span.set_status(Status(StatusCode.OK))
+
+ if disable_metrics is False:
+ attributes = {
+ TELEMETRY_SDK_NAME:
+ "openlit",
+ SemanticConvetion.GEN_AI_APPLICATION_NAME:
+ application_name,
+ SemanticConvetion.GEN_AI_SYSTEM:
+ SemanticConvetion.GEN_AI_SYSTEM_OLLAMA,
+ SemanticConvetion.GEN_AI_ENVIRONMENT:
+ environment,
+ SemanticConvetion.GEN_AI_TYPE:
+ SemanticConvetion.GEN_AI_TYPE_CHAT,
+ SemanticConvetion.GEN_AI_REQUEST_MODEL:
+ kwargs.get("model", "llama3")
+ }
+
+ metrics["genai_requests"].add(1, attributes)
+ metrics["genai_total_tokens"].add(total_tokens, attributes)
+ metrics["genai_completion_tokens"].add(completion_tokens, attributes)
+ metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)
+ metrics["genai_cost"].record(cost, attributes)
+
+ except Exception as e:
+ handle_exception(span, e)
+ logger.error("Error in trace creation: %s", e)
+
+ return stream_generator()
+
+ # Handling for non-streaming responses
+ else:
+ # pylint: disable=line-too-long
+ with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
+ response = await wrapped(*args, **kwargs)
+
+ try:
+ # Format 'messages' into a single string
+ message_prompt = kwargs.get("messages", "")
+ formatted_messages = []
+ for message in message_prompt:
+ role = message["role"]
+ content = message["content"]
+
+ if isinstance(content, list):
+ content_str = ", ".join(
+ # pylint: disable=line-too-long
+ f'{item["type"]}: {item["text"] if "text" in item else item["image_url"]}'
+ if "type" in item else f'text: {item["text"]}'
+ for item in content
+ )
+ formatted_messages.append(f"{role}: {content_str}")
+ else:
+ formatted_messages.append(f"{role}: {content}")
+ prompt = "\n".join(formatted_messages)
+
+ # Set base span attribues
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_OLLAMA)
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
+ SemanticConvetion.GEN_AI_TYPE_CHAT)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
+ gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,
+ environment)
+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
+ application_name)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ kwargs.get("model", "llama3"))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,
+ False)
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,
+ prompt)
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_COMPLETION,
+ response['message']['content'])
+
+ # Calculate cost of the operation
+ cost = 0
+ prompt_tokens = general_tokens(prompt)
+ completion_tokens = response["eval_count"]
+ total_tokens = prompt_tokens + completion_tokens
+
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
+ prompt_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,
+ completion_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ total_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,
+ response["done_reason"])
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+
+ span.set_status(Status(StatusCode.OK))
+
+ if disable_metrics is False:
+ attributes = {
+ TELEMETRY_SDK_NAME:
+ "openlit",
+ SemanticConvetion.GEN_AI_APPLICATION_NAME:
+ application_name,
+ SemanticConvetion.GEN_AI_SYSTEM:
+ SemanticConvetion.GEN_AI_SYSTEM_OLLAMA,
+ SemanticConvetion.GEN_AI_ENVIRONMENT:
+ environment,
+ SemanticConvetion.GEN_AI_TYPE:
+ SemanticConvetion.GEN_AI_TYPE_CHAT,
+ SemanticConvetion.GEN_AI_REQUEST_MODEL:
+ kwargs.get("model", "llama3")
+ }
+
+ metrics["genai_requests"].add(1, attributes)
+ metrics["genai_total_tokens"].add(total_tokens, attributes)
+ metrics["genai_completion_tokens"].add(completion_tokens, attributes)
+ metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)
+ metrics["genai_cost"].record(cost, attributes)
+
+ # Return original response
+ return response
+
+ except Exception as e:
+ handle_exception(span, e)
+ logger.error("Error in trace creation: %s", e)
+
+ # Return original response
+ return response
+
+ return wrapper
+
+def async_generate(gen_ai_endpoint, version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics):
+ """
+ Generates a telemetry wrapper for generate to collect metrics.
+
+ Args:
+ gen_ai_endpoint: Endpoint identifier for logging and tracing.
+ version: Version of the monitoring package.
+ environment: Deployment environment (e.g., production, staging).
+ application_name: Name of the application using the Ollama API.
+ tracer: OpenTelemetry tracer for creating spans.
+ pricing_info: Information used for calculating the cost of Ollama usage.
+ trace_content: Flag indicating whether to trace the actual content.
+
+ Returns:
+ A function that wraps the generate method to add telemetry.
+ """
+
+ async def wrapper(wrapped, instance, args, kwargs):
+ """
+ Wraps the 'generate' API call to add telemetry.
+
+ This collects metrics such as execution time, cost, and token usage, and handles errors
+ gracefully, adding details to the trace for observability.
+
+ Args:
+ wrapped: The original 'generate' method to be wrapped.
+ instance: The instance of the class where the original method is defined.
+ args: Positional arguments for the 'generate' method.
+ kwargs: Keyword arguments for the 'generate' method.
+
+ Returns:
+ The response from the original 'generate' method.
+ """
+
+ # Check if streaming is enabled for the API call
+ streaming = kwargs.get("stream", False)
+
+ # pylint: disable=no-else-return
+ if streaming:
+ # Special handling for streaming response to accommodate the nature of data flow
+ async def stream_generator():
+ with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
+ # Placeholder for aggregating streaming response
+ llmresponse = ""
+
+ # Loop through streaming events capturing relevant details
+ async for chunk in await wrapped(*args, **kwargs):
+ # Collect aggregated response from events
+ content = chunk['response']
+ llmresponse += content
+
+ if chunk['done'] is True:
+ completion_tokens = chunk["eval_count"]
+
+ yield chunk
+
+ # Handling exception ensure observability without disrupting operation
+ try:
+ # Calculate cost of the operation
+ cost = 0
+ prompt_tokens = general_tokens(kwargs.get("prompt", ""))
+ total_tokens = prompt_tokens + completion_tokens
+
+ # Set Span attributes
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_OLLAMA)
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
+ SemanticConvetion.GEN_AI_TYPE_CHAT)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
+ gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,
+ environment)
+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
+ application_name)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ kwargs.get("model", "llama3"))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,
+ True)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
+ prompt_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,
+ completion_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ total_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,
+ kwargs.get("prompt", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_COMPLETION,
+ llmresponse)
+
+ span.set_status(Status(StatusCode.OK))
+
+ if disable_metrics is False:
+ attributes = {
+ TELEMETRY_SDK_NAME:
+ "openlit",
+ SemanticConvetion.GEN_AI_APPLICATION_NAME:
+ application_name,
+ SemanticConvetion.GEN_AI_SYSTEM:
+ SemanticConvetion.GEN_AI_SYSTEM_OLLAMA,
+ SemanticConvetion.GEN_AI_ENVIRONMENT:
+ environment,
+ SemanticConvetion.GEN_AI_TYPE:
+ SemanticConvetion.GEN_AI_TYPE_CHAT,
+ SemanticConvetion.GEN_AI_REQUEST_MODEL:
+ kwargs.get("model", "llama3")
+ }
+
+ metrics["genai_requests"].add(1, attributes)
+ metrics["genai_total_tokens"].add(total_tokens, attributes)
+ metrics["genai_completion_tokens"].add(completion_tokens, attributes)
+ metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)
+ metrics["genai_cost"].record(cost, attributes)
+
+ except Exception as e:
+ handle_exception(span, e)
+ logger.error("Error in trace creation: %s", e)
+
+ return stream_generator()
+
+ # Handling for non-streaming responses
+ else:
+ # pylint: disable=line-too-long
+ with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
+ response = await wrapped(*args, **kwargs)
+
+ try:
+ # Set base span attribues
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_OLLAMA)
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
+ SemanticConvetion.GEN_AI_TYPE_CHAT)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
+ gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,
+ environment)
+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
+ application_name)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ kwargs.get("model", "llama3"))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,
+ False)
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,
+ kwargs.get("prompt", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_COMPLETION,
+ response['response'])
+
+ # Calculate cost of the operation
+ cost = 0
+ prompt_tokens = response["prompt_eval_count"]
+ completion_tokens = response["eval_count"]
+ total_tokens = prompt_tokens + completion_tokens
+
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
+ prompt_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,
+ completion_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ total_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,
+ response["done_reason"])
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+
+ span.set_status(Status(StatusCode.OK))
+
+ if disable_metrics is False:
+ attributes = {
+ TELEMETRY_SDK_NAME:
+ "openlit",
+ SemanticConvetion.GEN_AI_APPLICATION_NAME:
+ application_name,
+ SemanticConvetion.GEN_AI_SYSTEM:
+ SemanticConvetion.GEN_AI_SYSTEM_OLLAMA,
+ SemanticConvetion.GEN_AI_ENVIRONMENT:
+ environment,
+ SemanticConvetion.GEN_AI_TYPE:
+ SemanticConvetion.GEN_AI_TYPE_CHAT,
+ SemanticConvetion.GEN_AI_REQUEST_MODEL:
+ kwargs.get("model", "llama3")
+ }
+
+ metrics["genai_requests"].add(1, attributes)
+ metrics["genai_total_tokens"].add(total_tokens, attributes)
+ metrics["genai_completion_tokens"].add(completion_tokens, attributes)
+ metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)
+ metrics["genai_cost"].record(cost, attributes)
+
+ # Return original response
+ return response
+
+ except Exception as e:
+ handle_exception(span, e)
+ logger.error("Error in trace creation: %s", e)
+
+ # Return original response
+ return response
+
+ return wrapper
+
+def async_embeddings(gen_ai_endpoint, version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics):
+ """
+ Generates a telemetry wrapper for embeddings to collect metrics.
+
+ Args:
+ gen_ai_endpoint: Endpoint identifier for logging and tracing.
+ version: Version of the monitoring package.
+ environment: Deployment environment (e.g., production, staging).
+ application_name: Name of the application using the Ollama API.
+ tracer: OpenTelemetry tracer for creating spans.
+ pricing_info: Information used for calculating the cost of Ollama usage.
+ trace_content: Flag indicating whether to trace the actual content.
+
+ Returns:
+ A function that wraps the embeddings method to add telemetry.
+ """
+
+ async def wrapper(wrapped, instance, args, kwargs):
+ """
+ Wraps the 'embeddings' API call to add telemetry.
+
+ This collects metrics such as execution time, cost, and token usage, and handles errors
+ gracefully, adding details to the trace for observability.
+
+ Args:
+ wrapped: The original 'embeddings' method to be wrapped.
+ instance: The instance of the class where the original method is defined.
+ args: Positional arguments for the 'embeddings' method.
+ kwargs: Keyword arguments for the 'embeddings' method.
+
+ Returns:
+ The response from the original 'embeddings' method.
+ """
+
+ with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
+ response = await wrapped(*args, **kwargs)
+
+ try:
+ # Calculate cost of the operation
+ cost = 0
+ prompt_tokens = general_tokens(kwargs.get('prompt', ""))
+ # Set Span attributes
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_OLLAMA)
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
+ SemanticConvetion.GEN_AI_TYPE_EMBEDDING)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
+ gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,
+ environment)
+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
+ application_name)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ kwargs.get('model', "llama3"))
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
+ prompt_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ prompt_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,
+ kwargs.get('prompt', ""))
+
+ span.set_status(Status(StatusCode.OK))
+
+ if disable_metrics is False:
+ attributes = {
+ TELEMETRY_SDK_NAME:
+ "openlit",
+ SemanticConvetion.GEN_AI_APPLICATION_NAME:
+ application_name,
+ SemanticConvetion.GEN_AI_SYSTEM:
+ SemanticConvetion.GEN_AI_SYSTEM_OLLAMA,
+ SemanticConvetion.GEN_AI_ENVIRONMENT:
+ environment,
+ SemanticConvetion.GEN_AI_TYPE:
+ SemanticConvetion.GEN_AI_TYPE_EMBEDDING,
+ SemanticConvetion.GEN_AI_REQUEST_MODEL:
+ kwargs.get('model', "llama3")
+ }
+
+ metrics["genai_requests"].add(1, attributes)
+ metrics["genai_total_tokens"].add(prompt_tokens, attributes)
+ metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)
+ metrics["genai_cost"].record(cost, attributes)
+
+ # Return original response
+ return response
+
+ except Exception as e:
+ handle_exception(span, e)
+ logger.error("Error in trace creation: %s", e)
+
+ # Return original response
+ return response
+
+ return wrapper
diff --git a/sdk/python/src/openlit/instrumentation/ollama/ollama.py b/sdk/python/src/openlit/instrumentation/ollama/ollama.py
new file mode 100644
index 00000000..78e94e30
--- /dev/null
+++ b/sdk/python/src/openlit/instrumentation/ollama/ollama.py
@@ -0,0 +1,564 @@
+# pylint: disable=duplicate-code, broad-exception-caught, too-many-statements, unused-argument, possibly-used-before-assignment
+"""
+Module for monitoring Ollama API calls.
+"""
+
+import logging
+from opentelemetry.trace import SpanKind, Status, StatusCode
+from opentelemetry.sdk.resources import TELEMETRY_SDK_NAME
+from openlit.__helpers import handle_exception, general_tokens
+from openlit.semcov import SemanticConvetion
+
+# Initialize logger for logging potential issues and operations
+logger = logging.getLogger(__name__)
+
+def chat(gen_ai_endpoint, version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics):
+ """
+ Generates a telemetry wrapper for chat to collect metrics.
+
+ Args:
+ gen_ai_endpoint: Endpoint identifier for logging and tracing.
+ version: Version of the monitoring package.
+ environment: Deployment environment (e.g., production, staging).
+ application_name: Name of the application using the Ollama API.
+ tracer: OpenTelemetry tracer for creating spans.
+ pricing_info: Information used for calculating the cost of Ollama usage.
+ trace_content: Flag indicating whether to trace the actual content.
+
+ Returns:
+ A function that wraps the chat method to add telemetry.
+ """
+
+ def wrapper(wrapped, instance, args, kwargs):
+ """
+ Wraps the 'chat' API call to add telemetry.
+
+ This collects metrics such as execution time, cost, and token usage, and handles errors
+ gracefully, adding details to the trace for observability.
+
+ Args:
+ wrapped: The original 'chat' method to be wrapped.
+ instance: The instance of the class where the original method is defined.
+ args: Positional arguments for the 'chat' method.
+ kwargs: Keyword arguments for the 'chat' method.
+
+ Returns:
+ The response from the original 'chat' method.
+ """
+
+ # Check if streaming is enabled for the API call
+ streaming = kwargs.get("stream", False)
+
+ # pylint: disable=no-else-return
+ if streaming:
+ # Special handling for streaming response to accommodate the nature of data flow
+ def stream_generator():
+ with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
+ # Placeholder for aggregating streaming response
+ llmresponse = ""
+
+ # Loop through streaming events capturing relevant details
+ for chunk in wrapped(*args, **kwargs):
+ # Collect aggregated response from events
+ content = chunk['message']['content']
+ llmresponse += content
+
+ if chunk['done'] is True:
+ completion_tokens = chunk["eval_count"]
+
+ yield chunk
+
+ # Handling exception ensure observability without disrupting operation
+ try:
+ # Format 'messages' into a single string
+ message_prompt = kwargs.get("messages", "")
+ formatted_messages = []
+ for message in message_prompt:
+ role = message["role"]
+ content = message["content"]
+
+ if isinstance(content, list):
+ content_str = ", ".join(
+ # pylint: disable=line-too-long
+ f'{item["type"]}: {item["text"] if "text" in item else item["image_url"]}'
+ if "type" in item else f'text: {item["text"]}'
+ for item in content
+ )
+ formatted_messages.append(f"{role}: {content_str}")
+ else:
+ formatted_messages.append(f"{role}: {content}")
+ prompt = "\n".join(formatted_messages)
+
+ # Calculate cost of the operation
+ cost = 0
+ prompt_tokens = general_tokens(prompt)
+ total_tokens = prompt_tokens + completion_tokens
+
+ # Set Span attributes
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_OLLAMA)
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
+ SemanticConvetion.GEN_AI_TYPE_CHAT)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
+ gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,
+ environment)
+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
+ application_name)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ kwargs.get("model", "llama3"))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,
+ True)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
+ prompt_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,
+ completion_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ total_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,
+ prompt)
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_COMPLETION,
+ llmresponse)
+
+ span.set_status(Status(StatusCode.OK))
+
+ if disable_metrics is False:
+ attributes = {
+ TELEMETRY_SDK_NAME:
+ "openlit",
+ SemanticConvetion.GEN_AI_APPLICATION_NAME:
+ application_name,
+ SemanticConvetion.GEN_AI_SYSTEM:
+ SemanticConvetion.GEN_AI_SYSTEM_OLLAMA,
+ SemanticConvetion.GEN_AI_ENVIRONMENT:
+ environment,
+ SemanticConvetion.GEN_AI_TYPE:
+ SemanticConvetion.GEN_AI_TYPE_CHAT,
+ SemanticConvetion.GEN_AI_REQUEST_MODEL:
+ kwargs.get("model", "llama3")
+ }
+
+ metrics["genai_requests"].add(1, attributes)
+ metrics["genai_total_tokens"].add(total_tokens, attributes)
+ metrics["genai_completion_tokens"].add(completion_tokens, attributes)
+ metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)
+ metrics["genai_cost"].record(cost, attributes)
+
+ except Exception as e:
+ handle_exception(span, e)
+ logger.error("Error in trace creation: %s", e)
+
+ return stream_generator()
+
+ # Handling for non-streaming responses
+ else:
+ # pylint: disable=line-too-long
+ with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
+ response = wrapped(*args, **kwargs)
+
+ try:
+ # Format 'messages' into a single string
+ message_prompt = kwargs.get("messages", "")
+ formatted_messages = []
+ for message in message_prompt:
+ role = message["role"]
+ content = message["content"]
+
+ if isinstance(content, list):
+ content_str = ", ".join(
+ # pylint: disable=line-too-long
+ f'{item["type"]}: {item["text"] if "text" in item else item["image_url"]}'
+ if "type" in item else f'text: {item["text"]}'
+ for item in content
+ )
+ formatted_messages.append(f"{role}: {content_str}")
+ else:
+ formatted_messages.append(f"{role}: {content}")
+ prompt = "\n".join(formatted_messages)
+
+ # Set base span attribues
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_OLLAMA)
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
+ SemanticConvetion.GEN_AI_TYPE_CHAT)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
+ gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,
+ environment)
+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
+ application_name)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ kwargs.get("model", "llama3"))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,
+ False)
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,
+ prompt)
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_COMPLETION,
+ response['message']['content'])
+
+ # Calculate cost of the operation
+ cost = 0
+ prompt_tokens = general_tokens(prompt)
+ completion_tokens = response["eval_count"]
+ total_tokens = prompt_tokens + completion_tokens
+
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
+ prompt_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,
+ completion_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ total_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,
+ response["done_reason"])
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+
+ span.set_status(Status(StatusCode.OK))
+
+ if disable_metrics is False:
+ attributes = {
+ TELEMETRY_SDK_NAME:
+ "openlit",
+ SemanticConvetion.GEN_AI_APPLICATION_NAME:
+ application_name,
+ SemanticConvetion.GEN_AI_SYSTEM:
+ SemanticConvetion.GEN_AI_SYSTEM_OLLAMA,
+ SemanticConvetion.GEN_AI_ENVIRONMENT:
+ environment,
+ SemanticConvetion.GEN_AI_TYPE:
+ SemanticConvetion.GEN_AI_TYPE_CHAT,
+ SemanticConvetion.GEN_AI_REQUEST_MODEL:
+ kwargs.get("model", "llama3")
+ }
+
+ metrics["genai_requests"].add(1, attributes)
+ metrics["genai_total_tokens"].add(total_tokens, attributes)
+ metrics["genai_completion_tokens"].add(completion_tokens, attributes)
+ metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)
+ metrics["genai_cost"].record(cost, attributes)
+
+ # Return original response
+ return response
+
+ except Exception as e:
+ handle_exception(span, e)
+ logger.error("Error in trace creation: %s", e)
+
+ # Return original response
+ return response
+
+ return wrapper
+
+def generate(gen_ai_endpoint, version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics):
+ """
+ Generates a telemetry wrapper for generate to collect metrics.
+
+ Args:
+ gen_ai_endpoint: Endpoint identifier for logging and tracing.
+ version: Version of the monitoring package.
+ environment: Deployment environment (e.g., production, staging).
+ application_name: Name of the application using the Ollama API.
+ tracer: OpenTelemetry tracer for creating spans.
+ pricing_info: Information used for calculating the cost of Ollama usage.
+ trace_content: Flag indicating whether to trace the actual content.
+
+ Returns:
+ A function that wraps the generate method to add telemetry.
+ """
+
+ def wrapper(wrapped, instance, args, kwargs):
+ """
+ Wraps the 'generate' API call to add telemetry.
+
+ This collects metrics such as execution time, cost, and token usage, and handles errors
+ gracefully, adding details to the trace for observability.
+
+ Args:
+ wrapped: The original 'generate' method to be wrapped.
+ instance: The instance of the class where the original method is defined.
+ args: Positional arguments for the 'generate' method.
+ kwargs: Keyword arguments for the 'generate' method.
+
+ Returns:
+ The response from the original 'generate' method.
+ """
+
+ # Check if streaming is enabled for the API call
+ streaming = kwargs.get("stream", False)
+
+ # pylint: disable=no-else-return
+ if streaming:
+ # Special handling for streaming response to accommodate the nature of data flow
+ def stream_generator():
+ with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
+ # Placeholder for aggregating streaming response
+ llmresponse = ""
+
+ # Loop through streaming events capturing relevant details
+ for chunk in wrapped(*args, **kwargs):
+ # Collect aggregated response from events
+ content = chunk['response']
+ llmresponse += content
+
+ if chunk['done'] is True:
+ completion_tokens = chunk["eval_count"]
+
+ yield chunk
+
+ # Handling exception ensure observability without disrupting operation
+ try:
+ # Calculate cost of the operation
+ cost = 0
+ prompt_tokens = general_tokens(kwargs.get("prompt", ""))
+ total_tokens = prompt_tokens + completion_tokens
+
+ # Set Span attributes
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_OLLAMA)
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
+ SemanticConvetion.GEN_AI_TYPE_CHAT)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
+ gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,
+ environment)
+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
+ application_name)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ kwargs.get("model", "llama3"))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,
+ True)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
+ prompt_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,
+ completion_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ total_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,
+ kwargs.get("prompt", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_COMPLETION,
+ llmresponse)
+
+ span.set_status(Status(StatusCode.OK))
+
+ if disable_metrics is False:
+ attributes = {
+ TELEMETRY_SDK_NAME:
+ "openlit",
+ SemanticConvetion.GEN_AI_APPLICATION_NAME:
+ application_name,
+ SemanticConvetion.GEN_AI_SYSTEM:
+ SemanticConvetion.GEN_AI_SYSTEM_OLLAMA,
+ SemanticConvetion.GEN_AI_ENVIRONMENT:
+ environment,
+ SemanticConvetion.GEN_AI_TYPE:
+ SemanticConvetion.GEN_AI_TYPE_CHAT,
+ SemanticConvetion.GEN_AI_REQUEST_MODEL:
+ kwargs.get("model", "llama3")
+ }
+
+ metrics["genai_requests"].add(1, attributes)
+ metrics["genai_total_tokens"].add(total_tokens, attributes)
+ metrics["genai_completion_tokens"].add(completion_tokens, attributes)
+ metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)
+ metrics["genai_cost"].record(cost, attributes)
+
+ except Exception as e:
+ handle_exception(span, e)
+ logger.error("Error in trace creation: %s", e)
+
+ return stream_generator()
+
+ # Handling for non-streaming responses
+ else:
+ # pylint: disable=line-too-long
+ with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
+ response = wrapped(*args, **kwargs)
+
+ try:
+ # Set base span attribues
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_OLLAMA)
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
+ SemanticConvetion.GEN_AI_TYPE_CHAT)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
+ gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,
+ environment)
+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
+ application_name)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ kwargs.get("model", "llama3"))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,
+ False)
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,
+ kwargs.get("prompt", ""))
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_COMPLETION,
+ response['response'])
+
+ # Calculate cost of the operation
+ cost = 0
+ prompt_tokens = response["prompt_eval_count"]
+ completion_tokens = response["eval_count"]
+ total_tokens = prompt_tokens + completion_tokens
+
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
+ prompt_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,
+ completion_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ total_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,
+ response["done_reason"])
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+
+ span.set_status(Status(StatusCode.OK))
+
+ if disable_metrics is False:
+ attributes = {
+ TELEMETRY_SDK_NAME:
+ "openlit",
+ SemanticConvetion.GEN_AI_APPLICATION_NAME:
+ application_name,
+ SemanticConvetion.GEN_AI_SYSTEM:
+ SemanticConvetion.GEN_AI_SYSTEM_OLLAMA,
+ SemanticConvetion.GEN_AI_ENVIRONMENT:
+ environment,
+ SemanticConvetion.GEN_AI_TYPE:
+ SemanticConvetion.GEN_AI_TYPE_CHAT,
+ SemanticConvetion.GEN_AI_REQUEST_MODEL:
+ kwargs.get("model", "llama3")
+ }
+
+ metrics["genai_requests"].add(1, attributes)
+ metrics["genai_total_tokens"].add(total_tokens, attributes)
+ metrics["genai_completion_tokens"].add(completion_tokens, attributes)
+ metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)
+ metrics["genai_cost"].record(cost, attributes)
+
+ # Return original response
+ return response
+
+ except Exception as e:
+ handle_exception(span, e)
+ logger.error("Error in trace creation: %s", e)
+
+ # Return original response
+ return response
+
+ return wrapper
+
+def embeddings(gen_ai_endpoint, version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics):
+ """
+ Generates a telemetry wrapper for embeddings to collect metrics.
+
+ Args:
+ gen_ai_endpoint: Endpoint identifier for logging and tracing.
+ version: Version of the monitoring package.
+ environment: Deployment environment (e.g., production, staging).
+ application_name: Name of the application using the Ollama API.
+ tracer: OpenTelemetry tracer for creating spans.
+ pricing_info: Information used for calculating the cost of Ollama usage.
+ trace_content: Flag indicating whether to trace the actual content.
+
+ Returns:
+ A function that wraps the embeddings method to add telemetry.
+ """
+
+ def wrapper(wrapped, instance, args, kwargs):
+ """
+ Wraps the 'embeddings' API call to add telemetry.
+
+ This collects metrics such as execution time, cost, and token usage, and handles errors
+ gracefully, adding details to the trace for observability.
+
+ Args:
+ wrapped: The original 'embeddings' method to be wrapped.
+ instance: The instance of the class where the original method is defined.
+ args: Positional arguments for the 'embeddings' method.
+ kwargs: Keyword arguments for the 'embeddings' method.
+
+ Returns:
+ The response from the original 'embeddings' method.
+ """
+
+ with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
+ response = wrapped(*args, **kwargs)
+
+ try:
+ # Calculate cost of the operation
+ cost = 0
+ prompt_tokens = general_tokens(kwargs.get('prompt', ""))
+ # Set Span attributes
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_OLLAMA)
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
+ SemanticConvetion.GEN_AI_TYPE_EMBEDDING)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
+ gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,
+ environment)
+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
+ application_name)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ kwargs.get('model', "llama3"))
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
+ prompt_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ prompt_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,
+ kwargs.get('prompt', ""))
+
+ span.set_status(Status(StatusCode.OK))
+
+ if disable_metrics is False:
+ attributes = {
+ TELEMETRY_SDK_NAME:
+ "openlit",
+ SemanticConvetion.GEN_AI_APPLICATION_NAME:
+ application_name,
+ SemanticConvetion.GEN_AI_SYSTEM:
+ SemanticConvetion.GEN_AI_SYSTEM_OLLAMA,
+ SemanticConvetion.GEN_AI_ENVIRONMENT:
+ environment,
+ SemanticConvetion.GEN_AI_TYPE:
+ SemanticConvetion.GEN_AI_TYPE_EMBEDDING,
+ SemanticConvetion.GEN_AI_REQUEST_MODEL:
+ kwargs.get('model', "llama3")
+ }
+
+ metrics["genai_requests"].add(1, attributes)
+ metrics["genai_total_tokens"].add(prompt_tokens, attributes)
+ metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)
+ metrics["genai_cost"].record(cost, attributes)
+
+ # Return original response
+ return response
+
+ except Exception as e:
+ handle_exception(span, e)
+ logger.error("Error in trace creation: %s", e)
+
+ # Return original response
+ return response
+
+ return wrapper
diff --git a/sdk/python/src/openlit/semcov/__init__.py b/sdk/python/src/openlit/semcov/__init__.py
index bd93472d..4c757fb9 100644
--- a/sdk/python/src/openlit/semcov/__init__.py
+++ b/sdk/python/src/openlit/semcov/__init__.py
@@ -88,6 +88,7 @@ class SemanticConvetion:
GEN_AI_SYSTEM_BEDROCK = "bedrock"
GEN_AI_SYSTEM_VERTEXAI = "vertexai"
GEN_AI_SYSTEM_GROQ = "groq"
+ GEN_AI_SYSTEM_OLLAMA = "ollama"
GEN_AI_SYSTEM_LANGCHAIN = "langchain"
GEN_AI_SYSTEM_LLAMAINDEX = "llama_index"
GEN_AI_SYSTEM_HAYSTACK = "haystack"