Skip to content

Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.

[Feat]: Fix un-raised error and add wrapping for Ollama `generate` method (#853)

Gwenn-LRd0d3626845ca
diff --git a/sdk/python/pyproject.toml b/sdk/python/pyproject.toml
index 487aae14..8979e09d 100644
--- a/sdk/python/pyproject.toml
+++ b/sdk/python/pyproject.toml
@@ -1,6 +1,6 @@
[tool.poetry]
name = "openlit"
-version = "1.35.8"
+version = "1.35.7"
description = "OpenTelemetry-native Auto instrumentation library for monitoring LLM Applications and GPUs, facilitating the integration of observability into your GenAI-driven projects"
authors = ["OpenLIT"]
license = "Apache-2.0"
@@ -14,7 +14,7 @@ packages = [
]
[tool.poetry.dependencies]
-python = "^3.8.0"
+python = "^3.9.0"
requests = "^2.26.0"
schedule = "^1.2.2"
pydantic = "^2.0.0"
diff --git a/sdk/python/src/openlit/instrumentation/ollama/__init__.py b/sdk/python/src/openlit/instrumentation/ollama/__init__.py
index 876d597b..eb4a4670 100644
--- a/sdk/python/src/openlit/instrumentation/ollama/__init__.py
+++ b/sdk/python/src/openlit/instrumentation/ollama/__init__.py
@@ -7,19 +7,21 @@ import importlib.metadata
from opentelemetry.instrumentation.instrumentor import BaseInstrumentor
from wrapt import wrap_function_wrapper
-from openlit.instrumentation.ollama.ollama import chat, embeddings
-from openlit.instrumentation.ollama.async_ollama import async_chat, async_embeddings
+from openlit.instrumentation.ollama.ollama import chat, embeddings, generate
+from openlit.instrumentation.ollama.async_ollama import async_chat, async_embeddings, async_generate
_instruments = ("ollama >= 0.2.0",)
# Dispatch wrapper to route instrumentation to chat or embeddings based on path
-def _dispatch(sync_chat_wrap, sync_emb_wrap):
+def _dispatch(sync_chat_wrap, sync_generate_wrap, sync_emb_wrap):
def wrapper(wrapped, instance, args, kwargs):
if len(args) > 2 and isinstance(args[2], str):
op = args[2].rstrip("/").split("/")[-1]
if op == "chat":
return sync_chat_wrap(wrapped, instance, args, kwargs)
+ if op == "generate":
+ return sync_generate_wrap(wrapped, instance, args, kwargs)
if op == "embeddings":
return sync_emb_wrap(wrapped, instance, args, kwargs)
return wrapped(*args, **kwargs)
@@ -27,12 +29,14 @@ def _dispatch(sync_chat_wrap, sync_emb_wrap):
return wrapper
-def _dispatch_async(async_chat_wrap, async_emb_wrap):
+def _dispatch_async(async_chat_wrap, async_generate_wrap, async_emb_wrap):
async def wrapper(wrapped, instance, args, kwargs):
if len(args) > 2 and isinstance(args[2], str):
op = args[2].rstrip("/").split("/")[-1]
if op == "chat":
return await async_chat_wrap(wrapped, instance, args, kwargs)
+ if op == "generate":
+ return await async_generate_wrap(wrapped, instance, args, kwargs)
if op == "embeddings":
return await async_emb_wrap(wrapped, instance, args, kwargs)
return await wrapped(*args, **kwargs)
@@ -69,6 +73,16 @@ class OllamaInstrumentor(BaseInstrumentor):
metrics,
disable_metrics,
)
+ sync_generate_wrap = generate(
+ version,
+ environment,
+ application_name,
+ tracer,
+ pricing_info,
+ capture_message_content,
+ metrics,
+ disable_metrics,
+ )
sync_emb_wrap = embeddings(
version,
environment,
@@ -89,6 +103,16 @@ class OllamaInstrumentor(BaseInstrumentor):
metrics,
disable_metrics,
)
+ async_generate_wrap = async_generate(
+ version,
+ environment,
+ application_name,
+ tracer,
+ pricing_info,
+ capture_message_content,
+ metrics,
+ disable_metrics,
+ )
async_emb_wrap = async_embeddings(
version,
environment,
@@ -104,12 +128,12 @@ class OllamaInstrumentor(BaseInstrumentor):
wrap_function_wrapper(
"ollama._client",
"Client._request",
- _dispatch(sync_chat_wrap, sync_emb_wrap),
+ _dispatch(sync_chat_wrap, sync_generate_wrap, sync_emb_wrap),
)
wrap_function_wrapper(
"ollama._client",
"AsyncClient._request",
- _dispatch_async(async_chat_wrap, async_emb_wrap),
+ _dispatch_async(async_chat_wrap, async_generate_wrap, async_emb_wrap),
)
def _uninstrument(self, **kwargs):
diff --git a/sdk/python/src/openlit/instrumentation/ollama/async_ollama.py b/sdk/python/src/openlit/instrumentation/ollama/async_ollama.py
index c017ff91..250b4b08 100644
--- a/sdk/python/src/openlit/instrumentation/ollama/async_ollama.py
+++ b/sdk/python/src/openlit/instrumentation/ollama/async_ollama.py
@@ -9,6 +9,8 @@ from openlit.instrumentation.ollama.utils import (
process_chunk,
process_chat_response,
process_streaming_chat_response,
+ process_generate_response,
+ process_streaming_generate_response,
process_embedding_response,
)
from openlit.semcov import SemanticConvention
@@ -134,9 +136,9 @@ def async_chat(
with tracer.start_as_current_span(span_name, kind=SpanKind.CLIENT) as span:
start_time = time.monotonic()
- try:
- response = await wrapped(*args, **kwargs)
+ response = await wrapped(*args, **kwargs)
+ try:
response = process_chat_response(
response=response,
gen_ai_endpoint="ollama.chat",
@@ -162,6 +164,154 @@ def async_chat(
return wrapper
+def async_generate(
+ version,
+ environment,
+ application_name,
+ tracer,
+ pricing_info,
+ capture_message_content,
+ metrics,
+ disable_metrics,
+):
+ """
+ Generates a telemetry wrapper for Ollama async generate function call
+ """
+
+ class TracedAsyncStream:
+ """
+ Wrapper for streaming responses to collect telemetry.
+ """
+
+ def __init__(
+ self,
+ wrapped,
+ span,
+ span_name,
+ kwargs,
+ server_address,
+ server_port,
+ args,
+ ):
+ self.__wrapped__ = wrapped
+ self._span = span
+ self._llmresponse = ""
+ self._response_model = ""
+ self._finish_reason = ""
+ self._tools = []
+ self._input_tokens = 0
+ self._output_tokens = 0
+ self._response_role = ""
+ self._span_name = span_name
+ self._args = args
+ self._kwargs = kwargs
+ self._start_time = time.monotonic()
+ self._end_time = None
+ self._timestamps = []
+ self._ttft = 0
+ self._tbt = 0
+ self._server_address = server_address
+ self._server_port = server_port
+
+ async def __aenter__(self):
+ await self.__wrapped__.__aenter__()
+ return self
+
+ async def __aexit__(self, exc_type, exc_value, traceback):
+ await self.__wrapped__.__aexit__(exc_type, exc_value, traceback)
+
+ def __aiter__(self):
+ return self
+
+ async def __getattr__(self, name):
+ """Delegate attribute access to the wrapped object."""
+ return getattr(await self.__wrapped__, name)
+
+ async def __anext__(self):
+ try:
+ chunk = await self.__wrapped__.__anext__()
+ process_chunk(self, chunk)
+ return chunk
+ except StopAsyncIteration:
+ try:
+ with tracer.start_as_current_span(
+ self._span_name, kind=SpanKind.CLIENT
+ ) as self._span:
+ process_streaming_generate_response(
+ self,
+ pricing_info=pricing_info,
+ environment=environment,
+ application_name=application_name,
+ metrics=metrics,
+ capture_message_content=capture_message_content,
+ disable_metrics=disable_metrics,
+ version=version,
+ )
+ except Exception as e:
+ handle_exception(self._span, e)
+
+ raise
+
+ async def wrapper(wrapped, instance, args, kwargs):
+ """
+ Wraps the Ollama async chat function call.
+ """
+
+ streaming = kwargs.get("stream", False)
+
+ server_address, server_port = set_server_address_and_port(
+ instance, "127.0.0.1", 11434
+ )
+ json_body = kwargs.get("json", {}) or {}
+ request_model = json_body.get("model") or kwargs.get("model")
+
+ span_name = f"{SemanticConvention.GEN_AI_OPERATION_TYPE_TEXT_COMPLETION} {request_model}"
+
+ if streaming:
+ awaited_wrapped = await wrapped(*args, **kwargs)
+ span = tracer.start_span(span_name, kind=SpanKind.CLIENT)
+ return TracedAsyncStream(
+ awaited_wrapped,
+ span,
+ span_name,
+ kwargs,
+ server_address,
+ server_port,
+ args,
+ )
+
+ else:
+ with tracer.start_as_current_span(span_name, kind=SpanKind.CLIENT) as span:
+ start_time = time.monotonic()
+
+ response = await wrapped(*args, **kwargs)
+
+ try:
+ response = process_generate_response(
+ response=response,
+ gen_ai_endpoint="ollama.generate",
+ pricing_info=pricing_info,
+ server_port=server_port,
+ server_address=server_address,
+ environment=environment,
+ application_name=application_name,
+ metrics=metrics,
+ start_time=start_time,
+ span=span,
+ capture_message_content=capture_message_content,
+ disable_metrics=disable_metrics,
+ version=version,
+ **kwargs,
+ )
+
+ except Exception as e:
+ handle_exception(span, e)
+
+ return response
+
+ return wrapper
+
+
def async_embeddings(
version,
environment,
@@ -193,9 +343,9 @@ def async_embeddings(
with tracer.start_as_current_span(span_name, kind=SpanKind.CLIENT) as span:
start_time = time.monotonic()
- try:
- response = await wrapped(*args, **kwargs)
+ response = await wrapped(*args, **kwargs)
+ try:
response = process_embedding_response(
response=response,
gen_ai_endpoint="ollama.embeddings",
diff --git a/sdk/python/src/openlit/instrumentation/ollama/ollama.py b/sdk/python/src/openlit/instrumentation/ollama/ollama.py
index ce4016ab..d0d9387e 100644
--- a/sdk/python/src/openlit/instrumentation/ollama/ollama.py
+++ b/sdk/python/src/openlit/instrumentation/ollama/ollama.py
@@ -9,6 +9,8 @@ from openlit.instrumentation.ollama.utils import (
process_chunk,
process_chat_response,
process_streaming_chat_response,
+ process_generate_response,
+ process_streaming_generate_response,
process_embedding_response,
)
from openlit.semcov import SemanticConvention
@@ -134,9 +136,9 @@ def chat(
with tracer.start_as_current_span(span_name, kind=SpanKind.CLIENT) as span:
start_time = time.monotonic()
- try:
- response = wrapped(*args, **kwargs)
+ response = wrapped(*args, **kwargs)
+ try:
response = process_chat_response(
response=response,
gen_ai_endpoint="ollama.chat",
@@ -162,6 +164,154 @@ def chat(
return wrapper
+def generate(
+ version,
+ environment,
+ application_name,
+ tracer,
+ pricing_info,
+ capture_message_content,
+ metrics,
+ disable_metrics,
+):
+ """
+ Generates a telemetry wrapper for Ollama generate function call
+ """
+
+ class TracedSyncStream:
+ """
+ Wrapper for streaming responses to collect telemetry.
+ """
+
+ def __init__(
+ self,
+ wrapped,
+ span,
+ span_name,
+ kwargs,
+ server_address,
+ server_port,
+ args,
+ ):
+ self.__wrapped__ = wrapped
+ self._span = span
+ self._llmresponse = ""
+ self._response_model = ""
+ self._finish_reason = ""
+ self._tools = []
+ self._input_tokens = 0
+ self._output_tokens = 0
+ self._response_role = ""
+ self._span_name = span_name
+ self._args = args
+ self._kwargs = kwargs
+ self._start_time = time.monotonic()
+ self._end_time = None
+ self._timestamps = []
+ self._ttft = 0
+ self._tbt = 0
+ self._server_address = server_address
+ self._server_port = server_port
+
+ def __enter__(self):
+ self.__wrapped__.__enter__()
+ return self
+
+ def __exit__(self, exc_type, exc_value, traceback):
+ self.__wrapped__.__exit__(exc_type, exc_value, traceback)
+
+ def __iter__(self):
+ return self
+
+ def __getattr__(self, name):
+ """Delegate attribute access to the wrapped object."""
+ return getattr(self.__wrapped__, name)
+
+ def __next__(self):
+ try:
+ chunk = self.__wrapped__.__next__()
+ process_chunk(self, chunk)
+ return chunk
+ except StopIteration:
+ try:
+ with tracer.start_as_current_span(
+ self._span_name, kind=SpanKind.CLIENT
+ ) as self._span:
+ process_streaming_generate_response(
+ self,
+ pricing_info=pricing_info,
+ environment=environment,
+ application_name=application_name,
+ metrics=metrics,
+ capture_message_content=capture_message_content,
+ disable_metrics=disable_metrics,
+ version=version,
+ )
+ except Exception as e:
+ handle_exception(self._span, e)
+
+ raise
+
+ def wrapper(wrapped, instance, args, kwargs):
+ """
+ Wraps the Ollama chat function call.
+ """
+
+ streaming = kwargs.get("stream", False)
+
+ server_address, server_port = set_server_address_and_port(
+ instance, "127.0.0.1", 11434
+ )
+ json_body = kwargs.get("json", {}) or {}
+ request_model = json_body.get("model") or kwargs.get("model")
+
+ span_name = f"{SemanticConvention.GEN_AI_OPERATION_TYPE_TEXT_COMPLETION} {request_model}"
+
+ if streaming:
+ awaited_wrapped = wrapped(*args, **kwargs)
+ span = tracer.start_span(span_name, kind=SpanKind.CLIENT)
+ return TracedSyncStream(
+ awaited_wrapped,
+ span,
+ span_name,
+ kwargs,
+ server_address,
+ server_port,
+ args,
+ )
+
+ else:
+ with tracer.start_as_current_span(span_name, kind=SpanKind.CLIENT) as span:
+ start_time = time.monotonic()
+
+ response = wrapped(*args, **kwargs)
+
+ try:
+ response = process_generate_response(
+ response=response,
+ gen_ai_endpoint="ollama.generate",
+ pricing_info=pricing_info,
+ server_port=server_port,
+ server_address=server_address,
+ environment=environment,
+ application_name=application_name,
+ metrics=metrics,
+ start_time=start_time,
+ span=span,
+ capture_message_content=capture_message_content,
+ disable_metrics=disable_metrics,
+ version=version,
+ **kwargs,
+ )
+
+ except Exception as e:
+ handle_exception(span, e)
+
+ return response
+
+ return wrapper
+
+
def embeddings(
version,
environment,
@@ -193,9 +343,9 @@ def embeddings(
with tracer.start_as_current_span(span_name, kind=SpanKind.CLIENT) as span:
start_time = time.monotonic()
- try:
- response = wrapped(*args, **kwargs)
+ response = wrapped(*args, **kwargs)
+ try:
response = process_embedding_response(
response=response,
gen_ai_endpoint="ollama.embeddings",
diff --git a/sdk/python/src/openlit/instrumentation/ollama/utils.py b/sdk/python/src/openlit/instrumentation/ollama/utils.py
index 8f3c0e77..b58d9f9f 100644
--- a/sdk/python/src/openlit/instrumentation/ollama/utils.py
+++ b/sdk/python/src/openlit/instrumentation/ollama/utils.py
@@ -192,17 +192,187 @@ def common_chat_logic(
# Span Attributes for Tools
if scope._tools is not None:
+ if isinstance(scope._tools, dict):
+ scope._span.set_attribute(
+ SemanticConvention.GEN_AI_TOOL_NAME, scope._tools.get("function", "")
+ ).get("name", "")
+ scope._span.set_attribute(
+ SemanticConvention.GEN_AI_TOOL_CALL_ID, str(scope._tools.get("id", ""))
+ )
+ scope._span.set_attribute(
+ SemanticConvention.GEN_AI_TOOL_ARGS,
+ str(scope._tools.get("function", "").get("arguments", "")),
+ )
+ elif isinstance(scope._tools, list) and len(scope._tools) > 0:
+ for tool in scope._tools:
+ scope._span.set_attribute(
+ SemanticConvention.GEN_AI_TOOL_NAME,
+ tool.get("function", {}).get("name", "")
+ )
+ scope._span.set_attribute(
+ SemanticConvention.GEN_AI_TOOL_CALL_ID,
+ str(tool.get("id", ""))
+ )
+ scope._span.set_attribute(
+ SemanticConvention.GEN_AI_TOOL_ARGS,
+ str(tool.get("function", {}).get("arguments", ""))
+ )
+
+ # Span Attributes for Content
+ if capture_message_content:
+ scope._span.set_attribute(SemanticConvention.GEN_AI_CONTENT_PROMPT, prompt)
scope._span.set_attribute(
- SemanticConvention.GEN_AI_TOOL_NAME, scope._tools.get("function", "")
- ).get("name", "")
- scope._span.set_attribute(
- SemanticConvention.GEN_AI_TOOL_CALL_ID, str(scope._tools.get("id", ""))
+ SemanticConvention.GEN_AI_CONTENT_COMPLETION, scope._llmresponse
)
- scope._span.set_attribute(
- SemanticConvention.GEN_AI_TOOL_ARGS,
- str(scope._tools.get("function", "").get("arguments", "")),
+
+ # To be removed once the change to span_attributes (from span events) is complete
+ scope._span.add_event(
+ name=SemanticConvention.GEN_AI_CONTENT_PROMPT_EVENT,
+ attributes={
+ SemanticConvention.GEN_AI_CONTENT_PROMPT: prompt,
+ },
+ )
+ scope._span.add_event(
+ name=SemanticConvention.GEN_AI_CONTENT_COMPLETION_EVENT,
+ attributes={
+ SemanticConvention.GEN_AI_CONTENT_COMPLETION: scope._llmresponse,
+ },
+ )
+
+ scope._span.set_status(Status(StatusCode.OK))
+
+ # Metrics
+ if not disable_metrics:
+ record_completion_metrics(
+ metrics,
+ SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT,
+ SemanticConvention.GEN_AI_SYSTEM_OLLAMA,
+ scope._server_address,
+ scope._server_port,
+ request_model,
+ scope._response_model,
+ environment,
+ application_name,
+ scope._start_time,
+ scope._end_time,
+ cost,
+ scope._input_tokens,
+ scope._output_tokens,
+ scope._tbt,
+ scope._ttft,
)
+
+def common_generate_logic(
+ scope,
+ gen_ai_endpoint,
+ pricing_info,
+ environment,
+ application_name,
+ metrics,
+ capture_message_content,
+ disable_metrics,
+ version,
+):
+ """
+ Process generate request and generate Telemetry
+ """
+
+ scope._end_time = time.monotonic()
+ if len(scope._timestamps) > 1:
+ scope._tbt = calculate_tbt(scope._timestamps)
+ json_body = scope._kwargs.get("json", {}) or {}
+ prompt = json_body.get('prompt')
+ request_model = json_body.get("model") or scope._kwargs.get("model", "llama3.2")
+ is_stream = scope._kwargs.get("stream", False)
+
+ cost = get_chat_model_cost(
+ request_model, pricing_info, scope._input_tokens, scope._output_tokens
+ )
+
+ # Common Span Attributes
+ common_span_attributes(
+ scope,
+ SemanticConvention.GEN_AI_OPERATION_TYPE_TEXT_COMPLETION,
+ SemanticConvention.GEN_AI_SYSTEM_OLLAMA,
+ scope._server_address,
+ scope._server_port,
+ request_model,
+ request_model,
+ environment,
+ application_name,
+ is_stream,
+ scope._tbt,
+ scope._ttft,
+ version,
+ )
+
+ # Span Attributes for Request parameters
+ options = json_body.get("options", scope._kwargs.get("options", {}))
+ attributes = [
+ (SemanticConvention.GEN_AI_REQUEST_FREQUENCY_PENALTY, "repeat_penalty"),
+ (SemanticConvention.GEN_AI_REQUEST_MAX_TOKENS, "max_tokens"),
+ (SemanticConvention.GEN_AI_REQUEST_SEED, "seed"),
+ (SemanticConvention.GEN_AI_REQUEST_STOP_SEQUENCES, "stop"),
+ (SemanticConvention.GEN_AI_REQUEST_TEMPERATURE, "temperature"),
+ (SemanticConvention.GEN_AI_REQUEST_TOP_P, "top_p"),
+ (SemanticConvention.GEN_AI_REQUEST_TOP_K, "top_k"),
+ ]
+ for attribute, key in attributes:
+ value = options.get(key)
+ if value is not None:
+ scope._span.set_attribute(attribute, value)
+
+ # Span Attributes for Response parameters
+ scope._span.set_attribute(
+ SemanticConvention.GEN_AI_RESPONSE_FINISH_REASON, [scope._finish_reason]
+ )
+ scope._span.set_attribute(
+ SemanticConvention.GEN_AI_OUTPUT_TYPE,
+ "text" if isinstance(scope._llmresponse, str) else "json",
+ )
+
+ # Span Attributes for Cost and Tokens
+ scope._span.set_attribute(
+ SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS, scope._input_tokens
+ )
+ scope._span.set_attribute(
+ SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS, scope._output_tokens
+ )
+ scope._span.set_attribute(
+ SemanticConvention.GEN_AI_CLIENT_TOKEN_USAGE,
+ scope._input_tokens + scope._output_tokens,
+ )
+ scope._span.set_attribute(SemanticConvention.GEN_AI_USAGE_COST, cost)
+
+ # Span Attributes for Tools
+ if scope._tools is not None:
+ if isinstance(scope._tools, dict):
+ scope._span.set_attribute(
+ SemanticConvention.GEN_AI_TOOL_NAME, scope._tools.get("function", "")
+ ).get("name", "")
+ scope._span.set_attribute(
+ SemanticConvention.GEN_AI_TOOL_CALL_ID, str(scope._tools.get("id", ""))
+ )
+ scope._span.set_attribute(
+ SemanticConvention.GEN_AI_TOOL_ARGS,
+ str(scope._tools.get("function", "").get("arguments", "")),
+ )
+ elif isinstance(scope._tools, list) and len(scope._tools) > 0:
+ for tool in scope._tools:
+ scope._span.set_attribute(
+ SemanticConvention.GEN_AI_TOOL_NAME,
+ tool.get("function", {}).get("name", "")
+ )
+ scope._span.set_attribute(
+ SemanticConvention.GEN_AI_TOOL_CALL_ID,
+ str(tool.get("id", ""))
+ )
+ scope._span.set_attribute(
+ SemanticConvention.GEN_AI_TOOL_ARGS,
+ str(tool.get("function", {}).get("arguments", ""))
+ )
+
# Span Attributes for Content
if capture_message_content:
scope._span.set_attribute(SemanticConvention.GEN_AI_CONTENT_PROMPT, prompt)
@@ -230,7 +400,7 @@ def common_chat_logic(
if not disable_metrics:
record_completion_metrics(
metrics,
- SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT,
+ SemanticConvention.GEN_AI_OPERATION_TYPE_TEXT_COMPLETION,
SemanticConvention.GEN_AI_SYSTEM_OLLAMA,
scope._server_address,
scope._server_port,
@@ -409,6 +579,90 @@ def process_chat_response(
return response
+def process_streaming_generate_response(
+ self,
+ pricing_info,
+ environment,
+ application_name,
+ metrics,
+ capture_message_content=False,
+ disable_metrics=False,
+ version="",
+):
+ """
+ Process streaming generate request and generate Telemetry
+ """
+
+ common_generate_logic(
+ self,
+ "ollama.generate",
+ pricing_info,
+ environment,
+ application_name,
+ metrics,
+ capture_message_content,
+ disable_metrics,
+ version,
+ )
+
+def process_generate_response(
+ response,
+ gen_ai_endpoint,
+ pricing_info,
+ server_port,
+ server_address,
+ environment,
+ application_name,
+ metrics,
+ start_time,
+ span,
+ capture_message_content=False,
+ disable_metrics=False,
+ version="1.0.0",
+ **kwargs,
+):
+ """
+ Process generate request and generate Telemetry
+ """
+
+ scope = type("GenericScope", (), {})()
+ response_dict = response_as_dict(response)
+
+ scope._start_time = start_time
+ scope._end_time = time.monotonic()
+ scope._span = span
+ scope._llmresponse = response_dict.get("response", "")
+ scope._response_role = response_dict.get("message", {}).get("role", "assistant")
+ scope._input_tokens = response_dict.get("prompt_eval_count", 0)
+ scope._output_tokens = response_dict.get("eval_count", 0)
+ scope._response_model = response_dict.get("model", "llama3.2")
+ scope._finish_reason = response_dict.get("done_reason", "")
+ scope._timestamps = []
+ scope._ttft = scope._end_time - scope._start_time
+ scope._tbt = 0
+ scope._server_address, scope._server_port = server_address, server_port
+ scope._kwargs = kwargs
+
+ if scope._kwargs.get("tools"):
+ scope._tools = response_dict.get("choices")[0].get("message").get("tool_calls")
+ else:
+ scope._tools = None
+
+ common_generate_logic(
+ scope,
+ gen_ai_endpoint,
+ pricing_info,
+ environment,
+ application_name,
+ metrics,
+ capture_message_content,
+ disable_metrics,
+ version,
+ )
+
+ return response
+
+
def process_embedding_response(
response,
gen_ai_endpoint,
diff --git a/sdk/python/src/openlit/semcov/__init__.py b/sdk/python/src/openlit/semcov/__init__.py
index 7106735a..5b2c8327 100644
--- a/sdk/python/src/openlit/semcov/__init__.py
+++ b/sdk/python/src/openlit/semcov/__init__.py
@@ -70,6 +70,7 @@ class SemanticConvention:
GEN_AI_TOOL_ARGS = "gen_ai.tool.args"
# GenAI Operation Types (OTel Semconv)
+ GEN_AI_OPERATION_TYPE_TEXT_COMPLETION = "text_completion"
GEN_AI_OPERATION_TYPE_CHAT = "chat"
GEN_AI_OPERATION_TYPE_TOOLS = "execute_tool"
GEN_AI_OPERATION_TYPE_EMBEDDING = "embeddings"
diff --git a/sdk/python/tests/requirements.txt b/sdk/python/tests/requirements.txt
index 25e96190..9da73ed2 100644
--- a/sdk/python/tests/requirements.txt
+++ b/sdk/python/tests/requirements.txt
@@ -26,6 +26,7 @@ together
transformers
groq
elevenlabs
+ollama
# Vector DBs
chromadb
diff --git a/sdk/python/tests/test_ollama.py.holdup b/sdk/python/tests/test_ollama.py.holdup
new file mode 100644
index 00000000..86f58c54
--- /dev/null
+++ b/sdk/python/tests/test_ollama.py.holdup
@@ -0,0 +1,185 @@
+# pylint: disable=duplicate-code, no-name-in-module
+"""
+This module contains tests for Ollama functionality using the Ollama Python library.
+
+Tests cover various API endpoints, including embeddings, chat and generate.
+These tests validate integration with OpenLIT.
+
+Environment Variables:
+ - OLLAMA_HOST: Ollama server host.
+
+Note: Ensure the environment is properly configured for Ollama access and OpenLIT monitoring
+prior to running these tests.
+"""
+
+import os
+
+import pytest
+from ollama import Client, AsyncClient, EmbedResponse, ChatResponse, GenerateResponse, Message
+import openlit
+
+# Initialize Ollama
+EMBEDDING_MODEL_NAME = 'nomic-embed-text'
+LLM_MODEL_NAME = 'gemma2:2b'
+DEFAULT_OLLAMA_HOST = "http://localhost:11434"
+
+# Initialize environment and application name for OpenLIT monitoring
+openlit.init(environment="openlit-testing", application_name="openlit-python-ollama-test")
+
[email protected](scope="function")
+def sync_client_fixture():
+ "Sync Ollama client fixture with models pulling."
+ sync_client = Client(host=os.getenv("OLLAMA_HOST") or DEFAULT_OLLAMA_HOST)
+ sync_client.pull(EMBEDDING_MODEL_NAME)
+ sync_client.pull(LLM_MODEL_NAME)
+
+ return sync_client
+
[email protected]_fixture(scope="function")
+def async_client():
+ "Async Ollama client fixture."
+
+ return AsyncClient(host=os.getenv("OLLAMA_HOST") or DEFAULT_OLLAMA_HOST)
+
+
+def test_sync_ollama_embeddings(sync_client_fixture):
+ """
+ Tests synchronous embedding creation with the 'nomic-embed-text' model.
+
+ Raises:
+ AssertionError: If the embedding response object is not as expected.
+ """
+
+ response = sync_client_fixture.embed(
+ model=EMBEDDING_MODEL_NAME,
+ input=["Embed this sentence.", "OpenTelemetry LLM Observability"],
+ )
+ assert isinstance(response, EmbedResponse)
+ assert isinstance(response.embeddings, list)
+
+
+async def test_async_ollama_embeddings(async_client_fixture):
+ """
+ Tests asynchronous embedding creation with the 'nomic-embed-text' model.
+
+ Raises:
+ AssertionError: If the embedding response object is not as expected.
+ """
+
+ response = await async_client_fixture.embed(
+ model=EMBEDDING_MODEL_NAME,
+ input=["Embed this sentence.", "OpenTelemetry LLM Observability"],
+ )
+ assert isinstance(response, EmbedResponse)
+ assert isinstance(response.embeddings, list)
+
+
+def test_sync_ollama_chat(sync_client_fixture):
+ """
+ Tests synchronous Chat Completions with the 'gemma2:2b' model.
+
+ Raises:
+ AssertionError: If the Chat Completions response object is not as expected.
+ """
+
+ try:
+ chat_completions_resp = sync_client_fixture.chat(
+ messages=[
+ {
+ "role": "user",
+ "content": "What is LLM Observability?"
+ }
+ ],
+ model=LLM_MODEL_NAME,
+ )
+ assert isinstance(chat_completions_resp, ChatResponse)
+ assert isinstance(chat_completions_resp.message, Message)
+
+ # pylint: disable=broad-exception-caught
+ except Exception as e:
+ if "rate limit" in str(e).lower():
+ print("Rate Limited:", e)
+ else:
+ raise
+
+
+async def test_async_ollama_chat(async_client_fixture):
+ """
+ Tests asynchronous Chat Completions with the 'gemma2:2b' model.
+
+ Raises:
+ AssertionError: If the Chat Completions response object is not as expected.
+ """
+
+ try:
+ chat_completions_resp = await async_client_fixture.chat(
+ messages=[
+ {
+ "role": "user",
+ "content": "What is LLM Observability?",
+ }
+ ],
+ model=LLM_MODEL_NAME,
+ keep_alive=False
+ )
+ assert isinstance(chat_completions_resp, ChatResponse)
+ assert isinstance(chat_completions_resp.message, Message)
+
+ # pylint: disable=broad-exception-caught
+ except Exception as e:
+ if "rate limit" in str(e).lower():
+ print("Rate Limited:", e)
+ else:
+ raise
+
+
+def test_sync_ollama_generate(sync_client_fixture):
+ """
+ Tests synchronous Text Completions with the 'gemma2:2b' model.
+
+ Raises:
+ AssertionError: If the Text Completions response object is not as expected.
+ """
+
+ try:
+ chat_completions_resp = sync_client_fixture.generate(
+ prompt="What is LLM Observability?",
+ model=LLM_MODEL_NAME,
+ )
+ assert isinstance(chat_completions_resp, GenerateResponse)
+ assert isinstance(chat_completions_resp.response, str)
+
+ # pylint: disable=broad-exception-caught
+ except Exception as e:
+ if "rate limit" in str(e).lower():
+ print("Rate Limited:", e)
+ else:
+ raise
+
+
+async def test_async_ollama_generate(async_client_fixture):
+ """
+ Tests asynchronous Text Completions with the 'gemma2:2b' model.
+
+ Raises:
+ AssertionError: If the Text Completions response object is not as expected.
+ """
+
+ try:
+ chat_completions_resp = await async_client_fixture.generate(
+ prompt="What is LLM Observability?",
+ model=LLM_MODEL_NAME,
+ keep_alive=False
+ )
+ assert isinstance(chat_completions_resp, GenerateResponse)
+ assert isinstance(chat_completions_resp.response, str)
+
+ # pylint: disable=broad-exception-caught
+ except Exception as e:
+ if "rate limit" in str(e).lower():
+ print("Rate Limited:", e)
+ else:
+ raise