Python SDK for AI agent monitoring, LLM cost tracking, benchmarking, and more. Integrates with most LLMs and agent frameworks including CrewAI, Agno, OpenAI Agents SDK, Langchain, Autogen, AG2, and CamelAI
Python agentagentopsagents-sdkaianthropicautogencost-estimationcrewaievalsevaluation-metricsgroqlangchainllmmistralollamaopenaiopenai-agents
Ollama Support (#237)
Siddharth Prajosh4b86d9abee33
diff --git a/agentops/llm_tracker.py b/agentops/llm_tracker.pyindex dcead775..c07b2c9e 100644--- a/agentops/llm_tracker.py+++ b/agentops/llm_tracker.py@@ -1,15 +1,18 @@import functools+import inspect+import pprintimport sysfrom importlib import import_modulefrom importlib.metadata import version+from typing import Optional+from packaging.version import Version, parse++from .event import ActionEvent, ErrorEvent, LLMEvent+from .helpers import check_call_stack_for_agent_id, get_ISO_timefrom .log_config import logger-from .event import LLMEvent, ActionEvent, ToolEvent, ErrorEvent-from .helpers import get_ISO_time, check_call_stack_for_agent_id-import inspect-from typing import Optional-import pprint+original_func = {}original_create = Noneoriginal_create_async = None@@ -27,6 +30,7 @@ class LlmTracker:"cohere": {"5.4.0": ("chat", "chat_stream"),},+ "ollama": {"0.0.1": ("chat", "Client.chat", "AsyncClient.chat")},}def __init__(self, client):@@ -134,9 +138,9 @@ class LlmTracker:def _handle_response_v1_openai(self, response, kwargs, init_timestamp):"""Handle responses for OpenAI versions >v1.0.0"""- from openai import Stream, AsyncStream- from openai.types.chat import ChatCompletionChunk+ from openai import AsyncStream, Streamfrom openai.resources import AsyncCompletions+ from openai.types.chat import ChatCompletionChunkself.llm_event = LLMEvent(init_timestamp=init_timestamp, params=kwargs)@@ -151,9 +155,9 @@ class LlmTracker:self.llm_event.agent_id = check_call_stack_for_agent_id()self.llm_event.model = chunk.modelself.llm_event.prompt = kwargs["messages"]- choice = chunk.choices[- 0- ] # NOTE: We assume for completion only choices[0] is relevant++ # NOTE: We assume for completion only choices[0] is relevant+ choice = chunk.choices[0]if choice.delta.content:accumulated_delta.content += choice.delta.content@@ -261,7 +265,6 @@ class LlmTracker:)# from cohere.types.chat import ChatGenerationChunk-# NOTE: Cohere only returns one message and its role will be CHATBOT which we are coercing to "assistant"self.llm_event = LLMEvent(init_timestamp=init_timestamp, params=kwargs)@@ -417,6 +420,47 @@ class LlmTracker:return response+ def _handle_response_ollama(self, response, kwargs, init_timestamp):+ self.llm_event = LLMEvent(init_timestamp=init_timestamp, params=kwargs)++ def handle_stream_chunk(chunk: dict):+ message = chunk.get("message", {"role": None, "content": ""})++ if chunk.get("done"):+ self.llm_event.completion["content"] += message.get("content")+ self.llm_event.end_timestamp = get_ISO_time()+ self.llm_event.model = f'ollama/{chunk.get("model")}'+ self.llm_event.returns = chunk+ self.llm_event.returns["message"] = self.llm_event.completion+ self.llm_event.prompt = kwargs["messages"]+ self.llm_event.agent_id = check_call_stack_for_agent_id()+ self.client.record(self.llm_event)++ if self.llm_event.completion is None:+ self.llm_event.completion = message+ else:+ self.llm_event.completion["content"] += message.get("content")++ if inspect.isgenerator(response):++ def generator():+ for chunk in response:+ handle_stream_chunk(chunk)+ yield chunk++ return generator()++ self.llm_event.end_timestamp = get_ISO_time()++ self.llm_event.model = f'ollama/{response["model"]}'+ self.llm_event.returns = response+ self.llm_event.agent_id = check_call_stack_for_agent_id()+ self.llm_event.prompt = kwargs["messages"]+ self.llm_event.completion = response["message"]++ self.client.record(self.llm_event)+ return response+def override_openai_v1_completion(self):from openai.resources.chat import completions@@ -506,6 +550,48 @@ class LlmTracker:# Override the original method with the patched onecohere.Client.chat_stream = patched_function+ def override_ollama_chat(self):+ import ollama++ original_func["ollama.chat"] = ollama.chat++ def patched_function(*args, **kwargs):+ # Call the original function with its original arguments+ init_timestamp = get_ISO_time()+ result = original_func["ollama.chat"](*args, **kwargs)+ return self._handle_response_ollama(result, kwargs, init_timestamp)++ # Override the original method with the patched one+ ollama.chat = patched_function++ def override_ollama_chat_client(self):+ from ollama import Client++ original_func["ollama.Client.chat"] = Client.chat++ def patched_function(*args, **kwargs):+ # Call the original function with its original arguments+ init_timestamp = get_ISO_time()+ result = original_func["ollama.Client.chat"](*args, **kwargs)+ return self._handle_response_ollama(result, kwargs, init_timestamp)++ # Override the original method with the patched one+ Client.chat = patched_function++ def override_ollama_chat_async_client(self):+ from ollama import AsyncClient++ original_func["ollama.AsyncClient.chat"] = AsyncClient.chat++ async def patched_function(*args, **kwargs):+ # Call the original function with its original arguments+ init_timestamp = get_ISO_time()+ result = await original_func["ollama.AsyncClient.chat"](*args, **kwargs)+ return self._handle_response_ollama(result, kwargs, init_timestamp)++ # Override the original method with the patched one+ AsyncClient.chat = patched_function+def _override_method(self, api, method_path, module):def handle_response(result, kwargs, init_timestamp):if api == "openai":@@ -595,9 +681,22 @@ class LlmTracker:f"Only Cohere>=5.4.0 supported. v{module_version} found.")+ if api == "ollama":+ module_version = version(api)++ if Version(module_version) >= parse("0.0.1"):+ self.override_ollama_chat()+ self.override_ollama_chat_client()+ self.override_ollama_chat_async_client()+ else:+ logger.warning(+ f"Only Ollama>=0.0.1 supported. v{module_version} found."+ )+def stop_instrumenting(self):self.undo_override_openai_v1_async_completion()self.undo_override_openai_v1_completion()+ self.undo_override_ollama()def undo_override_openai_v1_completion(self):global original_create@@ -610,3 +709,10 @@ class LlmTracker:from openai.resources.chat import completionscompletions.AsyncCompletions.create = original_create_async++ def undo_override_ollama(self):+ import ollama++ ollama.chat = original_func["ollama.chat"]+ ollama.Client.chat = original_func["ollama.Client.chat"]+ ollama.AsyncClient.chat = original_func["ollama.AsyncClient.chat"]