Skip to content

Python SDK for AI agent monitoring, LLM cost tracking, benchmarking, and more. Integrates with most LLMs and agent frameworks including CrewAI, Agno, OpenAI Agents SDK, Langchain, Autogen, AG2, and CamelAI

Llmtracker fix (#554)

Alex Reibman5524c46d7d27
diff --git a/agentops/__init__.py b/agentops/__init__.py
index ccb5427d..4150f839 100755
--- a/agentops/__init__.py
+++ b/agentops/__init__.py
@@ -11,6 +11,7 @@ from .session import Session
import threading
from importlib.metadata import version as get_version
from packaging import version
+from .llms import tracker
try:
from .partners.langchain_callback_handler import (
diff --git a/agentops/client.py b/agentops/client.py
index d1bb51db..fb3e1793 100644
--- a/agentops/client.py
+++ b/agentops/client.py
@@ -23,7 +23,7 @@ from termcolor import colored
from .config import Configuration
from .event import ErrorEvent, Event
from .host_env import get_host_env
-from .llms import LlmTracker
+from .llms.tracker import LlmTracker
from .log_config import logger
from .meta_client import MetaClient
from .session import Session, active_sessions
diff --git a/agentops/llms/llama_stack_client.py b/agentops/llms/llama_stack_client.py
index 8379a6fe..50f733b1 100644
--- a/agentops/llms/llama_stack_client.py
+++ b/agentops/llms/llama_stack_client.py
@@ -9,7 +9,7 @@ from agentops.event import LLMEvent, ErrorEvent, ToolEvent
from agentops.session import Session
from agentops.log_config import logger
from agentops.helpers import get_ISO_time, check_call_stack_for_agent_id
-from agentops.llms.instrumented_provider import InstrumentedProvider
+from agentops.llms.providers.instrumented_provider import InstrumentedProvider
class LlamaStackClientProvider(InstrumentedProvider):
diff --git a/agentops/llms/ai21.py b/agentops/llms/providers/ai21.py
similarity index 96%
rename from agentops/llms/ai21.py
rename to agentops/llms/providers/ai21.py
index 4e080a5b..8c907d52 100644
--- a/agentops/llms/ai21.py
+++ b/agentops/llms/providers/ai21.py
@@ -2,14 +2,14 @@ import inspect
import pprint
from typing import Optional
-from agentops.llms.instrumented_provider import InstrumentedProvider
+from agentops.llms.providers.instrumented_provider import InstrumentedProvider
from agentops.time_travel import fetch_completion_override_from_time_travel_cache
-from ..event import ErrorEvent, LLMEvent, ActionEvent, ToolEvent
-from ..session import Session
-from ..log_config import logger
-from ..helpers import check_call_stack_for_agent_id, get_ISO_time
-from ..singleton import singleton
+from agentops.event import ErrorEvent, LLMEvent, ActionEvent, ToolEvent
+from agentops.session import Session
+from agentops.log_config import logger
+from agentops.helpers import check_call_stack_for_agent_id, get_ISO_time
+from agentops.singleton import singleton
@singleton
diff --git a/agentops/llms/anthropic.py b/agentops/llms/providers/anthropic.py
similarity index 97%
rename from agentops/llms/anthropic.py
rename to agentops/llms/providers/anthropic.py
index 0a729340..7ba523d8 100644
--- a/agentops/llms/anthropic.py
+++ b/agentops/llms/providers/anthropic.py
@@ -2,14 +2,14 @@ import json
import pprint
from typing import Optional
-from agentops.llms.instrumented_provider import InstrumentedProvider
+from agentops.llms.providers.instrumented_provider import InstrumentedProvider
from agentops.time_travel import fetch_completion_override_from_time_travel_cache
-from ..event import ErrorEvent, LLMEvent, ToolEvent
-from ..helpers import check_call_stack_for_agent_id, get_ISO_time
-from ..log_config import logger
-from ..session import Session
-from ..singleton import singleton
+from agentops.event import ErrorEvent, LLMEvent, ToolEvent
+from agentops.helpers import check_call_stack_for_agent_id, get_ISO_time
+from agentops.log_config import logger
+from agentops.session import Session
+from agentops.singleton import singleton
@singleton
diff --git a/agentops/llms/cohere.py b/agentops/llms/providers/cohere.py
similarity index 98%
rename from agentops/llms/cohere.py
rename to agentops/llms/providers/cohere.py
index 0756bc69..f5954bbf 100644
--- a/agentops/llms/cohere.py
+++ b/agentops/llms/providers/cohere.py
@@ -3,11 +3,11 @@ import pprint
from typing import Optional
from .instrumented_provider import InstrumentedProvider
-from ..event import ActionEvent, ErrorEvent, LLMEvent
-from ..session import Session
-from ..log_config import logger
+from agentops.event import ActionEvent, ErrorEvent, LLMEvent
+from agentops.session import Session
+from agentops.log_config import logger
from agentops.helpers import get_ISO_time, check_call_stack_for_agent_id
-from ..singleton import singleton
+from agentops.singleton import singleton
@singleton
diff --git a/agentops/llms/groq.py b/agentops/llms/providers/groq.py
similarity index 97%
rename from agentops/llms/groq.py
rename to agentops/llms/providers/groq.py
index 23d5b8db..226a9123 100644
--- a/agentops/llms/groq.py
+++ b/agentops/llms/providers/groq.py
@@ -2,11 +2,11 @@ import pprint
from typing import Optional
from .instrumented_provider import InstrumentedProvider
-from ..event import ErrorEvent, LLMEvent
-from ..session import Session
-from ..log_config import logger
+from agentops.event import ErrorEvent, LLMEvent
+from agentops.session import Session
+from agentops.log_config import logger
from agentops.helpers import get_ISO_time, check_call_stack_for_agent_id
-from ..singleton import singleton
+from agentops.singleton import singleton
@singleton
diff --git a/agentops/llms/instrumented_provider.py b/agentops/llms/providers/instrumented_provider.py
similarity index 91%
rename from agentops/llms/instrumented_provider.py
rename to agentops/llms/providers/instrumented_provider.py
index f4dc78cf..9b069c53 100644
--- a/agentops/llms/instrumented_provider.py
+++ b/agentops/llms/providers/instrumented_provider.py
@@ -1,8 +1,8 @@
from abc import ABC, abstractmethod
from typing import Optional
-from ..session import Session
-from ..event import LLMEvent
+from agentops.session import Session
+from agentops.event import LLMEvent
class InstrumentedProvider(ABC):
diff --git a/agentops/llms/litellm.py b/agentops/llms/providers/litellm.py
similarity index 97%
rename from agentops/llms/litellm.py
rename to agentops/llms/providers/litellm.py
index 0f73c22b..dff40765 100644
--- a/agentops/llms/litellm.py
+++ b/agentops/llms/providers/litellm.py
@@ -1,13 +1,13 @@
import pprint
from typing import Optional
-from ..log_config import logger
-from ..event import LLMEvent, ErrorEvent
-from ..session import Session
+from agentops.log_config import logger
+from agentops.event import LLMEvent, ErrorEvent
+from agentops.session import Session
from agentops.helpers import get_ISO_time, check_call_stack_for_agent_id
-from agentops.llms.instrumented_provider import InstrumentedProvider
+from agentops.llms.providers.instrumented_provider import InstrumentedProvider
from agentops.time_travel import fetch_completion_override_from_time_travel_cache
-from ..singleton import singleton
+from agentops.singleton import singleton
@singleton
diff --git a/agentops/llms/mistral.py b/agentops/llms/providers/mistral.py
similarity index 98%
rename from agentops/llms/mistral.py
rename to agentops/llms/providers/mistral.py
index 770d4a81..1754cae5 100644
--- a/agentops/llms/mistral.py
+++ b/agentops/llms/providers/mistral.py
@@ -3,9 +3,9 @@ import pprint
import sys
from typing import Optional
-from ..event import LLMEvent, ErrorEvent
-from ..session import Session
-from ..log_config import logger
+from agentops.event import LLMEvent, ErrorEvent
+from agentops.session import Session
+from agentops.log_config import logger
from agentops.helpers import get_ISO_time, check_call_stack_for_agent_id
from .instrumented_provider import InstrumentedProvider
diff --git a/agentops/llms/ollama.py b/agentops/llms/providers/ollama.py
similarity index 97%
rename from agentops/llms/ollama.py
rename to agentops/llms/providers/ollama.py
index 5c998897..480cae8b 100644
--- a/agentops/llms/ollama.py
+++ b/agentops/llms/providers/ollama.py
@@ -2,11 +2,11 @@ import inspect
import sys
from typing import Optional
-from ..event import LLMEvent
-from ..session import Session
+from agentops.event import LLMEvent
+from agentops.session import Session
from agentops.helpers import get_ISO_time, check_call_stack_for_agent_id
from .instrumented_provider import InstrumentedProvider
-from ..singleton import singleton
+from agentops.singleton import singleton
original_func = {}
diff --git a/agentops/llms/openai.py b/agentops/llms/providers/openai.py
similarity index 96%
rename from agentops/llms/openai.py
rename to agentops/llms/providers/openai.py
index 24756971..eb318581 100644
--- a/agentops/llms/openai.py
+++ b/agentops/llms/providers/openai.py
@@ -2,14 +2,14 @@ import inspect
import pprint
from typing import Optional
-from agentops.llms.instrumented_provider import InstrumentedProvider
+from agentops.llms.providers.instrumented_provider import InstrumentedProvider
from agentops.time_travel import fetch_completion_override_from_time_travel_cache
-from ..event import ActionEvent, ErrorEvent, LLMEvent
-from ..session import Session
-from ..log_config import logger
-from ..helpers import check_call_stack_for_agent_id, get_ISO_time
-from ..singleton import singleton
+from agentops.event import ActionEvent, ErrorEvent, LLMEvent
+from agentops.session import Session
+from agentops.log_config import logger
+from agentops.helpers import check_call_stack_for_agent_id, get_ISO_time
+from agentops.singleton import singleton
@singleton
diff --git a/agentops/llms/__init__.py b/agentops/llms/tracker.py
similarity index 94%
rename from agentops/llms/__init__.py
rename to agentops/llms/tracker.py
index b26cd123..06ed7de8 100644
--- a/agentops/llms/__init__.py
+++ b/agentops/llms/tracker.py
@@ -1,4 +1,3 @@
-import functools
import sys
from importlib import import_module
from importlib.metadata import version
@@ -9,14 +8,14 @@ from agentops.llms.llama_stack_client import LlamaStackClientProvider
from ..log_config import logger
-from .cohere import CohereProvider
-from .groq import GroqProvider
-from .litellm import LiteLLMProvider
-from .ollama import OllamaProvider
-from .openai import OpenAiProvider
-from .anthropic import AnthropicProvider
-from .mistral import MistralProvider
-from .ai21 import AI21Provider
+from .providers.cohere import CohereProvider
+from .providers.groq import GroqProvider
+from .providers.litellm import LiteLLMProvider
+from .providers.ollama import OllamaProvider
+from .providers.openai import OpenAiProvider
+from .providers.anthropic import AnthropicProvider
+from .providers.mistral import MistralProvider
+from .providers.ai21 import AI21Provider
original_func = {}
original_create = None
diff --git a/agentops/partners/autogen_logger.py b/agentops/partners/autogen_logger.py
index 9520801f..4b6088a9 100644
--- a/agentops/partners/autogen_logger.py
+++ b/agentops/partners/autogen_logger.py
@@ -61,7 +61,7 @@ class AutogenLogger(BaseLogger):
# Note: Autogen tokens are not included in the request and function call tokens are not counted in the completion
llm_event = LLMEvent(
prompt=request["messages"],
- completion=completion.message,
+ completion=completion.message.to_dict(),
model=response.model,
cost=cost,
returns=completion.message.to_json(),
diff --git a/agentops/partners/langchain_callback_handler.py b/agentops/partners/langchain_callback_handler.py
index 7d3ded9b..3803b488 100644
--- a/agentops/partners/langchain_callback_handler.py
+++ b/agentops/partners/langchain_callback_handler.py
@@ -1,6 +1,10 @@
from typing import Dict, Any, List, Optional, Sequence, Union
from collections import defaultdict
from uuid import UUID
+import logging
+import os
+
+from tenacity import RetryCallState
from langchain_core.agents import AgentFinish, AgentAction
from langchain_core.documents import Document
@@ -8,16 +12,12 @@ from langchain_core.outputs import ChatGenerationChunk, GenerationChunk, LLMResu
from langchain.callbacks.base import BaseCallbackHandler, AsyncCallbackHandler
from langchain_core.messages import BaseMessage
-from tenacity import RetryCallState
from agentops import Client as AOClient
from agentops import ActionEvent, LLMEvent, ToolEvent, ErrorEvent
-from agentops.helpers import get_ISO_time
+from agentops.helpers import get_ISO_time, debug_print_function_params
-from ..helpers import debug_print_function_params
-import os
from ..log_config import logger
-import logging
def get_model_from_kwargs(kwargs: any) -> str: