Python SDK for AI agent monitoring, LLM cost tracking, benchmarking, and more. Integrates with most LLMs and agent frameworks including CrewAI, Agno, OpenAI Agents SDK, Langchain, Autogen, AG2, and CamelAI
Python agentagentopsagents-sdkaianthropicautogencost-estimationcrewaievalsevaluation-metricsgroqlangchainllmmistralollamaopenaiopenai-agents
Llmtracker fix (#554)
Alex Reibman5524c46d7d27
diff --git a/agentops/__init__.py b/agentops/__init__.pyindex ccb5427d..4150f839 100755--- a/agentops/__init__.py+++ b/agentops/__init__.py@@ -11,6 +11,7 @@ from .session import Sessionimport threadingfrom importlib.metadata import version as get_versionfrom packaging import version+from .llms import trackertry:from .partners.langchain_callback_handler import (diff --git a/agentops/client.py b/agentops/client.pyindex d1bb51db..fb3e1793 100644--- a/agentops/client.py+++ b/agentops/client.py@@ -23,7 +23,7 @@ from termcolor import coloredfrom .config import Configurationfrom .event import ErrorEvent, Eventfrom .host_env import get_host_env-from .llms import LlmTracker+from .llms.tracker import LlmTrackerfrom .log_config import loggerfrom .meta_client import MetaClientfrom .session import Session, active_sessionsdiff --git a/agentops/llms/llama_stack_client.py b/agentops/llms/llama_stack_client.pyindex 8379a6fe..50f733b1 100644--- a/agentops/llms/llama_stack_client.py+++ b/agentops/llms/llama_stack_client.py@@ -9,7 +9,7 @@ from agentops.event import LLMEvent, ErrorEvent, ToolEventfrom agentops.session import Sessionfrom agentops.log_config import loggerfrom agentops.helpers import get_ISO_time, check_call_stack_for_agent_id-from agentops.llms.instrumented_provider import InstrumentedProvider+from agentops.llms.providers.instrumented_provider import InstrumentedProviderclass LlamaStackClientProvider(InstrumentedProvider):diff --git a/agentops/llms/ai21.py b/agentops/llms/providers/ai21.pysimilarity index 96%rename from agentops/llms/ai21.pyrename to agentops/llms/providers/ai21.pyindex 4e080a5b..8c907d52 100644--- a/agentops/llms/ai21.py+++ b/agentops/llms/providers/ai21.py@@ -2,14 +2,14 @@ import inspectimport pprintfrom typing import Optional-from agentops.llms.instrumented_provider import InstrumentedProvider+from agentops.llms.providers.instrumented_provider import InstrumentedProviderfrom agentops.time_travel import fetch_completion_override_from_time_travel_cache-from ..event import ErrorEvent, LLMEvent, ActionEvent, ToolEvent-from ..session import Session-from ..log_config import logger-from ..helpers import check_call_stack_for_agent_id, get_ISO_time-from ..singleton import singleton+from agentops.event import ErrorEvent, LLMEvent, ActionEvent, ToolEvent+from agentops.session import Session+from agentops.log_config import logger+from agentops.helpers import check_call_stack_for_agent_id, get_ISO_time+from agentops.singleton import singleton@singletondiff --git a/agentops/llms/anthropic.py b/agentops/llms/providers/anthropic.pysimilarity index 97%rename from agentops/llms/anthropic.pyrename to agentops/llms/providers/anthropic.pyindex 0a729340..7ba523d8 100644--- a/agentops/llms/anthropic.py+++ b/agentops/llms/providers/anthropic.py@@ -2,14 +2,14 @@ import jsonimport pprintfrom typing import Optional-from agentops.llms.instrumented_provider import InstrumentedProvider+from agentops.llms.providers.instrumented_provider import InstrumentedProviderfrom agentops.time_travel import fetch_completion_override_from_time_travel_cache-from ..event import ErrorEvent, LLMEvent, ToolEvent-from ..helpers import check_call_stack_for_agent_id, get_ISO_time-from ..log_config import logger-from ..session import Session-from ..singleton import singleton+from agentops.event import ErrorEvent, LLMEvent, ToolEvent+from agentops.helpers import check_call_stack_for_agent_id, get_ISO_time+from agentops.log_config import logger+from agentops.session import Session+from agentops.singleton import singleton@singletondiff --git a/agentops/llms/cohere.py b/agentops/llms/providers/cohere.pysimilarity index 98%rename from agentops/llms/cohere.pyrename to agentops/llms/providers/cohere.pyindex 0756bc69..f5954bbf 100644--- a/agentops/llms/cohere.py+++ b/agentops/llms/providers/cohere.py@@ -3,11 +3,11 @@ import pprintfrom typing import Optionalfrom .instrumented_provider import InstrumentedProvider-from ..event import ActionEvent, ErrorEvent, LLMEvent-from ..session import Session-from ..log_config import logger+from agentops.event import ActionEvent, ErrorEvent, LLMEvent+from agentops.session import Session+from agentops.log_config import loggerfrom agentops.helpers import get_ISO_time, check_call_stack_for_agent_id-from ..singleton import singleton+from agentops.singleton import singleton@singletondiff --git a/agentops/llms/groq.py b/agentops/llms/providers/groq.pysimilarity index 97%rename from agentops/llms/groq.pyrename to agentops/llms/providers/groq.pyindex 23d5b8db..226a9123 100644--- a/agentops/llms/groq.py+++ b/agentops/llms/providers/groq.py@@ -2,11 +2,11 @@ import pprintfrom typing import Optionalfrom .instrumented_provider import InstrumentedProvider-from ..event import ErrorEvent, LLMEvent-from ..session import Session-from ..log_config import logger+from agentops.event import ErrorEvent, LLMEvent+from agentops.session import Session+from agentops.log_config import loggerfrom agentops.helpers import get_ISO_time, check_call_stack_for_agent_id-from ..singleton import singleton+from agentops.singleton import singleton@singletondiff --git a/agentops/llms/instrumented_provider.py b/agentops/llms/providers/instrumented_provider.pysimilarity index 91%rename from agentops/llms/instrumented_provider.pyrename to agentops/llms/providers/instrumented_provider.pyindex f4dc78cf..9b069c53 100644--- a/agentops/llms/instrumented_provider.py+++ b/agentops/llms/providers/instrumented_provider.py@@ -1,8 +1,8 @@from abc import ABC, abstractmethodfrom typing import Optional-from ..session import Session-from ..event import LLMEvent+from agentops.session import Session+from agentops.event import LLMEventclass InstrumentedProvider(ABC):diff --git a/agentops/llms/litellm.py b/agentops/llms/providers/litellm.pysimilarity index 97%rename from agentops/llms/litellm.pyrename to agentops/llms/providers/litellm.pyindex 0f73c22b..dff40765 100644--- a/agentops/llms/litellm.py+++ b/agentops/llms/providers/litellm.py@@ -1,13 +1,13 @@import pprintfrom typing import Optional-from ..log_config import logger-from ..event import LLMEvent, ErrorEvent-from ..session import Session+from agentops.log_config import logger+from agentops.event import LLMEvent, ErrorEvent+from agentops.session import Sessionfrom agentops.helpers import get_ISO_time, check_call_stack_for_agent_id-from agentops.llms.instrumented_provider import InstrumentedProvider+from agentops.llms.providers.instrumented_provider import InstrumentedProviderfrom agentops.time_travel import fetch_completion_override_from_time_travel_cache-from ..singleton import singleton+from agentops.singleton import singleton@singletondiff --git a/agentops/llms/mistral.py b/agentops/llms/providers/mistral.pysimilarity index 98%rename from agentops/llms/mistral.pyrename to agentops/llms/providers/mistral.pyindex 770d4a81..1754cae5 100644--- a/agentops/llms/mistral.py+++ b/agentops/llms/providers/mistral.py@@ -3,9 +3,9 @@ import pprintimport sysfrom typing import Optional-from ..event import LLMEvent, ErrorEvent-from ..session import Session-from ..log_config import logger+from agentops.event import LLMEvent, ErrorEvent+from agentops.session import Session+from agentops.log_config import loggerfrom agentops.helpers import get_ISO_time, check_call_stack_for_agent_idfrom .instrumented_provider import InstrumentedProviderdiff --git a/agentops/llms/ollama.py b/agentops/llms/providers/ollama.pysimilarity index 97%rename from agentops/llms/ollama.pyrename to agentops/llms/providers/ollama.pyindex 5c998897..480cae8b 100644--- a/agentops/llms/ollama.py+++ b/agentops/llms/providers/ollama.py@@ -2,11 +2,11 @@ import inspectimport sysfrom typing import Optional-from ..event import LLMEvent-from ..session import Session+from agentops.event import LLMEvent+from agentops.session import Sessionfrom agentops.helpers import get_ISO_time, check_call_stack_for_agent_idfrom .instrumented_provider import InstrumentedProvider-from ..singleton import singleton+from agentops.singleton import singletonoriginal_func = {}diff --git a/agentops/llms/openai.py b/agentops/llms/providers/openai.pysimilarity index 96%rename from agentops/llms/openai.pyrename to agentops/llms/providers/openai.pyindex 24756971..eb318581 100644--- a/agentops/llms/openai.py+++ b/agentops/llms/providers/openai.py@@ -2,14 +2,14 @@ import inspectimport pprintfrom typing import Optional-from agentops.llms.instrumented_provider import InstrumentedProvider+from agentops.llms.providers.instrumented_provider import InstrumentedProviderfrom agentops.time_travel import fetch_completion_override_from_time_travel_cache-from ..event import ActionEvent, ErrorEvent, LLMEvent-from ..session import Session-from ..log_config import logger-from ..helpers import check_call_stack_for_agent_id, get_ISO_time-from ..singleton import singleton+from agentops.event import ActionEvent, ErrorEvent, LLMEvent+from agentops.session import Session+from agentops.log_config import logger+from agentops.helpers import check_call_stack_for_agent_id, get_ISO_time+from agentops.singleton import singleton@singletondiff --git a/agentops/llms/__init__.py b/agentops/llms/tracker.pysimilarity index 94%rename from agentops/llms/__init__.pyrename to agentops/llms/tracker.pyindex b26cd123..06ed7de8 100644--- a/agentops/llms/__init__.py+++ b/agentops/llms/tracker.py@@ -1,4 +1,3 @@-import functoolsimport sysfrom importlib import import_modulefrom importlib.metadata import version@@ -9,14 +8,14 @@ from agentops.llms.llama_stack_client import LlamaStackClientProviderfrom ..log_config import logger-from .cohere import CohereProvider-from .groq import GroqProvider-from .litellm import LiteLLMProvider-from .ollama import OllamaProvider-from .openai import OpenAiProvider-from .anthropic import AnthropicProvider-from .mistral import MistralProvider-from .ai21 import AI21Provider+from .providers.cohere import CohereProvider+from .providers.groq import GroqProvider+from .providers.litellm import LiteLLMProvider+from .providers.ollama import OllamaProvider+from .providers.openai import OpenAiProvider+from .providers.anthropic import AnthropicProvider+from .providers.mistral import MistralProvider+from .providers.ai21 import AI21Provideroriginal_func = {}original_create = Nonediff --git a/agentops/partners/autogen_logger.py b/agentops/partners/autogen_logger.pyindex 9520801f..4b6088a9 100644--- a/agentops/partners/autogen_logger.py+++ b/agentops/partners/autogen_logger.py@@ -61,7 +61,7 @@ class AutogenLogger(BaseLogger):# Note: Autogen tokens are not included in the request and function call tokens are not counted in the completionllm_event = LLMEvent(prompt=request["messages"],- completion=completion.message,+ completion=completion.message.to_dict(),model=response.model,cost=cost,returns=completion.message.to_json(),diff --git a/agentops/partners/langchain_callback_handler.py b/agentops/partners/langchain_callback_handler.pyindex 7d3ded9b..3803b488 100644--- a/agentops/partners/langchain_callback_handler.py+++ b/agentops/partners/langchain_callback_handler.py@@ -1,6 +1,10 @@from typing import Dict, Any, List, Optional, Sequence, Unionfrom collections import defaultdictfrom uuid import UUID+import logging+import os++from tenacity import RetryCallStatefrom langchain_core.agents import AgentFinish, AgentActionfrom langchain_core.documents import Document@@ -8,16 +12,12 @@ from langchain_core.outputs import ChatGenerationChunk, GenerationChunk, LLMResufrom langchain.callbacks.base import BaseCallbackHandler, AsyncCallbackHandlerfrom langchain_core.messages import BaseMessage-from tenacity import RetryCallStatefrom agentops import Client as AOClientfrom agentops import ActionEvent, LLMEvent, ToolEvent, ErrorEvent-from agentops.helpers import get_ISO_time+from agentops.helpers import get_ISO_time, debug_print_function_params-from ..helpers import debug_print_function_params-import osfrom ..log_config import logger-import loggingdef get_model_from_kwargs(kwargs: any) -> str: