Python SDK for AI agent monitoring, LLM cost tracking, benchmarking, and more. Integrates with most LLMs and agent frameworks including CrewAI, Agno, OpenAI Agents SDK, Langchain, Autogen, AG2, and CamelAI
Python agentagentopsagents-sdkaianthropicautogencost-estimationcrewaievalsevaluation-metricsgroqlangchainllmmistralollamaopenaiopenai-agents
Time Travel Debugging (#323)
Howard Gilde23413224f1
diff --git a/.gitignore b/.gitignoreindex 4f19a0a9..4db649aa 100644--- a/.gitignore+++ b/.gitignore@@ -161,4 +161,7 @@ cython_debug/.vscode/.benchmarks/-.DS_Store\ No newline at end of file+.DS_Store++agentops_time_travel.json+.agentops_time_travel.yaml\ No newline at end of filediff --git a/agentops/cli.py b/agentops/cli.pynew file mode 100644index 00000000..29b9e90b--- /dev/null+++ b/agentops/cli.py@@ -0,0 +1,41 @@+import argparse+from .time_travel import fetch_time_travel_id, set_time_travel_active_state+++def main():+ parser = argparse.ArgumentParser(description="AgentOps CLI")+ subparsers = parser.add_subparsers(dest="command")++ timetravel_parser = subparsers.add_parser(+ "timetravel", help="Time Travel Debugging commands", aliases=["tt"]+ )+ timetravel_parser.add_argument(+ "branch_name",+ type=str,+ nargs="?",+ help="Given a branch name, fetches the cache file for Time Travel Debugging. Turns on feature by default",+ )+ timetravel_parser.add_argument(+ "--on",+ action="store_true",+ help="Turns on Time Travel Debugging",+ )+ timetravel_parser.add_argument(+ "--off",+ action="store_true",+ help="Turns off Time Travel Debugging",+ )++ args = parser.parse_args()++ if args.command in ["timetravel", "tt"]:+ if args.branch_name:+ fetch_time_travel_id(args.branch_name)+ if args.on:+ set_time_travel_active_state("on")+ if args.off:+ set_time_travel_active_state("off")+++if __name__ == "__main__":+ main()diff --git a/agentops/llm_tracker.py b/agentops/llm_tracker.pyindex 475c5236..adadd886 100644--- a/agentops/llm_tracker.py+++ b/agentops/llm_tracker.py@@ -12,6 +12,15 @@ from .session import Sessionfrom .event import ActionEvent, ErrorEvent, LLMEventfrom .helpers import check_call_stack_for_agent_id, get_ISO_timefrom .log_config import logger+from .event import LLMEvent, ActionEvent, ToolEvent, ErrorEvent+from .helpers import get_ISO_time, check_call_stack_for_agent_id+import inspect+from typing import Optional+import pprint+from .time_travel import (+ fetch_completion_override_from_time_travel_cache,+ # fetch_prompt_override_from_time_travel_cache,+)original_func = {}original_create = None@@ -246,7 +255,7 @@ class LlmTracker:# v1.0.0+ responses are objectstry:- self.llm_event.returns = response.model_dump()+ self.llm_event.returns = responseself.llm_event.agent_id = check_call_stack_for_agent_id()self.llm_event.prompt = kwargs["messages"]self.llm_event.prompt_tokens = response.usage.prompt_tokens@@ -408,7 +417,7 @@ class LlmTracker:# Not enough to record StreamedChatResponse_ToolCallsGeneration because the tool may have not gotten calledtry:- self.llm_event.returns = response.dict()+ self.llm_event.returns = responseself.llm_event.agent_id = check_call_stack_for_agent_id()self.llm_event.prompt = []if response.chat_history:@@ -614,6 +623,7 @@ class LlmTracker:def override_openai_v1_completion(self):from openai.resources.chat import completions+ from openai.types.chat import ChatCompletion, ChatCompletionChunk# Store the original methodglobal original_create@@ -624,6 +634,37 @@ class LlmTracker:session = kwargs.get("session", None)if "session" in kwargs.keys():del kwargs["session"]++ completion_override = fetch_completion_override_from_time_travel_cache(+ kwargs+ )+ if completion_override:+ result_model = None+ pydantic_models = (ChatCompletion, ChatCompletionChunk)+ for pydantic_model in pydantic_models:+ try:+ result_model = pydantic_model.model_validate_json(+ completion_override+ )+ break+ except Exception as e:+ pass++ if result_model is None:+ logger.error(+ f"Time Travel: Pydantic validation failed for {pydantic_models} \n"+ f"Time Travel: Completion override was:\n"+ f"{pprint.pformat(completion_override)}"+ )+ return None+ return self._handle_response_v1_openai(+ result_model, kwargs, init_timestamp, session=session+ )++ # prompt_override = fetch_prompt_override_from_time_travel_cache(kwargs)+ # if prompt_override:+ # kwargs["messages"] = prompt_override["messages"]+# Call the original function with its original argumentsresult = original_create(*args, **kwargs)return self._handle_response_v1_openai(@@ -635,17 +676,51 @@ class LlmTracker:def override_openai_v1_async_completion(self):from openai.resources.chat import completions+ from openai.types.chat import ChatCompletion, ChatCompletionChunk# Store the original methodglobal original_create_asyncoriginal_create_async = completions.AsyncCompletions.createasync def patched_function(*args, **kwargs):- # Call the original function with its original arguments+init_timestamp = get_ISO_time()+session = kwargs.get("session", None)if "session" in kwargs.keys():del kwargs["session"]++ completion_override = fetch_completion_override_from_time_travel_cache(+ kwargs+ )+ if completion_override:+ result_model = None+ pydantic_models = (ChatCompletion, ChatCompletionChunk)+ for pydantic_model in pydantic_models:+ try:+ result_model = pydantic_model.model_validate_json(+ completion_override+ )+ break+ except Exception as e:+ pass++ if result_model is None:+ logger.error(+ f"Time Travel: Pydantic validation failed for {pydantic_models} \n"+ f"Time Travel: Completion override was:\n"+ f"{pprint.pformat(completion_override)}"+ )+ return None+ return self._handle_response_v1_openai(+ result_model, kwargs, init_timestamp, session=session+ )++ # prompt_override = fetch_prompt_override_from_time_travel_cache(kwargs)+ # if prompt_override:+ # kwargs["messages"] = prompt_override["messages"]++ # Call the original function with its original argumentsresult = await original_create_async(*args, **kwargs)return self._handle_response_v1_openai(result, kwargs, init_timestamp, session=session@@ -656,16 +731,34 @@ class LlmTracker:def override_litellm_completion(self):import litellm+ from openai.types.chat import (+ ChatCompletion,+ ) # Note: litellm calls all LLM APIs using the OpenAI formatoriginal_create = litellm.completiondef patched_function(*args, **kwargs):init_timestamp = get_ISO_time()+session = kwargs.get("session", None)if "session" in kwargs.keys():del kwargs["session"]++ completion_override = fetch_completion_override_from_time_travel_cache(+ kwargs+ )+ if completion_override:+ result_model = ChatCompletion.model_validate_json(completion_override)+ return self._handle_response_v1_openai(+ result_model, kwargs, init_timestamp, session=session+ )++ # prompt_override = fetch_prompt_override_from_time_travel_cache(kwargs)+ # if prompt_override:+ # kwargs["messages"] = prompt_override["messages"]++ # Call the original function with its original argumentsresult = original_create(*args, **kwargs)- # Note: litellm calls all LLM APIs using the OpenAI formatreturn self._handle_response_v1_openai(result, kwargs, init_timestamp, session=session)@@ -674,17 +767,34 @@ class LlmTracker:def override_litellm_async_completion(self):import litellm+ from openai.types.chat import (+ ChatCompletion,+ ) # Note: litellm calls all LLM APIs using the OpenAI format- original_create = litellm.acompletion+ original_create_async = litellm.acompletionasync def patched_function(*args, **kwargs):- # Call the original function with its original argumentsinit_timestamp = get_ISO_time()+session = kwargs.get("session", None)if "session" in kwargs.keys():del kwargs["session"]- result = await original_create(*args, **kwargs)- # Note: litellm calls all LLM APIs using the OpenAI format++ completion_override = fetch_completion_override_from_time_travel_cache(+ kwargs+ )+ if completion_override:+ result_model = ChatCompletion.model_validate_json(completion_override)+ return self._handle_response_v1_openai(+ result_model, kwargs, init_timestamp, session=session+ )++ # prompt_override = fetch_prompt_override_from_time_travel_cache(kwargs)+ # if prompt_override:+ # kwargs["messages"] = prompt_override["messages"]++ # Call the original function with its original arguments+ result = await original_create_async(*args, **kwargs)return self._handle_response_v1_openai(result, kwargs, init_timestamp, session=session)diff --git a/agentops/time_travel.py b/agentops/time_travel.pynew file mode 100644index 00000000..e7638438--- /dev/null+++ b/agentops/time_travel.py@@ -0,0 +1,138 @@+import json+import yaml+from .http_client import HttpClient+from .exceptions import ApiServerException+import os+from .helpers import singleton+from os import environ+++@singleton+class TimeTravel:+ def __init__(self):+ self._completion_overrides_map = {}+ self._prompt_override_map = {}++ script_dir = os.path.dirname(os.path.abspath(__file__))+ parent_dir = os.path.dirname(script_dir)+ cache_path = os.path.join(parent_dir, "agentops_time_travel.json")++ try:+ with open(cache_path, "r") as file:+ time_travel_cache_json = json.load(file)+ self._completion_overrides_map = time_travel_cache_json.get(+ "completion_overrides"+ )+ self._prompt_override_map = time_travel_cache_json.get(+ "prompt_override"+ )+ except FileNotFoundError:+ return+++def fetch_time_travel_id(ttd_id):+ try:+ endpoint = environ.get("AGENTOPS_API_ENDPOINT", "https://api.agentops.ai")+ payload = json.dumps({"ttd_id": ttd_id}).encode("utf-8")+ ttd_res = HttpClient.post(f"{endpoint}/v2/get_ttd", payload)+ if ttd_res.code != 200:+ raise Exception(+ f"Failed to fetch TTD with status code {ttd_res.status_code}"+ )++ prompt_to_returns_map = {+ "completion_overrides": {+ (+ str({"messages": item["prompt"]["messages"]})+ if item["prompt"].get("type") == "chatml"+ else str(item["prompt"])+ ): item["returns"]+ for item in ttd_res.body # TODO: rename returns to completion_override+ }+ }+ with open("agentops_time_travel.json", "w") as file:+ json.dump(prompt_to_returns_map, file, indent=4)++ set_time_travel_active_state(True)+ except ApiServerException as e:+ manage_time_travel_state(activated=False, error=e)+ except Exception as e:+ manage_time_travel_state(activated=False, error=e)+++def fetch_completion_override_from_time_travel_cache(kwargs):+ if not check_time_travel_active():+ return++ if TimeTravel()._completion_overrides_map:+ search_prompt = str({"messages": kwargs["messages"]})+ result_from_cache = TimeTravel()._completion_overrides_map.get(search_prompt)+ return result_from_cache+++def fetch_prompt_override_from_time_travel_cache(kwargs):+ if not check_time_travel_active():+ return++ if TimeTravel()._prompt_override_map:+ search_prompt = str({"messages": kwargs["messages"]})+ result_from_cache = TimeTravel()._prompt_override_map.get(search_prompt)+ return json.loads(result_from_cache)+++def check_time_travel_active():+ script_dir = os.path.dirname(os.path.abspath(__file__))+ parent_dir = os.path.dirname(script_dir)+ config_file_path = os.path.join(parent_dir, ".agentops_time_travel.yaml")++ with open(config_file_path, "r") as config_file:+ config = yaml.safe_load(config_file)+ if config.get("Time_Travel_Debugging_Active", True):+ manage_time_travel_state(activated=True)+ return True++ return False+++def set_time_travel_active_state(is_active: bool):+ config_path = ".agentops_time_travel.yaml"+ try:+ with open(config_path, "r") as config_file:+ config = yaml.safe_load(config_file) or {}+ except FileNotFoundError:+ config = {}++ config["Time_Travel_Debugging_Active"] = is_active++ with open(config_path, "w") as config_file:+ try:+ yaml.dump(config, config_file)+ except:+ print(+ f"🖇 AgentOps: Unable to write to {config_path}. Time Travel not activated"+ )+ return++ if is_active:+ manage_time_travel_state(activated=True)+ print("AgentOps: Time Travel Activated")+ else:+ manage_time_travel_state(activated=False)+ print("🖇 AgentOps: Time Travel Deactivated")+++def add_time_travel_terminal_indicator():+ print(f"🖇️ ⏰ | ", end="")+++def reset_terminal():+ print("\033[0m", end="")+++def manage_time_travel_state(activated=False, error=None):+ if activated:+ add_time_travel_terminal_indicator()+ else:+ reset_terminal()+ if error is not None:+ print(f"🖇 Deactivating Time Travel. Error with configuration: {error}")diff --git a/tests/core_manual_tests/time_travel.py b/tests/core_manual_tests/time_travel.pynew file mode 100644index 00000000..53eaa205--- /dev/null+++ b/tests/core_manual_tests/time_travel.py@@ -0,0 +1,136 @@+from openai import OpenAI, AsyncOpenAI+import openai+from openai.resources.chat import completions+import agentops+from dotenv import load_dotenv++load_dotenv()+client = OpenAI()++agentops.init(tags=["TTD Test", openai.__version__])++try:+ chat_completion_1 = client.chat.completions.create(+ messages=[+ {+ "content": "Come up with a random superpower that isn't time travel. Just return the superpower in the format: 'Superpower: [superpower]'",+ "role": "user",+ }+ ],+ model="gpt-3.5-turbo-0125",+ )+ content1 = chat_completion_1.choices[0].message.content+ print(content1)+ superpower = content1.split("Superpower:")[1].strip()++ chat_completion_2 = client.chat.completions.create(+ messages=[+ {+ "content": "Come up with a superhero name given this superpower: "+ + superpower+ + ". Just return the superhero name in this format: 'Superhero: [superhero name]'",+ "role": "user",+ }+ ],+ model="gpt-3.5-turbo-0125",+ )+ content2 = chat_completion_2.choices[0].message.content+ print(content2)+ superhero = content2.split("Superhero:")[1].strip()++ chat_completion_3 = client.chat.completions.create(+ messages=[+ {+ "content": "Come up with a fictional city for superhero "+ + superhero+ + ". Just return the city name in this format: 'City: [city name]'",+ "role": "user",+ }+ ],+ model="gpt-3.5-turbo-0125",+ )+ content3 = chat_completion_3.choices[0].message.content+ print(content3)+ city = content3.split("City:")[1].strip()++ chat_completion_4 = client.chat.completions.create(+ messages=[+ {+ "content": "Come up with a weakness for superhero "+ + superhero+ + " with superpower "+ + superpower+ + ". Just return the weakness in this format: 'Weakness: [weakness]'",+ "role": "user",+ }+ ],+ model="gpt-3.5-turbo-0125",+ )+ content4 = chat_completion_4.choices[0].message.content+ print(content4)+ weakness = content4.split("Weakness:")[1].strip()++ chat_completion_5 = client.chat.completions.create(+ messages=[+ {+ "content": "Come up with the superpower of superhero "+ + superhero+ + "'s arch nemesis. The superpower cannot be time travel. Just return the superpower in the format: 'Superpower: [superpower]'",+ "role": "user",+ }+ ],+ model="gpt-3.5-turbo-0125",+ )+ content5 = chat_completion_5.choices[0].message.content+ print(content5)+ arch_nemesis_superpower = content5.split("Superpower:")[1].strip()++ chat_completion_6 = client.chat.completions.create(+ messages=[+ {+ "content": "Given the following superpower of a supervillain - "+ + arch_nemesis_superpower+ + " - come up with the supervillain's name. Just return the supervillain's name in this format 'Supervillain: [supervillain name]'",+ "role": "user",+ }+ ],+ model="gpt-3.5-turbo-0125",+ )+ content6 = chat_completion_6.choices[0].message.content+ print(content6)+ supervillain = content6.split("Supervillain:")[1].strip()++ chat_completion_7 = client.chat.completions.create(+ messages=[+ {+ "content": "Write a 100 word superhero story about the feud between superhero "+ + superhero+ + " and his arch nemesis "+ + supervillain+ + " set in "+ + city+ + ". "+ + superhero+ + "'s superpower is "+ + superpower+ + " and his weakness is "+ + weakness+ + "."+ + supervillain+ + "'s superpower is "+ + arch_nemesis_superpower+ + ".",+ "role": "user",+ }+ ],+ model="gpt-3.5-turbo-0125",+ )+ content7 = chat_completion_7.choices[0].message.content+ print(content7)++ agentops.end_session("Success")++# TODO: This just fails more gracefully for the demo but might wanna refactor so split/strip don't except+except (IndexError, AttributeError) as e:+ agentops.logger.warning(f"An error occurred: {e}")+ agentops.end_session("Fail")diff --git a/tests/test_time_travel.py b/tests/test_time_travel.pynew file mode 100644index 00000000..da4c5f8b--- /dev/null+++ b/tests/test_time_travel.py@@ -0,0 +1,37 @@+import unittest+from unittest.mock import patch, mock_open, Mock++from agentops.time_travel import (+ TimeTravel,+ check_time_travel_active,+)+++class TestTimeTravel(unittest.TestCase):++ @patch("os.path.dirname")+ @patch("os.path.abspath")+ @patch(+ "builtins.open",+ new_callable=mock_open,+ read_data='{"completion_overrides": {}, "prompt_override": {}}',+ )+ def test_init(self, mock_open, mock_abspath, mock_dirname):+ mock_abspath.return_value = "/path/to/script"+ mock_dirname.return_value = "/path/to"+ instance = TimeTravel()+ self.assertEqual(instance._completion_overrides_map, {})+ self.assertEqual(instance._prompt_override_map, {})++ @patch("os.path.dirname")+ @patch("os.path.abspath")+ @patch(+ "builtins.open",+ new_callable=mock_open,+ read_data='{"Time_Travel_Debugging_Active": true}',+ )+ def test_check_time_travel_active(self, mock_open, mock_abspath, mock_dirname):+ mock_abspath.return_value = "/path/to/script"+ mock_dirname.return_value = "/path/to"+ result = check_time_travel_active()+ self.assertTrue(result)