Skip to content

Python SDK for AI agent monitoring, LLM cost tracking, benchmarking, and more. Integrates with most LLMs and agent frameworks including CrewAI, Agno, OpenAI Agents SDK, Langchain, Autogen, AG2, and CamelAI

setup logging config in callback handler (#223)

Braelyn Boynton63f6d9a33f5d
diff --git a/agentops/partners/langchain_callback_handler.py b/agentops/partners/langchain_callback_handler.py
index e2abf3cf..7e789aba 100644
--- a/agentops/partners/langchain_callback_handler.py
+++ b/agentops/partners/langchain_callback_handler.py
@@ -15,15 +15,18 @@ from agentops import ActionEvent, LLMEvent, ToolEvent, ErrorEvent
from agentops.helpers import get_ISO_time
from ..helpers import debug_print_function_params
+import os
+from ..log_config import logger
+import logging
def get_model_from_kwargs(kwargs: any) -> str:
- if 'model' in kwargs['invocation_params']:
- return kwargs['invocation_params']['model']
- elif '_type' in kwargs['invocation_params']:
- return kwargs['invocation_params']['_type']
+ if "model" in kwargs["invocation_params"]:
+ return kwargs["invocation_params"]["model"]
+ elif "_type" in kwargs["invocation_params"]:
+ return kwargs["invocation_params"]["_type"]
else:
- return 'unknown_model'
+ return "unknown_model"
class Events:
@@ -37,22 +40,36 @@ class Events:
class LangchainCallbackHandler(BaseCallbackHandler):
"""Callback handler for Langchain agents."""
- def __init__(self, api_key: Optional[str] = None,
- endpoint: Optional[str] = None,
- max_wait_time: Optional[int] = None,
- max_queue_size: Optional[int] = None,
- tags: Optional[List[str]] = None):
+ def __init__(
+ self,
+ api_key: Optional[str] = None,
+ endpoint: Optional[str] = None,
+ max_wait_time: Optional[int] = None,
+ max_queue_size: Optional[int] = None,
+ tags: Optional[List[str]] = None,
+ ):
+
+ logging_level = os.getenv("AGENTOPS_LOGGING_LEVEL")
+ log_levels = {
+ "CRITICAL": logging.CRITICAL,
+ "ERROR": logging.ERROR,
+ "INFO": logging.INFO,
+ "WARNING": logging.WARNING,
+ "DEBUG": logging.DEBUG,
+ }
+ logger.setLevel(log_levels.get(logging_level or "INFO", "INFO"))
client_params: Dict[str, Any] = {
- 'api_key': api_key,
- 'endpoint': endpoint,
- 'max_wait_time': max_wait_time,
- 'max_queue_size': max_queue_size,
- 'tags': tags
+ "api_key": api_key,
+ "endpoint": endpoint,
+ "max_wait_time": max_wait_time,
+ "max_queue_size": max_queue_size,
+ "tags": tags,
}
- self.ao_client = AOClient(**{k: v for k, v in client_params.items()
- if v is not None}, override=False)
+ self.ao_client = AOClient(
+ **{k: v for k, v in client_params.items() if v is not None}, override=False
+ )
self.agent_actions: Dict[UUID, List[ActionEvent]] = defaultdict(list)
self.events = Events()
@@ -69,22 +86,24 @@ class LangchainCallbackHandler(BaseCallbackHandler):
**kwargs: Any,
) -> Any:
self.events.llm[str(run_id)] = LLMEvent(
- params={**serialized,
- **({} if metadata is None else metadata),
- **kwargs}, # TODO: params is inconsistent, in ToolEvent we put it in logs
+ params={
+ **serialized,
+ **({} if metadata is None else metadata),
+ **kwargs,
+ }, # TODO: params is inconsistent, in ToolEvent we put it in logs
model=get_model_from_kwargs(kwargs),
- prompt=prompts[0]
+ prompt=prompts[0],
# tags=tags # TODO
)
@debug_print_function_params
def on_llm_error(
- self,
- error: BaseException,
- *,
- run_id: UUID,
- parent_run_id: Optional[UUID] = None,
- **kwargs: Any,
+ self,
+ error: BaseException,
+ *,
+ run_id: UUID,
+ parent_run_id: Optional[UUID] = None,
+ **kwargs: Any,
) -> Any:
llm_event: LLMEvent = self.events.llm[str(run_id)]
self.ao_client.record(llm_event)
@@ -104,19 +123,27 @@ class LangchainCallbackHandler(BaseCallbackHandler):
llm_event: LLMEvent = self.events.llm[str(run_id)]
llm_event.returns = {
"content": response.generations[0][0].text,
- "generations": response.generations
+ "generations": response.generations,
}
llm_event.end_timestamp = get_ISO_time()
llm_event.completion = response.generations[0][0].text
if response.llm_output is not None:
- llm_event.prompt_tokens = response.llm_output['token_usage']['prompt_tokens']
- llm_event.completion_tokens = response.llm_output['token_usage']['completion_tokens']
+ llm_event.prompt_tokens = response.llm_output["token_usage"][
+ "prompt_tokens"
+ ]
+ llm_event.completion_tokens = response.llm_output["token_usage"][
+ "completion_tokens"
+ ]
self.ao_client.record(llm_event)
if len(response.generations) == 0:
# TODO: more descriptive error
- error_event = ErrorEvent(trigger_event=self.events.llm[str(run_id)], error_type="NoGenerations", details="on_llm_end: No generations")
+ error_event = ErrorEvent(
+ trigger_event=self.events.llm[str(run_id)],
+ error_type="NoGenerations",
+ details="on_llm_end: No generations",
+ )
self.ao_client.record(error_event)
@debug_print_function_params
@@ -132,11 +159,13 @@ class LangchainCallbackHandler(BaseCallbackHandler):
**kwargs: Any,
) -> Any:
self.events.chain[str(run_id)] = ActionEvent(
- params={**serialized,
- **inputs,
- **({} if metadata is None else metadata),
- **kwargs},
- action_type="chain"
+ params={
+ **serialized,
+ **inputs,
+ **({} if metadata is None else metadata),
+ **kwargs,
+ },
+ action_type="chain",
)
@debug_print_function_params
@@ -183,22 +212,24 @@ class LangchainCallbackHandler(BaseCallbackHandler):
) -> Any:
self.events.tool[str(run_id)] = ToolEvent(
params=input_str if inputs is None else inputs,
- name=serialized['name'],
- logs={**serialized,
- 'tags': tags,
- **({} if metadata is None else metadata),
- **({} if inputs is None else inputs),
- **kwargs}
+ name=serialized["name"],
+ logs={
+ **serialized,
+ "tags": tags,
+ **({} if metadata is None else metadata),
+ **({} if inputs is None else inputs),
+ **kwargs,
+ },
)
@debug_print_function_params
def on_tool_end(
- self,
- output: str,
- *,
- run_id: UUID,
- parent_run_id: Optional[UUID] = None,
- **kwargs: Any,
+ self,
+ output: str,
+ *,
+ run_id: UUID,
+ parent_run_id: Optional[UUID] = None,
+ **kwargs: Any,
) -> Any:
tool_event: ToolEvent = self.events.tool[str(run_id)]
tool_event.end_timestamp = get_ISO_time()
@@ -207,18 +238,22 @@ class LangchainCallbackHandler(BaseCallbackHandler):
# Tools are capable of failing `on_tool_end` quietly.
# This is a workaround to make sure we can log it as an error.
- if kwargs.get('name') == '_Exception':
- error_event = ErrorEvent(trigger_event=tool_event, error_type="LangchainToolException", details=output)
+ if kwargs.get("name") == "_Exception":
+ error_event = ErrorEvent(
+ trigger_event=tool_event,
+ error_type="LangchainToolException",
+ details=output,
+ )
self.ao_client.record(error_event)
@debug_print_function_params
def on_tool_error(
- self,
- error: BaseException,
- *,
- run_id: UUID,
- parent_run_id: Optional[UUID] = None,
- **kwargs: Any,
+ self,
+ error: BaseException,
+ *,
+ run_id: UUID,
+ parent_run_id: Optional[UUID] = None,
+ **kwargs: Any,
) -> Any:
tool_event: ToolEvent = self.events.tool[str(run_id)]
self.ao_client.record(tool_event)
@@ -228,48 +263,52 @@ class LangchainCallbackHandler(BaseCallbackHandler):
@debug_print_function_params
def on_retriever_start(
- self,
- serialized: Dict[str, Any],
- query: str,
- *,
- run_id: UUID,
- parent_run_id: Optional[UUID] = None,
- tags: Optional[List[str]] = None,
- metadata: Optional[Dict[str, Any]] = None,
- **kwargs: Any,
+ self,
+ serialized: Dict[str, Any],
+ query: str,
+ *,
+ run_id: UUID,
+ parent_run_id: Optional[UUID] = None,
+ tags: Optional[List[str]] = None,
+ metadata: Optional[Dict[str, Any]] = None,
+ **kwargs: Any,
) -> None:
self.events.retriever[str(run_id)] = ActionEvent(
- params={**serialized,
- "query": query,
- **({} if metadata is None else metadata),
- ** kwargs},
- action_type="retriever"
+ params={
+ **serialized,
+ "query": query,
+ **({} if metadata is None else metadata),
+ **kwargs,
+ },
+ action_type="retriever",
)
@debug_print_function_params
def on_retriever_end(
- self,
- documents: Sequence[Document],
- *,
- run_id: UUID,
- parent_run_id: Optional[UUID] = None,
- tags: Optional[List[str]] = None,
- **kwargs: Any,
+ self,
+ documents: Sequence[Document],
+ *,
+ run_id: UUID,
+ parent_run_id: Optional[UUID] = None,
+ tags: Optional[List[str]] = None,
+ **kwargs: Any,
) -> None:
action_event: ActionEvent = self.events.retriever[str(run_id)]
- action_event.logs = documents # TODO: Adding this. Might want to add elsewhere e.g. params
+ action_event.logs = (
+ documents # TODO: Adding this. Might want to add elsewhere e.g. params
+ )
action_event.end_timestamp = get_ISO_time()
self.ao_client.record(action_event)
@debug_print_function_params
def on_retriever_error(
- self,
- error: BaseException,
- *,
- run_id: UUID,
- parent_run_id: Optional[UUID] = None,
- tags: Optional[List[str]] = None,
- **kwargs: Any,
+ self,
+ error: BaseException,
+ *,
+ run_id: UUID,
+ parent_run_id: Optional[UUID] = None,
+ tags: Optional[List[str]] = None,
+ **kwargs: Any,
) -> None:
action_event: ActionEvent = self.events.retriever[str(run_id)]
self.ao_client.record(action_event)
@@ -286,20 +325,18 @@ class LangchainCallbackHandler(BaseCallbackHandler):
parent_run_id: Optional[UUID] = None,
**kwargs: Any,
) -> Any:
- self.agent_actions[run_id].append(ActionEvent(
- params={"action": action,
- **kwargs},
- action_type="agent"
- ))
+ self.agent_actions[run_id].append(
+ ActionEvent(params={"action": action, **kwargs}, action_type="agent")
+ )
@debug_print_function_params
def on_agent_finish(
- self,
- finish: AgentFinish,
- *,
- run_id: UUID,
- parent_run_id: Optional[UUID] = None,
- **kwargs: Any,
+ self,
+ finish: AgentFinish,
+ *,
+ run_id: UUID,
+ parent_run_id: Optional[UUID] = None,
+ **kwargs: Any,
) -> Any:
# Need to attach finish to some on_agent_action so just choosing the last one
self.agent_actions[run_id][-1].returns = finish.to_json()
@@ -335,22 +372,26 @@ class LangchainCallbackHandler(BaseCallbackHandler):
class AsyncLangchainCallbackHandler(AsyncCallbackHandler):
"""Callback handler for Langchain agents."""
- def __init__(self, api_key: Optional[str] = None,
- endpoint: Optional[str] = None,
- max_wait_time: Optional[int] = None,
- max_queue_size: Optional[int] = None,
- tags: Optional[List[str]] = None):
+ def __init__(
+ self,
+ api_key: Optional[str] = None,
+ endpoint: Optional[str] = None,
+ max_wait_time: Optional[int] = None,
+ max_queue_size: Optional[int] = None,
+ tags: Optional[List[str]] = None,
+ ):
client_params: Dict[str, Any] = {
- 'api_key': api_key,
- 'endpoint': endpoint,
- 'max_wait_time': max_wait_time,
- 'max_queue_size': max_queue_size,
- 'tags': tags
+ "api_key": api_key,
+ "endpoint": endpoint,
+ "max_wait_time": max_wait_time,
+ "max_queue_size": max_queue_size,
+ "tags": tags,
}
- self.ao_client = AOClient(**{k: v for k, v in client_params.items()
- if v is not None}, override=False)
+ self.ao_client = AOClient(
+ **{k: v for k, v in client_params.items() if v is not None}, override=False
+ )
self.events = Events()
self.agent_actions: Dict[UUID, List[ActionEvent]] = defaultdict(list)
@@ -368,11 +409,13 @@ class AsyncLangchainCallbackHandler(AsyncCallbackHandler):
**kwargs: Any,
) -> Any:
self.events.llm[str(run_id)] = LLMEvent(
- params={**serialized,
- **({} if metadata is None else metadata),
- **kwargs}, # TODO: params is inconsistent, in ToolEvent we put it in logs
- model=kwargs['invocation_params']['model'],
- prompt=prompts[0]
+ params={
+ **serialized,
+ **({} if metadata is None else metadata),
+ **kwargs,
+ }, # TODO: params is inconsistent, in ToolEvent we put it in logs
+ model=kwargs["invocation_params"]["model"],
+ prompt=prompts[0],
)
@debug_print_function_params
@@ -404,12 +447,12 @@ class AsyncLangchainCallbackHandler(AsyncCallbackHandler):
@debug_print_function_params
async def on_llm_error(
- self,
- error: BaseException,
- *,
- run_id: UUID,
- parent_run_id: Optional[UUID] = None,
- **kwargs: Any,
+ self,
+ error: BaseException,
+ *,
+ run_id: UUID,
+ parent_run_id: Optional[UUID] = None,
+ **kwargs: Any,
) -> Any:
llm_event: LLMEvent = self.events.llm[str(run_id)]
self.ao_client.record(llm_event)
@@ -429,18 +472,26 @@ class AsyncLangchainCallbackHandler(AsyncCallbackHandler):
llm_event: LLMEvent = self.events.llm[str(run_id)]
llm_event.returns = {
"content": response.generations[0][0].text,
- "generations": response.generations
+ "generations": response.generations,
}
llm_event.end_timestamp = get_ISO_time()
llm_event.completion = response.generations[0][0].text
if response.llm_output is not None:
- llm_event.prompt_tokens = response.llm_output['token_usage']['prompt_tokens']
- llm_event.completion_tokens = response.llm_output['token_usage']['completion_tokens']
+ llm_event.prompt_tokens = response.llm_output["token_usage"][
+ "prompt_tokens"
+ ]
+ llm_event.completion_tokens = response.llm_output["token_usage"][
+ "completion_tokens"
+ ]
self.ao_client.record(llm_event)
if len(response.generations) == 0:
# TODO: more descriptive error
- error_event = ErrorEvent(trigger_event=self.events.llm[str(run_id)], error_type="NoGenerations", details="on_llm_end: No generations")
+ error_event = ErrorEvent(
+ trigger_event=self.events.llm[str(run_id)],
+ error_type="NoGenerations",
+ details="on_llm_end: No generations",
+ )
self.ao_client.record(error_event)
@debug_print_function_params
@@ -456,11 +507,13 @@ class AsyncLangchainCallbackHandler(AsyncCallbackHandler):
**kwargs: Any,
) -> Any:
self.events.chain[str(run_id)] = ActionEvent(
- params={**serialized,
- **inputs,
- **({} if metadata is None else metadata),
- **kwargs},
- action_type="chain"
+ params={
+ **serialized,
+ **inputs,
+ **({} if metadata is None else metadata),
+ **kwargs,
+ },
+ action_type="chain",
)
@debug_print_function_params
@@ -507,22 +560,24 @@ class AsyncLangchainCallbackHandler(AsyncCallbackHandler):
) -> Any:
self.events.tool[str(run_id)] = ToolEvent(
params=input_str if inputs is None else inputs,
- name=serialized['name'],
- logs={**serialized,
- 'tags': tags,
- **({} if metadata is None else metadata),
- **({} if inputs is None else inputs),
- **kwargs}
+ name=serialized["name"],
+ logs={
+ **serialized,
+ "tags": tags,
+ **({} if metadata is None else metadata),
+ **({} if inputs is None else inputs),
+ **kwargs,
+ },
)
@debug_print_function_params
async def on_tool_end(
- self,
- output: str,
- *,
- run_id: UUID,
- parent_run_id: Optional[UUID] = None,
- **kwargs: Any,
+ self,
+ output: str,
+ *,
+ run_id: UUID,
+ parent_run_id: Optional[UUID] = None,
+ **kwargs: Any,
) -> Any:
tool_event: ToolEvent = self.events.tool[str(run_id)]
tool_event.end_timestamp = get_ISO_time()
@@ -531,18 +586,22 @@ class AsyncLangchainCallbackHandler(AsyncCallbackHandler):
# Tools are capable of failing `on_tool_end` quietly.
# This is a workaround to make sure we can log it as an error.
- if kwargs.get('name') == '_Exception':
- error_event = ErrorEvent(trigger_event=tool_event, error_type="LangchainToolException", details=output)
+ if kwargs.get("name") == "_Exception":
+ error_event = ErrorEvent(
+ trigger_event=tool_event,
+ error_type="LangchainToolException",
+ details=output,
+ )
self.ao_client.record(error_event)
@debug_print_function_params
async def on_tool_error(
- self,
- error: BaseException,
- *,
- run_id: UUID,
- parent_run_id: Optional[UUID] = None,
- **kwargs: Any,
+ self,
+ error: BaseException,
+ *,
+ run_id: UUID,
+ parent_run_id: Optional[UUID] = None,
+ **kwargs: Any,
) -> Any:
tool_event: ToolEvent = self.events.tool[str(run_id)]
self.ao_client.record(tool_event)
@@ -552,48 +611,52 @@ class AsyncLangchainCallbackHandler(AsyncCallbackHandler):
@debug_print_function_params
async def on_retriever_start(
- self,
- serialized: Dict[str, Any],
- query: str,
- *,
- run_id: UUID,
- parent_run_id: Optional[UUID] = None,
- tags: Optional[List[str]] = None,
- metadata: Optional[Dict[str, Any]] = None,
- **kwargs: Any,
+ self,
+ serialized: Dict[str, Any],
+ query: str,
+ *,
+ run_id: UUID,
+ parent_run_id: Optional[UUID] = None,
+ tags: Optional[List[str]] = None,
+ metadata: Optional[Dict[str, Any]] = None,
+ **kwargs: Any,
) -> None:
self.events.retriever[str(run_id)] = ActionEvent(
- params={**serialized,
- "query": query,
- **({} if metadata is None else metadata),
- ** kwargs},
- action_type="retriever"
+ params={
+ **serialized,
+ "query": query,
+ **({} if metadata is None else metadata),
+ **kwargs,
+ },
+ action_type="retriever",
)
@debug_print_function_params
async def on_retriever_end(
- self,
- documents: Sequence[Document],
- *,
- run_id: UUID,
- parent_run_id: Optional[UUID] = None,
- tags: Optional[List[str]] = None,
- **kwargs: Any,
+ self,
+ documents: Sequence[Document],
+ *,
+ run_id: UUID,
+ parent_run_id: Optional[UUID] = None,
+ tags: Optional[List[str]] = None,
+ **kwargs: Any,
) -> None:
action_event: ActionEvent = self.events.retriever[str(run_id)]
- action_event.logs = documents # TODO: Adding this. Might want to add elsewhere e.g. params
+ action_event.logs = (
+ documents # TODO: Adding this. Might want to add elsewhere e.g. params
+ )
action_event.end_timestamp = get_ISO_time()
self.ao_client.record(action_event)
@debug_print_function_params
async def on_retriever_error(
- self,
- error: BaseException,
- *,
- run_id: UUID,
- parent_run_id: Optional[UUID] = None,
- tags: Optional[List[str]] = None,
- **kwargs: Any,
+ self,
+ error: BaseException,
+ *,
+ run_id: UUID,
+ parent_run_id: Optional[UUID] = None,
+ tags: Optional[List[str]] = None,
+ **kwargs: Any,
) -> None:
action_event: ActionEvent = self.events.retriever[str(run_id)]
self.ao_client.record(action_event)
@@ -610,20 +673,18 @@ class AsyncLangchainCallbackHandler(AsyncCallbackHandler):
parent_run_id: Optional[UUID] = None,
**kwargs: Any,
) -> Any:
- self.agent_actions[run_id].append(ActionEvent(
- params={"action": action,
- **kwargs},
- action_type="agent"
- ))
+ self.agent_actions[run_id].append(
+ ActionEvent(params={"action": action, **kwargs}, action_type="agent")
+ )
@debug_print_function_params
async def on_agent_finish(
- self,
- finish: AgentFinish,
- *,
- run_id: UUID,
- parent_run_id: Optional[UUID] = None,
- **kwargs: Any,
+ self,
+ finish: AgentFinish,
+ *,
+ run_id: UUID,
+ parent_run_id: Optional[UUID] = None,
+ **kwargs: Any,
) -> Any:
# Need to attach finish to some on_agent_action so just choosing the last one
self.agent_actions[run_id][-1].returns = finish.to_json()