Framework for orchestrating role-playing, autonomous AI agents. By fostering collaborative intelligence, CrewAI empowers agents to work together seamlessly, tackling complex tasks.
feat: Add LLM call events for improved observability (#2214)
Lorenze Jayc62fb615b146
diff --git a/src/crewai/llm.py b/src/crewai/llm.pyindex aac1af3b7..a0378cd2d 100644--- a/src/crewai/llm.py+++ b/src/crewai/llm.py@@ -21,6 +21,12 @@ from typing import (from dotenv import load_dotenvfrom pydantic import BaseModel+from crewai.utilities.events.llm_events import (+ LLMCallCompletedEvent,+ LLMCallFailedEvent,+ LLMCallStartedEvent,+ LLMCallType,+)from crewai.utilities.events.tool_usage_events import ToolExecutionErrorEventwith warnings.catch_warnings():@@ -259,6 +265,15 @@ class LLM:>>> print(response)"The capital of France is Paris.""""+ crewai_event_bus.emit(+ self,+ event=LLMCallStartedEvent(+ messages=messages,+ tools=tools,+ callbacks=callbacks,+ available_functions=available_functions,+ ),+ )# Validate parameters before proceeding with the call.self._validate_call_params()@@ -333,12 +348,13 @@ class LLM:# --- 4) If no tool calls, return the text responseif not tool_calls or not available_functions:+ self._handle_emit_call_events(text_response, LLMCallType.LLM_CALL)return text_response# --- 5) Handle the tool calltool_call = tool_calls[0]function_name = tool_call.function.name- print("function_name", function_name)+if function_name in available_functions:try:function_args = json.loads(tool_call.function.arguments)@@ -350,6 +366,7 @@ class LLM:try:# Call the actual tool functionresult = fn(**function_args)+ self._handle_emit_call_events(result, LLMCallType.TOOL_CALL)return resultexcept Exception as e:@@ -365,6 +382,12 @@ class LLM:error=str(e),),)+ crewai_event_bus.emit(+ self,+ event=LLMCallFailedEvent(+ error=f"Tool execution error: {str(e)}"+ ),+ )return text_responseelse:@@ -374,12 +397,28 @@ class LLM:return text_responseexcept Exception as e:+ crewai_event_bus.emit(+ self,+ event=LLMCallFailedEvent(error=str(e)),+ )if not LLMContextLengthExceededException(str(e))._is_context_limit_error(str(e)):logging.error(f"LiteLLM call failed: {str(e)}")raise+ def _handle_emit_call_events(self, response: Any, call_type: LLMCallType):+ """Handle the events for the LLM call.++ Args:+ response (str): The response from the LLM call.+ call_type (str): The type of call, either "tool_call" or "llm_call".+ """+ crewai_event_bus.emit(+ self,+ event=LLMCallCompletedEvent(response=response, call_type=call_type),+ )+def _format_messages_for_provider(self, messages: List[Dict[str, str]]) -> List[Dict[str, str]]:diff --git a/src/crewai/utilities/events/__init__.py b/src/crewai/utilities/events/__init__.pyindex 7f3442360..aa4a24ac5 100644--- a/src/crewai/utilities/events/__init__.py+++ b/src/crewai/utilities/events/__init__.py@@ -34,6 +34,7 @@ from .tool_usage_events import (ToolUsageEvent,ToolValidateInputErrorEvent,)+from .llm_events import LLMCallCompletedEvent, LLMCallFailedEvent, LLMCallStartedEvent# eventsfrom .event_listener import EventListenerdiff --git a/src/crewai/utilities/events/event_listener.py b/src/crewai/utilities/events/event_listener.pyindex 0dcefcd3d..793528240 100644--- a/src/crewai/utilities/events/event_listener.py+++ b/src/crewai/utilities/events/event_listener.py@@ -4,6 +4,11 @@ from crewai.telemetry.telemetry import Telemetryfrom crewai.utilities import Loggerfrom crewai.utilities.constants import EMITTER_COLORfrom crewai.utilities.events.base_event_listener import BaseEventListener+from crewai.utilities.events.llm_events import (+ LLMCallCompletedEvent,+ LLMCallFailedEvent,+ LLMCallStartedEvent,+)from .agent_events import AgentExecutionCompletedEvent, AgentExecutionStartedEventfrom .crew_events import (@@ -253,5 +258,28 @@ class EventListener(BaseEventListener):#)+ # ----------- LLM EVENTS -----------++ @crewai_event_bus.on(LLMCallStartedEvent)+ def on_llm_call_started(source, event: LLMCallStartedEvent):+ self.logger.log(+ f"🤖 LLM Call Started",+ event.timestamp,+ )++ @crewai_event_bus.on(LLMCallCompletedEvent)+ def on_llm_call_completed(source, event: LLMCallCompletedEvent):+ self.logger.log(+ f"✅ LLM Call Completed",+ event.timestamp,+ )++ @crewai_event_bus.on(LLMCallFailedEvent)+ def on_llm_call_failed(source, event: LLMCallFailedEvent):+ self.logger.log(+ f"❌ LLM Call Failed: '{event.error}'",+ event.timestamp,+ )+event_listener = EventListener()diff --git a/src/crewai/utilities/events/llm_events.py b/src/crewai/utilities/events/llm_events.pynew file mode 100644index 000000000..8c2554a21--- /dev/null+++ b/src/crewai/utilities/events/llm_events.py@@ -0,0 +1,36 @@+from enum import Enum+from typing import Any, Dict, List, Optional, Union++from crewai.utilities.events.base_events import CrewEvent+++class LLMCallType(Enum):+ """Type of LLM call being made"""++ TOOL_CALL = "tool_call"+ LLM_CALL = "llm_call"+++class LLMCallStartedEvent(CrewEvent):+ """Event emitted when a LLM call starts"""++ type: str = "llm_call_started"+ messages: Union[str, List[Dict[str, str]]]+ tools: Optional[List[dict]] = None+ callbacks: Optional[List[Any]] = None+ available_functions: Optional[Dict[str, Any]] = None+++class LLMCallCompletedEvent(CrewEvent):+ """Event emitted when a LLM call completes"""++ type: str = "llm_call_completed"+ response: Any+ call_type: LLMCallType+++class LLMCallFailedEvent(CrewEvent):+ """Event emitted when a LLM call fails"""++ error: str+ type: str = "llm_call_failed"diff --git a/src/crewai/utilities/events/task_events.py b/src/crewai/utilities/events/task_events.pyindex f69e77d6a..d81b7ce2d 100644--- a/src/crewai/utilities/events/task_events.py+++ b/src/crewai/utilities/events/task_events.py@@ -1,4 +1,4 @@-from typing import Any, Optional+from typing import Optionalfrom crewai.tasks.task_output import TaskOutputfrom crewai.utilities.events.base_events import CrewEventdiff --git a/tests/utilities/cassettes/test_llm_emits_call_failed_event.yaml b/tests/utilities/cassettes/test_llm_emits_call_failed_event.yamlnew file mode 100644index 000000000..2222ad933--- /dev/null+++ b/tests/utilities/cassettes/test_llm_emits_call_failed_event.yaml@@ -0,0 +1,103 @@+interactions:+- request:+ body: '{"messages": [{"role": "user", "content": "Hello, how are you?"}], "model":+ "gpt-4o-mini", "stop": []}'+ headers:+ accept:+ - application/json+ accept-encoding:+ - gzip, deflate+ connection:+ - keep-alive+ content-length:+ - '102'+ content-type:+ - application/json+ cookie:+ - _cfuvid=IY8ppO70AMHr2skDSUsGh71zqHHdCQCZ3OvkPi26NBc-1740424913267-0.0.1.1-604800000;+ __cf_bm=fU6K5KZoDmgcEuF8_yWAYKUO5fKHh6q5.wDPnna393g-1740424913-1.0.1.1-2iOaq3JVGWs439V0HxJee0IC9HdJm7dPkeJorD.AGw0YwkngRPM8rrTzn_7ht1BkbOauEezj.wPKcBz18gIYUg+ host:+ - api.openai.com+ user-agent:+ - OpenAI/Python 1.61.0+ x-stainless-arch:+ - arm64+ x-stainless-async:+ - 'false'+ x-stainless-lang:+ - python+ x-stainless-os:+ - MacOS+ x-stainless-package-version:+ - 1.61.0+ x-stainless-raw-response:+ - 'true'+ x-stainless-retry-count:+ - '0'+ x-stainless-runtime:+ - CPython+ x-stainless-runtime-version:+ - 3.12.8+ method: POST+ uri: https://api.openai.com/v1/chat/completions+ response:+ content: "{\n \"id\": \"chatcmpl-B4YLA2SrC2rwdVQ3U87G5a0P5lsLw\",\n \"object\":+ \"chat.completion\",\n \"created\": 1740425016,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n+ \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\":+ \"assistant\",\n \"content\": \"Hello! I'm just a computer program, so+ I don't have feelings, but I'm here and ready to help you. How can I assist+ you today?\",\n \"refusal\": null\n },\n \"logprobs\": null,\n+ \ \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\":+ 13,\n \"completion_tokens\": 30,\n \"total_tokens\": 43,\n \"prompt_tokens_details\":+ {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\":+ {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\":+ 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\":+ \"default\",\n \"system_fingerprint\": \"fp_709714d124\"\n}\n"+ headers:+ CF-RAY:+ - 9171d4c0ed44236e-SJC+ Connection:+ - keep-alive+ Content-Encoding:+ - gzip+ Content-Type:+ - application/json+ Date:+ - Mon, 24 Feb 2025 19:23:38 GMT+ Server:+ - cloudflare+ Transfer-Encoding:+ - chunked+ X-Content-Type-Options:+ - nosniff+ access-control-expose-headers:+ - X-Request-ID+ alt-svc:+ - h3=":443"; ma=86400+ cf-cache-status:+ - DYNAMIC+ openai-organization:+ - crewai-iuxna1+ openai-processing-ms:+ - '1954'+ openai-version:+ - '2020-10-01'+ strict-transport-security:+ - max-age=31536000; includeSubDomains; preload+ x-ratelimit-limit-requests:+ - '30000'+ x-ratelimit-limit-tokens:+ - '150000000'+ x-ratelimit-remaining-requests:+ - '29999'+ x-ratelimit-remaining-tokens:+ - '149999978'+ x-ratelimit-reset-requests:+ - 2ms+ x-ratelimit-reset-tokens:+ - 0s+ x-request-id:+ - req_ea2703502b8827e4297cd2a7bae9d9c8+ http_version: HTTP/1.1+ status_code: 200+version: 1diff --git a/tests/utilities/cassettes/test_llm_emits_call_started_event.yaml b/tests/utilities/cassettes/test_llm_emits_call_started_event.yamlnew file mode 100644index 000000000..0120aa1b3--- /dev/null+++ b/tests/utilities/cassettes/test_llm_emits_call_started_event.yaml@@ -0,0 +1,108 @@+interactions:+- request:+ body: '{"messages": [{"role": "user", "content": "Hello, how are you?"}], "model":+ "gpt-4o-mini", "stop": []}'+ headers:+ accept:+ - application/json+ accept-encoding:+ - gzip, deflate+ connection:+ - keep-alive+ content-length:+ - '102'+ content-type:+ - application/json+ cookie:+ - _cfuvid=GefCcEtb_Gem93E4a9Hvt3Xyof1YQZVJAXBb9I6pEUs-1739398417375-0.0.1.1-604800000+ host:+ - api.openai.com+ user-agent:+ - OpenAI/Python 1.61.0+ x-stainless-arch:+ - arm64+ x-stainless-async:+ - 'false'+ x-stainless-lang:+ - python+ x-stainless-os:+ - MacOS+ x-stainless-package-version:+ - 1.61.0+ x-stainless-raw-response:+ - 'true'+ x-stainless-retry-count:+ - '0'+ x-stainless-runtime:+ - CPython+ x-stainless-runtime-version:+ - 3.12.8+ method: POST+ uri: https://api.openai.com/v1/chat/completions+ response:+ content: "{\n \"id\": \"chatcmpl-B4YJU8IWKGyBQtAyPDRd3SFI2flYR\",\n \"object\":+ \"chat.completion\",\n \"created\": 1740424912,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n+ \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\":+ \"assistant\",\n \"content\": \"Hello! I'm just a computer program, so+ I don't have feelings, but I'm here and ready to help you. How can I assist+ you today?\",\n \"refusal\": null\n },\n \"logprobs\": null,\n+ \ \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\":+ 13,\n \"completion_tokens\": 30,\n \"total_tokens\": 43,\n \"prompt_tokens_details\":+ {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\":+ {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\":+ 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\":+ \"default\",\n \"system_fingerprint\": \"fp_7fcd609668\"\n}\n"+ headers:+ CF-RAY:+ - 9171d230d8ed7ae0-SJC+ Connection:+ - keep-alive+ Content-Encoding:+ - gzip+ Content-Type:+ - application/json+ Date:+ - Mon, 24 Feb 2025 19:21:53 GMT+ Server:+ - cloudflare+ Set-Cookie:+ - __cf_bm=fU6K5KZoDmgcEuF8_yWAYKUO5fKHh6q5.wDPnna393g-1740424913-1.0.1.1-2iOaq3JVGWs439V0HxJee0IC9HdJm7dPkeJorD.AGw0YwkngRPM8rrTzn_7ht1BkbOauEezj.wPKcBz18gIYUg;+ path=/; expires=Mon, 24-Feb-25 19:51:53 GMT; domain=.api.openai.com; HttpOnly;+ Secure; SameSite=None+ - _cfuvid=IY8ppO70AMHr2skDSUsGh71zqHHdCQCZ3OvkPi26NBc-1740424913267-0.0.1.1-604800000;+ path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None+ Transfer-Encoding:+ - chunked+ X-Content-Type-Options:+ - nosniff+ access-control-expose-headers:+ - X-Request-ID+ alt-svc:+ - h3=":443"; ma=86400+ cf-cache-status:+ - DYNAMIC+ openai-organization:+ - crewai-iuxna1+ openai-processing-ms:+ - '993'+ openai-version:+ - '2020-10-01'+ strict-transport-security:+ - max-age=31536000; includeSubDomains; preload+ x-ratelimit-limit-requests:+ - '30000'+ x-ratelimit-limit-tokens:+ - '150000000'+ x-ratelimit-remaining-requests:+ - '29999'+ x-ratelimit-remaining-tokens:+ - '149999978'+ x-ratelimit-reset-requests:+ - 2ms+ x-ratelimit-reset-tokens:+ - 0s+ x-request-id:+ - req_d9c4d49185e97b1797061efc1e55d811+ http_version: HTTP/1.1+ status_code: 200+version: 1diff --git a/tests/utilities/test_events.py b/tests/utilities/test_events.pyindex 68bda7bec..aa65d82b2 100644--- a/tests/utilities/test_events.py+++ b/tests/utilities/test_events.py@@ -1,6 +1,5 @@-import jsonfrom datetime import datetime-from unittest.mock import MagicMock, patch+from unittest.mock import patchimport pytestfrom pydantic import Field@@ -9,6 +8,7 @@ from crewai.agent import Agentfrom crewai.agents.crew_agent_executor import CrewAgentExecutorfrom crewai.crew import Crewfrom crewai.flow.flow import Flow, listen, start+from crewai.llm import LLMfrom crewai.task import Taskfrom crewai.tools.base_tool import BaseToolfrom crewai.tools.tool_usage import ToolUsage@@ -31,6 +31,12 @@ from crewai.utilities.events.flow_events import (MethodExecutionFailedEvent,MethodExecutionStartedEvent,)+from crewai.utilities.events.llm_events import (+ LLMCallCompletedEvent,+ LLMCallFailedEvent,+ LLMCallStartedEvent,+ LLMCallType,+)from crewai.utilities.events.task_events import (TaskCompletedEvent,TaskFailedEvent,@@ -495,3 +501,43 @@ def test_flow_emits_method_execution_failed_event():assert received_events[0].flow_name == "TestFlow"assert received_events[0].type == "method_execution_failed"assert received_events[0].error == error++[email protected](filter_headers=["authorization"])+def test_llm_emits_call_started_event():+ received_events = []++ @crewai_event_bus.on(LLMCallStartedEvent)+ def handle_llm_call_started(source, event):+ received_events.append(event)++ @crewai_event_bus.on(LLMCallCompletedEvent)+ def handle_llm_call_completed(source, event):+ received_events.append(event)++ llm = LLM(model="gpt-4o-mini")+ llm.call("Hello, how are you?")++ assert len(received_events) == 2+ assert received_events[0].type == "llm_call_started"+ assert received_events[1].type == "llm_call_completed"++[email protected](filter_headers=["authorization"])+def test_llm_emits_call_failed_event():+ received_events = []++ @crewai_event_bus.on(LLMCallFailedEvent)+ def handle_llm_call_failed(source, event):+ received_events.append(event)++ error_message = "Simulated LLM call failure"+ with patch.object(LLM, "_call_llm", side_effect=Exception(error_message)):+ llm = LLM(model="gpt-4o-mini")+ with pytest.raises(Exception) as exc_info:+ llm.call("Hello, how are you?")++ assert str(exc_info.value) == error_message+ assert len(received_events) == 1+ assert received_events[0].type == "llm_call_failed"+ assert received_events[0].error == error_message