Framework for orchestrating role-playing, autonomous AI agents. By fostering collaborative intelligence, CrewAI empowers agents to work together seamlessly, tackling complex tasks.
Cached prompt tokens on usage metrics
Thiago Moretto36aa4bcb4615
diff --git a/src/crewai/agents/agent_builder/utilities/base_token_process.py b/src/crewai/agents/agent_builder/utilities/base_token_process.pyindex e971d018e..91df8140e 100644--- a/src/crewai/agents/agent_builder/utilities/base_token_process.py+++ b/src/crewai/agents/agent_builder/utilities/base_token_process.py@@ -4,6 +4,7 @@ from crewai.types.usage_metrics import UsageMetricsclass TokenProcess:total_tokens: int = 0prompt_tokens: int = 0+ cached_prompt_tokens: int = 0completion_tokens: int = 0successful_requests: int = 0@@ -15,6 +16,10 @@ class TokenProcess:self.completion_tokens = self.completion_tokens + tokensself.total_tokens = self.total_tokens + tokens+ def sum_cached_prompt_tokens(self, tokens: int):+ self.cached_prompt_tokens = self.cached_prompt_tokens + tokens+ self.total_tokens = self.total_tokens + tokens+def sum_successful_requests(self, requests: int):self.successful_requests = self.successful_requests + requests@@ -22,6 +27,7 @@ class TokenProcess:return UsageMetrics(total_tokens=self.total_tokens,prompt_tokens=self.prompt_tokens,+ cached_prompt_tokens=self.cached_prompt_tokens,completion_tokens=self.completion_tokens,successful_requests=self.successful_requests,)diff --git a/src/crewai/types/usage_metrics.py b/src/crewai/types/usage_metrics.pyindex a5cee6a0f..e87a79e33 100644--- a/src/crewai/types/usage_metrics.py+++ b/src/crewai/types/usage_metrics.py@@ -8,6 +8,7 @@ class UsageMetrics(BaseModel):Attributes:total_tokens: Total number of tokens used.prompt_tokens: Number of tokens used in prompts.+ cached_prompt_tokens: Number of cached prompt tokens used.completion_tokens: Number of tokens used in completions.successful_requests: Number of successful requests made."""@@ -16,6 +17,9 @@ class UsageMetrics(BaseModel):prompt_tokens: int = Field(default=0, description="Number of tokens used in prompts.")+ cached_prompt_tokens: int = Field(+ default=0, description="Number of cached prompt tokens used."+ )completion_tokens: int = Field(default=0, description="Number of tokens used in completions.")@@ -32,5 +36,6 @@ class UsageMetrics(BaseModel):"""self.total_tokens += usage_metrics.total_tokensself.prompt_tokens += usage_metrics.prompt_tokens+ self.cached_prompt_tokens += usage_metrics.cached_prompt_tokensself.completion_tokens += usage_metrics.completion_tokensself.successful_requests += usage_metrics.successful_requestsdiff --git a/src/crewai/utilities/token_counter_callback.py b/src/crewai/utilities/token_counter_callback.pyindex 1b6215232..feaaa4c22 100644--- a/src/crewai/utilities/token_counter_callback.py+++ b/src/crewai/utilities/token_counter_callback.py@@ -1,5 +1,5 @@from litellm.integrations.custom_logger import CustomLogger-+from litellm.types.utils import Usagefrom crewai.agents.agent_builder.utilities.base_token_process import TokenProcess@@ -11,8 +11,11 @@ class TokenCalcHandler(CustomLogger):if self.token_cost_process is None:return+ usage : Usage = response_obj["usage"]self.token_cost_process.sum_successful_requests(1)- self.token_cost_process.sum_prompt_tokens(response_obj["usage"].prompt_tokens)- self.token_cost_process.sum_completion_tokens(- response_obj["usage"].completion_tokens- )+ self.token_cost_process.sum_prompt_tokens(usage.prompt_tokens)+ self.token_cost_process.sum_completion_tokens(usage.completion_tokens)+ if usage.prompt_tokens_details:+ self.token_cost_process.sum_cached_prompt_tokens(+ usage.prompt_tokens_details.cached_tokens+ )diff --git a/tests/cassettes/test_crew_kickoff_usage_metrics.yaml b/tests/cassettes/test_crew_kickoff_usage_metrics.yamlindex cc0863ee4..1851d0901 100644--- a/tests/cassettes/test_crew_kickoff_usage_metrics.yaml+++ b/tests/cassettes/test_crew_kickoff_usage_metrics.yaml@@ -10,7 +10,8 @@ interactions:criteria for your final answer: 1 bullet point about dog that''s under 15 words.\nyouMUST return the actual complete content as the final answer, not a summary.\n\nBegin!This is VERY important to you, use the tools available and give your best Final- Answer, your job depends on it!\n\nThought:"}], "model": "gpt-4o"}'+ Answer, your job depends on it!\n\nThought:"}], "model": "gpt-4o-mini", "stop":+ ["\nObservation:"], "stream": false}'headers:accept:- application/json@@ -19,49 +20,50 @@ interactions:connection:- keep-alivecontent-length:- - '869'+ - '919'content-type:- application/json- cookie:- - __cf_bm=9.8sBYBkvBR8R1K_bVF7xgU..80XKlEIg3N2OBbTSCU-1727214102-1.0.1.1-.qiTLXbPamYUMSuyNsOEB9jhGu.jOifujOrx9E2JZvStbIZ9RTIiE44xKKNfLPxQkOi6qAT3h6htK8lPDGV_5g;- _cfuvid=lbRdAddVWV6W3f5Dm9SaOPWDUOxqtZBSPr_fTW26nEA-1727213194587-0.0.1.1-604800000host:- api.openai.comuser-agent:- - OpenAI/Python 1.47.0+ - OpenAI/Python 1.52.1x-stainless-arch:- - arm64+ - x64x-stainless-async:- 'false'x-stainless-lang:- pythonx-stainless-os:- - MacOS+ - Linuxx-stainless-package-version:- - 1.47.0+ - 1.52.1x-stainless-raw-response:- 'true'+ x-stainless-retry-count:+ - '0'x-stainless-runtime:- CPythonx-stainless-runtime-version:- - 3.11.7+ - 3.11.9method: POSTuri: https://api.openai.com/v1/chat/completionsresponse:- content: "{\n \"id\": \"chatcmpl-AB7auGDrAVE0iXSBBhySZp3xE8gvP\",\n \"object\":- \"chat.completion\",\n \"created\": 1727214164,\n \"model\": \"gpt-4o-2024-05-13\",\n- \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\":- \"assistant\",\n \"content\": \"I now can give a great answer\\nFinal- Answer: Dogs are unparalleled in loyalty and companionship to humans.\",\n \"refusal\":- null\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n- \ }\n ],\n \"usage\": {\n \"prompt_tokens\": 175,\n \"completion_tokens\":- 21,\n \"total_tokens\": 196,\n \"completion_tokens_details\": {\n \"reasoning_tokens\":- 0\n }\n },\n \"system_fingerprint\": \"fp_e375328146\"\n}\n"+ body:+ string: !!binary |+ H4sIAAAAAAAAA4xSy27bMBC86ysWPEuB7ciV7VuAIkUObQ+59QFhTa0kttQuS9Jx08D/XkhyLAVJ+ gV4EaGdnMLPDpwRAmUrtQOkWo+6czW7u41q2t3+cvCvuPvxafSG+58XHTzXlxWeV9gzZ/yAdn1lX+ WjpnKRrhEdaeMFKvuiyul3m+uV7lA9BJRbanNS5muWSdYZOtFqs8WxTZcnNmt2I0BbWDrwkAwNPw+ 7X1yRb/VDhbp86SjELAhtbssASgvtp8oDMGEiBxVOoFaOBIP1u+A5QgaGRrzQIDQ9LYBORzJA3zj+ W8No4Wb438F7aQKgJ7DyiBb6zMhGOKRA3CJrww10xBEttIQ2toBcgTyQR2vhSNZmezLcXM39eKoP+ Afub8MHa8/x0CWilcV724Yxf5rVhE9rSEwbhPkyI4tSAnhKA78MhDy9uo5yXzsUyyk/iMHSzHvXU+ 1N+Ejo0BqCgR7Yy13aZv6JUVRTQ2zKpQGnVL1USdesNDZWQGJLPUr928pT0mN9z8j/wEaE0uUlU6+ T5XRLxNPa5765/2vtcuVB8MqPIZIXVkbbsg7b8bHVbuyXm9xs8xXRa2SU/IXAAD//wMAq2ZCBWoD+ AAA=headers:CF-Cache-Status:- DYNAMICCF-RAY:- - 8c85f22ddda01cf3-GRU+ - 8e19bf36db158761-GRUConnection:- keep-aliveContent-Encoding:@@ -69,19 +71,27 @@ interactions:Content-Type:- application/jsonDate:- - Tue, 24 Sep 2024 21:42:44 GMT+ - Tue, 12 Nov 2024 21:52:04 GMTServer:- cloudflare+ Set-Cookie:+ - __cf_bm=MkvcnvacGpTyn.y0OkFRoFXuAwg4oxjMhViZJTt9mw0-1731448324-1.0.1.1-oekkH_B0xOoPnIFw15LpqFCkZ2cu7VBTJVLDGylan4I67NjX.tlPvOiX9kvtP5Acewi28IE2IwlwtrZWzCH3vw;+ path=/; expires=Tue, 12-Nov-24 22:22:04 GMT; domain=.api.openai.com; HttpOnly;+ Secure; SameSite=None+ - _cfuvid=4.17346mfw5npZfYNbCx3Vj1VAVPy.tH0Jm2gkTteJ8-1731448324998-0.0.1.1-604800000;+ path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=NoneTransfer-Encoding:- chunkedX-Content-Type-Options:- nosniffaccess-control-expose-headers:- X-Request-ID+ alt-svc:+ - h3=":443"; ma=86400openai-organization:- - crewai-iuxna1+ - user-tqfegqsiobpvvjmn0giaipdqopenai-processing-ms:- - '349'+ - '601'openai-version:- '2020-10-01'strict-transport-security:@@ -89,19 +99,20 @@ interactions:x-ratelimit-limit-requests:- '10000'x-ratelimit-limit-tokens:- - '30000000'+ - '200000'x-ratelimit-remaining-requests:- '9999'x-ratelimit-remaining-tokens:- - '29999792'+ - '199793'x-ratelimit-reset-requests:- - 6ms+ - 8.64sx-ratelimit-reset-tokens:- - 0s+ - 62msx-request-id:- - req_4c8cd76fdfba7b65e5ce85397b33c22b- http_version: HTTP/1.1- status_code: 200+ - req_77fb166b4e272bfd45c37c08d2b93b0c+ status:+ code: 200+ message: OK- request:body: '{"messages": [{"role": "system", "content": "You are cat Researcher. Youhave a lot of experience with cat.\nYour personal goal is: Express hot takes@@ -113,7 +124,8 @@ interactions:criteria for your final answer: 1 bullet point about cat that''s under 15 words.\nyouMUST return the actual complete content as the final answer, not a summary.\n\nBegin!This is VERY important to you, use the tools available and give your best Final- Answer, your job depends on it!\n\nThought:"}], "model": "gpt-4o"}'+ Answer, your job depends on it!\n\nThought:"}], "model": "gpt-4o-mini", "stop":+ ["\nObservation:"], "stream": false}'headers:accept:- application/json@@ -122,49 +134,53 @@ interactions:connection:- keep-alivecontent-length:- - '869'+ - '919'content-type:- application/jsoncookie:- - __cf_bm=9.8sBYBkvBR8R1K_bVF7xgU..80XKlEIg3N2OBbTSCU-1727214102-1.0.1.1-.qiTLXbPamYUMSuyNsOEB9jhGu.jOifujOrx9E2JZvStbIZ9RTIiE44xKKNfLPxQkOi6qAT3h6htK8lPDGV_5g;- _cfuvid=lbRdAddVWV6W3f5Dm9SaOPWDUOxqtZBSPr_fTW26nEA-1727213194587-0.0.1.1-604800000+ - __cf_bm=MkvcnvacGpTyn.y0OkFRoFXuAwg4oxjMhViZJTt9mw0-1731448324-1.0.1.1-oekkH_B0xOoPnIFw15LpqFCkZ2cu7VBTJVLDGylan4I67NjX.tlPvOiX9kvtP5Acewi28IE2IwlwtrZWzCH3vw;+ _cfuvid=4.17346mfw5npZfYNbCx3Vj1VAVPy.tH0Jm2gkTteJ8-1731448324998-0.0.1.1-604800000host:- api.openai.comuser-agent:- - OpenAI/Python 1.47.0+ - OpenAI/Python 1.52.1x-stainless-arch:- - arm64+ - x64x-stainless-async:- 'false'x-stainless-lang:- pythonx-stainless-os:- - MacOS+ - Linuxx-stainless-package-version:- - 1.47.0+ - 1.52.1x-stainless-raw-response:- 'true'+ x-stainless-retry-count:+ - '0'x-stainless-runtime:- CPythonx-stainless-runtime-version:- - 3.11.7+ - 3.11.9method: POSTuri: https://api.openai.com/v1/chat/completionsresponse:- content: "{\n \"id\": \"chatcmpl-AB7auNbAqjT3rgBX92rhxBLuhaLBj\",\n \"object\":- \"chat.completion\",\n \"created\": 1727214164,\n \"model\": \"gpt-4o-2024-05-13\",\n- \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\":- \"assistant\",\n \"content\": \"Thought: I now can give a great answer\\nFinal- Answer: Cats are highly independent, agile, and intuitive creatures beloved- by millions worldwide.\",\n \"refusal\": null\n },\n \"logprobs\":- null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\":- 175,\n \"completion_tokens\": 28,\n \"total_tokens\": 203,\n \"completion_tokens_details\":- {\n \"reasoning_tokens\": 0\n }\n },\n \"system_fingerprint\": \"fp_e375328146\"\n}\n"+ body:+ string: !!binary |+ H4sIAAAAAAAAA4xSy27bMBC86ysWPFuB7MhN6ltQIGmBnlL00BcEmlxJ21JLhlzFLQL/eyH5IRlt+ gV4EaGZnMLPLlwxAkVUbUKbVYrrg8rsPsg4P+Orxs9XvPz0U8eP966dS6sdo3wa1GBR++x2NnFRX+ xnfBoZDnA20iasHBdXlzvSzL2+vVeiQ6b9ENsiZIXvq8I6Z8VazKvLjJl7dHdevJYFIb+JIBALyM+ 3yEnW/ypNlAsTkiHKekG1eY8BKCidwOidEqURLOoxUQaz4I8Rn8H7HdgNENDzwgamiE2aE47jABf+ +Z5YO7gb/zfwRksCHRGGGAHZIg/D1GmXFiBtpGfiBjyDtEgR/I5BMHYJNFvomZ56hIAxedaOhDBd+ zYNFrPukh+Vw79wR35+bOt+E6LfpyJ/xmphSW0XUyfPQKokPamT3GcC3caP9xZJUiL4LUon/gZzG+ I60Pfmo65MSuTqR40W6GF8c7XPpVFkWTS7ObKKNNi3aSTgfUvSU/I7JZ6z/T/M370Jy4+R/7iTAG+ g6CtQkRL5rLxNBZxeOf/GjtveQys0q8k2FU1cYMxRDq8sjpUxVYXdrkq66XK9tlvAAAA//8DAIjK+ KzJzAwAAheaders:CF-Cache-Status:- DYNAMICCF-RAY:- - 8c85f2321c1c1cf3-GRU+ - 8e19bf3fae118761-GRUConnection:- keep-aliveContent-Encoding:@@ -172,7 +188,7 @@ interactions:Content-Type:- application/jsonDate:- - Tue, 24 Sep 2024 21:42:45 GMT+ - Tue, 12 Nov 2024 21:52:05 GMTServer:- cloudflareTransfer-Encoding:@@ -181,10 +197,12 @@ interactions:- nosniffaccess-control-expose-headers:- X-Request-ID+ alt-svc:+ - h3=":443"; ma=86400openai-organization:- - crewai-iuxna1+ - user-tqfegqsiobpvvjmn0giaipdqopenai-processing-ms:- - '430'+ - '464'openai-version:- '2020-10-01'strict-transport-security:@@ -192,19 +210,20 @@ interactions:x-ratelimit-limit-requests:- '10000'x-ratelimit-limit-tokens:- - '30000000'+ - '200000'x-ratelimit-remaining-requests:- - '9999'+ - '9998'x-ratelimit-remaining-tokens:- - '29999792'+ - '199792'x-ratelimit-reset-requests:- - 6ms+ - 16.369sx-ratelimit-reset-tokens:- - 0s+ - 62msx-request-id:- - req_ace859b7d9e83d9fa7753ce23bb03716- http_version: HTTP/1.1- status_code: 200+ - req_91706b23d0ef23458ba63ec18304cd28+ status:+ code: 200+ message: OK- request:body: '{"messages": [{"role": "system", "content": "You are apple Researcher.You have a lot of experience with apple.\nYour personal goal is: Express hot@@ -217,7 +236,7 @@ interactions:under 15 words.\nyou MUST return the actual complete content as the final answer,not a summary.\n\nBegin! This is VERY important to you, use the tools availableand give your best Final Answer, your job depends on it!\n\nThought:"}], "model":- "gpt-4o"}'+ "gpt-4o-mini", "stop": ["\nObservation:"], "stream": false}'headers:accept:- application/json@@ -226,49 +245,53 @@ interactions:connection:- keep-alivecontent-length:- - '879'+ - '929'content-type:- application/jsoncookie:- - __cf_bm=9.8sBYBkvBR8R1K_bVF7xgU..80XKlEIg3N2OBbTSCU-1727214102-1.0.1.1-.qiTLXbPamYUMSuyNsOEB9jhGu.jOifujOrx9E2JZvStbIZ9RTIiE44xKKNfLPxQkOi6qAT3h6htK8lPDGV_5g;- _cfuvid=lbRdAddVWV6W3f5Dm9SaOPWDUOxqtZBSPr_fTW26nEA-1727213194587-0.0.1.1-604800000+ - __cf_bm=MkvcnvacGpTyn.y0OkFRoFXuAwg4oxjMhViZJTt9mw0-1731448324-1.0.1.1-oekkH_B0xOoPnIFw15LpqFCkZ2cu7VBTJVLDGylan4I67NjX.tlPvOiX9kvtP5Acewi28IE2IwlwtrZWzCH3vw;+ _cfuvid=4.17346mfw5npZfYNbCx3Vj1VAVPy.tH0Jm2gkTteJ8-1731448324998-0.0.1.1-604800000host:- api.openai.comuser-agent:- - OpenAI/Python 1.47.0+ - OpenAI/Python 1.52.1x-stainless-arch:- - arm64+ - x64x-stainless-async:- 'false'x-stainless-lang:- pythonx-stainless-os:- - MacOS+ - Linuxx-stainless-package-version:- - 1.47.0+ - 1.52.1x-stainless-raw-response:- 'true'+ x-stainless-retry-count:+ - '0'x-stainless-runtime:- CPythonx-stainless-runtime-version:- - 3.11.7+ - 3.11.9method: POSTuri: https://api.openai.com/v1/chat/completionsresponse:- content: "{\n \"id\": \"chatcmpl-AB7avZ0yqY18ukQS7SnLkZydsx72b\",\n \"object\":- \"chat.completion\",\n \"created\": 1727214165,\n \"model\": \"gpt-4o-2024-05-13\",\n- \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\":- \"assistant\",\n \"content\": \"I now can give a great answer.\\n\\nFinal- Answer: Apples are incredibly versatile, nutritious, and a staple in diets globally.\",\n- \ \"refusal\": null\n },\n \"logprobs\": null,\n \"finish_reason\":- \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 175,\n \"completion_tokens\":- 25,\n \"total_tokens\": 200,\n \"completion_tokens_details\": {\n \"reasoning_tokens\":- 0\n }\n },\n \"system_fingerprint\": \"fp_a5d11b2ef2\"\n}\n"+ body:+ string: !!binary |+ H4sIAAAAAAAAA4xSPW/bMBDd9SsOXLpIgeTITarNS4t26JJubSHQ5IliSh1ZHv0RBP7vhSTHctAU+ 6CJQ7909vHd3zxmAsFo0IFQvkxqCKzYPaf1b2/hhW+8PR9N9Kh9W5Zdhjebr4zeRjx1++4gqvXTd+ KD8Eh8l6mmkVUSYcVau726qu729X7ydi8Brd2GZCKmpfDJZssSpXdVHeFdX9ubv3ViGLBr5nAADP+ 03f0SRqPooEyf0EGZJYGRXMpAhDRuxERktlykpREvpDKU0KarH8G8gdQksDYPYIEM9oGSXzACPCD+ PlqSDjbTfwObEBy+Y0Dl+YkTDmApoYkyIUMvoz7IiDmw79L8kqSBMe7HMMAoB4fM7ikHpF6SsmRg+ xxgBjwGjRVJ4c+00YrdjOU6Lds6d8dMluvMmRL/lM3/BO0uW+zaiZE9jTE4+iIk9ZQA/pxHvXk1N+ hOiHkNrkfyHxtLX1rCeWzS7svEsAkXyS7govq/wNvVZjktbx1ZKEkqpHvbQuG5U7bf0VkV2l/tvN+ W9pzckvmf+QXQikMCXUbImqrXideyiKOh/+vssuUJ8NiPpO2s2Qwhmjns+tCW25lqatV3VUiO2V/+ AAAA//8DAPtpFJCEAwAAheaders:CF-Cache-Status:- DYNAMICCF-RAY:- - 8c85f2369a761cf3-GRU+ - 8e19bf447ba48761-GRUConnection:- keep-aliveContent-Encoding:@@ -276,7 +299,7 @@ interactions:Content-Type:- application/jsonDate:- - Tue, 24 Sep 2024 21:42:46 GMT+ - Tue, 12 Nov 2024 21:52:06 GMTServer:- cloudflareTransfer-Encoding:@@ -285,10 +308,12 @@ interactions:- nosniffaccess-control-expose-headers:- X-Request-ID+ alt-svc:+ - h3=":443"; ma=86400openai-organization:- - crewai-iuxna1+ - user-tqfegqsiobpvvjmn0giaipdqopenai-processing-ms:- - '389'+ - '655'openai-version:- '2020-10-01'strict-transport-security:@@ -296,17 +321,18 @@ interactions:x-ratelimit-limit-requests:- '10000'x-ratelimit-limit-tokens:- - '30000000'+ - '200000'x-ratelimit-remaining-requests:- - '9999'+ - '9997'x-ratelimit-remaining-tokens:- - '29999791'+ - '199791'x-ratelimit-reset-requests:- - 6ms+ - 24.239sx-ratelimit-reset-tokens:- - 0s+ - 62msx-request-id:- - req_0167388f0a7a7f1a1026409834ceb914- http_version: HTTP/1.1- status_code: 200+ - req_a228208b0e965ecee334a6947d6c9e7c+ status:+ code: 200+ message: OKversion: 1diff --git a/tests/crew_test.py b/tests/crew_test.pyindex 5f39557e0..8a7fc193a 100644--- a/tests/crew_test.py+++ b/tests/crew_test.py@@ -564,6 +564,7 @@ def test_crew_kickoff_usage_metrics():assert result.token_usage.prompt_tokens > 0assert result.token_usage.completion_tokens > 0assert result.token_usage.successful_requests > 0+ assert result.token_usage.cached_prompt_tokens == 0def test_agents_rpm_is_never_set_if_crew_max_RPM_is_not_set():@@ -1284,6 +1285,7 @@ def test_agent_usage_metrics_are_captured_for_hierarchical_process():prompt_tokens=1562,completion_tokens=111,successful_requests=3,+ cached_prompt_tokens=0)