Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

feat(ai): tracing / observability (#6272)

Eric Hayesa34ebdde96a1
diff --git a/.github/workspace-dep-closures.json b/.github/workspace-dep-closures.json
index fb2635c67e..c54d44c1f9 100644
--- a/.github/workspace-dep-closures.json
+++ b/.github/workspace-dep-closures.json
@@ -7,6 +7,7 @@
"crates/bot_id",
"crates/channel_sender",
"crates/cowlike",
+ "crates/genai_telemetry",
"crates/kafka_util",
"crates/macro_env",
"crates/macro_env_var",
@@ -24,6 +25,7 @@
"crates/bot_id",
"crates/bot_token",
"crates/cowlike",
+ "crates/genai_telemetry",
"crates/harness_id",
"crates/harness_token",
"crates/macro_auth",
@@ -74,6 +76,7 @@
"crates/foreign_entity",
"crates/frecency",
"crates/fusionauth",
+ "crates/genai_telemetry",
"crates/github",
"crates/harness_id",
"crates/harness_token",
@@ -176,6 +179,7 @@
"crates/foreign_entity",
"crates/frecency",
"crates/fusionauth",
+ "crates/genai_telemetry",
"crates/github",
"crates/harness_id",
"crates/harness_token",
@@ -283,6 +287,7 @@
"crates/foreign_entity",
"crates/frecency",
"crates/fusionauth",
+ "crates/genai_telemetry",
"crates/generic_email_domains",
"crates/github",
"crates/harness_id",
@@ -418,6 +423,7 @@
"crates/foreign_entity",
"crates/frecency",
"crates/fusionauth",
+ "crates/genai_telemetry",
"crates/generic_email_domains",
"crates/github",
"crates/harness_id",
@@ -532,6 +538,7 @@
"crates/entity_mutation",
"crates/filter_ast",
"crates/frecency",
+ "crates/genai_telemetry",
"crates/harness_id",
"crates/harness_token",
"crates/invite_email",
@@ -597,6 +604,7 @@
"crates/entity_mutation",
"crates/filter_ast",
"crates/frecency",
+ "crates/genai_telemetry",
"crates/harness_id",
"crates/harness_token",
"crates/invite_email",
@@ -663,6 +671,7 @@
"crates/entity_mutation",
"crates/filter_ast",
"crates/frecency",
+ "crates/genai_telemetry",
"crates/harness_id",
"crates/harness_token",
"crates/invite_email",
@@ -748,6 +757,7 @@
"crates/filter_ast",
"crates/foreign_entity",
"crates/frecency",
+ "crates/genai_telemetry",
"crates/generic_email_domains",
"crates/harness_id",
"crates/harness_token",
@@ -855,6 +865,7 @@
"crates/entity_mutation",
"crates/filter_ast",
"crates/frecency",
+ "crates/genai_telemetry",
"crates/harness_id",
"crates/harness_token",
"crates/invite_email",
@@ -946,6 +957,7 @@
"crates/filter_ast",
"crates/foreign_entity",
"crates/frecency",
+ "crates/genai_telemetry",
"crates/generic_email_domains",
"crates/harness_id",
"crates/harness_token",
@@ -1031,6 +1043,8 @@
"ai_toolset": [
"crates/ai_toolset",
"crates/cowlike",
+ "crates/genai_telemetry",
+ "crates/macro_env_var",
"crates/macro_user_id",
"crates/workspace-hack"
],
@@ -1060,6 +1074,7 @@
"crates/ai_toolset",
"crates/anthropic",
"crates/cowlike",
+ "crates/genai_telemetry",
"crates/macro_env_var",
"crates/macro_user_id",
"crates/workspace-hack"
@@ -1119,6 +1134,7 @@
"crates/foreign_entity",
"crates/frecency",
"crates/fusionauth",
+ "crates/genai_telemetry",
"crates/generic_email_domains",
"crates/github",
"crates/harness_id",
@@ -1208,6 +1224,7 @@
"crates/bot_token",
"crates/cowlike",
"crates/document_sub_type",
+ "crates/genai_telemetry",
"crates/harness_id",
"crates/harness_token",
"crates/macro_auth",
@@ -1261,6 +1278,7 @@
"crates/entity_mutation",
"crates/filter_ast",
"crates/frecency",
+ "crates/genai_telemetry",
"crates/harness_id",
"crates/harness_token",
"crates/invite_email",
@@ -1351,6 +1369,7 @@
"crates/filter_ast",
"crates/foreign_entity",
"crates/frecency",
+ "crates/genai_telemetry",
"crates/graphql_common",
"crates/graphql_soup_filter_input",
"crates/harness_id",
@@ -1421,6 +1440,7 @@
"crates/calendar_events",
"crates/channel_sender",
"crates/cowlike",
+ "crates/genai_telemetry",
"crates/harness_id",
"crates/harness_token",
"crates/invite_email",
@@ -1471,6 +1491,7 @@
"crates/entity_mutation",
"crates/filter_ast",
"crates/frecency",
+ "crates/genai_telemetry",
"crates/harness_id",
"crates/harness_token",
"crates/invite_email",
@@ -1561,6 +1582,7 @@
"crates/filter_ast",
"crates/foreign_entity",
"crates/frecency",
+ "crates/genai_telemetry",
"crates/generic_email_domains",
"crates/harness_id",
"crates/harness_token",
@@ -1672,6 +1694,7 @@
"crates/entity_mutation",
"crates/filter_ast",
"crates/frecency",
+ "crates/genai_telemetry",
"crates/harness_id",
"crates/harness_token",
"crates/invite_email",
@@ -1727,6 +1750,7 @@
"crates/entity_access_db_utils",
"crates/entity_access_management",
"crates/entity_mutation",
+ "crates/genai_telemetry",
"crates/harness_id",
"crates/harness_token",
"crates/kafka_util",
@@ -1792,6 +1816,7 @@
"crates/filter_ast",
"crates/foreign_entity",
"crates/frecency",
+ "crates/genai_telemetry",
"crates/harness_id",
"crates/harness_token",
"crates/harnesses",
@@ -1863,6 +1888,7 @@
"crates/database_env_vars",
"crates/document_sub_type",
"crates/entity_access",
+ "crates/genai_telemetry",
"crates/harness_id",
"crates/harness_token",
"crates/lexical_client",
@@ -1913,6 +1939,7 @@
"crates/entity_mutation",
"crates/filter_ast",
"crates/frecency",
+ "crates/genai_telemetry",
"crates/harness_id",
"crates/harness_token",
"crates/invite_email",
@@ -1990,6 +2017,7 @@
"crates/filter_ast",
"crates/foreign_entity",
"crates/frecency",
+ "crates/genai_telemetry",
"crates/graphql_activity",
"crates/graphql_channel",
"crates/graphql_common",
@@ -2081,6 +2109,7 @@
"crates/database_env_vars",
"crates/document_sub_type",
"crates/entity_access",
+ "crates/genai_telemetry",
"crates/harness_id",
"crates/harness_token",
"crates/macro_auth",
@@ -2119,6 +2148,7 @@
"crates/ensure_exists",
"crates/filter_ast",
"crates/frecency",
+ "crates/genai_telemetry",
"crates/harness_id",
"crates/harness_token",
"crates/item_filters",
@@ -2195,6 +2225,7 @@
"crates/cowlike",
"crates/database_env_vars",
"crates/document_sub_type",
+ "crates/genai_telemetry",
"crates/harness_id",
"crates/harness_token",
"crates/macro_auth",
@@ -2252,6 +2283,7 @@
"crates/entity_mutation",
"crates/filter_ast",
"crates/frecency",
+ "crates/genai_telemetry",
"crates/harness_id",
"crates/harness_token",
"crates/invite_email",
@@ -2332,6 +2364,7 @@
"crates/document_sub_type",
"crates/entity_access",
"crates/entity_access_db_utils",
+ "crates/genai_telemetry",
"crates/harness_id",
"crates/harness_token",
"crates/invite_email",
@@ -2401,6 +2434,7 @@
"crates/entity_mutation",
"crates/filter_ast",
"crates/frecency",
+ "crates/genai_telemetry",
"crates/harness_id",
"crates/harness_token",
"crates/invite_email",
@@ -2507,6 +2541,7 @@
"crates/filter_ast",
"crates/foreign_entity",
"crates/frecency",
+ "crates/genai_telemetry",
"crates/harness_id",
"crates/harness_token",
"crates/invite_email",
@@ -2615,6 +2650,7 @@
"crates/filter_ast",
"crates/foreign_entity",
"crates/frecency",
+ "crates/genai_telemetry",
"crates/generic_email_domains",
"crates/harness_id",
"crates/harness_token",
@@ -2761,6 +2797,7 @@
"crates/foreign_entity",
"crates/frecency",
"crates/fusionauth",
+ "crates/genai_telemetry",
"crates/generic_email_domains",
"crates/github",
"crates/graphql_activity",
@@ -2884,6 +2921,7 @@
"crates/cowlike",
"crates/document_storage_service_client",
"crates/document_sub_type",
+ "crates/genai_telemetry",
"crates/harness_id",
"crates/harness_token",
"crates/macro_auth",
@@ -2917,6 +2955,7 @@
"crates/bot_token",
"crates/cowlike",
"crates/document_sub_type",
+ "crates/genai_telemetry",
"crates/harness_id",
"crates/harness_token",
"crates/macro_auth",
@@ -2967,6 +3006,7 @@
"crates/entity_mutation",
"crates/filter_ast",
"crates/foreign_entity",
+ "crates/genai_telemetry",
"crates/harness_id",
"crates/harness_token",
"crates/item_filters",
@@ -3035,6 +3075,7 @@
"crates/entity_mutation",
"crates/filter_ast",
"crates/foreign_entity",
+ "crates/genai_telemetry",
"crates/harness_id",
"crates/harness_token",
"crates/item_filters",
@@ -3102,6 +3143,7 @@
"crates/entity_mutation",
"crates/filter_ast",
"crates/frecency",
+ "crates/genai_telemetry",
"crates/harness_id",
"crates/harness_token",
"crates/invite_email",
@@ -3190,6 +3232,7 @@
"crates/entity_mutation",
"crates/filter_ast",
"crates/frecency",
+ "crates/genai_telemetry",
"crates/harness_id",
"crates/harness_token",
"crates/invite_email",
@@ -3260,6 +3303,7 @@
"crates/channel_sender",
"crates/cowlike",
"crates/email_formatting",
+ "crates/genai_telemetry",
"crates/harness_id",
"crates/harness_token",
"crates/invite_email",
@@ -3311,6 +3355,7 @@
"crates/entity_mutation",
"crates/filter_ast",
"crates/frecency",
+ "crates/genai_telemetry",
"crates/harness_id",
"crates/harness_token",
"crates/invite_email",
@@ -3383,6 +3428,7 @@
"crates/entity_mutation",
"crates/filter_ast",
"crates/frecency",
+ "crates/genai_telemetry",
"crates/harness_id",
"crates/harness_token",
"crates/invite_email",
@@ -3462,6 +3508,7 @@
"crates/entity_mutation",
"crates/filter_ast",
"crates/frecency",
+ "crates/genai_telemetry",
"crates/gmail_client",
"crates/harness_id",
"crates/harness_token",
@@ -3552,6 +3599,7 @@
"crates/entity_mutation",
"crates/filter_ast",
"crates/frecency",
+ "crates/genai_telemetry",
"crates/harness_id",
"crates/harness_token",
"crates/invite_email",
@@ -3618,6 +3666,7 @@
"crates/entity_mutation",
"crates/filter_ast",
"crates/frecency",
+ "crates/genai_telemetry",
"crates/harness_id",
"crates/harness_token",
"crates/invite_email",
@@ -3675,6 +3724,7 @@
"crates/cowlike",
"crates/document_sub_type",
"crates/entity_access_db_utils",
+ "crates/genai_telemetry",
"crates/harness_id",
"crates/harness_token",
"crates/macro_auth",
@@ -3733,6 +3783,7 @@
"crates/database_env_vars",
"crates/document_sub_type",
"crates/entity_access",
+ "crates/genai_telemetry",
"crates/harness_id",
"crates/harness_token",
"crates/macro_auth",
@@ -3791,6 +3842,7 @@
"crates/document_sub_type",
"crates/entity_access",
"crates/entity_mutation",
+ "crates/genai_telemetry",
"crates/harness_id",
"crates/harness_token",
"crates/macro_auth",
@@ -3826,6 +3878,7 @@
"crates/document_sub_type",
"crates/entity_access",
"crates/favorites",
+ "crates/genai_telemetry",
"crates/harness_id",
"crates/harness_token",
"crates/macro_auth",
@@ -3865,6 +3918,7 @@
"crates/entity_access",
"crates/filter_ast",
"crates/foreign_entity",
+ "crates/genai_telemetry",
"crates/harness_id",
"crates/harness_token",
"crates/item_filters",
@@ -3907,6 +3961,11 @@
"crates/fusionauth",
"crates/workspace-hack"
],
+ "genai_telemetry": [
+ "crates/genai_telemetry",
+ "crates/macro_env_var",
+ "crates/workspace-hack"
+ ],
"generic_email_domains": [
"crates/generic_email_domains",
"crates/workspace-hack"
@@ -3938,6 +3997,7 @@
"crates/filter_ast",
"crates/foreign_entity",
"crates/fusionauth",
+ "crates/genai_telemetry",
"crates/github",
"crates/harness_id",
"crates/harness_token",
@@ -4006,6 +4066,7 @@
"crates/cowlike",
"crates/document_sub_type",
"crates/filter_ast",
+ "crates/genai_telemetry",
"crates/graphql_activity",
"crates/graphql_common",
"crates/harness_id",
@@ -4052,6 +4113,7 @@
"crates/entity_mutation",
"crates/filter_ast",
"crates/frecency",
+ "crates/genai_telemetry",
"crates/graphql_channel",
"crates/graphql_common",
"crates/harness_id",
@@ -4143,6 +4205,7 @@
"crates/entity_mutation",
"crates/filter_ast",
"crates/frecency",
+ "crates/genai_telemetry",
"crates/graphql_common",
"crates/graphql_email",
"crates/harness_id",
@@ -4221,6 +4284,7 @@
"crates/filter_ast",
"crates/foreign_entity",
"crates/frecency",
+ "crates/genai_telemetry",
"crates/graphql_common",
"crates/graphql_email",
"crates/graphql_entity_mutation",
@@ -4330,6 +4394,7 @@
"crates/filter_ast",
"crates/foreign_entity",
"crates/frecency",
+ "crates/genai_telemetry",
"crates/graphql_common",
"crates/graphql_email",
"crates/graphql_entity_mutation",
@@ -4412,6 +4477,7 @@
"crates/cowlike",
"crates/document_sub_type",
"crates/filter_ast",
+ "crates/genai_telemetry",
"crates/graphql_common",
"crates/graphql_notification",
"crates/harness_id",
@@ -4455,6 +4521,7 @@
"crates/database_env_vars",
"crates/document_sub_type",
"crates/entity_access",
+ "crates/genai_telemetry",
"crates/graphql_permission",
"crates/harness_id",
"crates/harness_token",
@@ -4495,6 +4562,7 @@
"crates/entity_access",
"crates/entity_access_db_utils",
"crates/filter_ast",
+ "crates/genai_telemetry",
"crates/graphql_common",
"crates/graphql_properties",
"crates/harness_id",
@@ -4571,6 +4639,7 @@
"crates/filter_ast",
"crates/foreign_entity",
"crates/frecency",
+ "crates/genai_telemetry",
"crates/graphql_common",
"crates/graphql_email",
"crates/graphql_permission",
@@ -4711,6 +4780,7 @@
"crates/cowlike",
"crates/document_sub_type",
"crates/entity_access_db_utils",
+ "crates/genai_telemetry",
"crates/harness_id",
"crates/harness_token",
"crates/ip_extractor",
@@ -4766,6 +4836,7 @@
"crates/document_sub_type",
"crates/entity_access",
"crates/entity_access_db_utils",
+ "crates/genai_telemetry",
"crates/harness_id",
"crates/harness_token",
"crates/import",
@@ -4815,6 +4886,7 @@
"crates/bot_token",
"crates/broadcast",
"crates/cowlike",
+ "crates/genai_telemetry",
"crates/harness_id",
"crates/harness_token",
"crates/invite_email",
@@ -4915,6 +4987,7 @@
"crates/entity_mutation",
"crates/filter_ast",
"crates/frecency",
+ "crates/genai_telemetry",
"crates/harness_id",
"crates/harness_token",
"crates/invite_email",
@@ -5044,6 +5117,7 @@
"crates/cowlike",
"crates/document_sub_type",
"crates/entity_access_db_utils",
+ "crates/genai_telemetry",
"crates/harness_id",
"crates/harness_token",
"crates/macro_auth",
@@ -5114,6 +5188,7 @@
"crates/cowlike",
"crates/document_sub_type",
"crates/entity_access_db_utils",
+ "crates/genai_telemetry",
"crates/harness_id",
"crates/harness_token",
"crates/ip_extractor",
@@ -5170,6 +5245,7 @@
"crates/bot_token",
"crates/cowlike",
"crates/document_sub_type",
+ "crates/genai_telemetry",
"crates/harness_id",
"crates/harness_token",
"crates/macro_auth",
@@ -5228,6 +5304,7 @@
"crates/bot_id",
"crates/bot_token",
"crates/cowlike",
+ "crates/genai_telemetry",
"crates/harness_id",
"crates/harness_token",
"crates/macro_auth",
@@ -5254,6 +5331,7 @@
"crates/bot_id",
"crates/bot_token",
"crates/cowlike",
+ "crates/genai_telemetry",
"crates/harness_id",
"crates/harness_token",
"crates/macro_auth",
@@ -5318,6 +5396,7 @@
"crates/foreign_entity",
"crates/frecency",
"crates/fusionauth",
+ "crates/genai_telemetry",
"crates/generic_email_domains",
"crates/harness_id",
"crates/harness_token",
@@ -5407,6 +5486,8 @@
"mcp_toolset": [
"crates/ai_toolset",
"crates/cowlike",
+ "crates/genai_telemetry",
+ "crates/macro_env_var",
"crates/macro_user_id",
"crates/mcp_toolset",
"crates/workspace-hack"
@@ -5453,6 +5534,7 @@
"crates/filter_ast",
"crates/foreign_entity",
"crates/frecency",
+ "crates/genai_telemetry",
"crates/generic_email_domains",
"crates/harness_id",
"crates/harness_token",
@@ -5561,6 +5643,7 @@
"crates/bot_token",
"crates/cowlike",
"crates/document_sub_type",
+ "crates/genai_telemetry",
"crates/harness_id",
"crates/harness_token",
"crates/macro_auth",
@@ -5601,6 +5684,7 @@
"crates/broadcast",
"crates/channel_sender",
"crates/cowlike",
+ "crates/genai_telemetry",
"crates/harness_id",
"crates/harness_token",
"crates/invite_email",
@@ -5719,6 +5803,7 @@
"crates/entity_mutation",
"crates/filter_ast",
"crates/frecency",
+ "crates/genai_telemetry",
"crates/harness_id",
"crates/harness_token",
"crates/invite_email",
@@ -5809,6 +5894,7 @@
"crates/entity_mutation",
"crates/filter_ast",
"crates/frecency",
+ "crates/genai_telemetry",
"crates/harness_id",
"crates/harness_token",
"crates/invite_email",
@@ -5881,6 +5967,7 @@
"crates/entity_mutation",
"crates/filter_ast",
"crates/frecency",
+ "crates/genai_telemetry",
"crates/harness_id",
"crates/harness_token",
"crates/invite_email",
@@ -5942,6 +6029,7 @@
"crates/bot_token",
"crates/broadcast",
"crates/cowlike",
+ "crates/genai_telemetry",
"crates/harness_id",
"crates/harness_token",
"crates/kafka_util",
@@ -5972,6 +6060,7 @@
"crates/broadcast",
"crates/channel_sender",
"crates/cowlike",
+ "crates/genai_telemetry",
"crates/harness_id",
"crates/harness_token",
"crates/invite_email",
@@ -6007,6 +6096,7 @@
"crates/channel_sender",
"crates/cowlike",
"crates/email_formatting",
+ "crates/genai_telemetry",
"crates/harness_id",
"crates/harness_token",
"crates/invite_email",
@@ -6049,6 +6139,7 @@
"crates/document_sub_type",
"crates/email_formatting",
"crates/entity_access_db_utils",
+ "crates/genai_telemetry",
"crates/harness_id",
"crates/harness_token",
"crates/invite_email",
@@ -6122,6 +6213,7 @@
"crates/document_sub_type",
"crates/entity_access",
"crates/entity_access_db_utils",
+ "crates/genai_telemetry",
"crates/generic_email_domains",
"crates/harness_id",
"crates/harness_token",
@@ -6203,6 +6295,7 @@
"crates/entity_mutation",
"crates/filter_ast",
"crates/frecency",
+ "crates/genai_telemetry",
"crates/harness_id",
"crates/harness_token",
"crates/invite_email",
@@ -6275,6 +6368,7 @@
"crates/entity_mutation",
"crates/filter_ast",
"crates/frecency",
+ "crates/genai_telemetry",
"crates/harness_id",
"crates/harness_token",
"crates/invite_email",
@@ -6327,6 +6421,7 @@
"crates/bot_id",
"crates/bot_token",
"crates/cowlike",
+ "crates/genai_telemetry",
"crates/harness_id",
"crates/harness_token",
"crates/macro_auth",
@@ -6376,6 +6471,7 @@
"crates/filter_ast",
"crates/foreign_entity",
"crates/frecency",
+ "crates/genai_telemetry",
"crates/harness_id",
"crates/harness_token",
"crates/invite_email",
@@ -6450,6 +6546,7 @@
"crates/document_sub_type",
"crates/entity_access",
"crates/entity_access_db_utils",
+ "crates/genai_telemetry",
"crates/harness_id",
"crates/harness_token",
"crates/invite_email",
@@ -6502,6 +6599,7 @@
"crates/document_sub_type",
"crates/entity_access",
"crates/entity_access_db_utils",
+ "crates/genai_telemetry",
"crates/harness_id",
"crates/harness_token",
"crates/invite_email",
@@ -6554,6 +6652,7 @@
"crates/bot_token",
"crates/broadcast",
"crates/cowlike",
+ "crates/genai_telemetry",
"crates/harness_id",
"crates/harness_token",
"crates/invite_email",
@@ -6593,6 +6692,7 @@
"crates/database_env_vars",
"crates/document_sub_type",
"crates/entity_access",
+ "crates/genai_telemetry",
"crates/harness_id",
"crates/harness_token",
"crates/invite_email",
@@ -6696,6 +6796,7 @@
"crates/filter_ast",
"crates/foreign_entity",
"crates/frecency",
+ "crates/genai_telemetry",
"crates/generic_email_domains",
"crates/harness_id",
"crates/harness_token",
@@ -6823,6 +6924,7 @@
"crates/filter_ast",
"crates/foreign_entity",
"crates/frecency",
+ "crates/genai_telemetry",
"crates/harness_id",
"crates/harness_token",
"crates/invite_email",
@@ -6934,6 +7036,7 @@
"crates/entity_mutation",
"crates/filter_ast",
"crates/frecency",
+ "crates/genai_telemetry",
"crates/harness_id",
"crates/harness_token",
"crates/invite_email",
@@ -7025,6 +7128,7 @@
"crates/entity_mutation",
"crates/filter_ast",
"crates/frecency",
+ "crates/genai_telemetry",
"crates/harness_id",
"crates/harness_token",
"crates/invite_email",
@@ -7087,6 +7191,7 @@
"crates/cowlike",
"crates/document_storage_service_client",
"crates/document_sub_type",
+ "crates/genai_telemetry",
"crates/harness_id",
"crates/harness_token",
"crates/macro_auth",
@@ -7147,6 +7252,7 @@
"crates/filter_ast",
"crates/frecency",
"crates/fusionauth",
+ "crates/genai_telemetry",
"crates/harness_id",
"crates/harness_token",
"crates/invite_email",
@@ -7262,6 +7368,7 @@
"crates/filter_ast",
"crates/foreign_entity",
"crates/frecency",
+ "crates/genai_telemetry",
"crates/harness_id",
"crates/harness_token",
"crates/invite_email",
@@ -7355,6 +7462,7 @@
"crates/filter_ast",
"crates/foreign_entity",
"crates/frecency",
+ "crates/genai_telemetry",
"crates/harness_id",
"crates/harness_token",
"crates/invite_email",
@@ -7437,6 +7545,7 @@
"crates/filter_ast",
"crates/foreign_entity",
"crates/frecency",
+ "crates/genai_telemetry",
"crates/graphql_common",
"crates/graphql_soup_filter_input",
"crates/harness_id",
@@ -7525,6 +7634,7 @@
"crates/filter_ast",
"crates/foreign_entity",
"crates/frecency",
+ "crates/genai_telemetry",
"crates/harness_id",
"crates/harness_token",
"crates/invite_email",
@@ -7612,6 +7722,7 @@
"crates/filter_ast",
"crates/foreign_entity",
"crates/frecency",
+ "crates/genai_telemetry",
"crates/harness_id",
"crates/harness_token",
"crates/invite_email",
@@ -7695,6 +7806,7 @@
"crates/entity_mutation",
"crates/filter_ast",
"crates/frecency",
+ "crates/genai_telemetry",
"crates/harness_id",
"crates/harness_token",
"crates/invite_email",
@@ -7762,6 +7874,7 @@
"crates/bot_token",
"crates/cowlike",
"crates/document_sub_type",
+ "crates/genai_telemetry",
"crates/harness_id",
"crates/harness_token",
"crates/macro_auth",
@@ -7823,6 +7936,7 @@
"crates/bot_token",
"crates/cowlike",
"crates/document_sub_type",
+ "crates/genai_telemetry",
"crates/harness_id",
"crates/harness_token",
"crates/macro_auth",
@@ -7891,6 +8005,7 @@
"crates/cowlike",
"crates/document_sub_type",
"crates/embedding",
+ "crates/genai_telemetry",
"crates/harness_id",
"crates/harness_token",
"crates/lexical_client",
@@ -7947,6 +8062,7 @@
"crates/entity_mutation",
"crates/filter_ast",
"crates/frecency",
+ "crates/genai_telemetry",
"crates/generic_email_domains",
"crates/harness_id",
"crates/harness_token",
@@ -8045,6 +8161,7 @@
"crates/entity_mutation",
"crates/filter_ast",
"crates/frecency",
+ "crates/genai_telemetry",
"crates/harness_id",
"crates/harness_token",
"crates/invite_email",
@@ -8118,6 +8235,7 @@
"crates/entity_mutation",
"crates/filter_ast",
"crates/frecency",
+ "crates/genai_telemetry",
"crates/harness_id",
"crates/harness_token",
"crates/invite_email",
@@ -8192,6 +8310,7 @@
"crates/cowlike",
"crates/document_sub_type",
"crates/entity_access_db_utils",
+ "crates/genai_telemetry",
"crates/harness_id",
"crates/harness_token",
"crates/macro_auth",
@@ -8263,6 +8382,7 @@
"crates/filter_ast",
"crates/foreign_entity",
"crates/frecency",
+ "crates/genai_telemetry",
"crates/harness_id",
"crates/harness_token",
"crates/invite_email",
diff --git a/Cargo.lock b/Cargo.lock
index 0a981f9bbd..9a7c664ced 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -100,10 +100,13 @@ dependencies = [
"async-trait",
"attachment",
"futures",
+ "genai_telemetry",
"macro_env_var",
"macro_user_id",
"model-entity",
"non_empty",
+ "opentelemetry",
+ "opentelemetry_sdk",
"prompt",
"regex",
"rig-agent",
@@ -116,6 +119,8 @@ dependencies = [
"tokio",
"tokio-util",
"tracing",
+ "tracing-opentelemetry",
+ "tracing-subscriber",
"utoipa",
"workspace-hack",
]
@@ -379,6 +384,7 @@ dependencies = [
"chat",
"dashmap",
"futures",
+ "genai_telemetry",
"http 1.4.0",
"http-body-util",
"macro_user_id",
@@ -386,6 +392,8 @@ dependencies = [
"mcp_select",
"mcp_toolset",
"memory",
+ "opentelemetry",
+ "opentelemetry_sdk",
"prompt",
"rig-agent",
"rmcp",
@@ -398,6 +406,8 @@ dependencies = [
"tokio",
"tokio-util",
"tracing",
+ "tracing-opentelemetry",
+ "tracing-subscriber",
"workspace-hack",
]
@@ -441,6 +451,7 @@ dependencies = [
"dashmap",
"entity_access",
"entity_access_db_utils",
+ "genai_telemetry",
"harness_id",
"macro_authorization",
"macro_db_migrator",
@@ -452,6 +463,8 @@ dependencies = [
"model-entity",
"model_user",
"non_empty",
+ "opentelemetry",
+ "opentelemetry_sdk",
"rootcause",
"serde",
"serde_json",
@@ -463,6 +476,8 @@ dependencies = [
"tokio-util",
"tower 0.5.3",
"tracing",
+ "tracing-opentelemetry",
+ "tracing-subscriber",
"utoipa",
"uuid",
"workspace-hack",
@@ -712,13 +727,19 @@ dependencies = [
"async-trait",
"axum",
"axum-macros",
+ "genai_telemetry",
"macro_user_id",
+ "opentelemetry",
+ "opentelemetry_sdk",
"schemars 1.2.1",
"serde",
"serde_json",
"thiserror 2.0.18",
+ "tokio",
"tokio-util",
"tracing",
+ "tracing-opentelemetry",
+ "tracing-subscriber",
"utoipa",
"workspace-hack",
]
@@ -6896,6 +6917,20 @@ dependencies = [
"byteorder",
]
+[[package]]
+name = "genai_telemetry"
+version = "0.1.0"
+dependencies = [
+ "macro_env_var",
+ "opentelemetry",
+ "opentelemetry_sdk",
+ "serde_json",
+ "tracing",
+ "tracing-opentelemetry",
+ "tracing-subscriber",
+ "workspace-hack",
+]
+
[[package]]
name = "generator"
version = "0.8.8"
diff --git a/Cargo.toml b/Cargo.toml
index 932425af2b..c555f01844 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -32,6 +32,7 @@ members = [
"crates/entity_mutation",
"crates/favorites",
"crates/foreign_entity",
+ "crates/genai_telemetry",
"crates/generic_email_domains",
"crates/gmail_client",
"crates/graphql_activity",
diff --git a/crates/agent/Cargo.toml b/crates/agent/Cargo.toml
index d4e2f529b0..139e5b0aaa 100644
--- a/crates/agent/Cargo.toml
+++ b/crates/agent/Cargo.toml
@@ -27,6 +27,7 @@ tracing = { workspace = true }
ai_usage = { path = "../ai_usage" }
ai_toolset = { path = "../ai_toolset" }
attachment = { path = "../attachment" }
+genai_telemetry = { path = "../genai_telemetry" }
macro_env_var = { path = "../macro_env_var" }
macro_user_id = { path = "../macro_user_id" }
prompt = { path = "../prompt" }
@@ -36,6 +37,10 @@ tokio-util.workspace = true
[dev-dependencies]
model-entity = { path = "../model-entity" }
non_empty = { path = "../non_empty" }
+opentelemetry = { workspace = true, features = ["trace"] }
+opentelemetry_sdk = { workspace = true, features = ["testing", "trace"] }
+tracing-opentelemetry = { workspace = true }
+tracing-subscriber = { workspace = true }
# Enable rig's scripted MockCompletionModel for tests (fake AI streams).
rig-core = { version = "0.41.0", features = ["test-utils"] }
rig-agent = { version = "0.41.0", features = ["test-utils"] }
diff --git a/crates/agent/src/agent_loop.rs b/crates/agent/src/agent_loop.rs
index 406c9175b4..243e63e95c 100644
--- a/crates/agent/src/agent_loop.rs
+++ b/crates/agent/src/agent_loop.rs
@@ -4,15 +4,18 @@ use crate::hook::{BridgeInputs, RegisterFn, ToolRouter, UserToolFinisher};
use crate::model::PredefinedModel;
use crate::model::router::{ModelRouter, ProviderAgent};
use crate::stream::ChatCompletionStream;
+use crate::telemetry::GenAiContext;
use crate::tool_adapter::DynToolSetAdapter;
use ai_toolset::{RequestContext, SearchableTool, ToolLoader, ToolSet as AiToolSet};
use ai_usage::{UsageContext, UsageRecorder};
+use genai_telemetry::ContentPolicy;
use rig_agent::tool::server::{ToolServer, ToolServerHandle};
use rig_core::message::Message;
use std::future::Future;
use std::pin::Pin;
use std::sync::{Arc, Mutex, RwLock};
use tokio_util::sync::CancellationToken;
+use tracing::Instrument as _;
const DEFAULT_MAX_TURNS: usize = 16;
const DEFAULT_MAX_TOKENS: u64 = 16_000;
@@ -31,6 +34,13 @@ pub struct AgentLoop {
max_tokens: u64,
recorder: Arc<dyn UsageRecorder>,
user_tool_finisher: Option<UserToolFinisher>,
+ /// The conversation (session) sessions belong to, for telemetry.
+ conversation_id: Option<String>,
+ /// The agent name spans carry; defaults to the usage context's feature.
+ agent_name: Option<String>,
+ /// Whether this loop enriches the runtime's GenAI spans (see
+ /// [`Self::with_genai_telemetry`]).
+ genai_telemetry: bool,
}
impl AgentLoop {
@@ -48,6 +58,9 @@ impl AgentLoop {
max_tokens: DEFAULT_MAX_TOKENS,
recorder,
user_tool_finisher: None,
+ conversation_id: None,
+ agent_name: None,
+ genai_telemetry: true,
}
}
@@ -86,6 +99,39 @@ impl AgentLoop {
self
}
+ /// Tag every span of the sessions created from this loop with the
+ /// conversation they belong to (`gen_ai.conversation.id`, e.g. the chat
+ /// id). Observability backends group the turns of one conversation into a
+ /// session by it, which is what session-level evaluations run over.
+ pub fn with_conversation_id<S: Into<String>>(mut self, conversation_id: S) -> Self {
+ self.conversation_id = Some(conversation_id.into());
+ self
+ }
+
+ /// Override the agent name spans carry (`gen_ai.agent.name`). Defaults to
+ /// the usage context's feature (`chat`, `automation`, …).
+ pub fn with_agent_name<S: Into<String>>(mut self, agent_name: S) -> Self {
+ self.agent_name = Some(agent_name.into());
+ self
+ }
+
+ /// Whether this loop records GenAI telemetry on the runtime's spans: the
+ /// `invoke_agent` span with the run's input, output and usage, the
+ /// content, tool definitions and conversation id on each `chat` span, and
+ /// the arguments and result on each `execute_tool` span. On by default.
+ ///
+ /// Off for a loop whose turns are already traced from outside - Macro's
+ /// in-process agent session runtime, whose ACP frames the session actor
+ /// projects onto GenAI spans for every harness alike. With it off the run
+ /// is still wrapped in a span (so the runtime adopts it rather than
+ /// opening an `invoke_agent` of its own), but that span carries no GenAI
+ /// fields, nothing is recorded onto the runtime's spans, and the tool
+ /// calls run with `RequestContext::genai_telemetry` off.
+ pub fn with_genai_telemetry(mut self, enabled: bool) -> Self {
+ self.genai_telemetry = enabled;
+ self
+ }
+
/// Start a new streaming session.
///
/// `toolset` is the combined tool set (static + MCP) for this request.
@@ -111,10 +157,17 @@ impl AgentLoop {
context,
system_prompt,
usage_ctx,
- |handle, prompt, max_turns, max_tokens| {
+ |handle, prompt, max_turns, max_tokens, telemetry| {
ModelRouter::shared()
.expect("failed to initialize model router")
- .agent(&self.model, handle, prompt, max_turns, max_tokens)
+ .agent(
+ &self.model,
+ handle,
+ prompt,
+ max_turns,
+ max_tokens,
+ telemetry,
+ )
},
)
.await
@@ -131,7 +184,7 @@ impl AgentLoop {
context: Arc<Context>,
system_prompt: &str,
usage_ctx: UsageContext,
- build: impl FnOnce(ToolServerHandle, &str, usize, u64) -> ProviderAgent,
+ build: impl FnOnce(ToolServerHandle, &str, usize, u64, GenAiContext) -> ProviderAgent,
) -> Session
where
Context: Clone + Send + Sync + 'static,
@@ -153,8 +206,9 @@ impl AgentLoop {
buffer.lock().expect("loaded_buffer poisoned").extend(tools)
})
};
- let request_context =
- RequestContext::new(usage_ctx.user.clone()).with_tool_search(Arc::new(catalog), loader);
+ let request_context = RequestContext::new(usage_ctx.user.clone())
+ .with_tool_search(Arc::new(catalog), loader)
+ .with_genai_telemetry(self.genai_telemetry);
// TODO this is cringe, make request context a RW lock newtype
let request_context_rw = Arc::new(RwLock::new(request_context.clone()));
@@ -238,7 +292,24 @@ impl AgentLoop {
system_prompt.push_str(&section);
}
- let agent = build(handle, &system_prompt, self.max_turns, self.max_tokens);
+ // Session-scoped GenAI telemetry: the conversation id ties the spans of
+ // every turn of this session together, the agent name labels them and
+ // the content policy governs what is recorded (see `crate::telemetry`).
+ let telemetry = GenAiContext::new(
+ self.conversation_id.clone(),
+ self.agent_name
+ .clone()
+ .unwrap_or_else(|| usage_ctx.feature.to_string()),
+ ContentPolicy::from_env(),
+ self.genai_telemetry,
+ );
+ let agent = build(
+ handle,
+ &system_prompt,
+ self.max_turns,
+ self.max_tokens,
+ telemetry.clone(),
+ );
Session {
agent,
@@ -254,6 +325,7 @@ impl AgentLoop {
usage_ctx,
model: self.model.clone(),
request_context,
+ telemetry,
}
}
@@ -279,8 +351,8 @@ impl AgentLoop {
context,
system_prompt,
usage_ctx,
- move |handle, prompt, max_turns, max_tokens| {
- ProviderAgent::test(model, prompt, max_turns, max_tokens, handle)
+ move |handle, prompt, max_turns, max_tokens, telemetry| {
+ ProviderAgent::test(model, prompt, max_turns, max_tokens, handle, telemetry)
},
)
.await
@@ -299,6 +371,7 @@ pub struct Session {
usage_ctx: UsageContext,
model: String,
request_context: RequestContext,
+ telemetry: GenAiContext,
}
impl Session {
@@ -311,10 +384,50 @@ impl Session {
///
/// The returned stream yields [`StreamPart`] items compatible with the
/// existing DCS consumer code.
- #[tracing::instrument(name = "invoke_agent", skip_all)]
+ ///
+ /// The run is wrapped in a span the runtime adopts instead of opening an
+ /// `invoke_agent` of its own. With GenAI telemetry on (the default) that
+ /// span *is* the run's `invoke_agent` span: the semconv fields are declared
+ /// on it and the run's usage and output are recorded onto it by the stream
+ /// driver (see `crate::telemetry`). With it off the span is a plain
+ /// `agent.turn`, traced from outside instead. Either way it stays open
+ /// until the returned stream ends.
pub async fn send_message(
&mut self,
messages: Vec<Message>,
+ ) -> Result<ChatCompletionStream<'_>, AgentError> {
+ let span = if self.telemetry.enabled() {
+ tracing::info_span!(
+ "invoke_agent",
+ gen_ai.operation.name = "invoke_agent",
+ gen_ai.agent.name = %self.telemetry.agent_name(),
+ gen_ai.conversation.id = self.telemetry.conversation_id(),
+ gen_ai.provider.name = self.telemetry.provider_name(),
+ gen_ai.request.model = self.telemetry.model_name(),
+ )
+ } else {
+ tracing::info_span!("agent.turn", agent.name = %self.telemetry.agent_name())
+ };
+ let telemetry = self.telemetry.clone();
+ let result = self
+ .send_message_in(messages)
+ .instrument(span.clone())
+ .await;
+ if let Err(error) = &result {
+ // The run never started; the span still says why.
+ telemetry.record_agent_failure(
+ &span,
+ "agent_error",
+ genai_telemetry::attr::finish_reason::ERROR,
+ &error.to_string(),
+ );
+ }
+ result
+ }
+
+ async fn send_message_in(
+ &mut self,
+ messages: Vec<Message>,
) -> Result<ChatCompletionStream<'_>, AgentError> {
self.history = messages;
@@ -335,6 +448,7 @@ impl Session {
self.usage_ctx.clone(),
self.model.clone(),
self.request_context.clone(),
+ self.telemetry.clone(),
)
.await;
diff --git a/crates/agent/src/completion.rs b/crates/agent/src/completion.rs
index e78ea1e5d5..7c021d59aa 100644
--- a/crates/agent/src/completion.rs
+++ b/crates/agent/src/completion.rs
@@ -2,8 +2,13 @@
//!
//! This is the non-streaming API layer: routing only resolves a model, and the
//! actual prompting lives here.
+#[cfg(test)]
+mod test;
+
use crate::model::router::{ModelRouter, RoutedModel};
+use crate::telemetry::{ChatSpanHook, GenAiContext, TracedModel};
use ai_usage::{UsageContext, UsageRecorder};
+use genai_telemetry::ContentPolicy;
use rig_agent::agent::{AgentBuilder, PromptResponse};
use rig_agent::completion::Prompt;
use rig_core::completion::CompletionModel;
@@ -28,15 +33,35 @@ pub async fn complete<M: ToString>(
ctx: UsageContext,
) -> anyhow::Result<String> {
let model = model.to_string();
- let response = match ModelRouter::shared()?.route_or_default(&model) {
+ let routed = ModelRouter::shared()?.route_or_default(&model);
+ let telemetry = telemetry_for(&ctx, &routed);
+ let response = match routed {
RoutedModel::Anthropic(m) => {
- prompt_once(m.completion(), system_prompt, user_message).await?
+ prompt_once(
+ TracedModel::new(m.completion(), telemetry.clone()),
+ system_prompt,
+ user_message,
+ telemetry,
+ )
+ .await?
}
RoutedModel::OpenAiChatCompletions(m) => {
- prompt_once(m.completion(), system_prompt, user_message).await?
+ prompt_once(
+ TracedModel::new(m.completion(), telemetry.clone()),
+ system_prompt,
+ user_message,
+ telemetry,
+ )
+ .await?
}
RoutedModel::OpenAiResponses(m) => {
- prompt_once(m.completion(), system_prompt, user_message).await?
+ prompt_once(
+ TracedModel::new(m.completion(), telemetry.clone()),
+ system_prompt,
+ user_message,
+ telemetry,
+ )
+ .await?
}
};
record(recorder, ctx, model, &response);
@@ -56,21 +81,54 @@ pub async fn complete_with_history<M: ToString>(
ctx: UsageContext,
) -> anyhow::Result<String> {
let model = model.to_string();
- let response = match ModelRouter::shared()?.route_or_default(&model) {
+ let routed = ModelRouter::shared()?.route_or_default(&model);
+ let telemetry = telemetry_for(&ctx, &routed);
+ let response = match routed {
RoutedModel::Anthropic(m) => {
- prompt_with_history(m.completion(), system_prompt, messages).await?
+ prompt_with_history(
+ TracedModel::new(m.completion(), telemetry.clone()),
+ system_prompt,
+ messages,
+ telemetry,
+ )
+ .await?
}
RoutedModel::OpenAiChatCompletions(m) => {
- prompt_with_history(m.completion(), system_prompt, messages).await?
+ prompt_with_history(
+ TracedModel::new(m.completion(), telemetry.clone()),
+ system_prompt,
+ messages,
+ telemetry,
+ )
+ .await?
}
RoutedModel::OpenAiResponses(m) => {
- prompt_with_history(m.completion(), system_prompt, messages).await?
+ prompt_with_history(
+ TracedModel::new(m.completion(), telemetry.clone()),
+ system_prompt,
+ messages,
+ telemetry,
+ )
+ .await?
}
};
record(recorder, ctx, model, &response);
Ok(response.output)
}
+/// GenAI telemetry for a one-shot completion: named after the feature making
+/// it, reporting the routed model, with no conversation (a one-shot has none).
+fn telemetry_for(ctx: &UsageContext, routed: &RoutedModel<'_>) -> GenAiContext {
+ let telemetry = GenAiContext::new(
+ None,
+ ctx.feature.to_string(),
+ ContentPolicy::from_env(),
+ true,
+ );
+ telemetry.set_model(routed.provider(), routed.model_name());
+ telemetry
+}
+
/// Record the usage of a one-shot completion.
fn record(
recorder: &dyn UsageRecorder,
@@ -86,17 +144,27 @@ fn record(
}
/// Build a toolless agent and prompt it with a single user message.
+///
+/// The model is a [`TracedModel`]; `telemetry` also drives the hook that
+/// records the model's output on the `chat` span.
async fn prompt_once<M: CompletionModel + 'static>(
completion_model: M,
system_prompt: &str,
user_message: &str,
+ telemetry: GenAiContext,
) -> anyhow::Result<PromptResponse> {
let agent = AgentBuilder::new(completion_model)
+ .name(telemetry.agent_name())
+ .record_content_telemetry(false)
.preamble(system_prompt)
.max_tokens(ONE_SHOT_MAX_TOKENS)
.build();
- Ok(agent.prompt(user_message).extended_details().await?)
+ Ok(agent
+ .prompt(user_message)
+ .extended_details()
+ .add_hook(ChatSpanHook(telemetry))
+ .await?)
}
/// Build a toolless agent and prompt it with the last message of `messages`,
@@ -105,8 +173,11 @@ async fn prompt_with_history<M: CompletionModel + 'static>(
completion_model: M,
system_prompt: &str,
messages: Vec<Message>,
+ telemetry: GenAiContext,
) -> anyhow::Result<PromptResponse> {
let agent = AgentBuilder::new(completion_model)
+ .name(telemetry.agent_name())
+ .record_content_telemetry(false)
.preamble(system_prompt)
.max_tokens(ONE_SHOT_MAX_TOKENS)
.build();
@@ -119,5 +190,6 @@ async fn prompt_with_history<M: CompletionModel + 'static>(
.prompt(prompt.clone())
.extended_details()
.history(history.to_vec())
+ .add_hook(ChatSpanHook(telemetry))
.await?)
}
diff --git a/crates/agent/src/completion/test.rs b/crates/agent/src/completion/test.rs
new file mode 100644
index 0000000000..d6d324d811
--- /dev/null
+++ b/crates/agent/src/completion/test.rs
@@ -0,0 +1,107 @@
+//! The one-shot paths trace their single model call like the agent loop does.
+use super::*;
+use genai_telemetry::attr;
+use opentelemetry::trace::{Status, TracerProvider as _};
+use opentelemetry_sdk::trace::{InMemorySpanExporter, SdkTracerProvider, SpanData};
+use rig_core::test_utils::{MockCompletionModel, MockTurn};
+use tracing_subscriber::layer::SubscriberExt as _;
+
+fn otel_test_pipeline() -> (
+ InMemorySpanExporter,
+ SdkTracerProvider,
+ tracing::subscriber::DefaultGuard,
+) {
+ let exporter = InMemorySpanExporter::default();
+ let provider = SdkTracerProvider::builder()
+ .with_simple_exporter(exporter.clone())
+ .build();
+ let layer = tracing_opentelemetry::layer().with_tracer(provider.tracer("test"));
+ let guard = tracing::subscriber::set_default(tracing_subscriber::registry().with(layer));
+ tracing::callsite::rebuild_interest_cache();
+ (exporter, provider, guard)
+}
+
+fn finished(exporter: &InMemorySpanExporter, provider: &SdkTracerProvider) -> Vec<SpanData> {
+ provider.force_flush().expect("flush");
+ exporter.get_finished_spans().expect("spans")
+}
+
+fn string_attribute(span: &SpanData, key: &str) -> Option<String> {
+ span.attributes
+ .iter()
+ .find(|kv| kv.key.as_str() == key)
+ .map(|kv| kv.value.to_string())
+}
+
+fn chat_spans(spans: &[SpanData]) -> Vec<&SpanData> {
+ spans
+ .iter()
+ .filter(|span| {
+ string_attribute(span, attr::OPERATION_NAME).as_deref() == Some(attr::operation::CHAT)
+ })
+ .collect()
+}
+
+#[tokio::test]
+async fn a_one_shot_completion_traces_its_model_call() {
+ let (exporter, provider, _guard) = otel_test_pipeline();
+ let telemetry = GenAiContext::new(None, "summarize".to_owned(), ContentPolicy::enabled(), true);
+ let model = TracedModel::new(
+ MockCompletionModel::new([MockTurn::text("a summary")]),
+ telemetry.clone(),
+ );
+
+ let response = prompt_once(model, "be brief", "summarize this", telemetry)
+ .await
+ .expect("the completion succeeds");
+ assert_eq!(response.output, "a summary");
+
+ let spans = finished(&exporter, &provider);
+ let chats = chat_spans(&spans);
+ assert_eq!(chats.len(), 1, "{spans:#?}");
+ let chat = chats[0];
+ assert_eq!(chat.status, Status::Unset);
+ assert!(string_attribute(chat, attr::INPUT_MESSAGES).is_some());
+ assert!(
+ string_attribute(chat, attr::OUTPUT_MESSAGES)
+ .is_some_and(|output| output.contains("a summary"))
+ );
+}
+
+/// A provider that refuses the call leaves no turn for the hook to record, so
+/// the traced model itself marks the call as failed.
+#[tokio::test]
+async fn a_failed_one_shot_completion_marks_its_chat_span() {
+ let (exporter, provider, _guard) = otel_test_pipeline();
+ let telemetry = GenAiContext::new(None, "summarize".to_owned(), ContentPolicy::enabled(), true);
+ let model = TracedModel::new(
+ MockCompletionModel::new([MockTurn::error("upstream is down")]),
+ telemetry.clone(),
+ );
+
+ let error = prompt_once(model, "be brief", "summarize this", telemetry)
+ .await
+ .expect_err("the completion fails");
+ assert!(error.to_string().contains("upstream is down"), "{error}");
+
+ let spans = finished(&exporter, &provider);
+ let chats = chat_spans(&spans);
+ assert_eq!(chats.len(), 1, "{spans:#?}");
+ let chat = chats[0];
+ match &chat.status {
+ Status::Error { description } => {
+ assert!(description.contains("upstream is down"), "{description}");
+ }
+ other => panic!("expected an error status, got {other:?}"),
+ }
+ assert_eq!(
+ string_attribute(chat, attr::ERROR_TYPE).as_deref(),
+ Some("provider_error")
+ );
+ assert!(
+ string_attribute(chat, attr::RESPONSE_FINISH_REASONS)
+ .is_some_and(|reasons| reasons.contains("error"))
+ );
+ assert!(string_attribute(chat, attr::INPUT_MESSAGES).is_some());
+ assert_eq!(string_attribute(chat, attr::OUTPUT_MESSAGES), None);
+}
diff --git a/crates/agent/src/lib.rs b/crates/agent/src/lib.rs
index 99154451b2..6599d821ad 100644
--- a/crates/agent/src/lib.rs
+++ b/crates/agent/src/lib.rs
@@ -12,6 +12,7 @@ mod model;
mod stream;
/// Structured output via prompted JSON generation.
pub mod structured_output;
+mod telemetry;
mod tool_adapter;
#[cfg(test)]
diff --git a/crates/agent/src/model/anthropic.rs b/crates/agent/src/model/anthropic.rs
index 4de920a80d..53a45b529a 100644
--- a/crates/agent/src/model/anthropic.rs
+++ b/crates/agent/src/model/anthropic.rs
@@ -18,6 +18,11 @@ impl<'a> AnthropicModel<'a> {
Self { model, client }
}
+ /// The routed id this model was bound to.
+ pub fn model(&self) -> &Model<'a> {
+ &self.model
+ }
+
/// The rig completion model for this id. The id is passed verbatim to the
/// Anthropic API.
pub fn completion(&self) -> anthropic::completion::CompletionModel {
diff --git a/crates/agent/src/model/openai.rs b/crates/agent/src/model/openai.rs
index 3ed0c22eef..0ac99b6bad 100644
--- a/crates/agent/src/model/openai.rs
+++ b/crates/agent/src/model/openai.rs
@@ -21,6 +21,11 @@ impl<'a> OpenAiChatCompletionsModel<'a> {
Self { model, client }
}
+ /// The routed id this model was bound to.
+ pub fn model(&self) -> &Model<'a> {
+ &self.model
+ }
+
/// The rig completion model for this id.
///
/// Unlike the Responses API, the Chat Completions API does not coerce tools
@@ -74,6 +79,11 @@ impl<'a> OpenAiResponsesModel<'a> {
Self { model, client }
}
+ /// The routed id this model was bound to.
+ pub fn model(&self) -> &Model<'a> {
+ &self.model
+ }
+
/// The rig Responses API completion model for this id.
///
/// Rig maps the generic `max_tokens` agent setting to the Responses API's
diff --git a/crates/agent/src/model/router.rs b/crates/agent/src/model/router.rs
index ddc856fbe7..fad7b8daa1 100644
--- a/crates/agent/src/model/router.rs
+++ b/crates/agent/src/model/router.rs
@@ -29,6 +29,7 @@ use rig_core::completion::{CompletionModel, GetTokenUsage};
use rig_core::message::Message;
use rig_core::providers::{anthropic, openai};
use rig_core::streaming::StreamedAssistantContent;
+use tracing::Instrument as _;
use super::PredefinedModel;
use super::anthropic::AnthropicModel;
@@ -37,6 +38,7 @@ use super::types::Model;
use crate::error::AgentError;
use crate::hook::{BridgeInputs, StreamBridge};
use crate::stream::{ChatCompletionStream, StreamPart};
+use crate::telemetry::{ChatSpanHook, GenAiContext, TracedModel};
env_var! {
struct ApiKeys {
@@ -67,6 +69,24 @@ pub(crate) enum RoutedModel<'a> {
}
impl<'a> RoutedModel<'a> {
+ /// The provider segment of the routed id (`anthropic`, `openai`, …).
+ pub(crate) fn provider(&self) -> &str {
+ match self {
+ RoutedModel::Anthropic(m) => m.model().provider(),
+ RoutedModel::OpenAiChatCompletions(m) => m.model().provider(),
+ RoutedModel::OpenAiResponses(m) => m.model().provider(),
+ }
+ }
+
+ /// The bare model name sent to the provider.
+ pub(crate) fn model_name(&self) -> &str {
+ match self {
+ RoutedModel::Anthropic(m) => m.model().name(),
+ RoutedModel::OpenAiChatCompletions(m) => m.model().name(),
+ RoutedModel::OpenAiResponses(m) => m.model().name(),
+ }
+ }
+
/// Build the rig agent for this model, applying provider-specific thinking
/// config. Pure construction — no model call is made here.
pub(crate) fn into_agent(
@@ -75,6 +95,7 @@ impl<'a> RoutedModel<'a> {
system_prompt: &str,
max_turns: usize,
max_tokens: u64,
+ telemetry: &GenAiContext,
) -> ProviderAgent {
match self {
RoutedModel::Anthropic(m) => {
@@ -86,6 +107,7 @@ impl<'a> RoutedModel<'a> {
system_prompt,
max_turns,
max_tokens,
+ telemetry,
))
}
RoutedModel::OpenAiChatCompletions(m) => {
@@ -97,6 +119,7 @@ impl<'a> RoutedModel<'a> {
system_prompt,
max_turns,
max_tokens,
+ telemetry,
))
}
RoutedModel::OpenAiResponses(m) => {
@@ -108,6 +131,7 @@ impl<'a> RoutedModel<'a> {
system_prompt,
max_turns,
max_tokens,
+ telemetry,
))
}
}
@@ -122,11 +146,11 @@ impl<'a> RoutedModel<'a> {
/// [`run_stream`]: ProviderAgent::run_stream
pub(crate) enum ProviderAgent {
/// An agent over Anthropic's native completion model.
- Anthropic(Agent<anthropic::completion::CompletionModel>),
+ Anthropic(Agent<TracedModel<anthropic::completion::CompletionModel>>),
/// An agent over the OpenAI Chat Completions model.
- OpenAiChatCompletions(Agent<openai::completion::CompletionModel>),
+ OpenAiChatCompletions(Agent<TracedModel<openai::completion::CompletionModel>>),
/// An agent over the OpenAI Responses model.
- OpenAiResponses(Agent<openai::responses_api::ResponsesCompletionModel>),
+ OpenAiResponses(Agent<TracedModel<openai::responses_api::ResponsesCompletionModel>>),
/// A test-only agent over an arbitrary completion model (e.g. a scripted
/// fake), type-erased so the enum itself stays non-generic.
#[cfg(test)]
@@ -147,6 +171,7 @@ impl ProviderAgent {
usage_ctx: UsageContext,
model: String,
request_context: RequestContext,
+ telemetry: GenAiContext,
) -> ChatCompletionStream<'static> {
match self {
ProviderAgent::Anthropic(agent) => {
@@ -160,6 +185,7 @@ impl ProviderAgent {
usage_ctx,
model,
request_context.clone(),
+ telemetry,
)
.await
}
@@ -174,6 +200,7 @@ impl ProviderAgent {
usage_ctx,
model,
request_context.clone(),
+ telemetry,
)
.await
}
@@ -188,6 +215,7 @@ impl ProviderAgent {
usage_ctx,
model,
request_context.clone(),
+ telemetry,
)
.await
}
@@ -203,6 +231,7 @@ impl ProviderAgent {
usage_ctx,
model,
request_context.clone(),
+ telemetry,
)
.await
}
@@ -306,7 +335,9 @@ impl ModelRouter {
}
/// Route + build the agent in one step, falling back to the default model on
- /// an unroutable id.
+ /// an unroutable id. Tells `telemetry` which provider and model the session
+ /// actually runs on, so its spans report the routed model, not the
+ /// requested id.
pub(crate) fn agent(
&self,
model: &str,
@@ -314,9 +345,11 @@ impl ModelRouter {
system_prompt: &str,
max_turns: usize,
max_tokens: u64,
+ telemetry: GenAiContext,
) -> ProviderAgent {
- self.route_or_default(model)
- .into_agent(handle, system_prompt, max_turns, max_tokens)
+ let routed = self.route_or_default(model);
+ telemetry.set_model(routed.provider(), routed.model_name());
+ routed.into_agent(handle, system_prompt, max_turns, max_tokens, &telemetry)
}
/// Route a `provider/model` id to the provider that serves it.
@@ -366,6 +399,10 @@ impl ModelRouter {
}
/// Build a rig agent from a completion model and per-session config.
+///
+/// The model is wrapped in [`TracedModel`] so every model call records its
+/// request on the `chat` span. rig's own content recording stays off — it is
+/// unbounded; `crate::telemetry` records bounded content instead.
fn build_agent<M: CompletionModel>(
model: M,
thinking: Option<serde_json::Value>,
@@ -373,8 +410,11 @@ fn build_agent<M: CompletionModel>(
system_prompt: &str,
max_turns: usize,
max_tokens: u64,
-) -> Agent<M> {
- let mut builder = AgentBuilder::new(model)
+ telemetry: &GenAiContext,
+) -> Agent<TracedModel<M>> {
+ let mut builder = AgentBuilder::new(TracedModel::new(model, telemetry.clone()))
+ .name(telemetry.agent_name())
+ .record_content_telemetry(false)
.tool_server_handle(handle)
.default_max_turns(max_turns)
.max_tokens(max_tokens)
@@ -398,11 +438,18 @@ async fn drive_stream<M>(
usage_ctx: UsageContext,
model: String,
request_context: RequestContext,
+ telemetry: GenAiContext,
) -> ChatCompletionStream<'static>
where
M: CompletionModel + 'static,
M::StreamingResponse: GetTokenUsage + Send + Sync,
{
+ // The caller's `invoke_agent` span (see `Session::send_message`): rig adopts
+ // it as the run's agent span but never records onto a span it did not
+ // open, so the run's input, output and usage are recorded here.
+ let agent_span = tracing::Span::current();
+ telemetry.record_agent_input(&agent_span, &prompt);
+
let (bridge, mut rx) = StreamBridge::channel(
inputs,
request_context.searchable_tools.clone(),
@@ -419,6 +466,7 @@ where
.max_turns(max_turns)
.max_invalid_tool_call_retries(crate::hook::MAX_INVALID_TOOL_CALL_RETRIES)
.add_hook(bridge)
+ .add_hook(ChatSpanHook(telemetry.clone()))
.await;
// Drive the rig stream on its own task. The hook emits a tool call the
@@ -430,49 +478,113 @@ where
// to the client as soon as it is produced — so a tool call renders in its
// pending state immediately and its response renders when execution
// finishes.
- let driver = tokio::spawn(async move {
- let mut thinking_buf = String::new();
-
- while let Some(item) = rig_stream.next().await {
- match item {
- Ok(MultiTurnStreamItem::StreamAssistantItem(
- StreamedAssistantContent::ReasoningDelta { reasoning, .. },
- )) => {
- thinking_buf.push_str(&reasoning);
- }
- other => {
- if !thinking_buf.is_empty() {
- let _ = driver_tx
- .send(Ok(StreamPart::Thinking(std::mem::take(&mut thinking_buf))));
+ // Whatever ends the driver - the stream running dry, a provider error, an
+ // abort when the consumer drops the stream - the model call's parked
+ // `chat` span is released with it (see `GenAiContext::finish_run`), and a
+ // run that never reached its final response or an error is recorded as
+ // cancelled: the consumer went away, and the run went with it.
+ struct FinishRun {
+ telemetry: GenAiContext,
+ agent_span: tracing::Span,
+ concluded: bool,
+ }
+ impl Drop for FinishRun {
+ fn drop(&mut self) {
+ if !self.concluded {
+ self.telemetry.record_agent_failure(
+ &self.agent_span,
+ "cancelled",
+ genai_telemetry::attr::finish_reason::CANCELLED,
+ "the run was cancelled before the agent answered",
+ );
+ }
+ self.telemetry.finish_run();
+ }
+ }
+ let mut finish_run = FinishRun {
+ telemetry: telemetry.clone(),
+ agent_span: agent_span.clone(),
+ concluded: false,
+ };
+ let driver_span = agent_span.clone();
+ let driver = tokio::spawn(
+ async move {
+ let mut thinking_buf = String::new();
+
+ while let Some(item) = rig_stream.next().await {
+ match item {
+ Ok(MultiTurnStreamItem::StreamAssistantItem(
+ StreamedAssistantContent::ReasoningDelta { reasoning, .. },
+ )) => {
+ thinking_buf.push_str(&reasoning);
}
- match other {
- Ok(MultiTurnStreamItem::FinalResponse(final_resp)) => {
- let usage = final_resp.usage;
- // Best-effort cost logging; never fails the stream.
- recorder.record(usage_ctx.clone().into_event(
- model.clone(),
- usage.input_tokens,
- usage.output_tokens,
- ));
- let _ = driver_tx.send(Ok(StreamPart::Usage(crate::stream::Usage {
- input_tokens: usage.input_tokens,
- output_tokens: usage.output_tokens,
- })));
+ other => {
+ if !thinking_buf.is_empty() {
+ let _ = driver_tx
+ .send(Ok(StreamPart::Thinking(std::mem::take(&mut thinking_buf))));
}
- Err(e) => {
- let _ = driver_tx.send(Err(AgentError::Streaming(e)));
+ match other {
+ Ok(MultiTurnStreamItem::FinalResponse(final_resp)) => {
+ let usage = final_resp.usage;
+ finish_run.concluded = true;
+ telemetry.record_agent_output(
+ &agent_span,
+ &final_resp.output,
+ &usage,
+ );
+ // Best-effort cost logging; never fails the stream.
+ recorder.record(usage_ctx.clone().into_event(
+ model.clone(),
+ usage.input_tokens,
+ usage.output_tokens,
+ ));
+ let _ =
+ driver_tx.send(Ok(StreamPart::Usage(crate::stream::Usage {
+ input_tokens: usage.input_tokens,
+ output_tokens: usage.output_tokens,
+ })));
+ }
+ Err(e) => {
+ finish_run.concluded = true;
+ let error = AgentError::Streaming(e);
+ if error.was_cancelled() {
+ // The caller stopped the run through its
+ // cancellation token: a stop, not a fault.
+ telemetry.record_agent_failure(
+ &agent_span,
+ "cancelled",
+ genai_telemetry::attr::finish_reason::CANCELLED,
+ "the run was cancelled",
+ );
+ } else {
+ // A provider error, or the runtime giving
+ // up (retries exhausted): the run failed.
+ telemetry.record_agent_failure(
+ &agent_span,
+ "streaming_error",
+ genai_telemetry::attr::finish_reason::ERROR,
+ &error.to_string(),
+ );
+ }
+ let _ = driver_tx.send(Err(error));
+ }
+ _ => {}
}
- _ => {}
}
}
}
+ if !thinking_buf.is_empty() {
+ let _ = driver_tx.send(Ok(StreamPart::Thinking(std::mem::take(&mut thinking_buf))));
+ }
+ // Dropping `rig_stream` (and with it the hook's sender) plus `driver_tx`
+ // here closes the channel, ending the consumer stream below.
+ drop(finish_run);
}
- if !thinking_buf.is_empty() {
- let _ = driver_tx.send(Ok(StreamPart::Thinking(std::mem::take(&mut thinking_buf))));
- }
- // Dropping `rig_stream` (and with it the hook's sender) plus `driver_tx`
- // here closes the channel, ending the consumer stream below.
- });
+ // Entered into the agent span: the runtime opens its `chat` and
+ // `execute_tool` spans from inside this task, and they belong under
+ // the run, not at the root of a trace of their own.
+ .instrument(driver_span),
+ );
// Abort the driver when the consumer drops the returned stream (e.g. on
// cancellation), which drops `rig_stream` and cancels any in-flight tool —
@@ -511,6 +623,7 @@ pub(crate) trait DynStreamAgent: Send + Sync {
usage_ctx: UsageContext,
model: String,
request_context: RequestContext,
+ telemetry: GenAiContext,
) -> std::pin::Pin<
Box<dyn std::future::Future<Output = ChatCompletionStream<'static>> + Send + 'a>,
>;
@@ -532,6 +645,7 @@ where
usage_ctx: UsageContext,
model: String,
request_context: RequestContext,
+ telemetry: GenAiContext,
) -> std::pin::Pin<
Box<dyn std::future::Future<Output = ChatCompletionStream<'static>> + Send + 'a>,
> {
@@ -545,6 +659,7 @@ where
usage_ctx,
model,
request_context,
+ telemetry,
))
}
}
@@ -559,11 +674,13 @@ impl ProviderAgent {
max_turns: usize,
max_tokens: u64,
handle: ToolServerHandle,
+ telemetry: GenAiContext,
) -> Self
where
M: CompletionModel + 'static,
M::StreamingResponse: GetTokenUsage + Send + Sync,
{
+ telemetry.set_model("test", "fake-model");
ProviderAgent::Test(Box::new(build_agent(
model,
None,
@@ -571,6 +688,7 @@ impl ProviderAgent {
system_prompt,
max_turns,
max_tokens,
+ &telemetry,
)))
}
}
diff --git a/crates/agent/src/telemetry.rs b/crates/agent/src/telemetry.rs
new file mode 100644
index 0000000000..d79042fcc3
--- /dev/null
+++ b/crates/agent/src/telemetry.rs
@@ -0,0 +1,558 @@
+//! GenAI span enrichment for the agent loop.
+//!
+//! rig opens the OpenTelemetry GenAI spans — `invoke_agent` (adopting ours, see
+//! [`crate::Session::send_message`]), one `chat` span per model call and one
+//! `execute_tool` span per tool call — and records the structural fields:
+//! provider, model, response id, token usage, tool name and call id. Its own
+//! content recording (`record_content_telemetry`) stays **off**: it is
+//! unbounded, so every `chat` span would carry the whole conversation so far.
+//! This module records what Datadog's evaluations need on top, under the
+//! content policy and size bounds of [`genai_telemetry`]:
+//!
+//! - on each `chat` span: the tool definitions offered (`gen_ai.tool.definitions`,
+//! what a tool-selection judge grades against), the input messages, the
+//! system instructions, request parameters, the conversation id and — once
+//! the turn finishes — the output messages and finish reason;
+//! - on the `invoke_agent` span: the run's token usage, its input (the prompt)
+//! and its output (the final answer), which a session-level judge reads.
+//!
+//! [`TracedModel`] sits between rig and the provider. rig instruments the
+//! model call with the `chat` span, so inside `completion` / `stream` the
+//! current span *is* that span: the request side is recorded there and the
+//! span is parked in the shared [`GenAiContext`]. The response side arrives
+//! through rig's per-turn hook ([`ChatSpanHook`]) after the provider stream
+//! ends, when the current span is the agent span again.
+
+#[cfg(test)]
+mod test;
+
+use std::sync::{Arc, Mutex, OnceLock};
+
+use genai_telemetry::messages::{self, MediaSource};
+use genai_telemetry::{
+ ContentPolicy, GenAiSpanExt as _, attr, bound_messages, bound_tool_definitions,
+};
+use rig_agent::agent::hook::{AgentHook, HookContext, ModelTurnAction, ModelTurnFinished};
+use rig_core::OneOrMany;
+use rig_core::completion::{
+ CompletionError, CompletionModel, CompletionRequest, CompletionResponse, ToolDefinition, Usage,
+};
+use rig_core::message::{
+ AssistantContent, DocumentSourceKind, Message, MimeType, Reasoning, ReasoningContent,
+ ToolResultContent, UserContent,
+};
+use rig_core::streaming::StreamingCompletionResponse;
+
+/// Session-scoped telemetry state shared by the traced model, the hooks and
+/// the agent span. Cheap to clone; all clones share one state.
+#[derive(Clone, Default)]
+pub(crate) struct GenAiContext(Arc<Inner>);
+
+#[derive(Default)]
+struct Inner {
+ /// The session (chat) every span of this run belongs to.
+ conversation_id: Option<String>,
+ /// Label for the agent, e.g. the AI feature driving it.
+ agent_name: String,
+ policy: ContentPolicy,
+ /// Whether anything is recorded at all. Off, every `record_*` is a no-op
+ /// and the model call's span is never parked.
+ enabled: bool,
+ /// The routed `(provider, model)`, set once routing has decided.
+ model: OnceLock<(String, String)>,
+ /// The `chat` span of the model call in flight: parked by
+ /// [`TracedModel`] on request, consumed by [`ChatSpanHook`] on response.
+ /// Holding the handle keeps the span open, so it is always taken out again.
+ chat_span: Mutex<Option<tracing::Span>>,
+}
+
+impl GenAiContext {
+ pub(crate) fn new(
+ conversation_id: Option<String>,
+ agent_name: String,
+ policy: ContentPolicy,
+ enabled: bool,
+ ) -> Self {
+ Self(Arc::new(Inner {
+ conversation_id,
+ agent_name,
+ policy,
+ enabled,
+ model: OnceLock::new(),
+ chat_span: Mutex::new(None),
+ }))
+ }
+
+ /// Whether this context records anything (see
+ /// [`crate::AgentLoop::with_genai_telemetry`]).
+ pub(crate) fn enabled(&self) -> bool {
+ self.0.enabled
+ }
+
+ pub(crate) fn conversation_id(&self) -> Option<&str> {
+ self.0.conversation_id.as_deref()
+ }
+
+ pub(crate) fn agent_name(&self) -> &str {
+ &self.0.agent_name
+ }
+
+ /// Record the provider and model routing settled on. First call wins.
+ pub(crate) fn set_model(&self, provider: &str, model: &str) {
+ let _ = self.0.model.set((provider.to_owned(), model.to_owned()));
+ }
+
+ pub(crate) fn provider_name(&self) -> Option<&str> {
+ self.0.model.get().map(|(provider, _)| provider.as_str())
+ }
+
+ pub(crate) fn model_name(&self) -> Option<&str> {
+ self.0.model.get().map(|(_, model)| model.as_str())
+ }
+
+ /// Record the request side of a model call on the current span (rig's
+ /// `chat` span when called from [`TracedModel`]) and park that span for
+ /// [`Self::record_response`].
+ fn record_request(&self, request: &CompletionRequest) {
+ if !self.0.enabled {
+ return;
+ }
+ let span = tracing::Span::current();
+ let policy = &self.0.policy;
+
+ if let Some(conversation_id) = self.conversation_id() {
+ span.set_str(attr::CONVERSATION_ID, conversation_id);
+ }
+ if let Some(max_tokens) = request.max_tokens {
+ span.set_u64(attr::REQUEST_MAX_TOKENS, max_tokens);
+ }
+ if let Some(temperature) = request.temperature {
+ span.set_f64(attr::REQUEST_TEMPERATURE, temperature);
+ }
+ // Tool definitions are our own code, not user content: always recorded.
+ if !request.tools.is_empty() {
+ let definitions = request.tools.iter().map(tool_definition_json).collect();
+ let (json, truncated) = bound_tool_definitions(definitions, &policy.limits);
+ span.set_str(attr::TOOL_DEFINITIONS, json);
+ if truncated {
+ span.set_bool(attr::MACRO_CONTENT_TRUNCATED, true);
+ }
+ }
+ if policy.capture {
+ let (system, input) = split_system(request);
+ if let Some(system) = system {
+ let bounded = bound_messages(messages::system_instructions(system), &policy.limits);
+ span.set_str(attr::SYSTEM_INSTRUCTIONS, bounded.json);
+ if bounded.truncated {
+ span.set_bool(attr::MACRO_CONTENT_TRUNCATED, true);
+ }
+ }
+ let bounded = bound_messages(input, &policy.limits);
+ span.set_str(attr::INPUT_MESSAGES, bounded.json);
+ if bounded.omitted > 0 {
+ span.set_u64(attr::MACRO_INPUT_MESSAGES_OMITTED, bounded.omitted as u64);
+ }
+ if bounded.truncated {
+ span.set_bool(attr::MACRO_CONTENT_TRUNCATED, true);
+ }
+ }
+
+ *self.0.chat_span.lock().expect("chat span slot poisoned") = Some(span);
+ }
+
+ /// Record the response side of the model call whose request was recorded
+ /// last, then release the parked span so it can close.
+ fn record_response(&self, content: &OneOrMany<AssistantContent>) {
+ let Some(span) = self
+ .0
+ .chat_span
+ .lock()
+ .expect("chat span slot poisoned")
+ .take()
+ else {
+ return;
+ };
+ let finish_reason = finish_reason(content);
+ span.set_str_array(attr::RESPONSE_FINISH_REASONS, [finish_reason]);
+ if self.0.policy.capture {
+ let message = messages::output_message(
+ messages::ROLE_ASSISTANT,
+ assistant_parts(content),
+ finish_reason,
+ );
+ let bounded = bound_messages(vec![message], &self.0.policy.limits);
+ span.set_str(attr::OUTPUT_MESSAGES, bounded.json);
+ if bounded.truncated {
+ span.set_bool(attr::MACRO_CONTENT_TRUNCATED, true);
+ }
+ }
+ }
+
+ /// Record that an agent run failed: `error_type` names the class of
+ /// failure, `finish_reason` how the run stopped (`error`, `cancelled`) and
+ /// `detail` the message, which is recorded bounded when content may be
+ /// captured and left out otherwise - a provider's error text can quote the
+ /// request.
+ ///
+ /// A model call still in flight - its `chat` span parked, its turn hook
+ /// never to fire - failed the same way: the span is marked and released,
+ /// so a failed call never reads as a successful one with no response.
+ pub(crate) fn record_agent_failure(
+ &self,
+ agent_span: &tracing::Span,
+ error_type: &str,
+ finish_reason: &'static str,
+ detail: &str,
+ ) {
+ if !self.0.enabled {
+ return;
+ }
+ let description = self.failure_description(error_type, detail);
+ agent_span.set_str_array(attr::RESPONSE_FINISH_REASONS, [finish_reason]);
+ agent_span.set_error(error_type, description.clone());
+ self.fail_parked_chat(error_type, finish_reason, description);
+ }
+
+ /// Record that the model call whose request was recorded last failed
+ /// before it produced a response: the provider refused or errored. The
+ /// parked `chat` span is marked and released. Called by [`TracedModel`]
+ /// on every path - a one-shot completion has no run driver to do it.
+ fn record_chat_failure(&self, error: &CompletionError) {
+ if !self.0.enabled {
+ return;
+ }
+ let description = self.failure_description("provider_error", &error.to_string());
+ self.fail_parked_chat("provider_error", attr::finish_reason::ERROR, description);
+ }
+
+ /// The description a failure's span status carries: the detail, bounded,
+ /// when content may be recorded - an error message can quote the request
+ /// - and a fixed summary otherwise.
+ fn failure_description(&self, error_type: &str, detail: &str) -> String {
+ if self.0.policy.capture {
+ genai_telemetry::truncate_chars(detail, self.0.policy.limits.max_part_chars)
+ .into_owned()
+ } else {
+ format!("the call ended with {error_type}")
+ }
+ }
+
+ /// Mark the parked `chat` span, if any, as ended by `error_type` and
+ /// release it. A no-op when no model call is in flight.
+ fn fail_parked_chat(&self, error_type: &str, finish_reason: &'static str, description: String) {
+ if let Some(chat_span) = self
+ .0
+ .chat_span
+ .lock()
+ .expect("chat span slot poisoned")
+ .take()
+ {
+ chat_span.set_str_array(attr::RESPONSE_FINISH_REASONS, [finish_reason]);
+ chat_span.set_error(error_type, description);
+ }
+ }
+
+ /// Release the parked `chat` span, if any, so it closes now. Called when
+ /// a run ends without its turn hook firing and without a recorded failure;
+ /// a no-op when the hook (or `record_agent_failure`) already released it.
+ pub(crate) fn finish_run(&self) {
+ self.0
+ .chat_span
+ .lock()
+ .expect("chat span slot poisoned")
+ .take();
+ }
+
+ /// Record the prompt of an agent run as the agent span's input.
+ pub(crate) fn record_agent_input(&self, agent_span: &tracing::Span, prompt: &Message) {
+ if !self.0.enabled || !self.0.policy.capture {
+ return;
+ }
+ let bounded = bound_messages(message_json(prompt), &self.0.policy.limits);
+ agent_span.set_str(attr::INPUT_MESSAGES, bounded.json);
+ if bounded.truncated {
+ agent_span.set_bool(attr::MACRO_CONTENT_TRUNCATED, true);
+ }
+ }
+
+ /// Record the final answer and aggregate usage of an agent run on the
+ /// agent span. Usage is structure and always recorded; the answer follows
+ /// the content policy.
+ pub(crate) fn record_agent_output(
+ &self,
+ agent_span: &tracing::Span,
+ output: &str,
+ usage: &Usage,
+ ) {
+ if !self.0.enabled {
+ return;
+ }
+ agent_span.set_u64(attr::USAGE_INPUT_TOKENS, usage.input_tokens);
+ agent_span.set_u64(attr::USAGE_OUTPUT_TOKENS, usage.output_tokens);
+ agent_span.set_str_array(attr::RESPONSE_FINISH_REASONS, [attr::finish_reason::STOP]);
+ if !self.0.policy.capture {
+ return;
+ }
+ let message = messages::output_message(
+ messages::ROLE_ASSISTANT,
+ vec![messages::text_part(output)],
+ attr::finish_reason::STOP,
+ );
+ let bounded = bound_messages(vec![message], &self.0.policy.limits);
+ agent_span.set_str(attr::OUTPUT_MESSAGES, bounded.json);
+ if bounded.truncated {
+ agent_span.set_bool(attr::MACRO_CONTENT_TRUNCATED, true);
+ }
+ }
+}
+
+/// A [`CompletionModel`] that records the request side of every model call
+/// on the current (`chat`) span before delegating to the provider model.
+#[derive(Clone)]
+pub(crate) struct TracedModel<M> {
+ inner: M,
+ telemetry: GenAiContext,
+}
+
+impl<M> TracedModel<M> {
+ pub(crate) fn new(inner: M, telemetry: GenAiContext) -> Self {
+ Self { inner, telemetry }
+ }
+}
+
+impl<M: CompletionModel> CompletionModel for TracedModel<M> {
+ type Response = M::Response;
+ type StreamingResponse = M::StreamingResponse;
+ type Client = M::Client;
+
+ fn make(client: &Self::Client, model: impl Into<String>) -> Self {
+ Self {
+ inner: M::make(client, model),
+ telemetry: GenAiContext::default(),
+ }
+ }
+
+ async fn completion(
+ &self,
+ request: CompletionRequest,
+ ) -> Result<CompletionResponse<Self::Response>, CompletionError> {
+ self.telemetry.record_request(&request);
+ let result = self.inner.completion(request).await;
+ if let Err(error) = &result {
+ self.telemetry.record_chat_failure(error);
+ }
+ result
+ }
+
+ async fn stream(
+ &self,
+ request: CompletionRequest,
+ ) -> Result<StreamingCompletionResponse<Self::StreamingResponse>, CompletionError> {
+ self.telemetry.record_request(&request);
+ let result = self.inner.stream(request).await;
+ if let Err(error) = &result {
+ self.telemetry.record_chat_failure(error);
+ }
+ result
+ }
+
+ fn composes_native_output_with_tools(&self) -> bool {
+ self.inner.composes_native_output_with_tools()
+ }
+}
+
+/// rig hook completing the `chat` span [`TracedModel`] parked: records the
+/// model's output and finish reason once the turn's response is in.
+///
+/// `ModelTurnFinished` is the one per-turn event both of rig's surfaces fire
+/// for every accepted turn, tool-only turns included (`StreamResponseFinish`
+/// skips those). A turn rig recovers from an invalid tool call fires nothing;
+/// its span is released, without output, when the next call parks its own.
+#[derive(Clone)]
+pub(crate) struct ChatSpanHook(pub(crate) GenAiContext);
+
+impl AgentHook for ChatSpanHook {
+ async fn on_model_turn_finished(
+ &self,
+ _ctx: &HookContext,
+ event: ModelTurnFinished<'_>,
+ ) -> ModelTurnAction {
+ self.0.record_response(event.content);
+ ModelTurnAction::Continue
+ }
+}
+
+/// Why the model stopped, as far as rig's normalized content tells: a turn
+/// with tool calls stopped to call them, anything else is a natural stop
+/// (rig does not retain a provider's `length` / `content_filter` reasons).
+fn finish_reason(content: &OneOrMany<AssistantContent>) -> &'static str {
+ if content
+ .iter()
+ .any(|content| matches!(content, AssistantContent::ToolCall(_)))
+ {
+ attr::finish_reason::TOOL_CALL
+ } else {
+ attr::finish_reason::STOP
+ }
+}
+
+/// Separate the system instructions (the preamble and any `System` messages,
+/// which rig prepends to the history) from the chat messages.
+fn split_system(request: &CompletionRequest) -> (Option<String>, Vec<serde_json::Value>) {
+ let mut system: Vec<String> = request.preamble.iter().cloned().collect();
+ let mut input = Vec::new();
+ for message in request.chat_history.iter() {
+ match message {
+ Message::System { content } => system.push(content.clone()),
+ other => input.extend(message_json(other)),
+ }
+ }
+ let system = (!system.is_empty()).then(|| system.join("\n\n"));
+ (system, input)
+}
+
+fn tool_definition_json(definition: &ToolDefinition) -> serde_json::Value {
+ messages::tool_definition(
+ &definition.name,
+ &definition.description,
+ definition.parameters.clone(),
+ )
+}
+
+/// One rig message as semconv messages. A user message carrying tool results
+/// yields a `tool` message for them (and a `user` message for anything else),
+/// matching the semconv role split.
+fn message_json(message: &Message) -> Vec<serde_json::Value> {
+ match message {
+ Message::System { content } => vec![messages::message(
+ messages::ROLE_SYSTEM,
+ vec![messages::text_part(content.clone())],
+ )],
+ Message::User { content } => {
+ let (tool_parts, user_parts): (Vec<_>, Vec<_>) = content
+ .iter()
+ .partition(|content| matches!(content, UserContent::ToolResult(_)));
+ let mut out = Vec::new();
+ if !tool_parts.is_empty() {
+ out.push(messages::message(
+ messages::ROLE_TOOL,
+ tool_parts.into_iter().map(user_part).collect(),
+ ));
+ }
+ if !user_parts.is_empty() {
+ out.push(messages::message(
+ messages::ROLE_USER,
+ user_parts.into_iter().map(user_part).collect(),
+ ));
+ }
+ out
+ }
+ Message::Assistant { content, .. } => vec![messages::message(
+ messages::ROLE_ASSISTANT,
+ assistant_parts(content),
+ )],
+ }
+}
+
+fn user_part(content: &UserContent) -> serde_json::Value {
+ match content {
+ UserContent::Text(text) => messages::text_part(text.text.clone()),
+ UserContent::ToolResult(result) => {
+ messages::tool_call_response_part(Some(&result.id), tool_result_json(&result.content))
+ }
+ UserContent::Image(image) => media_part(
+ "image",
+ image.media_type.as_ref().map(MimeType::to_mime_type),
+ &image.data,
+ ),
+ UserContent::Audio(audio) => media_part(
+ "audio",
+ audio.media_type.as_ref().map(MimeType::to_mime_type),
+ &audio.data,
+ ),
+ UserContent::Video(video) => media_part(
+ "video",
+ video.media_type.as_ref().map(MimeType::to_mime_type),
+ &video.data,
+ ),
+ UserContent::Document(document) => media_part(
+ "document",
+ document.media_type.as_ref().map(MimeType::to_mime_type),
+ &document.data,
+ ),
+ }
+}
+
+/// A tool result as the `response` of a `tool_call_response` part: a lone
+/// text or JSON item as itself, several items as an array.
+fn tool_result_json(content: &OneOrMany<ToolResultContent>) -> serde_json::Value {
+ let mut items: Vec<serde_json::Value> = content
+ .iter()
+ .map(|item| match item {
+ ToolResultContent::Text(text) => serde_json::Value::String(text.text.clone()),
+ ToolResultContent::Json { value } => value.clone(),
+ ToolResultContent::Image(image) => media_part(
+ "image",
+ image.media_type.as_ref().map(MimeType::to_mime_type),
+ &image.data,
+ ),
+ })
+ .collect();
+ if items.len() == 1 {
+ items.remove(0)
+ } else {
+ serde_json::Value::Array(items)
+ }
+}
+
+fn assistant_parts(content: &OneOrMany<AssistantContent>) -> Vec<serde_json::Value> {
+ content
+ .iter()
+ .filter_map(|content| match content {
+ AssistantContent::Text(text) => Some(messages::text_part(text.text.clone())),
+ AssistantContent::ToolCall(call) => Some(messages::tool_call_part(
+ Some(&call.id),
+ &call.function.name,
+ call.function.arguments.clone(),
+ )),
+ AssistantContent::Reasoning(reasoning) => {
+ reasoning_text(reasoning).map(messages::reasoning_part)
+ }
+ AssistantContent::Image(image) => Some(media_part(
+ "image",
+ image.media_type.as_ref().map(MimeType::to_mime_type),
+ &image.data,
+ )),
+ })
+ .collect()
+}
+
+/// The readable text of a reasoning block; `None` when it is entirely
+/// encrypted or redacted.
+fn reasoning_text(reasoning: &Reasoning) -> Option<String> {
+ let text: Vec<&str> = reasoning
+ .content
+ .iter()
+ .filter_map(|content| match content {
+ ReasoningContent::Text { text, .. } => Some(text.as_str()),
+ ReasoningContent::Summary(summary) => Some(summary.as_str()),
+ // Encrypted, redacted, or a variant added later: nothing readable.
+ _ => None,
+ })
+ .collect();
+ (!text.is_empty()).then(|| text.join("\n"))
+}
+
+fn media_part(
+ modality: &str,
+ mime_type: Option<&str>,
+ data: &DocumentSourceKind,
+) -> serde_json::Value {
+ let source = match data {
+ DocumentSourceKind::Url(uri) => MediaSource::Uri(uri.clone()),
+ DocumentSourceKind::FileId(file_id) => MediaSource::FileId(file_id.clone()),
+ // Base64, raw bytes, inline strings: never copied onto a span.
+ _ => MediaSource::Inline,
+ };
+ messages::media_part(modality, mime_type, source)
+}
diff --git a/crates/agent/src/telemetry/test.rs b/crates/agent/src/telemetry/test.rs
new file mode 100644
index 0000000000..f5b5153fdd
--- /dev/null
+++ b/crates/agent/src/telemetry/test.rs
@@ -0,0 +1,126 @@
+//! Unit tests for the rig → semconv message encoding.
+use super::*;
+use rig_core::message::{Text, ToolCall, ToolFunction, ToolResult};
+use serde_json::json;
+
+#[test]
+fn user_text_becomes_a_user_message_with_a_text_part() {
+ let encoded = message_json(&Message::user("hello"));
+ assert_eq!(
+ encoded,
+ vec![json!({"role": "user", "parts": [{"type": "text", "content": "hello"}]})]
+ );
+}
+
+#[test]
+fn tool_results_become_a_tool_message() {
+ let message = Message::User {
+ content: OneOrMany::many([
+ UserContent::ToolResult(ToolResult {
+ id: "call-1".to_string(),
+ call_id: None,
+ content: OneOrMany::one(ToolResultContent::Json {
+ value: json!({ "echo": "a" }),
+ }),
+ }),
+ UserContent::Text(Text {
+ text: "and more".to_string(),
+ additional_params: None,
+ }),
+ ])
+ .expect("two items"),
+ };
+ let encoded = message_json(&message);
+ assert_eq!(
+ encoded,
+ vec![
+ json!({"role": "tool", "parts": [{"type": "tool_call_response", "id": "call-1", "response": {"echo": "a"}}]}),
+ json!({"role": "user", "parts": [{"type": "text", "content": "and more"}]}),
+ ]
+ );
+}
+
+#[test]
+fn assistant_tool_calls_and_text_are_parts_and_decide_the_finish_reason() {
+ let content = OneOrMany::many([
+ AssistantContent::Text(Text {
+ text: "let me check".to_string(),
+ additional_params: None,
+ }),
+ AssistantContent::ToolCall(ToolCall {
+ id: "call-1".to_string(),
+ call_id: None,
+ function: ToolFunction {
+ name: "echo_tool".to_string(),
+ arguments: json!({ "value": "a" }),
+ },
+ signature: None,
+ additional_params: None,
+ }),
+ ])
+ .expect("two items");
+ assert_eq!(
+ assistant_parts(&content),
+ vec![
+ json!({"type": "text", "content": "let me check"}),
+ json!({"type": "tool_call", "id": "call-1", "name": "echo_tool", "arguments": {"value": "a"}}),
+ ]
+ );
+ assert_eq!(finish_reason(&content), attr::finish_reason::TOOL_CALL);
+ assert_eq!(
+ finish_reason(&OneOrMany::one(AssistantContent::text("done"))),
+ attr::finish_reason::STOP
+ );
+}
+
+#[test]
+fn a_lone_text_tool_result_is_a_plain_string_and_several_items_an_array() {
+ assert_eq!(
+ tool_result_json(&OneOrMany::one(ToolResultContent::Text(Text {
+ text: "ok".to_string(),
+ additional_params: None,
+ }))),
+ json!("ok")
+ );
+ assert_eq!(
+ tool_result_json(
+ &OneOrMany::many([
+ ToolResultContent::Text(Text {
+ text: "ok".to_string(),
+ additional_params: None,
+ }),
+ ToolResultContent::Json { value: json!(1) },
+ ])
+ .expect("two items")
+ ),
+ json!(["ok", 1])
+ );
+}
+
+#[test]
+fn finishing_a_run_releases_the_parked_chat_span() {
+ use opentelemetry::trace::TracerProvider as _;
+ use opentelemetry_sdk::trace::{InMemorySpanExporter, SdkTracerProvider};
+ use tracing_subscriber::layer::SubscriberExt as _;
+
+ let exporter = InMemorySpanExporter::default();
+ let provider = SdkTracerProvider::builder()
+ .with_simple_exporter(exporter.clone())
+ .build();
+ let layer = tracing_opentelemetry::layer().with_tracer(provider.tracer("test"));
+ let _guard = tracing::subscriber::set_default(tracing_subscriber::registry().with(layer));
+ tracing::callsite::rebuild_interest_cache();
+
+ let telemetry = GenAiContext::new(None, "chat".to_owned(), ContentPolicy::enabled(), true);
+ // A model call whose turn never finished: its span sits parked.
+ *telemetry.0.chat_span.lock().unwrap() = Some(tracing::info_span!("chat"));
+ provider.force_flush().unwrap();
+ assert!(exporter.get_finished_spans().unwrap().is_empty());
+
+ telemetry.finish_run();
+ provider.force_flush().unwrap();
+ assert_eq!(exporter.get_finished_spans().unwrap().len(), 1);
+ assert!(telemetry.0.chat_span.lock().unwrap().is_none());
+ // Idempotent.
+ telemetry.finish_run();
+}
diff --git a/crates/agent/src/test/agent_loop/mod.rs b/crates/agent/src/test/agent_loop/mod.rs
index cf5a1931d4..6816d89245 100644
--- a/crates/agent/src/test/agent_loop/mod.rs
+++ b/crates/agent/src/test/agent_loop/mod.rs
@@ -7,6 +7,7 @@ mod util;
mod test_cooperative_cancellation;
mod test_eager_tools;
+mod test_telemetry;
mod test_tool;
mod test_tool_search;
mod test_user_tools;
diff --git a/crates/agent/src/test/agent_loop/test_telemetry.rs b/crates/agent/src/test/agent_loop/test_telemetry.rs
new file mode 100644
index 0000000000..f14a7f93ae
--- /dev/null
+++ b/crates/agent/src/test/agent_loop/test_telemetry.rs
@@ -0,0 +1,588 @@
+//! GenAI telemetry: what the agent loop adds to rig's `invoke_agent`, `chat`
+//! and `execute_tool` spans so an observability backend can evaluate a run
+//! (tool selection against the offered tools, goal completion across a
+//! conversation).
+use super::util;
+use ai_toolset::{
+ AsyncTool, RequestContext, ServiceContext, ToolAnnotated, ToolAnnotations, ToolResult,
+};
+use async_trait::async_trait;
+use genai_telemetry::attr;
+use opentelemetry::trace::TracerProvider as _;
+use opentelemetry::{Array, Value};
+use opentelemetry_sdk::trace::{InMemorySpanExporter, SdkTracerProvider, SpanData};
+use rig_core::test_utils::{MockCompletionModel, MockStreamEvent};
+use schemars::JsonSchema;
+use serde::Deserialize;
+use serde_json::json;
+use std::sync::Arc;
+use tracing_subscriber::layer::SubscriberExt as _;
+
+/// A tool that echoes its input.
+#[derive(Deserialize, JsonSchema)]
+#[schemars(title = "echo_tool", description = "Echoes its input back.")]
+struct EchoTool {
+ value: String,
+}
+
+impl ToolAnnotated for EchoTool {
+ const ANNOTATIONS: ToolAnnotations = ToolAnnotations::read_only("Echo");
+}
+
+#[async_trait]
+impl AsyncTool<()> for EchoTool {
+ type Output = serde_json::Value;
+
+ async fn call(
+ &self,
+ _service_context: ServiceContext<()>,
+ _request_context: RequestContext,
+ ) -> ToolResult<Self::Output> {
+ Ok(json!({ "echo": self.value }))
+ }
+}
+
+/// An OpenTelemetry pipeline that keeps finished spans in memory, installed as
+/// the thread's default subscriber for the returned guard's lifetime. The
+/// tests run on tokio's current-thread runtime, so the driver task the agent
+/// loop spawns sees the same subscriber.
+fn otel_test_pipeline() -> (
+ InMemorySpanExporter,
+ SdkTracerProvider,
+ tracing::subscriber::DefaultGuard,
+) {
+ let exporter = InMemorySpanExporter::default();
+ let provider = SdkTracerProvider::builder()
+ .with_simple_exporter(exporter.clone())
+ .build();
+ let layer = tracing_opentelemetry::layer().with_tracer(provider.tracer("test"));
+ let guard = tracing::subscriber::set_default(tracing_subscriber::registry().with(layer));
+ // Other tests in this binary run the same code paths with no subscriber
+ // installed, which can cache "never interested" for a span callsite on
+ // another thread; recompute so this subscriber sees every span.
+ tracing::callsite::rebuild_interest_cache();
+ (exporter, provider, guard)
+}
+
+fn finished(exporter: &InMemorySpanExporter, provider: &SdkTracerProvider) -> Vec<SpanData> {
+ provider.force_flush().expect("flush");
+ exporter.get_finished_spans().expect("finished spans")
+}
+
+fn attribute<'a>(span: &'a SpanData, key: &str) -> Option<&'a Value> {
+ span.attributes
+ .iter()
+ .find(|kv| kv.key.as_str() == key)
+ .map(|kv| &kv.value)
+}
+
+fn string_attribute(span: &SpanData, key: &str) -> Option<String> {
+ match attribute(span, key)? {
+ Value::String(s) => Some(s.as_str().to_string()),
+ _ => None,
+ }
+}
+
+fn int_attribute(span: &SpanData, key: &str) -> Option<i64> {
+ match attribute(span, key)? {
+ Value::I64(i) => Some(*i),
+ _ => None,
+ }
+}
+
+fn string_array_attribute(span: &SpanData, key: &str) -> Option<Vec<String>> {
+ match attribute(span, key)? {
+ Value::Array(Array::String(values)) => {
+ Some(values.iter().map(|v| v.as_str().to_string()).collect())
+ }
+ _ => None,
+ }
+}
+
+fn json_attribute(span: &SpanData, key: &str) -> Vec<serde_json::Value> {
+ let raw = string_attribute(span, key).unwrap_or_else(|| panic!("{key} on {}", span.name));
+ serde_json::from_str(&raw).unwrap_or_else(|e| panic!("{key} is not a JSON array: {e}: {raw}"))
+}
+
+/// Spans with the given `gen_ai.operation.name`, in the order they finished.
+fn spans_with_operation<'a>(spans: &'a [SpanData], operation: &str) -> Vec<&'a SpanData> {
+ spans
+ .iter()
+ .filter(|span| string_attribute(span, attr::OPERATION_NAME).as_deref() == Some(operation))
+ .collect()
+}
+
+fn names(spans: &[SpanData]) -> Vec<&str> {
+ spans.iter().map(|span| span.name.as_ref()).collect()
+}
+
+/// Every exported span with its attribute keys, for assertion messages.
+fn describe(spans: &[SpanData]) -> String {
+ spans
+ .iter()
+ .map(|span| {
+ let keys: Vec<&str> = span.attributes.iter().map(|kv| kv.key.as_str()).collect();
+ format!("{} {:?}", span.name, keys)
+ })
+ .collect::<Vec<_>>()
+ .join("\n")
+}
+
+/// One run: the model calls `echo_tool`, then answers "done".
+#[tokio::test]
+async fn a_run_records_the_attributes_evaluations_read() {
+ let (exporter, provider, _guard) = otel_test_pipeline();
+ let model = MockCompletionModel::from_stream_turns([
+ vec![
+ MockStreamEvent::tool_call("call-1", "echo_tool", json!({ "value": "a" })),
+ MockStreamEvent::final_response_with_default_usage(),
+ ],
+ vec![
+ MockStreamEvent::text("done"),
+ MockStreamEvent::final_response_with_default_usage(),
+ ],
+ ]);
+ let toolset = util::single_tool_set::<EchoTool, ()>();
+ let mut session = util::test_loop()
+ .with_conversation_id("chat-123")
+ .test_session(
+ toolset,
+ Arc::new(()),
+ "test preamble",
+ util::usage_ctx(),
+ model,
+ )
+ .await;
+
+ let result = util::drive(&mut session, "call echo").await;
+ assert_eq!(result.content(), "done");
+ drop(session);
+
+ let spans = finished(&exporter, &provider);
+
+ // The agent span: ours, adopted by rig, carrying the session and the run's
+ // input, output and usage.
+ let agents = spans_with_operation(&spans, attr::operation::INVOKE_AGENT);
+ assert_eq!(agents.len(), 1, "one agent span, got {:?}", names(&spans));
+ let agent = agents[0];
+ assert_eq!(agent.name, "invoke_agent");
+ assert_eq!(
+ string_attribute(agent, attr::CONVERSATION_ID).as_deref(),
+ Some("chat-123")
+ );
+ assert_eq!(
+ string_attribute(agent, attr::AGENT_NAME).as_deref(),
+ Some("chat"),
+ "the agent is named after the usage context's feature"
+ );
+ assert_eq!(
+ string_attribute(agent, attr::PROVIDER_NAME).as_deref(),
+ Some("test")
+ );
+ assert_eq!(
+ string_attribute(agent, attr::REQUEST_MODEL).as_deref(),
+ Some("fake-model")
+ );
+ assert!(int_attribute(agent, attr::USAGE_INPUT_TOKENS).is_some());
+ assert!(int_attribute(agent, attr::USAGE_OUTPUT_TOKENS).is_some());
+ let input = json_attribute(agent, attr::INPUT_MESSAGES);
+ assert_eq!(input[0]["role"], "user");
+ assert_eq!(input[0]["parts"][0]["content"], "call echo");
+ let output = json_attribute(agent, attr::OUTPUT_MESSAGES);
+ assert_eq!(output[0]["parts"][0]["content"], "done");
+ assert_eq!(output[0]["finish_reason"], "stop");
+
+ // One chat span per model call, each with the tool set it was offered, the
+ // history it saw, the session, and what it produced.
+ let chats = spans_with_operation(&spans, attr::operation::CHAT);
+ assert_eq!(
+ chats.len(),
+ 2,
+ "two model calls, got:\n{}",
+ describe(&spans)
+ );
+ let (first, second) = (chats[0], chats[1]);
+ for chat in &chats {
+ assert_eq!(
+ string_attribute(chat, attr::CONVERSATION_ID).as_deref(),
+ Some("chat-123")
+ );
+ assert!(int_attribute(chat, attr::REQUEST_MAX_TOKENS).is_some());
+ let definitions = json_attribute(chat, attr::TOOL_DEFINITIONS);
+ assert!(
+ definitions
+ .iter()
+ .any(|d| d["type"] == "function" && d["name"] == "echo_tool"),
+ "{definitions:?}"
+ );
+ let system = json_attribute(chat, attr::SYSTEM_INSTRUCTIONS);
+ assert!(
+ system[0]["content"]
+ .as_str()
+ .is_some_and(|s| s.contains("test preamble")),
+ "{system:?}"
+ );
+ }
+
+ let input = json_attribute(first, attr::INPUT_MESSAGES);
+ assert!(
+ input.iter().all(|m| m["role"] != "system"),
+ "system instructions stay out of the history: {input:?}"
+ );
+ assert_eq!(input.last().unwrap()["parts"][0]["content"], "call echo");
+ assert_eq!(
+ string_array_attribute(first, attr::RESPONSE_FINISH_REASONS),
+ Some(vec!["tool_call".to_string()])
+ );
+ let output = json_attribute(first, attr::OUTPUT_MESSAGES);
+ assert_eq!(output[0]["role"], "assistant");
+ assert_eq!(output[0]["finish_reason"], "tool_call");
+ assert_eq!(output[0]["parts"][0]["type"], "tool_call");
+ assert_eq!(output[0]["parts"][0]["name"], "echo_tool");
+ assert_eq!(output[0]["parts"][0]["arguments"], json!({ "value": "a" }));
+
+ let input = json_attribute(second, attr::INPUT_MESSAGES);
+ let tool_message = input
+ .iter()
+ .find(|m| m["role"] == "tool")
+ .expect("the second call sees the tool result as a tool message");
+ assert_eq!(tool_message["parts"][0]["type"], "tool_call_response");
+ assert_eq!(tool_message["parts"][0]["response"], json!({ "echo": "a" }));
+ assert_eq!(
+ string_array_attribute(second, attr::RESPONSE_FINISH_REASONS),
+ Some(vec!["stop".to_string()])
+ );
+ let output = json_attribute(second, attr::OUTPUT_MESSAGES);
+ assert_eq!(output[0]["parts"][0]["content"], "done");
+
+ // The tool span: rig's, with the model's call id, plus our arguments and
+ // result — and only one of them per call.
+ let tools = spans_with_operation(&spans, attr::operation::EXECUTE_TOOL);
+ assert_eq!(
+ tools.len(),
+ 1,
+ "one tool span per call, got {:?}",
+ names(&spans)
+ );
+ let tool = tools[0];
+ assert_eq!(
+ string_attribute(tool, attr::TOOL_NAME).as_deref(),
+ Some("echo_tool")
+ );
+ assert_eq!(
+ string_attribute(tool, attr::TOOL_CALL_ID).as_deref(),
+ Some("call-1")
+ );
+ assert_eq!(
+ string_attribute(tool, attr::TOOL_CALL_ARGUMENTS).as_deref(),
+ Some(r#"{"value":"a"}"#)
+ );
+ assert_eq!(
+ string_attribute(tool, attr::TOOL_CALL_RESULT).as_deref(),
+ Some(r#"{"echo":"a"}"#)
+ );
+
+ // Everything is one trace under the agent span: the model calls hang off
+ // it directly, and the tool call off the run too (rig opens it from the
+ // driver, which runs inside the agent span).
+ let trace_id = agent.span_context.trace_id();
+ assert!(
+ spans
+ .iter()
+ .all(|span| span.span_context.trace_id() == trace_id),
+ "{:?}",
+ names(&spans)
+ );
+ let agent_id = agent.span_context.span_id();
+ for chat in &chats {
+ assert_eq!(chat.parent_span_id, agent_id, "{}", describe(&spans));
+ }
+ assert_eq!(tool.parent_span_id, agent_id, "{}", describe(&spans));
+}
+
+/// Without a conversation id nothing is invented, and the run still traces.
+#[tokio::test]
+async fn a_run_without_a_conversation_id_has_no_session_attribute() {
+ let (exporter, provider, _guard) = otel_test_pipeline();
+ let model = MockCompletionModel::from_stream_turns([vec![
+ MockStreamEvent::text("hi"),
+ MockStreamEvent::final_response_with_default_usage(),
+ ]]);
+ let toolset = util::single_tool_set::<EchoTool, ()>();
+ let mut session = util::session(toolset, Arc::new(()), model).await;
+
+ let result = util::drive(&mut session, "hello").await;
+ assert_eq!(result.content(), "hi");
+ drop(session);
+
+ let spans = finished(&exporter, &provider);
+ let agent = spans_with_operation(&spans, attr::operation::INVOKE_AGENT)[0];
+ assert_eq!(string_attribute(agent, attr::CONVERSATION_ID), None);
+ let chat = spans_with_operation(&spans, attr::operation::CHAT)[0];
+ assert_eq!(string_attribute(chat, attr::CONVERSATION_ID), None);
+ assert_eq!(
+ string_array_attribute(chat, attr::RESPONSE_FINISH_REASONS),
+ Some(vec!["stop".to_string()])
+ );
+}
+
+/// A run the provider fails is a failed run on the agent span, not a quiet
+/// success.
+#[tokio::test]
+async fn a_provider_error_marks_the_agent_span_failed() {
+ let (exporter, provider, _guard) = otel_test_pipeline();
+ let model = MockCompletionModel::from_stream_turns([vec![
+ MockStreamEvent::text("partial"),
+ MockStreamEvent::error("upstream exploded"),
+ ]]);
+ let toolset = util::single_tool_set::<EchoTool, ()>();
+ let mut session = util::session(toolset, Arc::new(()), model).await;
+
+ let stream = session
+ .send_message(vec![rig_core::message::Message::user("hello")])
+ .await
+ .expect("the stream starts");
+ let collected = util::collect(stream).await;
+ assert!(
+ collected.error.is_some(),
+ "the failure reaches the consumer"
+ );
+ drop(session);
+
+ let spans = finished(&exporter, &provider);
+ let agent = spans_with_operation(&spans, attr::operation::INVOKE_AGENT)[0];
+ assert!(
+ matches!(agent.status, opentelemetry::trace::Status::Error { .. }),
+ "{agent:#?}"
+ );
+ assert_eq!(
+ string_attribute(agent, attr::ERROR_TYPE).as_deref(),
+ Some("streaming_error")
+ );
+ assert_eq!(
+ string_array_attribute(agent, attr::RESPONSE_FINISH_REASONS),
+ Some(vec!["error".to_string()])
+ );
+
+ // The model call that was in flight failed the same way: its span carries
+ // the request, the failure, and no output.
+ let chats = spans_with_operation(&spans, attr::operation::CHAT);
+ assert_eq!(chats.len(), 1, "{}", describe(&spans));
+ let chat = chats[0];
+ assert!(
+ matches!(chat.status, opentelemetry::trace::Status::Error { .. }),
+ "{chat:#?}"
+ );
+ assert_eq!(
+ string_attribute(chat, attr::ERROR_TYPE).as_deref(),
+ Some("streaming_error")
+ );
+ assert_eq!(
+ string_array_attribute(chat, attr::RESPONSE_FINISH_REASONS),
+ Some(vec!["error".to_string()])
+ );
+ assert!(string_attribute(chat, attr::INPUT_MESSAGES).is_some());
+ assert_eq!(string_attribute(chat, attr::OUTPUT_MESSAGES), None);
+}
+
+/// A tool that never returns, so a run can be abandoned mid-flight.
+#[derive(Deserialize, JsonSchema)]
+#[schemars(title = "hang_tool", description = "Never finishes.")]
+struct HangTool {}
+
+impl ToolAnnotated for HangTool {
+ const ANNOTATIONS: ToolAnnotations = ToolAnnotations::read_only("Hang");
+}
+
+#[async_trait]
+impl AsyncTool<()> for HangTool {
+ type Output = serde_json::Value;
+
+ async fn call(
+ &self,
+ _service_context: ServiceContext<()>,
+ _request_context: RequestContext,
+ ) -> ToolResult<Self::Output> {
+ std::future::pending().await
+ }
+}
+
+/// A consumer that walks away from the stream cancels the run, and the agent
+/// span says so rather than closing as if the agent had answered.
+#[tokio::test]
+async fn dropping_the_stream_marks_the_agent_span_cancelled() {
+ let (exporter, provider, _guard) = otel_test_pipeline();
+ let model = MockCompletionModel::from_stream_turns([vec![
+ MockStreamEvent::tool_call("call-1", "hang_tool", json!({})),
+ MockStreamEvent::final_response_with_default_usage(),
+ ]]);
+ let toolset = util::single_tool_set::<HangTool, ()>();
+ let mut session = util::session(toolset, Arc::new(()), model).await;
+
+ let mut stream = session
+ .send_message(vec![rig_core::message::Message::user("hang")])
+ .await
+ .expect("the stream starts");
+ // The tool call is announced the moment the model emits it, while the
+ // tool itself hangs; walking away here abandons the run mid-tool.
+ let first = util::next_within(&mut stream, std::time::Duration::from_secs(5)).await;
+ assert!(first.is_some(), "the pending tool call is streamed");
+ drop(stream);
+ drop(session);
+ // The aborted driver's destructors run on the runtime's next turns.
+ for _ in 0..20 {
+ tokio::task::yield_now().await;
+ }
+
+ let spans = finished(&exporter, &provider);
+ let agent = spans_with_operation(&spans, attr::operation::INVOKE_AGENT)[0];
+ assert_eq!(
+ string_attribute(agent, attr::ERROR_TYPE).as_deref(),
+ Some("cancelled"),
+ "{}",
+ describe(&spans)
+ );
+ assert_eq!(
+ string_array_attribute(agent, attr::RESPONSE_FINISH_REASONS),
+ Some(vec!["cancelled".to_string()])
+ );
+}
+
+/// A tool that cancels the request it runs in, then returns: the runtime
+/// notices the cancellation before its next model call and ends the run with
+/// a cancellation error.
+#[derive(Deserialize, JsonSchema)]
+#[schemars(title = "cancel_tool", description = "Cancels the request.")]
+struct CancelTool {}
+
+impl ToolAnnotated for CancelTool {
+ const ANNOTATIONS: ToolAnnotations = ToolAnnotations::read_only("Cancel");
+}
+
+#[async_trait]
+impl AsyncTool<()> for CancelTool {
+ type Output = serde_json::Value;
+
+ async fn call(
+ &self,
+ _service_context: ServiceContext<()>,
+ request_context: RequestContext,
+ ) -> ToolResult<Self::Output> {
+ request_context.cancel.cancel();
+ Ok(json!({ "status": "cancelling" }))
+ }
+}
+
+/// A user stopping the run is a cancelled run, not a failed one, even though
+/// the runtime reports it as an error item on the stream.
+#[tokio::test]
+async fn a_cancelled_run_is_recorded_as_cancelled_not_failed() {
+ let (exporter, provider, _guard) = otel_test_pipeline();
+ let model = MockCompletionModel::from_stream_turns([
+ vec![
+ MockStreamEvent::tool_call("call-1", "cancel_tool", json!({})),
+ MockStreamEvent::final_response_with_default_usage(),
+ ],
+ vec![
+ MockStreamEvent::text("never reached"),
+ MockStreamEvent::final_response_with_default_usage(),
+ ],
+ ]);
+ let toolset = util::single_tool_set::<CancelTool, ()>();
+ let mut session = util::session(toolset, Arc::new(()), model).await;
+
+ let stream = session
+ .send_message(vec![rig_core::message::Message::user("stop soon")])
+ .await
+ .expect("the stream starts");
+ let collected = util::collect(stream).await;
+ assert!(
+ collected
+ .error
+ .as_ref()
+ .is_some_and(|error| error.was_cancelled()),
+ "the run ends with a cancellation: {:?}",
+ collected.error
+ );
+ drop(session);
+
+ let spans = finished(&exporter, &provider);
+ let agent = spans_with_operation(&spans, attr::operation::INVOKE_AGENT)[0];
+ assert_eq!(
+ string_attribute(agent, attr::ERROR_TYPE).as_deref(),
+ Some("cancelled"),
+ "{}",
+ describe(&spans)
+ );
+ assert_eq!(
+ string_array_attribute(agent, attr::RESPONSE_FINISH_REASONS),
+ Some(vec!["cancelled".to_string()])
+ );
+}
+
+/// With GenAI telemetry off - a loop whose turns another layer reports - the
+/// run records nothing GenAI onto any span, tool calls included; the runtime's
+/// structural spans still nest under a plain `agent.turn`.
+#[tokio::test]
+async fn a_loop_with_telemetry_off_records_no_genai_content() {
+ let (exporter, provider, _guard) = otel_test_pipeline();
+ let model = MockCompletionModel::from_stream_turns([
+ vec![
+ MockStreamEvent::tool_call("call-1", "echo_tool", json!({ "value": "a" })),
+ MockStreamEvent::final_response_with_default_usage(),
+ ],
+ vec![
+ MockStreamEvent::text("done"),
+ MockStreamEvent::final_response_with_default_usage(),
+ ],
+ ]);
+ let toolset = util::single_tool_set::<EchoTool, ()>();
+ let mut session = util::test_loop()
+ .with_genai_telemetry(false)
+ .test_session(
+ toolset,
+ Arc::new(()),
+ "test preamble",
+ util::usage_ctx(),
+ model,
+ )
+ .await;
+
+ let result = util::drive(&mut session, "call echo").await;
+ assert_eq!(result.content(), "done");
+ drop(session);
+
+ let spans = finished(&exporter, &provider);
+ assert!(
+ spans.iter().any(|span| span.name == "agent.turn"),
+ "{:?}",
+ names(&spans)
+ );
+ for span in &spans {
+ assert_eq!(
+ string_attribute(span, attr::INPUT_MESSAGES),
+ None,
+ "{span:#?}"
+ );
+ assert_eq!(
+ string_attribute(span, attr::OUTPUT_MESSAGES),
+ None,
+ "{span:#?}"
+ );
+ assert_eq!(
+ string_attribute(span, attr::TOOL_CALL_ARGUMENTS),
+ None,
+ "{span:#?}"
+ );
+ assert_eq!(
+ string_attribute(span, attr::TOOL_CALL_RESULT),
+ None,
+ "{span:#?}"
+ );
+ assert_eq!(
+ string_attribute(span, attr::TOOL_DEFINITIONS),
+ None,
+ "{span:#?}"
+ );
+ }
+}
diff --git a/crates/agent/src/test/agent_loop/test_tool_search.rs b/crates/agent/src/test/agent_loop/test_tool_search.rs
index ab68910144..3ddfdb1558 100644
--- a/crates/agent/src/test/agent_loop/test_tool_search.rs
+++ b/crates/agent/src/test/agent_loop/test_tool_search.rs
@@ -108,7 +108,7 @@ impl FakeCombined {
}
impl AiToolSet<()> for FakeCombined {
- fn try_tool_call<'a>(
+ fn dispatch_tool_call<'a>(
&'a self,
context: (),
request_context: RequestContext,
@@ -129,7 +129,7 @@ impl AiToolSet<()> for FakeCombined {
});
}
self.static_tools
- .try_tool_call(context, request_context, tool_name, json)
+ .dispatch_tool_call(context, request_context, tool_name, json)
}
fn request_schemas(&self) -> Option<Vec<RequestSchema>> {
diff --git a/crates/agent/src/tool_adapter.rs b/crates/agent/src/tool_adapter.rs
index e8936034fd..e4133a5b75 100644
--- a/crates/agent/src/tool_adapter.rs
+++ b/crates/agent/src/tool_adapter.rs
@@ -1,8 +1,13 @@
/// Adapts `ai_toolset` tool types into RIG [`DynamicTool`] objects.
+use ai_toolset::telemetry::ToolCallSpan;
use ai_toolset::tool_object::ToolSetCallable;
-use ai_toolset::{AsyncToolCollection, RequestContext, RequestSchema, ToolSet as AiToolSet};
+use ai_toolset::{
+ AsyncToolCollection, RequestContext, RequestSchema, ToolResult, ToolSet as AiToolSet,
+ ToolSetError,
+};
use rig_agent::tool::{DynamicTool, ToolExecutionError, ToolOutput};
use std::sync::{Arc, RwLock};
+use tracing::Instrument as _;
/// Ensure every object schema carries an explicit `properties` map.
///
@@ -90,6 +95,7 @@ impl ToolsetToolAdapter {
);
let context = context.clone();
let request_context = request_context.clone();
+ let tool_name = name.clone();
DynamicTool::new(
name,
@@ -99,20 +105,41 @@ impl ToolsetToolAdapter {
let deserializer = deserializer.clone();
let context = context.clone();
let request_context = request_context.clone();
+ let tool_name = tool_name.clone();
Box::pin(async move {
- let callable =
- (deserializer)(&args).map_err(invalid_args)?;
- let ctx = (*context).clone();
let req_ctx = request_context
.read()
.expect("request_context lock poisoned")
.clone();
- match callable.call(ctx, req_ctx).await {
- Ok(value) => Ok(ToolOutput::json(value)),
- Err(e) => {
+ // Bypasses `ToolSet::try_tool_call`, so it carries
+ // the same `execute_tool` telemetry itself - under
+ // the same request-level opt-out.
+ let telemetry = req_ctx
+ .genai_telemetry
+ .then(|| ToolCallSpan::begin(&tool_name, &args));
+ let span = telemetry.as_ref().map_or_else(tracing::Span::none, |t| {
+ t.span().clone()
+ });
+ let result: Result<ToolResult<serde_json::Value>, ToolSetError> =
+ async {
+ let callable = (deserializer)(&args)
+ .map_err(ToolSetError::Deserialization)?;
+ let ctx = (*context).clone();
+ Ok(callable.call(ctx, req_ctx).await)
+ }
+ .instrument(span)
+ .await;
+ if let Some(telemetry) = &telemetry {
+ telemetry.finish(&result);
+ }
+ match result {
+ Ok(Ok(value)) => Ok(ToolOutput::json(value)),
+ Ok(Err(e)) => {
tracing::error!(error = ?e.internal_error, "toolset tool error");
Err(ToolExecutionError::other(e.description))
}
+ Err(ToolSetError::Deserialization(e)) => Err(invalid_args(e)),
+ Err(e) => Err(ToolExecutionError::other(e.to_string())),
}
})
},
diff --git a/crates/agent_inmem/Cargo.toml b/crates/agent_inmem/Cargo.toml
index 02eb2bb4ac..df4c13052d 100644
--- a/crates/agent_inmem/Cargo.toml
+++ b/crates/agent_inmem/Cargo.toml
@@ -22,6 +22,7 @@ axum = { workspace = true }
chat = { path = "../chat", default-features = false }
dashmap = { workspace = true }
futures = { workspace = true }
+genai_telemetry = { path = "../genai_telemetry" }
macro_user_id = { path = "../macro_user_id" }
http = { workspace = true }
http-body-util = { workspace = true }
@@ -48,4 +49,8 @@ agent_session = { path = "../agent_session", features = ["test-utils"] }
bot_id = { path = "../bot_id", features = ["test-utils"] }
futures = { workspace = true }
rig-agent = "0.41.0"
+opentelemetry = { workspace = true, features = ["trace"] }
+opentelemetry_sdk = { workspace = true, features = ["testing", "trace"] }
+tracing-opentelemetry = { workspace = true }
+tracing-subscriber = { workspace = true }
tokio = { workspace = true, features = ["macros", "rt-multi-thread", "time"] }
diff --git a/crates/agent_inmem/src/domain/agent.rs b/crates/agent_inmem/src/domain/agent.rs
index ddc973dd7f..8d7342095a 100644
--- a/crates/agent_inmem/src/domain/agent.rs
+++ b/crates/agent_inmem/src/domain/agent.rs
@@ -38,6 +38,7 @@ use ai_tools::user_tool_review::{
use async_trait::async_trait;
use macro_user_id::user_id::MacroUserIdStr;
use tokio_util::sync::CancellationToken;
+use tracing::Instrument as _;
use crate::domain::engine::{AgentIdentity, TurnEngine, TurnRequest};
use crate::domain::mcp::{DynMcpToolConnector, dialable_servers};
@@ -552,6 +553,13 @@ pub async fn serve(state: Arc<AgentState>, acp: AcpChannel) -> Result<(), AcpErr
if let Err(error) = state.expect_session(&request.session_id) {
return responder.respond_with_error(error);
}
+ let span = tracing::info_span!(
+ parent: None,
+ "agent.acp.prompt",
+ agent.session.id = %state.session_id,
+ gen_ai.conversation.id = %state.session_id,
+ );
+ genai_telemetry::propagation::set_parent(&span, request.meta.as_ref());
let prompt = prompt_text(&request);
if prompt.trim() == COMPACT_COMMAND {
state.clear_history();
@@ -585,6 +593,7 @@ pub async fn serve(state: Arc<AgentState>, acp: AcpChannel) -> Result<(), AcpErr
let _ = responder.respond(PromptResponse::new(stop));
Ok(())
}
+ .instrument(span)
})?;
return Ok(());
}
@@ -602,6 +611,7 @@ pub async fn serve(state: Arc<AgentState>, acp: AcpChannel) -> Result<(), AcpErr
let _ = responder.respond(PromptResponse::new(stop));
Ok(())
}
+ .instrument(span)
})?;
Ok(())
}
diff --git a/crates/agent_inmem/src/domain/agent/test.rs b/crates/agent_inmem/src/domain/agent/test.rs
index 05c0508453..103a877d8b 100644
--- a/crates/agent_inmem/src/domain/agent/test.rs
+++ b/crates/agent_inmem/src/domain/agent/test.rs
@@ -1024,3 +1024,5 @@ async fn ask_without_form_support_explains_instead_of_asking() {
spoken(&notifications)
);
}
+
+mod telemetry;
diff --git a/crates/agent_inmem/src/domain/agent/test/telemetry.rs b/crates/agent_inmem/src/domain/agent/test/telemetry.rs
new file mode 100644
index 0000000000..16bdbf4b55
--- /dev/null
+++ b/crates/agent_inmem/src/domain/agent/test/telemetry.rs
@@ -0,0 +1,83 @@
+use super::*;
+
+/// The ACP dispatcher and spawned prompt task must restore context from each
+/// request, rather than inheriting the connection's context or the last turn.
+#[tokio::test]
+async fn prompt_trace_context_reaches_the_engine_for_each_turn() {
+ use opentelemetry::trace::{TraceContextExt as _, TracerProvider as _};
+ use opentelemetry_sdk::trace::{InMemorySpanExporter, SdkTracerProvider};
+ use tracing_opentelemetry::OpenTelemetrySpanExt as _;
+ use tracing_subscriber::layer::SubscriberExt as _;
+
+ struct TracedEngine;
+ impl TurnEngine for TracedEngine {
+ fn supported_models(&self) -> &[&str] {
+ crate::testing::TEST_MODELS
+ }
+
+ fn run_turn(
+ &self,
+ _request: TurnRequest,
+ ) -> tokio::sync::mpsc::Receiver<Result<StreamPart, agent::AgentError>> {
+ let _work = tracing::info_span!("engine.work");
+ let (_sender, receiver) = tokio::sync::mpsc::channel(1);
+ receiver
+ }
+ }
+
+ opentelemetry::global::set_text_map_propagator(
+ opentelemetry_sdk::propagation::TraceContextPropagator::new(),
+ );
+ let exporter = InMemorySpanExporter::default();
+ let provider = SdkTracerProvider::builder()
+ .with_simple_exporter(exporter.clone())
+ .build();
+ let subscriber = tracing_subscriber::registry()
+ .with(tracing_opentelemetry::layer().with_tracer(provider.tracer("test")));
+ let _guard = tracing::subscriber::set_default(subscriber);
+ // While exactly one dispatcher is registered, tracing-core caches each
+ // new span callsite's interest from the registering thread's default
+ // subscriber. Other tests in this binary hit the prompt callsite with no
+ // subscriber, which would cache "never interested" and disable the span
+ // on this thread too. A second registered dispatcher makes tracing-core
+ // combine every registered subscriber's interest instead, so the cache
+ // stays "sometimes" and each thread's own default subscriber decides.
+ let _pin_interest = tracing::Dispatch::new(tracing::subscriber::NoSubscriber::default());
+ let (_, _, parents) = with_agent(Arc::new(TracedEngine), async |connection, session| {
+ let mut parents = Vec::new();
+ for _ in 0..2 {
+ let parent = tracing::info_span!(parent: None, "invocation");
+ parents.push(parent.context().span().span_context().clone());
+ let mut request = text_prompt(&session, "test prompt");
+ let mut meta = Meta::new();
+ genai_telemetry::propagation::inject(&parent, &mut meta);
+ request.meta = Some(meta);
+ connection
+ .send_request(request)
+ .block_task()
+ .await
+ .expect("prompt response");
+ }
+ parents
+ })
+ .await;
+ provider.force_flush().expect("flush");
+ let spans = exporter.get_finished_spans().expect("spans");
+ assert_ne!(parents[0].trace_id(), parents[1].trace_id());
+ for parent in parents {
+ let prompt = spans
+ .iter()
+ .find(|span| {
+ span.name == "agent.acp.prompt" && span.span_context.trace_id() == parent.trace_id()
+ })
+ .expect("ACP prompt stays in its caller's trace");
+ assert_eq!(prompt.parent_span_id, parent.span_id());
+ let work = spans
+ .iter()
+ .find(|span| {
+ span.name == "engine.work" && span.span_context.trace_id() == parent.trace_id()
+ })
+ .expect("engine runs in the prompt trace");
+ assert_eq!(work.parent_span_id, prompt.span_context.span_id());
+ }
+}
diff --git a/crates/agent_inmem/src/outbound/mod.rs b/crates/agent_inmem/src/outbound/mod.rs
index 2449f6e45c..3eaa8e7833 100644
--- a/crates/agent_inmem/src/outbound/mod.rs
+++ b/crates/agent_inmem/src/outbound/mod.rs
@@ -4,3 +4,4 @@ pub mod acp_mcp;
pub mod egress_mcp;
pub mod log_frames;
pub mod manager;
+pub mod tool_catalog;
diff --git a/crates/agent_inmem/src/outbound/tool_catalog.rs b/crates/agent_inmem/src/outbound/tool_catalog.rs
new file mode 100644
index 0000000000..247e63eec3
--- /dev/null
+++ b/crates/agent_inmem/src/outbound/tool_catalog.rs
@@ -0,0 +1,71 @@
+//! Listing a session's MCP tools for its telemetry.
+//!
+//! The session actor records, on each turn's span, the tools the agent had to
+//! choose from (`gen_ai.tool.definitions`). A sandboxed harness learns those
+//! by dialing the MCP servers it was handed in `session/new`; this lists the
+//! same servers the same way, in process through the egress proxy, so the
+//! definitions on the span are the ones the agent saw - Macro's own tools and
+//! the owner's connected apps alike, under the names the harness calls them
+//! by. Which is also why Macro's server is *not* skipped here the way
+//! [`dialable_servers`](crate::domain::mcp::dialable_servers) skips it for the
+//! in-process runtime's own toolset: every other harness reaches Macro's tools
+//! over that server, and the listing has to say so.
+
+#[cfg(test)]
+mod test;
+
+use std::pin::Pin;
+use std::sync::Arc;
+
+use agent_client_protocol::schema::v1::McpServer;
+use agent_session::domain::ports::{SessionToolCatalog, SessionToolDefinition};
+
+use crate::domain::mcp::McpToolConnector;
+
+/// A [`SessionToolCatalog`] over the connector the in-process runtime dials
+/// its MCP servers with.
+pub struct McpToolCatalog<Connector> {
+ connector: Arc<Connector>,
+}
+
+impl<Connector> McpToolCatalog<Connector> {
+ /// A catalog dialing through `connector`.
+ pub fn new(connector: Arc<Connector>) -> Self {
+ Self { connector }
+ }
+}
+
+impl<Connector> SessionToolCatalog for McpToolCatalog<Connector>
+where
+ Connector: McpToolConnector,
+{
+ fn tool_definitions(
+ &self,
+ servers: Vec<McpServer>,
+ ) -> Pin<Box<dyn Future<Output = Vec<SessionToolDefinition>> + Send + '_>> {
+ Box::pin(async move {
+ // HTTP entries only: the egress proxy serves nothing else, and a
+ // stdio or SSE entry is not something this process can dial.
+ let http = servers
+ .into_iter()
+ .filter_map(|server| match server {
+ McpServer::Http(http) => Some(http),
+ _ => None,
+ })
+ .collect();
+ let Some(tools) = self.connector.connect(http).await else {
+ return Vec::new();
+ };
+ tools
+ .searchable_catalog()
+ .into_iter()
+ .map(|tool| SessionToolDefinition {
+ name: tool.name,
+ description: tool.description,
+ parameters: serde_json::to_value(tool.schema)
+ .unwrap_or(serde_json::Value::Null),
+ })
+ .collect()
+ })
+ }
+}
diff --git a/crates/agent_inmem/src/outbound/tool_catalog/test.rs b/crates/agent_inmem/src/outbound/tool_catalog/test.rs
new file mode 100644
index 0000000000..2361fd53df
--- /dev/null
+++ b/crates/agent_inmem/src/outbound/tool_catalog/test.rs
@@ -0,0 +1,50 @@
+use super::*;
+use agent_client_protocol::schema::v1::{HttpHeader, McpServerHttp, McpServerStdio};
+use mcp_toolset::RemoteMcpToolSet;
+use std::sync::Mutex;
+
+/// A connector that records which servers it was asked to dial and finds no
+/// tools on them.
+#[derive(Default)]
+struct RecordingConnector {
+ dialed: Mutex<Vec<String>>,
+}
+
+impl McpToolConnector for RecordingConnector {
+ async fn connect(&self, servers: Vec<McpServerHttp>) -> Option<RemoteMcpToolSet> {
+ self.dialed
+ .lock()
+ .expect("not poisoned")
+ .extend(servers.into_iter().map(|server| server.name));
+ None
+ }
+}
+
+fn http(name: &str) -> McpServer {
+ McpServer::Http(
+ McpServerHttp::new(name, format!("https://egress.test/mcp/{name}")).headers(vec![
+ HttpHeader::new("Authorization", "Bearer session-token"),
+ ]),
+ )
+}
+
+#[tokio::test]
+async fn every_http_server_is_listed_including_macros_own() {
+ let connector = Arc::new(RecordingConnector::default());
+ let catalog = McpToolCatalog::new(Arc::clone(&connector));
+
+ let definitions = catalog
+ .tool_definitions(vec![
+ http("macro"),
+ McpServer::Stdio(McpServerStdio::new("local", "some-binary")),
+ http("linear"),
+ ])
+ .await;
+
+ assert!(definitions.is_empty(), "no server yielded a tool");
+ assert_eq!(
+ connector.dialed.lock().expect("not poisoned").as_slice(),
+ ["macro", "linear"],
+ "HTTP servers are dialed, Macro's included; stdio entries are not"
+ );
+}
diff --git a/crates/agent_inmem/src/rig_engine.rs b/crates/agent_inmem/src/rig_engine.rs
index 80f3bf034d..55b9ffccf7 100644
--- a/crates/agent_inmem/src/rig_engine.rs
+++ b/crates/agent_inmem/src/rig_engine.rs
@@ -35,6 +35,7 @@ use memory::domain::service::MemoryServiceImpl;
use memory::outbound::pg_memory_repo::PgMemoryRepo;
use sqlx::PgPool;
use tokio::sync::mpsc;
+use tracing::Instrument as _;
use crate::domain::engine::{AgentIdentity, TurnEngine, TurnRequest};
use crate::inbound::ask_user::{AskUser, AskUserContext};
@@ -102,11 +103,14 @@ impl TurnEngine for RigTurnEngine {
let (parts, receiver) = mpsc::channel(PART_BUFFER);
let db = self.db.clone();
let tool_context = self.tool_context.clone();
- tokio::spawn(async move {
- if let Err(error) = drive_turn(db, tool_context, request, &parts).await {
- let _ = parts.send(Err(error)).await;
+ tokio::spawn(
+ async move {
+ if let Err(error) = drive_turn(db, tool_context, request, &parts).await {
+ let _ = parts.send(Err(error)).await;
+ }
}
- });
+ .in_current_span(),
+ );
receiver
}
}
@@ -158,7 +162,13 @@ async fn drive_turn(
},
};
- let mut agent_loop = AgentLoop::new(base_context.recorder.clone()).with_model(&model);
+ // GenAI telemetry stays off here: this runtime's turns and tool calls
+ // reach the session actor as ACP frames, and the actor projects those onto
+ // `invoke_agent` / `execute_tool` spans for every harness alike. Enriching
+ // rig's spans too would report each turn twice.
+ let mut agent_loop = AgentLoop::new(base_context.recorder.clone())
+ .with_model(&model)
+ .with_genai_telemetry(false);
if let Some(reviewer) = reviewer {
agent_loop = agent_loop.with_user_tool_finisher(user_tool_finisher(
Arc::clone(&toolset),
diff --git a/crates/agent_session/Cargo.toml b/crates/agent_session/Cargo.toml
index 1ab79578a9..fb6dba7743 100644
--- a/crates/agent_session/Cargo.toml
+++ b/crates/agent_session/Cargo.toml
@@ -56,6 +56,7 @@ entity_access = { path = "../entity_access", default-features = false, features
"ports",
] }
entity_access_db_utils = { path = "../entity_access_db_utils", optional = true }
+genai_telemetry = { path = "../genai_telemetry" }
model-entity = { path = "../model-entity", optional = true }
clap = { workspace = true, optional = true }
macro_authorization = { path = "../macro_authorization", default-features = false, optional = true }
@@ -89,6 +90,10 @@ non_empty = { path = "../non_empty" }
agent_fold = { path = "../agent_fold", features = ["test-utils"] }
axum = { workspace = true, features = ["ws"] }
macro_db_migrator = { path = "../macro_db_migrator" }
+opentelemetry = { workspace = true, features = ["trace"] }
+opentelemetry_sdk = { workspace = true, features = ["testing", "trace"] }
+tracing-opentelemetry = { workspace = true }
+tracing-subscriber = { workspace = true }
tokio = { workspace = true, features = ["fs", "io-std", "io-util", "macros", "net", "rt-multi-thread", "time"] }
tokio-tungstenite = { workspace = true }
tower = { workspace = true, features = ["util"] }
diff --git a/crates/agent_session/src/domain/ports.rs b/crates/agent_session/src/domain/ports.rs
index ccddeab4be..1a4027d75b 100644
--- a/crates/agent_session/src/domain/ports.rs
+++ b/crates/agent_session/src/domain/ports.rs
@@ -4,7 +4,7 @@ use super::error::{AgentSessionError, Result};
use super::model::*;
use super::session::StopReason;
use crate::domain::events::AgentSessionLifecycleEvent;
-use agent_client_protocol::schema::v1::SessionId;
+use agent_client_protocol::schema::v1::{McpServer, SessionId};
use agent_fold::domain::model::TurnSignal;
use agent_runtime_protocol::domain::action::{AgentAction, AgentActionId};
use agent_runtime_protocol::domain::ports::Transport;
@@ -509,6 +509,57 @@ pub trait AgentSessionLogRepo: Send + Sync + 'static {
) -> impl Future<Output = Result<Vec<MacroUserIdStr<'static>>>> + Send;
}
+/// One tool a session's agent may call, as the MCP server offering it
+/// describes it - the same name, description and parameter schema the agent
+/// itself is shown.
+#[derive(Debug, Clone, PartialEq)]
+pub struct SessionToolDefinition {
+ /// The name the agent calls the tool by, exactly as the server registers
+ /// it for this session (an MCP tool arrives as `mcp__<server>__<tool>`).
+ pub name: String,
+ /// The description the agent chooses the tool on.
+ pub description: String,
+ /// The JSON schema of the tool's arguments.
+ pub parameters: serde_json::Value,
+}
+
+/// Lists the tools a session's MCP servers advertise, so the session's
+/// telemetry can say what the agent had to choose from
+/// (`gen_ai.tool.definitions`).
+///
+/// Best-effort by contract: a listing that fails, or takes too long, costs
+/// a span its tool definitions and nothing else. The one production
+/// implementation dials the same egress-proxy URLs the agent is handed, in
+/// process, so what it lists is what the agent sees - Macro's own tools and
+/// the owner's connected apps alike. A harness's built-in tools (its shell,
+/// its file editor) are not on any server and are not listed; the
+/// convention does not require them.
+///
+/// Object-safe on purpose: the session service stores it erased so wiring it
+/// is not another type parameter.
+pub trait SessionToolCatalog: Send + Sync + 'static {
+ /// Every tool the given servers offer. Empty when none do, or when the
+ /// listing failed.
+ fn tool_definitions(
+ &self,
+ servers: Vec<McpServer>,
+ ) -> std::pin::Pin<Box<dyn Future<Output = Vec<SessionToolDefinition>> + Send + '_>>;
+}
+
+/// A [`SessionToolCatalog`] that lists nothing: tests, offline tooling, and
+/// deployments with no in-process MCP client.
+#[derive(Debug, Clone, Copy, Default)]
+pub struct NoOpToolCatalog;
+
+impl SessionToolCatalog for NoOpToolCatalog {
+ fn tool_definitions(
+ &self,
+ _servers: Vec<McpServer>,
+ ) -> std::pin::Pin<Box<dyn Future<Output = Vec<SessionToolDefinition>> + Send + '_>> {
+ Box::pin(async { Vec::new() })
+ }
+}
+
/// One frame appended: its durable identity, and what the fold made of it.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Appended {
diff --git a/crates/agent_session/src/domain/service.rs b/crates/agent_session/src/domain/service.rs
index 7ca89426d3..1b6ba31089 100644
--- a/crates/agent_session/src/domain/service.rs
+++ b/crates/agent_session/src/domain/service.rs
@@ -58,7 +58,8 @@ use super::model::{
use super::ports::{
AgentConnector, AgentSessionLifecyclePublisher, AgentSessionLogRepo, AgentSessionLogWriter,
AgentSessionNameGenerator, AgentSessionQueueChanged, AgentSessionRealtime, AgentSessionRepo,
- Appended, NoOpAgentSessionNameGenerator, SessionOwnership, SessionTurnObserver,
+ Appended, NoOpAgentSessionNameGenerator, NoOpToolCatalog, SessionOwnership, SessionToolCatalog,
+ SessionTurnObserver,
};
use super::session::actors::{SessionActor, SessionCommand, Stepped};
use super::session::{CloseReason, Input};
@@ -285,6 +286,9 @@ pub struct AgentSessionServiceImpl<R, Folds, Rt, Namer = NoOpAgentSessionNameGen
/// Told when a session's turn ends or its actor stops - the harness's
/// prompt-queue gate. Erased so wiring it is not another type parameter.
turn_observer: Arc<dyn SessionTurnObserver>,
+ /// Lists a session's MCP tools for its telemetry. Erased like the
+ /// observer, for the same reason.
+ tool_catalog: Arc<dyn SessionToolCatalog>,
/// Where lifecycle facts go - renames, from here; everything else from
/// the harness. Erased for the same reason as the observer.
lifecycle_publisher: Arc<dyn AgentSessionLifecyclePublisher>,
@@ -306,7 +310,10 @@ impl<R, Folds, Rt, Namer> AgentSessionServiceImpl<R, Folds, Rt, Namer> {
/// it passes [`NoOpTurnObserver`], one with nothing downstream passes
/// [`NoopLifecyclePublisher`], and one that is the only service instance
/// in its process mints its own [`ReplicaId`] - each choice visible at the
- /// call site rather than hidden in a builder's default.
+ /// call site rather than hidden in a builder's default. The one
+ /// exception is the tool catalog: it starts as [`NoOpToolCatalog`] and
+ /// [`Self::with_tool_catalog`] swaps in a real one, since only a process
+ /// with an in-process MCP client can list anything.
///
/// `replica` is this service's identity in the session-management lease.
/// A restarted process is a new replica whose claims are recovered by
@@ -329,6 +336,7 @@ impl<R, Folds, Rt, Namer> AgentSessionServiceImpl<R, Folds, Rt, Namer> {
name_generator,
turn_observer,
lifecycle_publisher,
+ tool_catalog: Arc::new(NoOpToolCatalog),
active: Arc::new(DashMap::new()),
replica,
tasks: TaskTracker::new(),
@@ -337,6 +345,14 @@ impl<R, Folds, Rt, Namer> AgentSessionServiceImpl<R, Folds, Rt, Namer> {
}
}
+ /// Replace the no-op tool catalog with one that lists a session's MCP
+ /// tools, so its turns' spans carry the tools the agent could choose from.
+ #[must_use]
+ pub fn with_tool_catalog(mut self, tool_catalog: Arc<dyn SessionToolCatalog>) -> Self {
+ self.tool_catalog = tool_catalog;
+ self
+ }
+
/// This service's identity in the session-management lease, for the
/// composition root to heartbeat while the process lives.
#[must_use]
@@ -437,6 +453,7 @@ impl<R, Folds, Rt, Namer> AgentSessionServiceImpl<R, Folds, Rt, Namer> {
command_rx,
attachment.handshake,
Arc::clone(&self.turn_observer),
+ Arc::clone(&self.tool_catalog),
);
self.tasks.spawn(
run_session(
diff --git a/crates/agent_session/src/domain/service/test.rs b/crates/agent_session/src/domain/service/test.rs
index e02d650d9b..42ade3811b 100644
--- a/crates/agent_session/src/domain/service/test.rs
+++ b/crates/agent_session/src/domain/service/test.rs
@@ -842,6 +842,7 @@ async fn cancellation_does_not_drop_an_effect_batch_after_machine_mutation() {
command_rx,
handshake,
Arc::new(crate::domain::ports::NoOpTurnObserver),
+ Arc::new(crate::domain::ports::NoOpToolCatalog),
);
let active = Arc::new(ActiveSessions::new());
let cancellation = CancellationToken::new();
@@ -922,6 +923,7 @@ async fn live_inbound_logs_do_not_reuse_the_expired_handshake_deadline() {
command_rx,
handshake,
Arc::new(crate::domain::ports::NoOpTurnObserver),
+ Arc::new(crate::domain::ports::NoOpToolCatalog),
);
let active = Arc::new(ActiveSessions::new());
let cancellation = CancellationToken::new();
@@ -1384,6 +1386,7 @@ async fn shared_transport_copies_durable_initialization_before_load() {
commands,
handshake.clone(),
Arc::new(crate::domain::ports::NoOpTurnObserver),
+ Arc::new(crate::domain::ports::NoOpToolCatalog),
);
actor
.dispatch(Input::Inbound(ToServerMessage::Event {
@@ -1436,6 +1439,7 @@ async fn shared_transport_copies_durable_initialization_before_load() {
commands,
handshake,
Arc::new(crate::domain::ports::NoOpTurnObserver),
+ Arc::new(crate::domain::ports::NoOpToolCatalog),
);
let ready = actor.next_input().await;
assert!(matches!(ready, Input::SharedReady { .. }));
@@ -1540,6 +1544,7 @@ async fn assert_restore_persistence_failure_does_not_send_prompt(failure: Restor
command_rx,
handshake.clone(),
Arc::new(crate::domain::ports::NoOpTurnObserver),
+ Arc::new(crate::domain::ports::NoOpToolCatalog),
);
let (completed, completion) = oneshot::channel();
commands
@@ -1647,6 +1652,156 @@ async fn assert_restore_persistence_failure_does_not_send_prompt(failure: Restor
));
}
+/// The actor's GenAI projection, end to end: a prompt delivered through the
+/// actor opens an `invoke_agent` span under the command that carried it, and
+/// the runtime's answer closes it with what the agent said.
+#[tokio::test]
+async fn a_prompt_turn_is_traced_as_an_agent_span_under_its_command() {
+ use genai_telemetry::attr;
+ use opentelemetry::trace::{TraceContextExt as _, TracerProvider as _};
+ use opentelemetry_sdk::trace::{InMemorySpanExporter, SdkTracerProvider};
+ use tracing_opentelemetry::OpenTelemetrySpanExt as _;
+ use tracing_subscriber::layer::SubscriberExt as _;
+
+ let exporter = InMemorySpanExporter::default();
+ let provider = SdkTracerProvider::builder()
+ .with_simple_exporter(exporter.clone())
+ .build();
+ let layer = tracing_opentelemetry::layer().with_tracer(provider.tracer("test"));
+ let _guard = tracing::subscriber::set_default(tracing_subscriber::registry().with(layer));
+ tracing::callsite::rebuild_interest_cache();
+
+ let repo = InMemoryAgentSessionRepo::new();
+ let session = test_session();
+ repo.insert_session(test_agent_session(session));
+ let release = Arc::new(Notify::new());
+ let logs = BlockingPromptLogs {
+ repo: repo.clone(),
+ entered: Arc::new(Notify::new()),
+ release: release.clone(),
+ hang_disconnect: false,
+ fail_restore_log: None,
+ };
+ let (outbound_tx, mut outbound_rx) = mpsc::channel(8);
+ let (inbound_tx, inbound_rx) = mpsc::channel(8);
+ let (commands, command_rx) = mpsc::channel(8);
+ let (handshake, _) = watch::channel(HandshakeStatus::Pending);
+ let actor = SessionActor::new(
+ session,
+ None,
+ "/workspace".to_owned(),
+ Vec::new(),
+ RecordingTransport {
+ outbound: outbound_tx,
+ inbound: inbound_rx,
+ },
+ logs,
+ command_rx,
+ handshake,
+ Arc::new(crate::domain::ports::NoOpTurnObserver),
+ Arc::new(crate::domain::ports::NoOpToolCatalog),
+ );
+ let active = Arc::new(ActiveSessions::new());
+ let cancellation = CancellationToken::new();
+ let (stopped_tx, _) = watch::channel(false);
+ let claim = claim_for_test(&repo, session).await;
+ let task = tokio::spawn(
+ run_session(
+ actor,
+ Arc::downgrade(&active),
+ Arc::new(()),
+ stopped_tx,
+ cancellation.clone(),
+ repo.clone(),
+ claim,
+ Arc::new(crate::domain::ports::NoOpTurnObserver),
+ )
+ .with_current_subscriber(),
+ );
+ open_test_session(&inbound_tx, &mut outbound_rx, session).await;
+
+ release.notify_one();
+ let command_span = tracing::info_span!("agent.session.command");
+ let command_id = command_span.context().span().span_context().span_id();
+ let action_id = AgentActionId::mint();
+ let (completed, result) = oneshot::channel();
+ commands
+ .send(SessionCommand {
+ user_id: None,
+ action: AgentAction::prompt("what time is it?"),
+ action_id,
+ completed,
+ span: command_span,
+ enqueued_at: tokio::time::Instant::now(),
+ })
+ .await
+ .unwrap();
+ let prompt = outbound_rx.recv().await.expect("prompt is dispatched");
+ assert!(matches!(
+ &prompt,
+ ToRuntimeMessage::Acp(AcpMessage(
+ agent_client_protocol::RawJsonRpcMessage::Request(request)
+ )) if request.method.as_ref() == "session/prompt"
+ ));
+ result.await.unwrap().expect("delivery completes");
+
+ let update = agent_client_protocol::RawJsonRpcMessage::notification(
+ "session/update".to_owned(),
+ serde_json::json!({
+ "sessionId": "acp-1",
+ "update": {
+ "sessionUpdate": "agent_message_chunk",
+ "content": { "type": "text", "text": "It is noon." }
+ }
+ }),
+ )
+ .unwrap();
+ inbound_tx
+ .send(ToServerMessage::Acp(AcpMessage(update)))
+ .await
+ .unwrap();
+ inbound_tx
+ .send(ToServerMessage::Acp(AcpMessage(
+ agent_client_protocol::RawJsonRpcMessage::response(
+ action_id.to_request_id(),
+ Ok(serde_json::json!({ "stopReason": "end_turn" })),
+ ),
+ )))
+ .await
+ .unwrap();
+ // The transport closing stops the actor, which flushes anything open.
+ drop(inbound_tx);
+ task.await.unwrap();
+
+ provider.force_flush().expect("flush");
+ let spans = exporter.get_finished_spans().expect("finished spans");
+ let agent: Vec<_> = spans
+ .iter()
+ .filter(|span| span.name == "invoke_agent")
+ .collect();
+ assert_eq!(agent.len(), 1, "one prompt, one agent span: {spans:#?}");
+ assert_eq!(agent[0].parent_span_id, command_id);
+ let attribute = |key: &str| {
+ agent[0]
+ .attributes
+ .iter()
+ .find(|kv| kv.key.as_str() == key)
+ .map(|kv| kv.value.to_string())
+ };
+ assert_eq!(
+ attribute(attr::CONVERSATION_ID).as_deref(),
+ Some(session.to_string().as_str())
+ );
+ assert_eq!(
+ attribute(attr::RESPONSE_FINISH_REASONS).as_deref(),
+ Some(r#"["stop"]"#)
+ );
+ let output = attribute(attr::OUTPUT_MESSAGES).expect("output recorded");
+ assert!(output.contains("It is noon."), "{output}");
+ let input = attribute(attr::INPUT_MESSAGES).expect("input recorded");
+ assert!(input.contains("what time is it?"), "{input}");
+}
+
mod owner_binding;
/// The live writer's fold says what each appended frame meant for the turn;
diff --git a/crates/agent_session/src/domain/session/actors.rs b/crates/agent_session/src/domain/session/actors.rs
index ee43461719..272b14f4a1 100644
--- a/crates/agent_session/src/domain/session/actors.rs
+++ b/crates/agent_session/src/domain/session/actors.rs
@@ -12,7 +12,7 @@
//! loop (see [`super::super::service`]), while tests exercise its input and
//! dispatch halves directly.
-use std::{collections::VecDeque, time::Duration};
+use std::{collections::VecDeque, sync::Arc, time::Duration};
use agent_client_protocol::RawJsonRpcMessage;
use agent_client_protocol::schema::v1::{McpServer, SessionId};
@@ -30,10 +30,12 @@ use crate::domain::model::{AgentSessionId, AgentSessionLog, Message};
use agent_runtime_protocol::domain::ports::{TransportReceiver, TransportSender};
use crate::domain::ports::{
- AgentConnector, AgentSessionLogWriter, AgentSessionRepo, SessionTurnObserver,
+ AgentConnector, AgentSessionLogWriter, AgentSessionRepo, SessionToolCatalog,
+ SessionTurnObserver,
};
use agent_fold::domain::model::TurnSignal;
+use super::telemetry::GenAiProjector;
use super::{
CloseReason, Effect, HandshakeStatus, Input, RuntimeStatus, SessionMachine, StopReason,
};
@@ -42,6 +44,9 @@ use super::{
const HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(60);
/// How long one caller's action has to reach the runtime.
const COMMAND_DELIVERY_TIMEOUT: Duration = Duration::from_secs(60);
+/// How long the session's MCP servers have to list their tools for telemetry
+/// before the listing is given up on. Off the actor's own path, so generous.
+const TOOL_LISTING_TIMEOUT: Duration = Duration::from_secs(60);
/// A caller's request to deliver one action, and the wire back to them.
pub(crate) struct SessionCommand {
@@ -90,7 +95,17 @@ pub(crate) struct SessionActor<Connector: AgentConnector, Logs> {
handshake_deadline: Instant,
/// Told when the machine reports a turn over, so the harness can dispatch
/// its next queued prompt.
- turn_observer: std::sync::Arc<dyn SessionTurnObserver>,
+ turn_observer: Arc<dyn SessionTurnObserver>,
+ /// GenAI spans for the turns and tool calls this connection carries, fed
+ /// every frame the effects below send or log. See [`super::telemetry`].
+ telemetry: GenAiProjector,
+ /// Lists the session's MCP tools once the connection is live, for the
+ /// projector's `gen_ai.tool.definitions`.
+ tool_catalog: Arc<dyn SessionToolCatalog>,
+ /// The servers to list, as the machine sends them on `session/new`.
+ mcp_servers: Vec<McpServer>,
+ /// Whether the listing has been started; it runs once per connection.
+ tools_listed: bool,
}
impl<Connector, Logs> SessionActor<Connector, Logs>
@@ -110,7 +125,8 @@ where
logs: Logs,
commands: mpsc::Receiver<SessionCommand>,
handshake: watch::Sender<HandshakeStatus>,
- turn_observer: std::sync::Arc<dyn SessionTurnObserver>,
+ turn_observer: Arc<dyn SessionTurnObserver>,
+ tool_catalog: Arc<dyn SessionToolCatalog>,
) -> Self {
// Marked unseen so the first wait reports the *current* state rather
// than only later changes: a session binding after the handshake
@@ -134,8 +150,10 @@ where
handshake_seen,
handshake,
machine: match acp_session_id {
- None => SessionMachine::new(id, workspace, mcp_servers),
- Some(session_id) => SessionMachine::resume(id, session_id, workspace, mcp_servers),
+ None => SessionMachine::new(id, workspace, mcp_servers.clone()),
+ Some(session_id) => {
+ SessionMachine::resume(id, session_id, workspace, mcp_servers.clone())
+ }
}
.with_connection_context(macro_uuid::generate_uuid_v7()),
logs,
@@ -143,6 +161,14 @@ where
handshake_span: Some(handshake_span),
handshake_deadline: Instant::now() + HANDSHAKE_TIMEOUT,
turn_observer,
+ telemetry: GenAiProjector::new(
+ id,
+ genai_telemetry::ContentPolicy::from_env(),
+ Default::default(),
+ ),
+ tool_catalog,
+ mcp_servers,
+ tools_listed: false,
}
}
@@ -240,11 +266,16 @@ where
while let Some(effect) = effects.pop_front() {
match effect {
- Effect::Send { from, message } => {
+ Effect::Send { from, mut message } => {
let command_span = effects.front().and_then(|effect| match effect {
Effect::Complete { token, .. } => Some(token.span.clone()),
_ => None,
});
+ // Before delivery, so the turn's span starts when the
+ // prompt leaves. A delivery that fails closes the
+ // connection, which ends the turn in error below.
+ self.telemetry
+ .on_outbound(&mut message, command_span.as_ref());
let delivery = self.deliver(from, message);
let (result, close_reason) =
match (command_span.as_ref(), handshake_span.as_ref()) {
@@ -302,6 +333,7 @@ where
}
}
Effect::Log { message, boundary } => {
+ self.telemetry.on_inbound(&message);
if let Some(ToRuntimeMessage::Acp(AcpMessage(RawJsonRpcMessage::Request(
request,
)))) = &self.initialization_request
@@ -424,6 +456,7 @@ where
let _ = token.completed.send(result);
}
Effect::Stop { reason } => {
+ self.telemetry.on_stopped(&reason);
// The one place a session's `disconnected` event is
// written. The frame itself carries no cause, so a
// routine idle teardown and a runtime lost mid-turn are
@@ -585,6 +618,7 @@ where
match self.machine.status() {
RuntimeStatus::Live { .. } => {
self.handshake_span.take();
+ self.list_tools_once();
}
RuntimeStatus::Dead => {
if let Some(span) = self.handshake_span.take() {
@@ -596,6 +630,40 @@ where
}
}
+ /// List the session's MCP tools for its telemetry, once, off this actor's
+ /// path: the listing dials the same servers the agent was just handed and
+ /// must never hold up a prompt. Whatever it finds is published for the
+ /// projector to pick up on the next turn it records.
+ fn list_tools_once(&mut self) {
+ if self.tools_listed {
+ return;
+ }
+ self.tools_listed = true;
+ if self.mcp_servers.is_empty() {
+ return;
+ }
+ let catalog = Arc::clone(&self.tool_catalog);
+ let servers = self.mcp_servers.clone();
+ let definitions = self.telemetry.tool_definitions();
+ let id = self.machine.id();
+ tokio::spawn(
+ async move {
+ match tokio::time::timeout(TOOL_LISTING_TIMEOUT, catalog.tool_definitions(servers))
+ .await
+ {
+ Ok(listed) => {
+ tracing::debug!(%id, tools = listed.len(), "listed an agent session's tools");
+ definitions.publish(listed);
+ }
+ Err(_) => {
+ tracing::warn!(%id, "timed out listing an agent session's tools");
+ }
+ }
+ }
+ .in_current_span(),
+ );
+ }
+
/// The failing effect's own caller gets `cause` itself; anything still
/// queued behind it never reached the runtime, so it gets `Disconnected`.
fn fail_remaining_completions(
diff --git a/crates/agent_session/src/domain/session/mod.rs b/crates/agent_session/src/domain/session/mod.rs
index 5eb2e3d175..4418597fee 100644
--- a/crates/agent_session/src/domain/session/mod.rs
+++ b/crates/agent_session/src/domain/session/mod.rs
@@ -5,6 +5,7 @@ pub(crate) mod actors;
// choice: logic in `session`, vocabulary in `types`.
#[allow(clippy::module_inception)]
mod session;
+pub(crate) mod telemetry;
mod types;
#[cfg(test)]
diff --git a/crates/agent_session/src/domain/session/telemetry.rs b/crates/agent_session/src/domain/session/telemetry.rs
new file mode 100644
index 0000000000..ba6797f994
--- /dev/null
+++ b/crates/agent_session/src/domain/session/telemetry.rs
@@ -0,0 +1,870 @@
+//! GenAI spans projected from a live session's ACP frames.
+//!
+//! Every harness a session can run on - a sandboxed OpenCode, a Cursor cloud
+//! agent, Macro's own in-process agent, an external `macrod` - speaks ACP
+//! through the [`SessionActor`](super::actors::SessionActor), and the actor
+//! sees every frame in both directions exactly once. So this is the one place
+//! a session's turns and tool calls can be traced the same way for all of
+//! them, in the OpenTelemetry GenAI vocabulary an evaluation backend reads:
+//!
+//! - one `invoke_agent` span per prompt turn, from the `session/prompt` going
+//! out to its response coming back: the prompt as `gen_ai.input.messages`,
+//! the agent's streamed prose, reasoning and tool calls as
+//! `gen_ai.output.messages`, the stop reason as the finish reason, the
+//! turn's token usage, the tools the session's MCP servers offered as
+//! `gen_ai.tool.definitions`, the harness as the agent name and the session
+//! as the conversation id;
+//! - one `execute_tool` span per tool call, from its `tool_call` to the
+//! `tool_call_update` that finishes it, with the tool's name, arguments and
+//! result read through the harness conventions `agent_fold` knows, and an
+//! error status when the call failed.
+//!
+//! What ACP does not carry is not invented: there is no `chat` span, because
+//! the model calls happen inside the harness and never cross the wire.
+//!
+//! Content follows the [`ContentPolicy`] and size limits of
+//! [`genai_telemetry`]; structure (names, ids, status, usage) is always
+//! recorded. Everything here is best-effort and total: a frame the projector
+//! cannot read changes nothing, and never the session.
+//!
+//! Only live frames are projected. The actor feeds this from its effect loop,
+//! not from the log writer, so a reconnecting actor catching its fold up on
+//! the stored log re-emits nothing.
+
+#[cfg(test)]
+mod test;
+
+use std::collections::{HashMap, HashSet};
+use std::sync::{Arc, OnceLock};
+
+use agent_client_protocol::schema::v1::{
+ ContentBlock, EmbeddedResourceResource, InitializeRequest, InitializeResponse,
+ LoadSessionRequest, NewSessionRequest, PromptRequest, PromptResponse, RequestId, Response,
+ ResumeSessionRequest, SessionConfigKind, SessionConfigOption, SessionNotification,
+ SessionUpdate, SetSessionConfigOptionRequest, StopReason as AcpStopReason, ToolCall,
+ ToolCallUpdate, ToolKind,
+};
+use agent_client_protocol::{JsonRpcMessage, RawJsonRpcMessage, RawJsonRpcParams};
+use agent_fold::domain::harness::{ToolFrame, tool_name};
+use agent_fold::domain::model::{Harness, ToolName, ToolStatus};
+use agent_runtime_protocol::domain::action::MODEL_CONFIG_ID;
+use agent_runtime_protocol::domain::schema::v0::{AcpMessage, ToRuntimeMessage, ToServerMessage};
+use genai_telemetry::messages::{self, MediaSource};
+use genai_telemetry::{
+ ContentPolicy, GenAiSpanExt as _, attr, bound_messages, bound_tool_definitions,
+ bounded_json_string, truncate_chars,
+};
+use serde::Deserialize;
+use serde_json::Value;
+
+use crate::domain::model::AgentSessionId;
+use crate::domain::ports::SessionToolDefinition;
+
+use super::StopReason;
+
+/// The tools a session's MCP servers advertise, listed once per connection by
+/// a task the actor spawns and read by the projector when it records a turn.
+///
+/// A write-once cell rather than a channel: the listing is one fact, whoever
+/// reads it wants the latest and only value, and a turn that opens before the
+/// listing lands picks it up when it closes instead.
+#[derive(Clone, Default)]
+pub(crate) struct SharedToolDefinitions(Arc<OnceLock<Vec<SessionToolDefinition>>>);
+
+impl SharedToolDefinitions {
+ /// Publish the listing. A second publish is dropped; the first stands.
+ pub(crate) fn publish(&self, definitions: Vec<SessionToolDefinition>) {
+ let _ = self.0.set(definitions);
+ }
+
+ fn get(&self) -> Option<&[SessionToolDefinition]> {
+ self.0.get().map(Vec::as_slice)
+ }
+}
+
+/// Projects one session connection's ACP frames onto GenAI spans.
+pub(crate) struct GenAiProjector {
+ session: AgentSessionId,
+ policy: ContentPolicy,
+ /// The harness producing the frames: announced by `initialize`, or
+ /// recognized from the first tool frame of a log with no `initialize`.
+ harness: Harness,
+ /// The model the session runs on, per the runtime's own config options.
+ model: Option<String>,
+ tool_definitions: SharedToolDefinitions,
+ /// The `initialize` request whose response names the harness.
+ initialize: Option<RequestId>,
+ /// Requests whose responses carry the session's config options (and so
+ /// its model): `session/new`, `session/load`, `session/resume`, and every
+ /// `session/set_config_option`.
+ config_requests: HashSet<RequestId>,
+ /// The turn in flight, if a `session/prompt` is awaiting its answer.
+ turn: Option<Turn>,
+ /// Tool calls opened and not yet finished, by ACP tool call id. Under the
+ /// turn in flight when there is one; a call streamed outside any turn (a
+ /// resumed session whose prompt this connection never sent) gets a root
+ /// span of its own.
+ tools: HashMap<String, OpenTool>,
+ /// The cumulative token totals the last prompt response reported, so
+ /// each turn records its own share rather than the running total.
+ usage_totals: Option<(u64, u64)>,
+}
+
+/// One prompt turn's span and the output it has streamed so far.
+struct Turn {
+ request_id: RequestId,
+ span: tracing::Span,
+ /// The agent's output, in order: prose, reasoning and tool calls.
+ parts: Vec<OutputPart>,
+ /// Facts recorded at open so they are not recorded twice at close. Each
+ /// is recorded as soon as it is known, which may be at open or at close.
+ agent_name_recorded: bool,
+ model_recorded: bool,
+ definitions_recorded: bool,
+}
+
+enum OutputPart {
+ Text(String),
+ Reasoning(String),
+ ToolCall {
+ id: String,
+ name: String,
+ arguments: Value,
+ },
+}
+
+impl OutputPart {
+ fn into_json(self) -> Value {
+ match self {
+ Self::Text(text) => messages::text_part(text),
+ Self::Reasoning(text) => messages::reasoning_part(text),
+ Self::ToolCall {
+ id,
+ name,
+ arguments,
+ } => messages::tool_call_part(Some(&id), &name, arguments),
+ }
+ }
+}
+
+/// One tool call's span and what its frames have said so far.
+struct OpenTool {
+ span: tracing::Span,
+ name: ToolName,
+ status: ToolStatus,
+ /// The latest `rawInput` seen: a harness may stream the arguments over
+ /// several patches, and the last complete value is the call's.
+ arguments: Option<Value>,
+ /// Index of this call's part in the turn's output, when it has one.
+ part: Option<usize>,
+}
+
+/// How a turn ended.
+enum Outcome<'outcome> {
+ /// The runtime answered the prompt; the result carries the stop reason.
+ Answered(&'outcome Value),
+ /// The runtime refused the prompt with a JSON-RPC error.
+ Refused(&'outcome str),
+ /// Another prompt went out before this one was answered.
+ Superseded,
+ /// The connection ended with the turn in flight.
+ Stopped(&'outcome StopReason),
+}
+
+impl GenAiProjector {
+ /// A projector for `session`, recording content under `policy` and tool
+ /// definitions from `tool_definitions` once they are published.
+ pub(crate) fn new(
+ session: AgentSessionId,
+ policy: ContentPolicy,
+ tool_definitions: SharedToolDefinitions,
+ ) -> Self {
+ Self {
+ session,
+ policy,
+ harness: Harness::Unknown,
+ model: None,
+ tool_definitions,
+ initialize: None,
+ config_requests: HashSet::new(),
+ turn: None,
+ tools: HashMap::new(),
+ usage_totals: None,
+ }
+ }
+
+ /// The cell the connection's tool listing is published into.
+ pub(crate) fn tool_definitions(&self) -> SharedToolDefinitions {
+ self.tool_definitions.clone()
+ }
+
+ /// A frame this side is sending to the runtime. A `session/prompt` opens a
+ /// turn, as a child of `parent` - the span of the command that carried
+ /// the prompt - when there is one.
+ pub(crate) fn on_outbound(
+ &mut self,
+ message: &mut ToRuntimeMessage,
+ parent: Option<&tracing::Span>,
+ ) {
+ let ToRuntimeMessage::Acp(AcpMessage(RawJsonRpcMessage::Request(request))) = message else {
+ return;
+ };
+ let method = &request.method;
+ if InitializeRequest::matches_method(method) {
+ self.initialize = Some(request.id.clone());
+ } else if NewSessionRequest::matches_method(method)
+ || LoadSessionRequest::matches_method(method)
+ || ResumeSessionRequest::matches_method(method)
+ || SetSessionConfigOptionRequest::matches_method(method)
+ {
+ self.config_requests.insert(request.id.clone());
+ } else if PromptRequest::matches_method(method) {
+ let prompt: Option<PromptRequest> = deserialize_params(request.params.as_ref());
+ self.begin_turn(request.id.clone(), prompt, parent);
+ if let Some(turn) = &self.turn
+ && let Some(RawJsonRpcParams::Object(params)) = &mut request.params
+ && let Some(meta) = params
+ .entry("_meta")
+ .or_insert_with(|| Value::Object(Default::default()))
+ .as_object_mut()
+ {
+ genai_telemetry::propagation::inject(&turn.span, meta);
+ }
+ }
+ }
+
+ /// A frame the runtime sent.
+ pub(crate) fn on_inbound(&mut self, message: &ToServerMessage) {
+ let ToServerMessage::Acp(AcpMessage(frame)) = message else {
+ return;
+ };
+ match frame {
+ RawJsonRpcMessage::Response(Response::Result { id, result }) => {
+ self.on_result(id, result);
+ }
+ RawJsonRpcMessage::Response(Response::Error { id, error }) => {
+ if self
+ .turn
+ .as_ref()
+ .is_some_and(|turn| &turn.request_id == id)
+ && let Some(turn) = self.turn.take()
+ {
+ self.end_turn(turn, Outcome::Refused(&error.message));
+ }
+ }
+ RawJsonRpcMessage::Notification(notification)
+ if SessionNotification::matches_method(&notification.method) =>
+ {
+ if let Some(notification) =
+ deserialize_params::<SessionNotification>(notification.params.as_ref())
+ {
+ self.on_update(notification.update);
+ }
+ }
+ RawJsonRpcMessage::Request(_) | RawJsonRpcMessage::Notification(_) => {}
+ }
+ }
+
+ /// The connection is over. A turn in flight ends in error; tool calls
+ /// still open are marked abandoned.
+ pub(crate) fn on_stopped(&mut self, reason: &StopReason) {
+ if let Some(turn) = self.turn.take() {
+ self.end_turn(turn, Outcome::Stopped(reason));
+ }
+ for (_, tool) in self.tools.drain() {
+ tool.span.set_error("session_stopped", reason.to_string());
+ }
+ }
+
+ fn on_result(&mut self, id: &RequestId, result: &Value) {
+ if self.initialize.as_ref() == Some(id) {
+ self.initialize = None;
+ if let Ok(response) = serde_json::from_value::<InitializeResponse>(result.clone())
+ && let Some(info) = response.agent_info
+ {
+ self.harness = Harness::from_agent_info(&info.name);
+ }
+ return;
+ }
+ if self.config_requests.remove(id) {
+ #[derive(Deserialize)]
+ #[serde(rename_all = "camelCase")]
+ struct ConfigCarrier {
+ #[serde(default)]
+ config_options: Vec<SessionConfigOption>,
+ }
+ if let Ok(carrier) = serde_json::from_value::<ConfigCarrier>(result.clone()) {
+ self.apply_config_options(carrier.config_options);
+ }
+ return;
+ }
+ if self
+ .turn
+ .as_ref()
+ .is_some_and(|turn| &turn.request_id == id)
+ && let Some(turn) = self.turn.take()
+ {
+ self.end_turn(turn, Outcome::Answered(result));
+ }
+ }
+
+ fn on_update(&mut self, update: SessionUpdate) {
+ match update {
+ SessionUpdate::AgentMessageChunk(chunk) => self.append_output(&chunk.content, false),
+ SessionUpdate::AgentThoughtChunk(chunk) => self.append_output(&chunk.content, true),
+ SessionUpdate::ToolCall(call) => self.open_tool(call),
+ SessionUpdate::ToolCallUpdate(update) => self.patch_tool(update),
+ SessionUpdate::ConfigOptionUpdate(update) => {
+ self.apply_config_options(update.config_options);
+ }
+ SessionUpdate::UsageUpdate(usage) => {
+ if let Some(turn) = &self.turn {
+ turn.span
+ .set_u64(attr::MACRO_CONTEXT_USED_TOKENS, usage.used);
+ turn.span
+ .set_u64(attr::MACRO_CONTEXT_SIZE_TOKENS, usage.size);
+ }
+ }
+ // Plans, modes, commands, titles and the user's own echoed
+ // chunks: nothing a judge of the turn needs.
+ _ => {}
+ }
+ }
+
+ fn apply_config_options(&mut self, options: Vec<SessionConfigOption>) {
+ let model = options
+ .into_iter()
+ .find(|option| option.id.to_string() == MODEL_CONFIG_ID);
+ if let Some(SessionConfigOption {
+ kind: SessionConfigKind::Select(select),
+ ..
+ }) = model
+ {
+ self.model = Some(select.current_value.to_string());
+ }
+ }
+
+ fn begin_turn(
+ &mut self,
+ request_id: RequestId,
+ prompt: Option<PromptRequest>,
+ parent: Option<&tracing::Span>,
+ ) {
+ if let Some(previous) = self.turn.take() {
+ self.end_turn(previous, Outcome::Superseded);
+ }
+ // The literal stays in step with `attr::span_name::INVOKE_AGENT`;
+ // `tracing` bakes the name into static metadata.
+ let span = match parent {
+ Some(parent) => tracing::info_span!(
+ parent: parent,
+ "invoke_agent",
+ gen_ai.operation.name = attr::operation::INVOKE_AGENT,
+ gen_ai.conversation.id = %self.session,
+ agent.session.id = %self.session,
+ ),
+ None => tracing::info_span!(
+ "invoke_agent",
+ gen_ai.operation.name = attr::operation::INVOKE_AGENT,
+ gen_ai.conversation.id = %self.session,
+ agent.session.id = %self.session,
+ ),
+ };
+ let mut turn = Turn {
+ request_id,
+ span,
+ parts: Vec::new(),
+ agent_name_recorded: false,
+ model_recorded: false,
+ definitions_recorded: false,
+ };
+ self.record_turn_facts(&mut turn);
+ if self.policy.capture
+ && let Some(prompt) = prompt
+ {
+ let parts: Vec<Value> = prompt.prompt.iter().filter_map(prompt_part).collect();
+ let bounded = bound_messages(
+ vec![messages::message(messages::ROLE_USER, parts)],
+ &self.policy.limits,
+ );
+ turn.span.set_str(attr::INPUT_MESSAGES, bounded.json);
+ if bounded.truncated {
+ turn.span.set_bool(attr::MACRO_CONTENT_TRUNCATED, true);
+ }
+ }
+ self.turn = Some(turn);
+ }
+
+ /// Record what is known about the session onto a turn's span: the
+ /// harness, the model and the offered tools, each once, as soon as known.
+ fn record_turn_facts(&self, turn: &mut Turn) {
+ if !turn.agent_name_recorded && self.harness != Harness::Unknown {
+ turn.span
+ .set_str(attr::AGENT_NAME, harness_name(self.harness));
+ turn.agent_name_recorded = true;
+ }
+ if !turn.model_recorded
+ && let Some(model) = &self.model
+ {
+ turn.span.set_str(attr::REQUEST_MODEL, model.clone());
+ turn.model_recorded = true;
+ }
+ if !turn.definitions_recorded
+ && let Some(definitions) = self.tool_definitions.get()
+ {
+ turn.definitions_recorded = true;
+ if !definitions.is_empty() {
+ let definitions = definitions
+ .iter()
+ .map(|definition| {
+ messages::tool_definition(
+ &self.definition_name(&definition.name),
+ &definition.description,
+ definition.parameters.clone(),
+ )
+ })
+ .collect();
+ let (json, truncated) = bound_tool_definitions(definitions, &self.policy.limits);
+ turn.span.set_str(attr::TOOL_DEFINITIONS, json);
+ if truncated {
+ turn.span.set_bool(attr::MACRO_CONTENT_TRUNCATED, true);
+ }
+ }
+ }
+ }
+
+ /// A listed tool's name as this harness calls it. Macro's in-process agent
+ /// calls Macro's tools natively, by their bare names, while the listing
+ /// sees them through Macro's MCP server as `mcp__macro__<tool>`; every
+ /// other harness calls them by the listed name.
+ fn definition_name(&self, listed: &str) -> String {
+ let name: ToolName = listed.parse().unwrap_or_else(|never| match never {});
+ if self.harness == Harness::Macro
+ && let ToolName::Mcp { .. } = &name
+ && let Some(tool) = self.harness.reader().macro_tool(&name)
+ {
+ return tool.to_owned();
+ }
+ listed.to_owned()
+ }
+
+ /// The listed description of the tool behind `name`, if it was listed.
+ fn tool_description(&self, name: &ToolName) -> Option<String> {
+ let display = name.display();
+ let mangled = match name {
+ ToolName::Mcp { server, tool } => Some(format!("mcp__{server}__{tool}")),
+ ToolName::Native { .. } => None,
+ };
+ self.tool_definitions.get()?.iter().find_map(|definition| {
+ (definition.name == display
+ || mangled.as_deref() == Some(definition.name.as_str())
+ || self.definition_name(&definition.name) == display)
+ .then(|| definition.description.clone())
+ })
+ }
+
+ fn end_turn(&mut self, mut turn: Turn, outcome: Outcome<'_>) {
+ let finish_reason = match outcome {
+ Outcome::Answered(result) => {
+ self.record_usage(&turn.span, result);
+ let stop_reason = serde_json::from_value::<PromptResponse>(result.clone())
+ .ok()
+ .map(|response| response.stop_reason);
+ finish_reason(stop_reason)
+ }
+ Outcome::Refused(message) => {
+ turn.span.set_error(
+ "prompt_refused",
+ self.error_description(message, "the runtime refused the prompt"),
+ );
+ attr::finish_reason::ERROR
+ }
+ Outcome::Superseded => {
+ turn.span.set_error(
+ "superseded",
+ "another prompt was sent before this turn was answered",
+ );
+ attr::finish_reason::ERROR
+ }
+ Outcome::Stopped(reason) => {
+ turn.span.set_error("session_stopped", reason.to_string());
+ attr::finish_reason::ERROR
+ }
+ };
+
+ // A call this turn opened and the turn ended on is over for the
+ // judge's purposes, whatever the harness would have said next: mark
+ // it rather than leak its span. A call opened outside any turn (its
+ // `part` is `None`) belongs to no turn and stays open for its own
+ // finish.
+ let abandoned = if finish_reason == attr::finish_reason::CANCELLED {
+ "the turn was cancelled before the tool call finished"
+ } else {
+ "the turn ended before the tool call finished"
+ };
+ self.tools.retain(|_, tool| {
+ if tool.part.is_some() {
+ tool.span.set_error("abandoned", abandoned);
+ false
+ } else {
+ true
+ }
+ });
+
+ self.record_turn_facts(&mut turn);
+ if !turn.agent_name_recorded {
+ turn.span
+ .set_str(attr::AGENT_NAME, harness_name(self.harness));
+ }
+ turn.span
+ .set_str_array(attr::RESPONSE_FINISH_REASONS, [finish_reason]);
+ if self.policy.capture {
+ let parts = turn.parts.into_iter().map(OutputPart::into_json).collect();
+ let bounded = bound_messages(
+ vec![messages::output_message(
+ messages::ROLE_ASSISTANT,
+ parts,
+ finish_reason,
+ )],
+ &self.policy.limits,
+ );
+ turn.span.set_str(attr::OUTPUT_MESSAGES, bounded.json);
+ if bounded.truncated {
+ turn.span.set_bool(attr::MACRO_CONTENT_TRUNCATED, true);
+ }
+ }
+ // Dropping the last handle closes the span.
+ }
+
+ /// The turn's token usage from the prompt response. ACP reports running
+ /// session totals, so each turn records the increase over the previous
+ /// response; the first response a connection sees records the total,
+ /// which for a resumed session includes earlier turns.
+ fn record_usage(&mut self, span: &tracing::Span, result: &Value) {
+ let input = result.pointer("/usage/inputTokens").and_then(Value::as_u64);
+ let output = result
+ .pointer("/usage/outputTokens")
+ .and_then(Value::as_u64);
+ let (Some(input), Some(output)) = (input, output) else {
+ return;
+ };
+ let (turn_input, turn_output) = match self.usage_totals {
+ Some((previous_input, previous_output))
+ if input >= previous_input && output >= previous_output =>
+ {
+ (input - previous_input, output - previous_output)
+ }
+ _ => (input, output),
+ };
+ self.usage_totals = Some((input, output));
+ span.set_u64(attr::USAGE_INPUT_TOKENS, turn_input);
+ span.set_u64(attr::USAGE_OUTPUT_TOKENS, turn_output);
+ }
+
+ fn append_output(&mut self, content: &ContentBlock, reasoning: bool) {
+ let ContentBlock::Text(text) = content else {
+ return;
+ };
+ let Some(turn) = &mut self.turn else {
+ return;
+ };
+ match (turn.parts.last_mut(), reasoning) {
+ (Some(OutputPart::Text(existing)), false)
+ | (Some(OutputPart::Reasoning(existing)), true) => existing.push_str(&text.text),
+ (_, false) => turn.parts.push(OutputPart::Text(text.text.clone())),
+ (_, true) => turn.parts.push(OutputPart::Reasoning(text.text.clone())),
+ }
+ }
+
+ fn open_tool(&mut self, call: ToolCall) {
+ let frame = ToolFrame::of_call(&call);
+ self.sniff_harness(&frame);
+ let reader = self.harness.reader();
+ let name = tool_name(reader, &frame);
+ let id = call.tool_call_id.0.to_string();
+ let tool_display = name.display().to_owned();
+
+ // The literal stays in step with `attr::span_name::EXECUTE_TOOL`;
+ // `otel.name` gives the exported span the semconv `execute_tool
+ // {name}` name.
+ let otel_name = format!("{} {tool_display}", attr::operation::EXECUTE_TOOL);
+ let span = match &self.turn {
+ Some(turn) => tracing::info_span!(
+ parent: &turn.span,
+ "execute_tool",
+ otel.name = %otel_name,
+ gen_ai.operation.name = attr::operation::EXECUTE_TOOL,
+ gen_ai.tool.type = attr::tool_type::FUNCTION,
+ gen_ai.tool.name = %tool_display,
+ gen_ai.tool.call.id = %id,
+ gen_ai.conversation.id = %self.session,
+ agent.session.id = %self.session,
+ ),
+ None => tracing::info_span!(
+ "execute_tool",
+ otel.name = %otel_name,
+ gen_ai.operation.name = attr::operation::EXECUTE_TOOL,
+ gen_ai.tool.type = attr::tool_type::FUNCTION,
+ gen_ai.tool.name = %tool_display,
+ gen_ai.tool.call.id = %id,
+ gen_ai.conversation.id = %self.session,
+ agent.session.id = %self.session,
+ ),
+ };
+ span.set_str(attr::MACRO_TOOL_KIND, tool_kind_name(call.kind));
+ if !call.title.is_empty() {
+ span.set_str(attr::MACRO_TOOL_TITLE, call.title.clone());
+ }
+ if let ToolName::Mcp { server, .. } = &name {
+ span.set_str(attr::MACRO_MCP_SERVER, server.clone());
+ }
+ if let Some(description) = self.tool_description(&name) {
+ span.set_str(attr::TOOL_DESCRIPTION, description);
+ }
+
+ let arguments = call.raw_input.clone().filter(|input| !input.is_null());
+ let part = self.turn.as_mut().map(|turn| {
+ turn.parts.push(OutputPart::ToolCall {
+ id: id.clone(),
+ name: tool_display,
+ arguments: arguments.clone().unwrap_or(Value::Null),
+ });
+ turn.parts.len() - 1
+ });
+ let status = frame.status.unwrap_or_default();
+ if let Some(replaced) = self.tools.insert(
+ id.clone(),
+ OpenTool {
+ span,
+ name,
+ status,
+ arguments,
+ part,
+ },
+ ) {
+ replaced
+ .span
+ .set_error("superseded", "the harness reopened this tool call id");
+ }
+ if status.is_finished() {
+ self.finish_tool(&id, &frame);
+ }
+ }
+
+ fn patch_tool(&mut self, update: ToolCallUpdate) {
+ let id = update.tool_call_id.0.to_string();
+ let mut frame = ToolFrame::of_update(&update);
+ self.sniff_harness(&frame);
+ let Some(tool) = self.tools.get_mut(&id) else {
+ // A call this connection never saw open: a resumed session
+ // mid-call. Nothing to attach it to.
+ return;
+ };
+ if frame.status.is_none() {
+ frame = frame.with_status(tool.status);
+ }
+ if let Some(status) = frame.status {
+ tool.status = status;
+ }
+ if let Some(input) = frame.raw_input
+ && !input.is_null()
+ {
+ tool.arguments = Some(input.clone());
+ }
+ // A harness may name the tool only once it knows what it is running.
+ if tool.name.is_empty() {
+ let named = tool_name(self.harness.reader(), &frame);
+ if !named.is_empty() {
+ let display = named.display().to_owned();
+ tool.span.record("gen_ai.tool.name", display.as_str());
+ tool.span.record(
+ "otel.name",
+ format!("{} {display}", attr::operation::EXECUTE_TOOL).as_str(),
+ );
+ tool.name = named;
+ }
+ }
+ if frame.finished() {
+ self.finish_tool(&id, &frame);
+ }
+ }
+
+ fn finish_tool(&mut self, id: &str, frame: &ToolFrame<'_>) {
+ let Some(tool) = self.tools.remove(id) else {
+ return;
+ };
+ let reader = self.harness.reader();
+ let (result, mut error) = match frame.raw_output {
+ Some(raw) => {
+ let (value, error) = reader.unwrap_tool_output(raw);
+ (Some(value), error)
+ }
+ None => (
+ frame
+ .content_text()
+ .or_else(|| reader.terminal_output(frame))
+ .map(Value::String),
+ None,
+ ),
+ };
+ if error.is_none()
+ && let Some(code) = reader.terminal_exit_code(frame)
+ && code != 0
+ {
+ error = Some(format!("the command exited with code {code}"));
+ }
+
+ if self.policy.capture {
+ if let Some(arguments) = &tool.arguments {
+ let (json, truncated) = bounded_json_string(arguments, &self.policy.limits);
+ tool.span.set_str(attr::TOOL_CALL_ARGUMENTS, json);
+ if truncated {
+ tool.span.set_bool(attr::MACRO_CONTENT_TRUNCATED, true);
+ }
+ }
+ if let Some(result) = &result {
+ let (json, truncated) = bounded_json_string(result, &self.policy.limits);
+ tool.span.set_str(attr::TOOL_CALL_RESULT, json);
+ if truncated {
+ tool.span.set_bool(attr::MACRO_CONTENT_TRUNCATED, true);
+ }
+ }
+ }
+ // The error text is the tool's own output and so content: recorded
+ // under the policy, bounded.
+ match (tool.status, error) {
+ (ToolStatus::Failed, error) => tool.span.set_error(
+ "tool_error",
+ error.map_or_else(
+ || "the tool call failed".to_owned(),
+ |error| self.error_description(&error, "the tool call failed"),
+ ),
+ ),
+ (_, Some(error)) => tool.span.set_error(
+ "tool_error",
+ self.error_description(&error, "the tool reported an error"),
+ ),
+ (_, None) => {}
+ }
+
+ // The turn's record of the call gets the final arguments and name.
+ if let (Some(turn), Some(index)) = (&mut self.turn, tool.part)
+ && let Some(OutputPart::ToolCall {
+ name, arguments, ..
+ }) = turn.parts.get_mut(index)
+ {
+ if let Some(final_arguments) = tool.arguments {
+ *arguments = final_arguments;
+ }
+ if name.is_empty() {
+ *name = tool.name.display().to_owned();
+ }
+ }
+ // Dropping the last handle closes the span.
+ }
+
+ /// An error description fit for a span's status: the detail, bounded,
+ /// when content may be recorded; a fixed summary otherwise.
+ fn error_description(&self, detail: &str, summary: &'static str) -> String {
+ if self.policy.capture {
+ truncate_chars(detail, self.policy.limits.max_part_chars).into_owned()
+ } else {
+ summary.to_owned()
+ }
+ }
+
+ /// Recognize the harness from a tool frame when no `initialize` named it.
+ fn sniff_harness(&mut self, frame: &ToolFrame<'_>) {
+ if self.harness == Harness::Unknown
+ && let Some(harness) = Harness::sniff(frame)
+ {
+ self.harness = harness;
+ }
+ }
+}
+
+/// Deserialize a request's or notification's params as a specific ACP type.
+/// `None` for positional params or a shape mismatch: less is derived from the
+/// frame, nothing fails.
+fn deserialize_params<T: serde::de::DeserializeOwned>(
+ params: Option<&RawJsonRpcParams>,
+) -> Option<T> {
+ match params? {
+ RawJsonRpcParams::Object(map) => serde_json::from_value(Value::Object(map.clone())).ok(),
+ RawJsonRpcParams::Array(_) => None,
+ }
+}
+
+/// A prompt content block as a semconv message part. Only the user's own text
+/// is copied onto a span; attached resources - inline bytes and embedded text
+/// alike - are named by their URI, since a document the user attached is not
+/// the prompt and can be arbitrarily large.
+fn prompt_part(block: &ContentBlock) -> Option<Value> {
+ Some(match block {
+ ContentBlock::Text(text) => messages::text_part(text.text.clone()),
+ ContentBlock::Image(image) => messages::media_part(
+ "image",
+ Some(&image.mime_type),
+ image
+ .uri
+ .clone()
+ .map_or(MediaSource::Inline, MediaSource::Uri),
+ ),
+ ContentBlock::Audio(audio) => {
+ messages::media_part("audio", Some(&audio.mime_type), MediaSource::Inline)
+ }
+ ContentBlock::ResourceLink(link) => messages::media_part(
+ "document",
+ link.mime_type.as_deref(),
+ MediaSource::Uri(link.uri.clone()),
+ ),
+ ContentBlock::Resource(embedded) => match &embedded.resource {
+ EmbeddedResourceResource::TextResourceContents(text) => messages::media_part(
+ "document",
+ text.mime_type.as_deref(),
+ MediaSource::Uri(text.uri.clone()),
+ ),
+ EmbeddedResourceResource::BlobResourceContents(blob) => messages::media_part(
+ "document",
+ blob.mime_type.as_deref(),
+ MediaSource::Uri(blob.uri.clone()),
+ ),
+ // `#[non_exhaustive]`: a resource shape this build does not know.
+ _ => return None,
+ },
+ // `#[non_exhaustive]`: a block this build does not know.
+ _ => return None,
+ })
+}
+
+/// The semconv finish reason for how a turn stopped.
+fn finish_reason(stop_reason: Option<AcpStopReason>) -> &'static str {
+ match stop_reason {
+ Some(AcpStopReason::EndTurn) => attr::finish_reason::STOP,
+ Some(AcpStopReason::MaxTokens | AcpStopReason::MaxTurnRequests) => {
+ attr::finish_reason::LENGTH
+ }
+ Some(AcpStopReason::Refusal) => attr::finish_reason::CONTENT_FILTER,
+ Some(AcpStopReason::Cancelled) => attr::finish_reason::CANCELLED,
+ // `#[non_exhaustive]`, and a response this build cannot read: the
+ // turn was answered, which is all that is known.
+ Some(_) | None => attr::finish_reason::STOP,
+ }
+}
+
+/// The harness as the agent name spans carry, in the fold's own snake_case
+/// spelling so it matches the `harness` field a session's readers show.
+fn harness_name(harness: Harness) -> String {
+ serde_json::to_value(harness)
+ .ok()
+ .and_then(|value| value.as_str().map(str::to_owned))
+ .unwrap_or_else(|| "unknown".to_owned())
+}
+
+/// ACP's coarse tool kind, in its wire spelling.
+fn tool_kind_name(kind: ToolKind) -> String {
+ serde_json::to_value(kind)
+ .ok()
+ .and_then(|value| value.as_str().map(str::to_owned))
+ .unwrap_or_else(|| "unknown".to_owned())
+}
diff --git a/crates/agent_session/src/domain/session/telemetry/test.rs b/crates/agent_session/src/domain/session/telemetry/test.rs
new file mode 100644
index 0000000000..f048f2c3ec
--- /dev/null
+++ b/crates/agent_session/src/domain/session/telemetry/test.rs
@@ -0,0 +1,776 @@
+//! The projector over recorded sessions: which spans come out, with which
+//! attributes, from the frames a live actor would feed it.
+
+use super::*;
+use crate::domain::model::Message;
+use crate::domain::ports::SessionToolDefinition;
+use crate::domain::session::CloseReason;
+use agent_fold::testing::{fixtures, parse_log};
+use opentelemetry::trace::{SpanId, Status, TraceContextExt as _, TracerProvider as _};
+use opentelemetry::{Array, Value as OtelValue};
+use opentelemetry_sdk::trace::{InMemorySpanExporter, SdkTracerProvider, SpanData};
+use serde_json::json;
+use tracing_opentelemetry::OpenTelemetrySpanExt as _;
+use tracing_subscriber::layer::SubscriberExt as _;
+
+/// An OpenTelemetry pipeline that keeps finished spans in memory, installed as
+/// the thread's default subscriber for the returned guard's lifetime.
+fn otel_test_pipeline() -> (
+ InMemorySpanExporter,
+ SdkTracerProvider,
+ tracing::subscriber::DefaultGuard,
+) {
+ let exporter = InMemorySpanExporter::default();
+ let provider = SdkTracerProvider::builder()
+ .with_simple_exporter(exporter.clone())
+ .build();
+ let layer = tracing_opentelemetry::layer().with_tracer(provider.tracer("test"));
+ let guard = tracing::subscriber::set_default(tracing_subscriber::registry().with(layer));
+ // Other tests in this binary run the same code paths with no subscriber
+ // installed, which can cache "never interested" for a span callsite on
+ // another thread; recompute so this subscriber sees every span.
+ tracing::callsite::rebuild_interest_cache();
+ (exporter, provider, guard)
+}
+
+fn finished(exporter: &InMemorySpanExporter, provider: &SdkTracerProvider) -> Vec<SpanData> {
+ provider.force_flush().expect("flush");
+ exporter.get_finished_spans().expect("finished spans")
+}
+
+fn attribute<'a>(span: &'a SpanData, key: &str) -> Option<&'a OtelValue> {
+ span.attributes
+ .iter()
+ .find(|kv| kv.key.as_str() == key)
+ .map(|kv| &kv.value)
+}
+
+fn string_attribute(span: &SpanData, key: &str) -> Option<String> {
+ match attribute(span, key)? {
+ OtelValue::String(s) => Some(s.to_string()),
+ other => panic!("{key} is not a string: {other:?}"),
+ }
+}
+
+fn int_attribute(span: &SpanData, key: &str) -> Option<i64> {
+ match attribute(span, key)? {
+ OtelValue::I64(i) => Some(*i),
+ other => panic!("{key} is not an integer: {other:?}"),
+ }
+}
+
+fn string_array_attribute(span: &SpanData, key: &str) -> Option<Vec<String>> {
+ match attribute(span, key)? {
+ OtelValue::Array(Array::String(values)) => {
+ Some(values.iter().map(ToString::to_string).collect())
+ }
+ other => panic!("{key} is not a string array: {other:?}"),
+ }
+}
+
+fn json_attribute(span: &SpanData, key: &str) -> Option<Value> {
+ string_attribute(span, key).map(|json| serde_json::from_str(&json).expect("valid JSON"))
+}
+
+fn spans_named<'a>(spans: &'a [SpanData], name: &str) -> Vec<&'a SpanData> {
+ spans.iter().filter(|span| span.name == name).collect()
+}
+
+fn projector(policy: ContentPolicy) -> GenAiProjector {
+ GenAiProjector::new(
+ AgentSessionId::TEST_A,
+ policy,
+ SharedToolDefinitions::default(),
+ )
+}
+
+/// Feed a recorded log through the projector the way the actor does: frames
+/// to the runtime through `on_outbound`, frames from it through `on_inbound`.
+fn replay(projector: &mut GenAiProjector, jsonl: &str) {
+ for mut entry in parse_log(jsonl) {
+ match &mut entry.content {
+ Message::ToRuntime(message) => projector.on_outbound(message, None),
+ Message::ToServer(message) => projector.on_inbound(message),
+ }
+ }
+}
+
+#[test]
+fn a_turn_becomes_an_agent_span_with_a_tool_span_per_call() {
+ let (exporter, provider, _guard) = otel_test_pipeline();
+ let mut projector = projector(ContentPolicy::enabled());
+ replay(&mut projector, fixtures::TURN);
+ let spans = finished(&exporter, &provider);
+
+ let agent = spans_named(&spans, "invoke_agent");
+ assert_eq!(agent.len(), 1, "one prompt, one agent span: {spans:#?}");
+ let agent = agent[0];
+ assert_eq!(
+ string_attribute(agent, attr::OPERATION_NAME).as_deref(),
+ Some("invoke_agent")
+ );
+ assert_eq!(
+ string_attribute(agent, attr::CONVERSATION_ID).as_deref(),
+ Some(AgentSessionId::TEST_A.to_string().as_str())
+ );
+ // The fixture's `initialize` names no agent; the harness is recognized
+ // from the `_meta.claudeCode` namespace on its tool frames.
+ assert_eq!(
+ string_attribute(agent, attr::AGENT_NAME).as_deref(),
+ Some("claude_code")
+ );
+ assert_eq!(
+ string_array_attribute(agent, attr::RESPONSE_FINISH_REASONS),
+ Some(vec!["stop".to_owned()])
+ );
+ assert_eq!(int_attribute(agent, attr::USAGE_INPUT_TOKENS), Some(10));
+ assert_eq!(int_attribute(agent, attr::USAGE_OUTPUT_TOKENS), Some(8));
+ assert_eq!(
+ int_attribute(agent, attr::MACRO_CONTEXT_USED_TOKENS),
+ Some(10)
+ );
+ assert_eq!(
+ int_attribute(agent, attr::MACRO_CONTEXT_SIZE_TOKENS),
+ Some(1_000_000)
+ );
+
+ let input = json_attribute(agent, attr::INPUT_MESSAGES).expect("input recorded");
+ assert_eq!(
+ input,
+ json!([{ "role": "user", "parts": [{ "type": "text", "content": "list the examples and write a file" }] }])
+ );
+ let output = json_attribute(agent, attr::OUTPUT_MESSAGES).expect("output recorded");
+ assert_eq!(output[0]["role"], "assistant");
+ assert_eq!(output[0]["finish_reason"], "stop");
+ let parts = output[0]["parts"].as_array().expect("parts");
+ assert_eq!(
+ parts[0],
+ json!({ "type": "text", "content": "Sure, one moment." })
+ );
+ assert_eq!(parts[1]["type"], "tool_call");
+ assert_eq!(parts[1]["id"], "tc-run");
+ assert_eq!(parts[1]["name"], "Bash");
+ assert_eq!(parts[1]["arguments"], json!({ "command": "ls examples" }));
+ assert_eq!(parts[2]["type"], "tool_call");
+ assert_eq!(parts[2]["name"], "Write");
+ assert_eq!(parts[3], json!({ "type": "text", "content": "Done." }));
+
+ let tools = spans_named(&spans, "execute_tool Bash");
+ assert_eq!(tools.len(), 1, "one Bash call: {spans:#?}");
+ let bash = tools[0];
+ assert_eq!(
+ bash.parent_span_id,
+ agent.span_context.span_id(),
+ "tool spans hang off the turn"
+ );
+ assert_eq!(
+ string_attribute(bash, attr::TOOL_NAME).as_deref(),
+ Some("Bash")
+ );
+ assert_eq!(
+ string_attribute(bash, attr::TOOL_CALL_ID).as_deref(),
+ Some("tc-run")
+ );
+ assert_eq!(
+ string_attribute(bash, attr::MACRO_TOOL_KIND).as_deref(),
+ Some("execute")
+ );
+ assert_eq!(
+ json_attribute(bash, attr::TOOL_CALL_ARGUMENTS),
+ Some(json!({ "command": "ls examples" }))
+ );
+ // No `rawOutput`; the terminal output in `_meta` is the result.
+ let result = string_attribute(bash, attr::TOOL_CALL_RESULT).expect("result recorded");
+ assert!(
+ result.contains("events.rs"),
+ "terminal output recorded: {result}"
+ );
+ assert_eq!(bash.status, Status::Unset);
+
+ let write = spans_named(&spans, "execute_tool Write");
+ assert_eq!(write.len(), 1);
+ assert_eq!(write[0].parent_span_id, agent.span_context.span_id());
+ // The tool spans close before the turn does.
+ assert!(bash.end_time <= agent.end_time);
+}
+
+#[test]
+fn the_harness_announced_by_initialize_names_the_agent() {
+ let (exporter, provider, _guard) = otel_test_pipeline();
+ let mut projector = projector(ContentPolicy::enabled());
+ replay(
+ &mut projector,
+ include_str!("../../../../../agent_fold/fixtures/real/real_single_turn.jsonl"),
+ );
+ let spans = finished(&exporter, &provider);
+
+ let agent = spans_named(&spans, "invoke_agent");
+ assert_eq!(agent.len(), 1);
+ assert_eq!(
+ string_attribute(agent[0], attr::AGENT_NAME).as_deref(),
+ Some("claude_code")
+ );
+ assert_eq!(int_attribute(agent[0], attr::USAGE_INPUT_TOKENS), Some(2));
+ assert_eq!(int_attribute(agent[0], attr::USAGE_OUTPUT_TOKENS), Some(29));
+ let output = json_attribute(agent[0], attr::OUTPUT_MESSAGES).expect("output");
+ assert_eq!(
+ output[0]["parts"][0]["content"],
+ "Hey! What are we working on today?"
+ );
+}
+
+#[test]
+fn usage_is_recorded_per_turn_from_the_running_totals() {
+ let (exporter, provider, _guard) = otel_test_pipeline();
+ let mut projector = projector(ContentPolicy::enabled());
+ replay(
+ &mut projector,
+ include_str!("../../../../../agent_fold/fixtures/real/real_multi_turn.jsonl"),
+ );
+ let spans = finished(&exporter, &provider);
+
+ let agents = spans_named(&spans, "invoke_agent");
+ assert!(agents.len() > 1, "a multi-turn recording: {}", agents.len());
+ let first = agents
+ .iter()
+ .min_by_key(|span| span.start_time)
+ .expect("a first turn");
+ assert_eq!(int_attribute(first, attr::USAGE_OUTPUT_TOKENS), Some(2161));
+ // Every later turn reports its increase, never the total so far.
+ let total: i64 = agents
+ .iter()
+ .filter_map(|span| int_attribute(span, attr::USAGE_OUTPUT_TOKENS))
+ .sum();
+ assert_eq!(total, 48946, "the turns' shares add up to the final total");
+}
+
+#[test]
+fn a_failed_mcp_call_marks_its_span_and_unwraps_the_envelope() {
+ let (exporter, provider, _guard) = otel_test_pipeline();
+ let mut projector = projector(ContentPolicy::enabled());
+ replay(&mut projector, fixtures::MACRO_MCP);
+ let spans = finished(&exporter, &provider);
+
+ let tools: Vec<_> = spans
+ .iter()
+ .filter(|span| span.name.starts_with("execute_tool "))
+ .collect();
+ assert!(!tools.is_empty(), "tool spans: {spans:#?}");
+ for tool in &tools {
+ assert_eq!(
+ string_attribute(tool, attr::MACRO_MCP_SERVER).as_deref(),
+ Some("macro"),
+ "Macro's tools are reached over its MCP server: {tool:#?}"
+ );
+ // The MCP `CallToolResult` envelope is not what the tool returned:
+ // a text result is recorded as its bare text, a structured one as
+ // its own JSON, never `{"content": [...], "isError": ...}`.
+ if let Some(result) = string_attribute(tool, attr::TOOL_CALL_RESULT) {
+ assert!(
+ !result.contains("\"isError\""),
+ "envelope unwrapped: {result}"
+ );
+ }
+ }
+ let failed: Vec<_> = tools
+ .iter()
+ .filter(|span| matches!(span.status, Status::Error { .. }))
+ .collect();
+ assert_eq!(failed.len(), 1, "one failed call: {tools:#?}");
+ assert_eq!(
+ string_attribute(failed[0], attr::ERROR_TYPE).as_deref(),
+ Some("tool_error")
+ );
+}
+
+#[test]
+fn a_stopped_connection_ends_the_turn_and_its_tools_in_error() {
+ let (exporter, provider, _guard) = otel_test_pipeline();
+ let mut projector = projector(ContentPolicy::enabled());
+ // Everything up to, not including, the prompt's answer.
+ let frames: Vec<&str> = fixtures::TURN.lines().collect();
+ let truncated = frames[..frames.len() - 3].join("\n");
+ replay(&mut projector, &truncated);
+ projector.on_stopped(&StopReason::Closed(CloseReason::TransportClosed));
+ let spans = finished(&exporter, &provider);
+
+ let agent = spans_named(&spans, "invoke_agent");
+ assert_eq!(agent.len(), 1);
+ assert!(matches!(agent[0].status, Status::Error { .. }));
+ assert_eq!(
+ string_attribute(agent[0], attr::ERROR_TYPE).as_deref(),
+ Some("session_stopped")
+ );
+ assert_eq!(
+ string_array_attribute(agent[0], attr::RESPONSE_FINISH_REASONS),
+ Some(vec!["error".to_owned()])
+ );
+ // The Write call had not finished when the connection dropped.
+ let write = spans_named(&spans, "execute_tool Write");
+ assert_eq!(write.len(), 1);
+ assert!(matches!(write[0].status, Status::Error { .. }));
+ assert_eq!(
+ string_attribute(write[0], attr::ERROR_TYPE).as_deref(),
+ Some("abandoned")
+ );
+}
+
+#[test]
+fn a_refused_prompt_ends_the_turn_in_error() {
+ let (exporter, provider, _guard) = otel_test_pipeline();
+ let mut projector = projector(ContentPolicy::enabled());
+ let prompt = RawJsonRpcMessage::request(
+ "session/prompt".to_owned(),
+ json!({ "sessionId": "s1", "prompt": [{ "type": "text", "text": "hi" }] }),
+ RequestId::Str("p1".to_owned()),
+ )
+ .expect("a request");
+ projector.on_outbound(&mut ToRuntimeMessage::Acp(AcpMessage(prompt)), None);
+ projector.on_inbound(&ToServerMessage::Acp(AcpMessage(
+ RawJsonRpcMessage::response(
+ RequestId::Str("p1".to_owned()),
+ Err(agent_client_protocol::Error::internal_error()),
+ ),
+ )));
+ let spans = finished(&exporter, &provider);
+
+ let agent = spans_named(&spans, "invoke_agent");
+ assert_eq!(agent.len(), 1);
+ assert_eq!(
+ string_attribute(agent[0], attr::ERROR_TYPE).as_deref(),
+ Some("prompt_refused")
+ );
+ assert_eq!(
+ string_array_attribute(agent[0], attr::RESPONSE_FINISH_REASONS),
+ Some(vec!["error".to_owned()])
+ );
+}
+
+#[test]
+fn content_is_not_recorded_when_capture_is_off() {
+ let (exporter, provider, _guard) = otel_test_pipeline();
+ let mut projector = projector(ContentPolicy::disabled());
+ replay(&mut projector, fixtures::TURN);
+ let spans = finished(&exporter, &provider);
+
+ let agent = spans_named(&spans, "invoke_agent")[0];
+ assert!(attribute(agent, attr::INPUT_MESSAGES).is_none());
+ assert!(attribute(agent, attr::OUTPUT_MESSAGES).is_none());
+ // Structure stays.
+ assert_eq!(
+ string_array_attribute(agent, attr::RESPONSE_FINISH_REASONS),
+ Some(vec!["stop".to_owned()])
+ );
+ assert_eq!(int_attribute(agent, attr::USAGE_OUTPUT_TOKENS), Some(8));
+
+ let bash = spans_named(&spans, "execute_tool Bash")[0];
+ assert!(attribute(bash, attr::TOOL_CALL_ARGUMENTS).is_none());
+ assert!(attribute(bash, attr::TOOL_CALL_RESULT).is_none());
+ assert_eq!(
+ string_attribute(bash, attr::TOOL_NAME).as_deref(),
+ Some("Bash")
+ );
+}
+
+#[test]
+fn published_tool_definitions_reach_the_turn_and_describe_its_calls() {
+ let (exporter, provider, _guard) = otel_test_pipeline();
+ let definitions = SharedToolDefinitions::default();
+ let mut projector = GenAiProjector::new(
+ AgentSessionId::TEST_A,
+ ContentPolicy::enabled(),
+ definitions.clone(),
+ );
+ definitions.publish(vec![
+ SessionToolDefinition {
+ name: "mcp__macro__ReadContent".to_owned(),
+ description: "Read a document.".to_owned(),
+ parameters: json!({ "type": "object", "properties": { "id": { "type": "string" } } }),
+ },
+ SessionToolDefinition {
+ name: "mcp__linear__create_issue".to_owned(),
+ description: "Create a Linear issue.".to_owned(),
+ parameters: json!({ "type": "object" }),
+ },
+ ]);
+ replay(&mut projector, fixtures::MACRO_MCP);
+ let spans = finished(&exporter, &provider);
+
+ let agent = spans_named(&spans, "invoke_agent");
+ assert!(!agent.is_empty());
+ let listed = json_attribute(agent[0], attr::TOOL_DEFINITIONS).expect("definitions recorded");
+ assert_eq!(listed[0]["name"], "mcp__macro__ReadContent");
+ assert_eq!(listed[0]["description"], "Read a document.");
+ assert_eq!(listed[1]["name"], "mcp__linear__create_issue");
+
+ let read = spans
+ .iter()
+ .find(|span| string_attribute(span, attr::TOOL_NAME).as_deref() == Some("ReadContent"))
+ .expect("a ReadContent call in the fixture");
+ assert_eq!(
+ string_attribute(read, attr::TOOL_DESCRIPTION).as_deref(),
+ Some("Read a document.")
+ );
+}
+
+#[test]
+fn definitions_published_after_the_prompt_are_recorded_when_the_turn_closes() {
+ let (exporter, provider, _guard) = otel_test_pipeline();
+ let definitions = SharedToolDefinitions::default();
+ let mut projector = GenAiProjector::new(
+ AgentSessionId::TEST_A,
+ ContentPolicy::enabled(),
+ definitions.clone(),
+ );
+ let frames: Vec<&str> = fixtures::TURN.lines().collect();
+ // Up to and including the prompt.
+ replay(&mut projector, &frames[..4].join("\n"));
+ definitions.publish(vec![SessionToolDefinition {
+ name: "mcp__linear__create_issue".to_owned(),
+ description: "Create a Linear issue.".to_owned(),
+ parameters: json!({ "type": "object" }),
+ }]);
+ replay(&mut projector, &frames[4..].join("\n"));
+ let spans = finished(&exporter, &provider);
+
+ let agent = spans_named(&spans, "invoke_agent")[0];
+ let listed = json_attribute(agent, attr::TOOL_DEFINITIONS).expect("definitions recorded");
+ assert_eq!(listed.as_array().map(Vec::len), Some(1));
+}
+
+#[test]
+fn macro_tools_are_listed_by_their_bare_names_for_the_in_process_agent() {
+ let (exporter, provider, _guard) = otel_test_pipeline();
+ let definitions = SharedToolDefinitions::default();
+ let mut projector = GenAiProjector::new(
+ AgentSessionId::TEST_A,
+ ContentPolicy::enabled(),
+ definitions.clone(),
+ );
+ definitions.publish(vec![
+ SessionToolDefinition {
+ name: "mcp__macro__ReadContent".to_owned(),
+ description: "Read a document.".to_owned(),
+ parameters: json!({ "type": "object" }),
+ },
+ SessionToolDefinition {
+ name: "mcp__linear__create_issue".to_owned(),
+ description: "Create a Linear issue.".to_owned(),
+ parameters: json!({ "type": "object" }),
+ },
+ ]);
+ replay(&mut projector, fixtures::SUBAGENT_MACRO_INMEM);
+ let spans = finished(&exporter, &provider);
+
+ let agent = spans_named(&spans, "invoke_agent");
+ assert!(!agent.is_empty(), "{spans:#?}");
+ assert_eq!(
+ string_attribute(agent[0], attr::AGENT_NAME).as_deref(),
+ Some("macro")
+ );
+ let listed = json_attribute(agent[0], attr::TOOL_DEFINITIONS).expect("definitions recorded");
+ assert_eq!(listed[0]["name"], "ReadContent");
+ assert_eq!(listed[1]["name"], "mcp__linear__create_issue");
+}
+
+#[test]
+fn the_turn_hangs_off_the_command_that_carried_the_prompt() {
+ let (exporter, provider, _guard) = otel_test_pipeline();
+ opentelemetry::global::set_text_map_propagator(
+ opentelemetry_sdk::propagation::TraceContextPropagator::new(),
+ );
+ let mut projector = projector(ContentPolicy::enabled());
+ let command = tracing::info_span!("agent.session.command");
+ let command_id = command.context().span().span_context().span_id();
+ assert_ne!(command_id, SpanId::INVALID);
+ let prompt = RawJsonRpcMessage::request(
+ "session/prompt".to_owned(),
+ json!({ "sessionId": "s1", "prompt": [{ "type": "text", "text": "hi" }] }),
+ RequestId::Str("p1".to_owned()),
+ )
+ .expect("a request");
+ let mut message = ToRuntimeMessage::Acp(AcpMessage(prompt));
+ projector.on_outbound(&mut message, Some(&command));
+ let wire = serde_json::to_value(&message).expect("serialize prompt");
+ let request: PromptRequest = serde_json::from_value(wire["params"].clone()).expect("prompt");
+ let runtime = tracing::info_span!(parent: None, "runtime");
+ genai_telemetry::propagation::set_parent(&runtime, request.meta.as_ref());
+ drop(runtime);
+ projector.on_inbound(&ToServerMessage::Acp(AcpMessage(
+ RawJsonRpcMessage::response(
+ RequestId::Str("p1".to_owned()),
+ Ok(json!({ "stopReason": "cancelled" })),
+ ),
+ )));
+ drop(command);
+ let spans = finished(&exporter, &provider);
+
+ let agent = spans_named(&spans, "invoke_agent");
+ assert_eq!(agent.len(), 1);
+ assert_eq!(agent[0].parent_span_id, command_id);
+ let runtime = spans_named(&spans, "runtime");
+ assert_eq!(runtime.len(), 1);
+ assert_eq!(runtime[0].parent_span_id, agent[0].span_context.span_id());
+ assert_eq!(
+ runtime[0].span_context.trace_id(),
+ agent[0].span_context.trace_id()
+ );
+ assert_eq!(
+ string_array_attribute(agent[0], attr::RESPONSE_FINISH_REASONS),
+ Some(vec!["cancelled".to_owned()])
+ );
+ // No harness ever announced itself and no tool frame gave it away.
+ assert_eq!(
+ string_attribute(agent[0], attr::AGENT_NAME).as_deref(),
+ Some("unknown")
+ );
+}
+
+#[test]
+fn the_model_comes_from_the_runtime_config_options() {
+ let (exporter, provider, _guard) = otel_test_pipeline();
+ let mut projector = projector(ContentPolicy::enabled());
+ let open = RawJsonRpcMessage::request(
+ "session/new".to_owned(),
+ json!({ "cwd": "/w", "mcpServers": [] }),
+ RequestId::Str("n1".to_owned()),
+ )
+ .expect("a request");
+ projector.on_outbound(&mut ToRuntimeMessage::Acp(AcpMessage(open)), None);
+ projector.on_inbound(&ToServerMessage::Acp(AcpMessage(
+ RawJsonRpcMessage::response(
+ RequestId::Str("n1".to_owned()),
+ Ok(json!({
+ "sessionId": "s1",
+ "configOptions": [{
+ "id": "model",
+ "name": "Model",
+ "category": "model",
+ "type": "select",
+ "currentValue": "claude-opus-5",
+ "options": [{ "value": "claude-opus-5", "name": "Opus" }]
+ }]
+ })),
+ ),
+ )));
+ let prompt = RawJsonRpcMessage::request(
+ "session/prompt".to_owned(),
+ json!({ "sessionId": "s1", "prompt": [{ "type": "text", "text": "hi" }] }),
+ RequestId::Str("p1".to_owned()),
+ )
+ .expect("a request");
+ projector.on_outbound(&mut ToRuntimeMessage::Acp(AcpMessage(prompt)), None);
+ projector.on_inbound(&ToServerMessage::Acp(AcpMessage(
+ RawJsonRpcMessage::response(
+ RequestId::Str("p1".to_owned()),
+ Ok(json!({ "stopReason": "max_tokens" })),
+ ),
+ )));
+ let spans = finished(&exporter, &provider);
+
+ let agent = spans_named(&spans, "invoke_agent")[0];
+ assert_eq!(
+ string_attribute(agent, attr::REQUEST_MODEL).as_deref(),
+ Some("claude-opus-5")
+ );
+ assert_eq!(
+ string_array_attribute(agent, attr::RESPONSE_FINISH_REASONS),
+ Some(vec!["length".to_owned()])
+ );
+}
+
+#[test]
+fn oversized_tool_results_are_cut_and_marked() {
+ let (exporter, provider, _guard) = otel_test_pipeline();
+ let mut projector = projector(ContentPolicy {
+ capture: true,
+ limits: genai_telemetry::Limits {
+ max_part_chars: 16,
+ max_attribute_bytes: 64,
+ },
+ });
+ let call = RawJsonRpcMessage::notification(
+ "session/update".to_owned(),
+ json!({
+ "sessionId": "s1",
+ "update": {
+ "sessionUpdate": "tool_call",
+ "toolCallId": "t1",
+ "title": "Read",
+ "kind": "read",
+ "status": "completed",
+ "rawInput": { "path": "/a" },
+ "rawOutput": { "text": "x".repeat(500) }
+ }
+ }),
+ )
+ .expect("a notification");
+ projector.on_inbound(&ToServerMessage::Acp(AcpMessage(call)));
+ let spans = finished(&exporter, &provider);
+
+ let read = spans_named(&spans, "execute_tool Read");
+ assert_eq!(read.len(), 1, "a call outside any turn still gets a span");
+ assert_eq!(read[0].parent_span_id, SpanId::INVALID);
+ let result = string_attribute(read[0], attr::TOOL_CALL_RESULT).expect("result");
+ assert!(result.len() <= 64, "cut to the budget: {}", result.len());
+ assert_eq!(
+ attribute(read[0], attr::MACRO_CONTENT_TRUNCATED),
+ Some(&OtelValue::Bool(true))
+ );
+}
+
+#[test]
+fn a_tool_call_opened_outside_any_turn_survives_a_later_turn_ending() {
+ let (exporter, provider, _guard) = otel_test_pipeline();
+ let mut projector = projector(ContentPolicy::enabled());
+ // A call streamed by a resumed session, before this connection prompts.
+ let open = RawJsonRpcMessage::notification(
+ "session/update".to_owned(),
+ json!({
+ "sessionId": "s1",
+ "update": {
+ "sessionUpdate": "tool_call",
+ "toolCallId": "bg-1",
+ "title": "Bash",
+ "kind": "execute",
+ "status": "in_progress",
+ "rawInput": { "command": "sleep 60" }
+ }
+ }),
+ )
+ .expect("a notification");
+ projector.on_inbound(&ToServerMessage::Acp(AcpMessage(open)));
+
+ // A whole turn passes.
+ let prompt = RawJsonRpcMessage::request(
+ "session/prompt".to_owned(),
+ json!({ "sessionId": "s1", "prompt": [{ "type": "text", "text": "hi" }] }),
+ RequestId::Str("p1".to_owned()),
+ )
+ .expect("a request");
+ projector.on_outbound(&mut ToRuntimeMessage::Acp(AcpMessage(prompt)), None);
+ projector.on_inbound(&ToServerMessage::Acp(AcpMessage(
+ RawJsonRpcMessage::response(
+ RequestId::Str("p1".to_owned()),
+ Ok(json!({ "stopReason": "end_turn" })),
+ ),
+ )));
+
+ // Then the background call finishes, and its span records that.
+ let done = RawJsonRpcMessage::notification(
+ "session/update".to_owned(),
+ json!({
+ "sessionId": "s1",
+ "update": {
+ "sessionUpdate": "tool_call_update",
+ "toolCallId": "bg-1",
+ "status": "completed",
+ "rawOutput": { "exit": 0 }
+ }
+ }),
+ )
+ .expect("a notification");
+ projector.on_inbound(&ToServerMessage::Acp(AcpMessage(done)));
+ let spans = finished(&exporter, &provider);
+
+ let bash = spans_named(&spans, "execute_tool Bash");
+ assert_eq!(bash.len(), 1, "{spans:#?}");
+ assert_eq!(bash[0].status, Status::Unset, "not abandoned by the turn");
+ assert_eq!(
+ json_attribute(bash[0], attr::TOOL_CALL_RESULT),
+ Some(json!({ "exit": 0 }))
+ );
+ assert!(bash[0].end_time >= spans_named(&spans, "invoke_agent")[0].end_time);
+}
+
+#[test]
+fn attached_resources_are_named_not_copied() {
+ let (exporter, provider, _guard) = otel_test_pipeline();
+ let mut projector = projector(ContentPolicy::enabled());
+ let prompt = RawJsonRpcMessage::request(
+ "session/prompt".to_owned(),
+ json!({
+ "sessionId": "s1",
+ "prompt": [
+ { "type": "text", "text": "summarize this" },
+ { "type": "resource", "resource": {
+ "uri": "https://user:[email protected]/doc.md?sig=abc",
+ "mimeType": "text/markdown",
+ "text": "CONFIDENTIAL BODY"
+ } },
+ { "type": "resource_link", "name": "notes", "uri": "macro://doc/42#p3" }
+ ]
+ }),
+ RequestId::Str("p1".to_owned()),
+ )
+ .expect("a request");
+ projector.on_outbound(&mut ToRuntimeMessage::Acp(AcpMessage(prompt)), None);
+ projector.on_inbound(&ToServerMessage::Acp(AcpMessage(
+ RawJsonRpcMessage::response(
+ RequestId::Str("p1".to_owned()),
+ Ok(json!({ "stopReason": "end_turn" })),
+ ),
+ )));
+ let spans = finished(&exporter, &provider);
+
+ let agent = spans_named(&spans, "invoke_agent")[0];
+ let input = string_attribute(agent, attr::INPUT_MESSAGES).expect("input recorded");
+ assert!(input.contains("summarize this"));
+ assert!(!input.contains("CONFIDENTIAL BODY"), "{input}");
+ assert!(!input.contains("secret"), "{input}");
+ assert!(!input.contains("sig=abc"), "{input}");
+ let parsed: Value = serde_json::from_str(&input).expect("valid JSON");
+ assert_eq!(parsed[0]["parts"][1]["type"], "uri");
+ assert_eq!(
+ parsed[0]["parts"][1]["uri"],
+ "https://files.example.com/doc.md"
+ );
+ assert_eq!(parsed[0]["parts"][2]["uri"], "macro://doc/42");
+}
+
+#[test]
+fn tool_error_text_follows_the_content_policy() {
+ let failing_call = |id: &str| {
+ RawJsonRpcMessage::notification(
+ "session/update".to_owned(),
+ json!({
+ "sessionId": "s1",
+ "update": {
+ "sessionUpdate": "tool_call",
+ "toolCallId": id,
+ "title": "Read",
+ "kind": "read",
+ "status": "failed",
+ "rawOutput": {
+ "content": [{ "type": "text", "text": "permission denied: /home/eric/.ssh/id_rsa" }],
+ "isError": true
+ }
+ }
+ }),
+ )
+ .expect("a notification")
+ };
+
+ let (exporter, provider, guard) = otel_test_pipeline();
+ let mut quiet = projector(ContentPolicy::disabled());
+ quiet.on_inbound(&ToServerMessage::Acp(AcpMessage(failing_call("t1"))));
+ let spans = finished(&exporter, &provider);
+ match &spans_named(&spans, "execute_tool Read")[0].status {
+ Status::Error { description } => {
+ assert!(!description.contains("id_rsa"), "{description}");
+ }
+ other => panic!("expected an error status, got {other:?}"),
+ }
+ drop(guard);
+
+ let (exporter, provider, _guard) = otel_test_pipeline();
+ let mut verbose = projector(ContentPolicy::enabled());
+ verbose.on_inbound(&ToServerMessage::Acp(AcpMessage(failing_call("t2"))));
+ let spans = finished(&exporter, &provider);
+ match &spans_named(&spans, "execute_tool Read")[0].status {
+ Status::Error { description } => {
+ assert!(description.contains("permission denied"), "{description}");
+ }
+ other => panic!("expected an error status, got {other:?}"),
+ }
+}
diff --git a/crates/ai_toolset/Cargo.toml b/crates/ai_toolset/Cargo.toml
index 1eb7115baf..f8bb8d2787 100644
--- a/crates/ai_toolset/Cargo.toml
+++ b/crates/ai_toolset/Cargo.toml
@@ -13,8 +13,16 @@ async-trait = { workspace = true }
thiserror = { workspace = true }
anyhow = { workspace = true }
macro_user_id = { path = "../macro_user_id" }
+genai_telemetry = { path = "../genai_telemetry" }
axum = { workspace = true }
axum-macros = { workspace = true }
tracing = { workspace = true }
workspace-hack = { version = "0.1", path = "../workspace-hack" }
tokio-util.workspace = true
+
+[dev-dependencies]
+opentelemetry = { workspace = true, features = ["trace"] }
+opentelemetry_sdk = { workspace = true, features = ["testing", "trace"] }
+tokio = { workspace = true }
+tracing-opentelemetry = { workspace = true }
+tracing-subscriber = { workspace = true }
diff --git a/crates/ai_toolset/TOOL_DESIGN.md b/crates/ai_toolset/TOOL_DESIGN.md
index 90ff1dbce5..7ae4d20bf7 100644
--- a/crates/ai_toolset/TOOL_DESIGN.md
+++ b/crates/ai_toolset/TOOL_DESIGN.md
@@ -131,3 +131,21 @@ Before adding a tool, verify:
- Anthropic's guide: https://www.anthropic.com/engineering/writing-tools-for-agents
- Crate documentation: `src/lib.rs`
- Schema requirements: `src/schema/`
+
+## Telemetry
+
+Every tool call dispatched through `ToolSet::try_tool_call` runs inside an
+OpenTelemetry GenAI `execute_tool` span (see `ai_toolset::telemetry`). The span
+carries the tool name, the arguments as input and the result as output — the
+attributes an LLM-as-a-judge evaluation of tool selection reads — and an error
+status when the call fails. Two rules follow:
+
+- **Tools and toolsets do not open tool telemetry themselves.** Implement
+ `ToolSet::dispatch_tool_call` (routing only) and let `try_tool_call` wrap it;
+ a toolset that delegates to children calls *their* `dispatch_tool_call`, so
+ one call yields exactly one tool span. Inside a tool, ordinary
+ `#[tracing::instrument]` spans for its own work are fine and nest under it.
+- **Arguments and results are content**, recorded under the content policy
+ (`OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT`, on unless set to
+ `false`) and cut to a size budget. Do not record them as span fields or log
+ them in tools: that bypasses the policy and repeats them on every log line.
diff --git a/crates/ai_toolset/src/context.rs b/crates/ai_toolset/src/context.rs
index 15102cf8e3..94398f705b 100644
--- a/crates/ai_toolset/src/context.rs
+++ b/crates/ai_toolset/src/context.rs
@@ -40,6 +40,12 @@ pub struct RequestContext {
/// user has requested stream stop
/// long-running tools should comsume this
pub cancel: CancellationToken,
+ /// Whether tool calls in this request record `execute_tool` GenAI
+ /// telemetry (arguments, result, failure) on their span. On by default;
+ /// off when another layer already reports the request's tool calls - the
+ /// agent session actor does, for the in-process runtime - so each call is
+ /// counted once.
+ pub genai_telemetry: bool,
}
impl RequestContext {
@@ -51,9 +57,17 @@ impl RequestContext {
searchable_tools: Arc::new(Vec::new()),
tool_loader: None,
cancel: CancellationToken::new(),
+ genai_telemetry: true,
}
}
+ /// Turn `execute_tool` GenAI telemetry for this request's tool calls on or
+ /// off (see [`Self::genai_telemetry`]).
+ pub fn with_genai_telemetry(mut self, enabled: bool) -> Self {
+ self.genai_telemetry = enabled;
+ self
+ }
+
/// Attach the searchable-tool catalog and loader that power `SearchTools`.
pub fn with_tool_search(
mut self,
diff --git a/crates/ai_toolset/src/lib.rs b/crates/ai_toolset/src/lib.rs
index 5602b11277..0856c30719 100644
--- a/crates/ai_toolset/src/lib.rs
+++ b/crates/ai_toolset/src/lib.rs
@@ -144,12 +144,20 @@
//!
//! assert!(parent_toolset.tools.contains_key("SubTool"));
//! ```
+//!
+//! # Telemetry
+//!
+//! Every dispatch through [`ToolSet::try_tool_call`] runs inside an
+//! OpenTelemetry GenAI `execute_tool` span carrying the tool name, arguments
+//! and result (see [`telemetry`]). Implementors provide
+//! [`ToolSet::dispatch_tool_call`] and never open tool telemetry themselves.
#![deny(missing_docs)]
mod annotations;
mod context;
pub mod schema;
+pub mod telemetry;
mod tool;
pub mod tool_search;
mod toolset;
@@ -159,6 +167,6 @@ pub use context::{RequestContext, ServiceContext};
pub use tool::{AsyncTool, NoContext, ToolCallError, ToolResult};
pub use tool_search::{SearchableTool, ToolLoader};
pub use toolset::{
- AsyncToolCollection, RequestSchema, ToolCollection, ToolInfo, ToolSchema, ToolSet,
- ToolSetCreationError, ToolSetError, tool_object,
+ AsyncToolCollection, RequestSchema, ToolCallFuture, ToolCollection, ToolInfo, ToolSchema,
+ ToolSet, ToolSetCreationError, ToolSetError, tool_object,
};
diff --git a/crates/ai_toolset/src/telemetry.rs b/crates/ai_toolset/src/telemetry.rs
new file mode 100644
index 0000000000..bd6c629da1
--- /dev/null
+++ b/crates/ai_toolset/src/telemetry.rs
@@ -0,0 +1,153 @@
+//! Enforced `execute_tool` telemetry for every tool dispatch.
+//!
+//! [`ToolSet::try_tool_call`](crate::ToolSet::try_tool_call) is the one door a
+//! tool call goes through — the agent loop, user-executed tools and pipelines
+//! all dispatch via the trait — and it wraps every call in a [`ToolCallSpan`].
+//! Two situations:
+//!
+//! - The agent runtime already opened the semconv `execute_tool` span around
+//! the dispatch (it carries the tool name and the provider's call id). The
+//! guard **enriches** that span rather than nesting a duplicate.
+//! - Nothing did — a user confirming a deferred tool, a pipeline calling a
+//! tool directly, a test. The guard **opens** `execute_tool {name}` itself.
+//!
+//! Either way the span ends up with what Datadog's tool evaluations read:
+//! `gen_ai.operation.name`, `gen_ai.tool.name`, the arguments as input and the
+//! result as output (both subject to the content policy and size bounds in
+//! [`genai_telemetry`]), and an error status when the call failed.
+
+#[cfg(test)]
+mod test;
+
+use crate::{ToolResult, ToolSetError};
+use genai_telemetry::{
+ ContentPolicy, GenAiSpanExt as _, attr, bounded_json_string, current_span_is_named,
+ truncate_chars,
+};
+use serde::Serialize;
+
+/// Telemetry guard for one tool call.
+///
+/// Create it with [`ToolCallSpan::begin`], run the call inside
+/// [`ToolCallSpan::span`] (`fut.instrument(guard.span().clone())`), then hand
+/// the outcome to [`ToolCallSpan::finish`].
+#[derive(Debug)]
+pub struct ToolCallSpan {
+ span: tracing::Span,
+ policy: ContentPolicy,
+ tool_name: String,
+}
+
+impl ToolCallSpan {
+ /// Begin telemetry for a call of `tool_name` with `arguments`, under the
+ /// process-wide [`ContentPolicy`].
+ pub fn begin(tool_name: &str, arguments: &serde_json::Value) -> Self {
+ Self::begin_with_policy(tool_name, arguments, ContentPolicy::from_env())
+ }
+
+ /// [`Self::begin`] with an explicit content policy.
+ pub fn begin_with_policy(
+ tool_name: &str,
+ arguments: &serde_json::Value,
+ policy: ContentPolicy,
+ ) -> Self {
+ let span = if current_span_is_named(attr::span_name::EXECUTE_TOOL) {
+ // The runtime's span already names the tool and carries the call
+ // id; only the content and outcome are missing.
+ tracing::Span::current()
+ } else {
+ // The literal must stay in step with `attr::span_name::EXECUTE_TOOL`
+ // (`tracing` bakes the name into static metadata); `otel.name`
+ // gives the exported span the semconv `execute_tool {name}` name.
+ let otel_name = format!("{} {tool_name}", attr::operation::EXECUTE_TOOL);
+ tracing::info_span!(
+ "execute_tool",
+ otel.name = %otel_name,
+ gen_ai.operation.name = attr::operation::EXECUTE_TOOL,
+ gen_ai.tool.type = attr::tool_type::FUNCTION,
+ gen_ai.tool.name = %tool_name,
+ )
+ };
+ if policy.capture {
+ let (arguments, truncated) = bounded_json_string(arguments, &policy.limits);
+ span.set_str(attr::TOOL_CALL_ARGUMENTS, arguments);
+ if truncated {
+ span.set_bool(attr::MACRO_CONTENT_TRUNCATED, true);
+ }
+ }
+ Self {
+ span,
+ policy,
+ tool_name: tool_name.to_owned(),
+ }
+ }
+
+ /// The span the call must run inside.
+ pub fn span(&self) -> &tracing::Span {
+ &self.span
+ }
+
+ /// Record the outcome: the result as the span's output on success, an
+ /// error status (and the model-facing description as output) otherwise.
+ pub fn finish<T: Serialize>(&self, result: &Result<ToolResult<T>, ToolSetError>) {
+ match result {
+ Ok(Ok(output)) => {
+ if self.policy.capture {
+ match serde_json::to_value(output) {
+ Ok(value) => self.record_result(&value),
+ Err(error) => tracing::debug!(
+ error = ?error,
+ tool_name = %self.tool_name,
+ "tool result is not serializable; not recorded on the span"
+ ),
+ }
+ }
+ }
+ Ok(Err(error)) => {
+ // The tool ran and failed; `description` is what the model
+ // sees, so it doubles as the span's output - and is content,
+ // recorded under the same policy.
+ self.span.set_error(
+ "tool_error",
+ self.error_description(&error.description, "the tool reported an error"),
+ );
+ if self.policy.capture {
+ self.record_result(&serde_json::Value::String(error.description.clone()));
+ }
+ }
+ Err(error @ ToolSetError::NotFound(_)) => {
+ tracing::warn!(tool_name = %self.tool_name, error = %error, "tool call could not be dispatched");
+ self.span.set_error("tool_not_found", "no such tool");
+ }
+ Err(error @ ToolSetError::Deserialization(_)) => {
+ tracing::warn!(tool_name = %self.tool_name, error = %error, "tool call could not be dispatched");
+ // The message quotes the arguments it could not read.
+ self.span.set_error(
+ "invalid_arguments",
+ self.error_description(
+ &error.to_string(),
+ "the arguments did not match the tool's schema",
+ ),
+ );
+ }
+ }
+ }
+
+ /// An error description fit for the span's status: the detail, bounded,
+ /// when content may be recorded; a fixed summary otherwise.
+ fn error_description(&self, detail: &str, summary: &'static str) -> String {
+ if self.policy.capture {
+ truncate_chars(detail, self.policy.limits.max_part_chars).into_owned()
+ } else {
+ summary.to_owned()
+ }
+ }
+
+ fn record_result(&self, value: &serde_json::Value) {
+ let (result, truncated) = bounded_json_string(value, &self.policy.limits);
+ self.span.set_str(attr::TOOL_CALL_RESULT, result);
+ if truncated {
+ self.span.set_bool(attr::MACRO_CONTENT_TRUNCATED, true);
+ }
+ }
+}
diff --git a/crates/ai_toolset/src/telemetry/test.rs b/crates/ai_toolset/src/telemetry/test.rs
new file mode 100644
index 0000000000..bf1d8fdff6
--- /dev/null
+++ b/crates/ai_toolset/src/telemetry/test.rs
@@ -0,0 +1,369 @@
+use super::*;
+use crate::{
+ AsyncTool, AsyncToolCollection, RequestContext, ServiceContext, ToolAnnotated, ToolAnnotations,
+ ToolCallError, ToolSet as _,
+};
+use async_trait::async_trait;
+use macro_user_id::user_id::MacroUserIdStr;
+use opentelemetry::Value;
+use opentelemetry::trace::{Status, TracerProvider as _};
+use opentelemetry_sdk::trace::{InMemorySpanExporter, SdkTracerProvider, SpanData};
+use schemars::JsonSchema;
+use serde::Deserialize;
+use serde_json::json;
+use tracing::Instrument as _;
+use tracing_subscriber::layer::SubscriberExt as _;
+
+/// A tool that echoes its input.
+#[derive(Deserialize, JsonSchema)]
+#[schemars(title = "echo_tool", description = "Echoes its input back.")]
+struct EchoTool {
+ value: String,
+}
+
+impl ToolAnnotated for EchoTool {
+ const ANNOTATIONS: ToolAnnotations = ToolAnnotations::read_only("Echo");
+}
+
+#[async_trait]
+impl AsyncTool<()> for EchoTool {
+ type Output = serde_json::Value;
+
+ async fn call(
+ &self,
+ _service_context: ServiceContext<()>,
+ _request_context: RequestContext,
+ ) -> ToolResult<Self::Output> {
+ Ok(json!({ "echo": self.value }))
+ }
+}
+
+/// A tool that always fails.
+#[derive(Deserialize, JsonSchema)]
+#[schemars(title = "boom_tool", description = "Always fails.")]
+struct BoomTool {}
+
+impl ToolAnnotated for BoomTool {
+ const ANNOTATIONS: ToolAnnotations = ToolAnnotations::read_only("Boom");
+}
+
+#[async_trait]
+impl AsyncTool<()> for BoomTool {
+ type Output = serde_json::Value;
+
+ async fn call(
+ &self,
+ _service_context: ServiceContext<()>,
+ _request_context: RequestContext,
+ ) -> ToolResult<Self::Output> {
+ Err(ToolCallError {
+ description: "boom failed".to_string(),
+ internal_error: anyhow::anyhow!("boom"),
+ })
+ }
+}
+
+fn toolset() -> AsyncToolCollection<()> {
+ AsyncToolCollection::<()>::new()
+ .add_tool::<EchoTool, ()>()
+ .add_tool::<BoomTool, ()>()
+}
+
+/// `echo_tool` registered as a user-executed (deferred) tool instead.
+fn user_toolset() -> AsyncToolCollection<()> {
+ AsyncToolCollection::<()>::new().add_user_tool::<EchoTool, ()>()
+}
+
+fn request_context() -> RequestContext {
+ RequestContext::new(MacroUserIdStr::try_from_email("[email protected]").expect("valid user id"))
+}
+
+/// An OpenTelemetry pipeline that keeps finished spans in memory, installed as
+/// the thread's default subscriber for the returned guard's lifetime.
+fn otel_test_pipeline() -> (
+ InMemorySpanExporter,
+ SdkTracerProvider,
+ tracing::subscriber::DefaultGuard,
+) {
+ let exporter = InMemorySpanExporter::default();
+ let provider = SdkTracerProvider::builder()
+ .with_simple_exporter(exporter.clone())
+ .build();
+ let layer = tracing_opentelemetry::layer().with_tracer(provider.tracer("test"));
+ let guard = tracing::subscriber::set_default(tracing_subscriber::registry().with(layer));
+ // Other tests in this binary run the same code paths with no subscriber
+ // installed, which can cache "never interested" for a span callsite on
+ // another thread; recompute so this subscriber sees every span.
+ tracing::callsite::rebuild_interest_cache();
+ (exporter, provider, guard)
+}
+
+fn finished(exporter: &InMemorySpanExporter, provider: &SdkTracerProvider) -> Vec<SpanData> {
+ provider.force_flush().expect("flush");
+ exporter.get_finished_spans().expect("finished spans")
+}
+
+fn string_attribute(span: &SpanData, key: &str) -> Option<String> {
+ span.attributes
+ .iter()
+ .find(|kv| kv.key.as_str() == key)
+ .and_then(|kv| match &kv.value {
+ Value::String(s) => Some(s.as_str().to_string()),
+ _ => None,
+ })
+}
+
+fn tool_spans(spans: &[SpanData]) -> Vec<&SpanData> {
+ spans
+ .iter()
+ .filter(|span| span.name.starts_with("execute_tool"))
+ .collect()
+}
+
+#[tokio::test]
+async fn try_tool_call_opens_an_execute_tool_span_with_arguments_and_result() {
+ let (exporter, provider, _guard) = otel_test_pipeline();
+ let toolset = toolset();
+
+ let result = toolset
+ .try_tool_call((), request_context(), "echo_tool", &json!({ "value": "a" }))
+ .await
+ .expect("dispatch")
+ .expect("tool success");
+ assert_eq!(result, json!({ "echo": "a" }));
+
+ let spans = finished(&exporter, &provider);
+ let tool_spans = tool_spans(&spans);
+ assert_eq!(tool_spans.len(), 1, "exactly one tool span, got {spans:#?}");
+ let span = tool_spans[0];
+ assert_eq!(span.name, "execute_tool echo_tool");
+ assert_eq!(
+ string_attribute(span, attr::OPERATION_NAME).as_deref(),
+ Some("execute_tool")
+ );
+ assert_eq!(
+ string_attribute(span, attr::TOOL_TYPE).as_deref(),
+ Some("function")
+ );
+ assert_eq!(
+ string_attribute(span, attr::TOOL_NAME).as_deref(),
+ Some("echo_tool")
+ );
+ assert_eq!(
+ string_attribute(span, attr::TOOL_CALL_ARGUMENTS).as_deref(),
+ Some(r#"{"value":"a"}"#)
+ );
+ assert_eq!(
+ string_attribute(span, attr::TOOL_CALL_RESULT).as_deref(),
+ Some(r#"{"echo":"a"}"#)
+ );
+ assert_eq!(span.status, Status::Unset);
+}
+
+#[tokio::test]
+async fn try_tool_call_enriches_an_open_execute_tool_span_instead_of_nesting() {
+ let (exporter, provider, _guard) = otel_test_pipeline();
+ let toolset = toolset();
+
+ // The agent runtime opens `execute_tool` around the dispatch; the toolset
+ // must add to it, not open a second tool span underneath.
+ let runtime_span = tracing::info_span!("execute_tool");
+ toolset
+ .try_tool_call((), request_context(), "echo_tool", &json!({ "value": "b" }))
+ .instrument(runtime_span)
+ .await
+ .expect("dispatch")
+ .expect("tool success");
+
+ let spans = finished(&exporter, &provider);
+ let tool_spans = tool_spans(&spans);
+ assert_eq!(tool_spans.len(), 1, "no nested tool span, got {spans:#?}");
+ let span = tool_spans[0];
+ assert_eq!(
+ span.name, "execute_tool",
+ "the runtime's span is kept as-is"
+ );
+ assert_eq!(
+ string_attribute(span, attr::TOOL_CALL_ARGUMENTS).as_deref(),
+ Some(r#"{"value":"b"}"#)
+ );
+ assert_eq!(
+ string_attribute(span, attr::TOOL_CALL_RESULT).as_deref(),
+ Some(r#"{"echo":"b"}"#)
+ );
+}
+
+#[tokio::test]
+async fn a_failing_tool_marks_the_span_failed_with_its_description_as_output() {
+ let (exporter, provider, _guard) = otel_test_pipeline();
+ let toolset = toolset();
+
+ let result = toolset
+ .try_tool_call((), request_context(), "boom_tool", &json!({}))
+ .await
+ .expect("dispatch");
+ assert!(result.is_err());
+
+ let spans = finished(&exporter, &provider);
+ let span = tool_spans(&spans)[0];
+ assert!(
+ matches!(&span.status, Status::Error { description } if description == "boom failed"),
+ "{:?}",
+ span.status
+ );
+ assert_eq!(
+ string_attribute(span, attr::ERROR_TYPE).as_deref(),
+ Some("tool_error")
+ );
+ assert_eq!(
+ string_attribute(span, attr::TOOL_CALL_RESULT).as_deref(),
+ Some("boom failed")
+ );
+}
+
+#[tokio::test]
+async fn dispatch_errors_are_classified() {
+ let (exporter, provider, _guard) = otel_test_pipeline();
+ let toolset = toolset();
+
+ assert!(matches!(
+ toolset
+ .try_tool_call((), request_context(), "missing_tool", &json!({}))
+ .await,
+ Err(ToolSetError::NotFound(_))
+ ));
+ assert!(matches!(
+ toolset
+ .try_tool_call((), request_context(), "echo_tool", &json!({ "wrong": 1 }))
+ .await,
+ Err(ToolSetError::Deserialization(_))
+ ));
+
+ let spans = finished(&exporter, &provider);
+ let tool_spans = tool_spans(&spans);
+ assert_eq!(tool_spans.len(), 2);
+ assert_eq!(tool_spans[0].name, "execute_tool missing_tool");
+ assert_eq!(
+ string_attribute(tool_spans[0], attr::ERROR_TYPE).as_deref(),
+ Some("tool_not_found")
+ );
+ assert_eq!(
+ string_attribute(tool_spans[1], attr::ERROR_TYPE).as_deref(),
+ Some("invalid_arguments")
+ );
+ assert!(
+ tool_spans
+ .iter()
+ .all(|span| matches!(span.status, Status::Error { .. }))
+ );
+}
+
+#[tokio::test]
+async fn user_tool_calls_are_traced_too() {
+ let (exporter, provider, _guard) = otel_test_pipeline();
+ let toolset = user_toolset();
+
+ toolset
+ .try_user_tool_call((), request_context(), "echo_tool", &json!({ "value": "c" }))
+ .await
+ .expect("dispatch")
+ .expect("tool success");
+
+ let spans = finished(&exporter, &provider);
+ let span = tool_spans(&spans)[0];
+ assert_eq!(span.name, "execute_tool echo_tool");
+ assert_eq!(
+ string_attribute(span, attr::TOOL_CALL_ARGUMENTS).as_deref(),
+ Some(r#"{"value":"c"}"#)
+ );
+ // The tool's own output, not the `UserToolResponse` API wrapper.
+ assert_eq!(
+ string_attribute(span, attr::TOOL_CALL_RESULT).as_deref(),
+ Some(r#"{"echo":"c"}"#)
+ );
+}
+
+#[tokio::test]
+async fn content_capture_off_records_structure_only() {
+ let (exporter, provider, _guard) = otel_test_pipeline();
+
+ let guard = ToolCallSpan::begin_with_policy(
+ "echo_tool",
+ &json!({ "value": "secret" }),
+ ContentPolicy::disabled(),
+ );
+ let result: Result<ToolResult<serde_json::Value>, ToolSetError> = Ok(Err(ToolCallError {
+ description: "private failure".to_string(),
+ internal_error: anyhow::anyhow!("boom"),
+ }));
+ guard.finish(&result);
+ drop(guard);
+
+ let spans = finished(&exporter, &provider);
+ let span = tool_spans(&spans)[0];
+ assert_eq!(
+ string_attribute(span, attr::TOOL_NAME).as_deref(),
+ Some("echo_tool")
+ );
+ assert_eq!(string_attribute(span, attr::TOOL_CALL_ARGUMENTS), None);
+ assert_eq!(string_attribute(span, attr::TOOL_CALL_RESULT), None);
+ // The failure itself is not content, but its description is.
+ match &span.status {
+ Status::Error { description } => {
+ assert!(!description.contains("private failure"), "{description}");
+ }
+ other => panic!("expected an error status, got {other:?}"),
+ }
+}
+
+#[tokio::test]
+async fn oversized_content_is_bounded_and_flagged() {
+ let (exporter, provider, _guard) = otel_test_pipeline();
+ let toolset = toolset();
+
+ let big = "x".repeat(20_000);
+ toolset
+ .try_tool_call((), request_context(), "echo_tool", &json!({ "value": big }))
+ .await
+ .expect("dispatch")
+ .expect("tool success");
+
+ let spans = finished(&exporter, &provider);
+ let span = tool_spans(&spans)[0];
+ let arguments = string_attribute(span, attr::TOOL_CALL_ARGUMENTS).expect("arguments");
+ assert!(
+ arguments.len() < 10_000,
+ "arguments should be cut, got {} bytes",
+ arguments.len()
+ );
+ assert!(arguments.contains("[truncated"), "{arguments}");
+ assert_eq!(
+ span.attributes
+ .iter()
+ .find(|kv| kv.key.as_str() == attr::MACRO_CONTENT_TRUNCATED)
+ .map(|kv| kv.value.clone()),
+ Some(Value::Bool(true))
+ );
+}
+
+/// A request that opted out - another layer reports its tool calls - runs
+/// its tools without `execute_tool` telemetry of their own.
+#[tokio::test]
+async fn a_request_that_opted_out_records_no_tool_span() {
+ let (exporter, provider, _guard) = otel_test_pipeline();
+ let toolset = toolset();
+
+ toolset
+ .try_tool_call(
+ (),
+ request_context().with_genai_telemetry(false),
+ "echo_tool",
+ &json!({ "value": "quiet" }),
+ )
+ .await
+ .expect("dispatch")
+ .expect("tool success");
+
+ let spans = finished(&exporter, &provider);
+ assert!(tool_spans(&spans).is_empty(), "{spans:#?}");
+}
diff --git a/crates/ai_toolset/src/toolset/mod.rs b/crates/ai_toolset/src/toolset/mod.rs
index 65c7c6c591..c27b132e46 100644
--- a/crates/ai_toolset/src/toolset/mod.rs
+++ b/crates/ai_toolset/src/toolset/mod.rs
@@ -4,7 +4,7 @@ pub mod tool_object;
mod traits;
mod types;
-pub use traits::ToolSet;
+pub use traits::{ToolCallFuture, ToolSet};
pub use types::{
AsyncToolCollection, RequestSchema, ToolCollection, ToolInfo, ToolSchema, ToolSetCreationError,
ToolSetError,
diff --git a/crates/ai_toolset/src/toolset/tool_object/tool_async.rs b/crates/ai_toolset/src/toolset/tool_object/tool_async.rs
index e51bb7dab7..1a67ffef0e 100644
--- a/crates/ai_toolset/src/toolset/tool_object/tool_async.rs
+++ b/crates/ai_toolset/src/toolset/tool_object/tool_async.rs
@@ -123,7 +123,10 @@ pub type AsyncToolObject<ToolSetContext> = ToolObject<AsyncDeserializer<ToolSetC
impl<ToolSetContext> ToolObject<AsyncDeserializer<ToolSetContext>> {
/// Attempts to deserialize JSON input into a callable async tool instance.
- #[tracing::instrument(err, skip(self))]
+ ///
+ /// `data` is the model's arguments: content, recorded on the tool span
+ /// under the content policy rather than as a field here.
+ #[tracing::instrument(err, skip_all)]
pub fn try_deserialize(
&self,
data: &serde_json::Value,
diff --git a/crates/ai_toolset/src/toolset/traits.rs b/crates/ai_toolset/src/toolset/traits.rs
index fb84991cb2..345538a121 100644
--- a/crates/ai_toolset/src/toolset/traits.rs
+++ b/crates/ai_toolset/src/toolset/traits.rs
@@ -1,20 +1,64 @@
//! The core ToolSet trait implemented by the ToolSet and AsyncToolset
+use crate::telemetry::ToolCallSpan;
use crate::toolset::types::{RequestSchema, ToolInfo};
use crate::{AsyncToolCollection, RequestContext, SearchableTool, ToolResult, ToolSetError};
use std::pin::Pin;
+use tracing::Instrument as _;
+
+/// The future a tool dispatch resolves to.
+pub type ToolCallFuture<'a> =
+ Pin<Box<dyn Future<Output = Result<ToolResult<serde_json::Value>, ToolSetError>> + 'a + Send>>;
/// An object with a set of tools
pub trait ToolSet<Context>: Send + Sync {
- /// Try to call a tool indexed by name with the provided raw json
+ /// Route a tool call by name to its implementation and return the result.
+ ///
+ /// This is the implementor's hook and carries **no telemetry**:
+ /// [`Self::try_tool_call`] wraps it in the `execute_tool` span (see
+ /// [`crate::telemetry`]). A toolset that delegates to child toolsets calls
+ /// the children's `dispatch_tool_call`, so one tool call yields one span.
+ fn dispatch_tool_call<'a>(
+ &'a self,
+ context: Context,
+ request_context: RequestContext,
+ tool_name: &'a str,
+ json: &'a serde_json::Value,
+ ) -> ToolCallFuture<'a>;
+
+ /// Try to call a tool indexed by name with the provided raw json.
+ ///
+ /// This is what callers use — the agent loop, user-executed tools,
+ /// pipelines. Every call runs inside `execute_tool` telemetry: the span is
+ /// enriched when the agent runtime already opened it, opened otherwise, and
+ /// records the arguments, the result and any failure (see
+ /// [`crate::telemetry::ToolCallSpan`]) - unless the request opted out
+ /// ([`RequestContext::genai_telemetry`]) because another layer reports its
+ /// tool calls.
fn try_tool_call<'a>(
&'a self,
context: Context,
request_context: RequestContext,
tool_name: &'a str,
json: &'a serde_json::Value,
- ) -> Pin<
- Box<dyn Future<Output = Result<ToolResult<serde_json::Value>, ToolSetError>> + 'a + Send>,
- >;
+ ) -> ToolCallFuture<'a>
+ where
+ Context: Send + 'a,
+ {
+ Box::pin(async move {
+ if !request_context.genai_telemetry {
+ return self
+ .dispatch_tool_call(context, request_context, tool_name, json)
+ .await;
+ }
+ let telemetry = ToolCallSpan::begin(tool_name, json);
+ let result = self
+ .dispatch_tool_call(context, request_context, tool_name, json)
+ .instrument(telemetry.span().clone())
+ .await;
+ telemetry.finish(&result);
+ result
+ })
+ }
/// Returns the input schemas for the tools sent to the provider on every
/// request. Tools loaded on demand via tool search are excluded — they are
@@ -45,15 +89,13 @@ impl<Context> ToolSet<Context> for AsyncToolCollection<Context>
where
Context: Send + Sync + 'static,
{
- fn try_tool_call<'a>(
+ fn dispatch_tool_call<'a>(
&'a self,
context: Context,
request_context: RequestContext,
tool_name: &'a str,
json: &'a serde_json::Value,
- ) -> Pin<
- Box<dyn Future<Output = Result<ToolResult<serde_json::Value>, ToolSetError>> + 'a + Send>,
- > {
+ ) -> ToolCallFuture<'a> {
Box::pin(self.try_tool_call_internal(context, request_context, tool_name, json))
}
diff --git a/crates/ai_toolset/src/toolset/types.rs b/crates/ai_toolset/src/toolset/types.rs
index f1ec7fbc51..bb0be177d7 100644
--- a/crates/ai_toolset/src/toolset/types.rs
+++ b/crates/ai_toolset/src/toolset/types.rs
@@ -3,6 +3,7 @@ use super::tool_object::{AsyncToolObject, UserTool, UserToolResponse};
use crate::RequestContext;
use crate::annotations::ToolAnnotated;
use crate::schema::ValidationError;
+use crate::telemetry::ToolCallSpan;
use crate::{AsyncTool, ToolResult};
use axum::extract::FromRef;
use schemars::{JsonSchema, Schema};
@@ -10,6 +11,7 @@ use serde::Serialize;
use serde::de::Deserialize;
use std::collections::BTreeMap;
use thiserror::Error;
+use tracing::Instrument as _;
#[cfg(test)]
mod test;
@@ -200,7 +202,10 @@ where
/// was added.
///
/// Returns an error if the tool is not found or if deserialization fails.
- #[tracing::instrument(err, skip(self, context, request_context))]
+ ///
+ /// No span of its own: it runs inside the `execute_tool` span that
+ /// [`ToolSet::try_tool_call`](crate::ToolSet::try_tool_call) opens, which
+ /// records the arguments and result under the content policy.
pub(crate) async fn try_tool_call_internal(
&self,
context: ToolSetContext,
@@ -220,7 +225,9 @@ where
}
/// this isn't called in the tool loop it's called by the user-facing API
- #[tracing::instrument(err, skip(self, context, request_context))]
+ ///
+ /// A user-executed tool is still a tool execution, so it gets the same
+ /// `execute_tool` telemetry as a model-initiated call.
pub async fn try_user_tool_call(
&self,
context: ToolSetContext,
@@ -228,22 +235,35 @@ where
tool_name: &str,
json: &serde_json::Value,
) -> Result<ToolResult<UserToolResponse<serde_json::Value>>, ToolSetError> {
- let tool = self
- .user_tools
- .get(tool_name)
- .ok_or_else(|| ToolSetError::NotFound(tool_name.to_owned()))
- .and_then(|tool| {
- tool.try_deserialize(json)
- .map_err(ToolSetError::Deserialization)
- })?;
- Ok(tool
- .call(context, request_context)
- .await
- .map(UserToolResponse::UserAction))
+ let telemetry = request_context
+ .genai_telemetry
+ .then(|| ToolCallSpan::begin(tool_name, json));
+ let span = telemetry
+ .as_ref()
+ .map_or_else(tracing::Span::none, |telemetry| telemetry.span().clone());
+ let result: Result<ToolResult<serde_json::Value>, ToolSetError> = async {
+ let tool = self
+ .user_tools
+ .get(tool_name)
+ .ok_or_else(|| ToolSetError::NotFound(tool_name.to_owned()))
+ .and_then(|tool| {
+ tool.try_deserialize(json)
+ .map_err(ToolSetError::Deserialization)
+ })?;
+ Ok(tool.call(context, request_context).await)
+ }
+ .instrument(span)
+ .await;
+ // Recorded before the API wrapper goes on, so the span carries the
+ // tool's own output rather than `{"UserAction": ...}`.
+ if let Some(telemetry) = &telemetry {
+ telemetry.finish(&result);
+ }
+ result.map(|result| result.map(UserToolResponse::UserAction))
}
/// check if json + name matches a known tool in the toolset
- #[tracing::instrument(skip(self))]
+ #[tracing::instrument(skip_all, fields(tool_name = %tool_name))]
pub fn is_valid_tool(&self, tool_name: &str, json: &serde_json::Value) -> bool {
let Some(tool) = self
.user_tools
diff --git a/crates/genai_telemetry/Cargo.toml b/crates/genai_telemetry/Cargo.toml
new file mode 100644
index 0000000000..fe1588158c
--- /dev/null
+++ b/crates/genai_telemetry/Cargo.toml
@@ -0,0 +1,17 @@
+[package]
+edition = "2024"
+name = "genai_telemetry"
+publish = false
+version = "0.1.0"
+
+[dependencies]
+macro_env_var = { path = "../macro_env_var" }
+opentelemetry = { workspace = true, features = ["trace"] }
+serde_json = { workspace = true }
+tracing = { workspace = true }
+tracing-opentelemetry = { workspace = true }
+workspace-hack = { version = "0.1", path = "../workspace-hack" }
+
+[dev-dependencies]
+opentelemetry_sdk = { workspace = true, features = ["testing", "trace"] }
+tracing-subscriber = { workspace = true }
diff --git a/crates/genai_telemetry/src/attr.rs b/crates/genai_telemetry/src/attr.rs
new file mode 100644
index 0000000000..45e5de5afa
--- /dev/null
+++ b/crates/genai_telemetry/src/attr.rs
@@ -0,0 +1,122 @@
+//! OpenTelemetry GenAI semantic-convention attribute names and well-known values.
+//!
+//! Mirrors the `gen_ai.*` registry of the OpenTelemetry GenAI semantic
+//! conventions, plus the two `macro.genai.*` attributes this workspace adds to
+//! describe its own truncation. Datadog maps these onto its LLM Observability
+//! schema: `gen_ai.operation.name` picks the span kind (`chat` → llm,
+//! `execute_tool` → tool, `invoke_agent` → agent), `gen_ai.conversation.id` is
+//! the session, `gen_ai.tool.definitions` the tool set an evaluation judges a
+//! tool choice against, and the `*.messages` / `tool.call.*` attributes are the
+//! span's input and output.
+
+/// The operation performed: see [`operation`].
+pub const OPERATION_NAME: &str = "gen_ai.operation.name";
+/// The GenAI provider serving the request (`anthropic`, `openai`, …).
+pub const PROVIDER_NAME: &str = "gen_ai.provider.name";
+/// The model requested.
+pub const REQUEST_MODEL: &str = "gen_ai.request.model";
+/// The `max_tokens` request parameter.
+pub const REQUEST_MAX_TOKENS: &str = "gen_ai.request.max_tokens";
+/// The `temperature` request parameter.
+pub const REQUEST_TEMPERATURE: &str = "gen_ai.request.temperature";
+/// The model that actually answered.
+pub const RESPONSE_MODEL: &str = "gen_ai.response.model";
+/// The provider-assigned response id.
+pub const RESPONSE_ID: &str = "gen_ai.response.id";
+/// Why generation stopped, one entry per choice: see [`finish_reason`].
+pub const RESPONSE_FINISH_REASONS: &str = "gen_ai.response.finish_reasons";
+/// Input tokens consumed.
+pub const USAGE_INPUT_TOKENS: &str = "gen_ai.usage.input_tokens";
+/// Output tokens produced.
+pub const USAGE_OUTPUT_TOKENS: &str = "gen_ai.usage.output_tokens";
+/// The messages sent to the model / agent, as a JSON array of
+/// `{"role", "parts"}` objects (see [`crate::messages`]).
+pub const INPUT_MESSAGES: &str = "gen_ai.input.messages";
+/// The messages produced by the model / agent, as a JSON array of
+/// `{"role", "parts", "finish_reason"}` objects (see [`crate::messages`]).
+pub const OUTPUT_MESSAGES: &str = "gen_ai.output.messages";
+/// The system instructions, as a JSON array of parts, kept separate from the
+/// chat history.
+pub const SYSTEM_INSTRUCTIONS: &str = "gen_ai.system_instructions";
+/// The tools offered to the model, as a JSON array of
+/// `{"type": "function", "name", "description", "parameters"}` objects.
+pub const TOOL_DEFINITIONS: &str = "gen_ai.tool.definitions";
+/// The name of the tool executed.
+pub const TOOL_NAME: &str = "gen_ai.tool.name";
+/// The description of the tool executed.
+pub const TOOL_DESCRIPTION: &str = "gen_ai.tool.description";
+/// The kind of tool executed: see [`tool_type`].
+pub const TOOL_TYPE: &str = "gen_ai.tool.type";
+/// The provider-assigned tool call id.
+pub const TOOL_CALL_ID: &str = "gen_ai.tool.call.id";
+/// The arguments the tool was called with (opt-in content).
+pub const TOOL_CALL_ARGUMENTS: &str = "gen_ai.tool.call.arguments";
+/// The result the tool returned (opt-in content).
+pub const TOOL_CALL_RESULT: &str = "gen_ai.tool.call.result";
+/// The name of the agent.
+pub const AGENT_NAME: &str = "gen_ai.agent.name";
+/// The conversation (session) this operation belongs to.
+pub const CONVERSATION_ID: &str = "gen_ai.conversation.id";
+/// Low-cardinality error class when the operation failed.
+pub const ERROR_TYPE: &str = "error.type";
+
+/// Macro-specific: how many of the oldest messages were dropped from
+/// [`INPUT_MESSAGES`] to fit the size budget (see [`crate::Limits`]).
+pub const MACRO_INPUT_MESSAGES_OMITTED: &str = "macro.genai.input_messages_omitted";
+/// Macro-specific: `true` when any content attribute on the span was cut to fit
+/// the size budget (see [`crate::Limits`]).
+pub const MACRO_CONTENT_TRUNCATED: &str = "macro.genai.content_truncated";
+/// Macro-specific: tokens in the agent's context window after a turn, as the
+/// harness reported it (ACP `usage_update.used`).
+pub const MACRO_CONTEXT_USED_TOKENS: &str = "macro.genai.context.used_tokens";
+/// Macro-specific: the agent's context window size in tokens (ACP
+/// `usage_update.size`).
+pub const MACRO_CONTEXT_SIZE_TOKENS: &str = "macro.genai.context.size_tokens";
+/// Macro-specific: the coarse ACP tool kind (`execute`, `edit`, `read`, …).
+pub const MACRO_TOOL_KIND: &str = "macro.genai.tool.kind";
+/// Macro-specific: the human-readable title the harness gave a tool call.
+pub const MACRO_TOOL_TITLE: &str = "macro.genai.tool.title";
+/// Macro-specific: the MCP server a tool was reached through, when it was.
+pub const MACRO_MCP_SERVER: &str = "macro.genai.mcp.server";
+
+/// Well-known [`OPERATION_NAME`] values.
+pub mod operation {
+ /// An agent run (a turn of the agent loop).
+ pub const INVOKE_AGENT: &str = "invoke_agent";
+ /// A model completion.
+ pub const CHAT: &str = "chat";
+ /// A tool execution.
+ pub const EXECUTE_TOOL: &str = "execute_tool";
+}
+
+/// Well-known [`TOOL_TYPE`] values.
+pub mod tool_type {
+ /// A client-side function the agent executes.
+ pub const FUNCTION: &str = "function";
+}
+
+/// Well-known [`RESPONSE_FINISH_REASONS`] values.
+pub mod finish_reason {
+ /// The model finished naturally.
+ pub const STOP: &str = "stop";
+ /// The model stopped to call one or more tools.
+ pub const TOOL_CALL: &str = "tool_call";
+ /// The model hit a token or request limit.
+ pub const LENGTH: &str = "length";
+ /// The model refused to continue.
+ pub const CONTENT_FILTER: &str = "content_filter";
+ /// The run failed before the model finished.
+ pub const ERROR: &str = "error";
+ /// The run was cancelled by the caller.
+ pub const CANCELLED: &str = "cancelled";
+}
+
+/// Span names used for the semconv spans this workspace opens or adopts. The
+/// agent runtime names its spans identically, which is how the toolset detects
+/// an already-open tool span (see [`crate::current_span_is_named`]).
+pub mod span_name {
+ /// The per-tool-call span.
+ pub const EXECUTE_TOOL: &str = "execute_tool";
+ /// The per-agent-run span.
+ pub const INVOKE_AGENT: &str = "invoke_agent";
+}
diff --git a/crates/genai_telemetry/src/content.rs b/crates/genai_telemetry/src/content.rs
new file mode 100644
index 0000000000..92fc82d021
--- /dev/null
+++ b/crates/genai_telemetry/src/content.rs
@@ -0,0 +1,317 @@
+//! Content-capture policy and size bounding for GenAI span attributes.
+//!
+//! Prompts, tool arguments, tool results and model output are the most useful
+//! and the most dangerous span attributes: an LLM-as-a-judge evaluation needs
+//! them, but they carry user data and can be arbitrarily large. Two controls
+//! apply to every content attribute this workspace records:
+//!
+//! - **Capture** — [`ContentPolicy::capture`], read once from the
+//! OpenTelemetry-standard `OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT`
+//! variable. It defaults to **on** here (the spec's default is off): the point
+//! of the instrumentation is evaluation, which is impossible without content,
+//! and traces already reach the same backend as the service logs. Set it to
+//! `false` to export structure (spans, names, usage, ids) only.
+//! - **Size** — [`Limits`]. Every string leaf is cut to `max_part_chars`, and
+//! a whole attribute to `max_attribute_bytes`; message lists shed their
+//! oldest entries first so the most recent turn survives. Unbounded content
+//! would blow the OTLP batch past the agent's receive limit (dropping the
+//! whole batch) and pay for the same conversation once per model call.
+
+#[cfg(test)]
+mod test;
+
+use macro_env_var::maybe_env_vars;
+use std::borrow::Cow;
+use std::sync::OnceLock;
+
+maybe_env_vars! {
+ pub struct OtelInstrumentationGenaiCaptureMessageContent;
+}
+
+/// Size budgets for content attributes.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct Limits {
+ /// Maximum characters of a single string leaf — a text part, a tool
+ /// argument value, a tool response. Longer values are cut and marked.
+ pub max_part_chars: usize,
+ /// Maximum bytes of one serialized attribute value. Message lists drop
+ /// their oldest messages to fit; tool definitions drop their parameter
+ /// schemas; anything else is cut and marked.
+ pub max_attribute_bytes: usize,
+}
+
+impl Default for Limits {
+ fn default() -> Self {
+ Self {
+ max_part_chars: 8_000,
+ max_attribute_bytes: 64_000,
+ }
+ }
+}
+
+/// Whether, and how much, content may be recorded on GenAI spans.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct ContentPolicy {
+ /// Record prompts, tool arguments/results and model output.
+ pub capture: bool,
+ /// Size budgets applied to everything recorded.
+ pub limits: Limits,
+}
+
+impl Default for ContentPolicy {
+ /// Capture on, default [`Limits`] — the same as an unset environment.
+ fn default() -> Self {
+ Self::enabled()
+ }
+}
+
+impl ContentPolicy {
+ /// Capture content with the default [`Limits`].
+ pub const fn enabled() -> Self {
+ Self {
+ capture: true,
+ limits: Limits {
+ max_part_chars: 8_000,
+ max_attribute_bytes: 64_000,
+ },
+ }
+ }
+
+ /// Record structure only.
+ pub const fn disabled() -> Self {
+ Self {
+ capture: false,
+ ..Self::enabled()
+ }
+ }
+
+ /// The process-wide policy, read from
+ /// `OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT` once and cached.
+ /// Unset means capture is on; see [`Self::parse`].
+ pub fn from_env() -> Self {
+ static POLICY: OnceLock<ContentPolicy> = OnceLock::new();
+ *POLICY.get_or_init(|| {
+ Self::parse(OtelInstrumentationGenaiCaptureMessageContent::new().as_deref())
+ })
+ }
+
+ /// Interpret the capture switch: `false`, `0`, `no` and `off`
+ /// (case-insensitive) disable capture; anything else, including unset,
+ /// enables it.
+ pub fn parse(value: Option<&str>) -> Self {
+ let off = value.is_some_and(|value| {
+ matches!(
+ value.trim().to_ascii_lowercase().as_str(),
+ "false" | "0" | "no" | "off"
+ )
+ });
+ if off {
+ Self::disabled()
+ } else {
+ Self::enabled()
+ }
+ }
+}
+
+/// Cut `s` to at most `max_chars` characters, appending a marker naming how
+/// many characters were dropped. Borrows when nothing needs cutting.
+pub fn truncate_chars(s: &str, max_chars: usize) -> Cow<'_, str> {
+ // Bytes are never fewer than chars, so this cheap check settles the common case.
+ if s.len() <= max_chars {
+ return Cow::Borrowed(s);
+ }
+ let total = s.chars().count();
+ if total <= max_chars {
+ return Cow::Borrowed(s);
+ }
+ let mut cut: String = s.chars().take(max_chars).collect();
+ cut.push_str(&format!("…[truncated {} chars]", total - max_chars));
+ Cow::Owned(cut)
+}
+
+/// Room kept for the marker [`truncate_bytes`] appends, so a cut value never
+/// overshoots its byte budget.
+const TRUNCATION_MARKER_RESERVE: usize = 40;
+
+/// Cut `s` to at most `max_bytes` bytes of UTF-8, on a character boundary,
+/// appending a marker naming how many bytes were dropped. The marker is
+/// counted against the budget. Borrows when nothing needs cutting.
+pub fn truncate_bytes(s: &str, max_bytes: usize) -> Cow<'_, str> {
+ if s.len() <= max_bytes {
+ return Cow::Borrowed(s);
+ }
+ // A budget too small to hold the marker keeps half of itself; the marker
+ // then overshoots, which beats recording nothing at all.
+ let mut keep = max_bytes.saturating_sub(TRUNCATION_MARKER_RESERVE.min(max_bytes / 2));
+ while keep > 0 && !s.is_char_boundary(keep) {
+ keep -= 1;
+ }
+ let mut cut = s[..keep].to_owned();
+ cut.push_str(&format!("…[truncated {} bytes]", s.len() - keep));
+ Cow::Owned(cut)
+}
+
+/// Cut every string leaf in `value` to `max_part_chars` characters, in place.
+/// Returns whether anything was cut.
+fn bound_json_strings(value: &mut serde_json::Value, max_part_chars: usize) -> bool {
+ match value {
+ serde_json::Value::String(s) => match truncate_chars(s, max_part_chars) {
+ Cow::Borrowed(_) => false,
+ Cow::Owned(cut) => {
+ *s = cut;
+ true
+ }
+ },
+ serde_json::Value::Array(items) => items.iter_mut().fold(false, |cut, item| {
+ bound_json_strings(item, max_part_chars) || cut
+ }),
+ serde_json::Value::Object(map) => map.values_mut().fold(false, |cut, item| {
+ bound_json_strings(item, max_part_chars) || cut
+ }),
+ _ => false,
+ }
+}
+
+/// Serialize `value` as an opaque string attribute within `limits`: string
+/// leaves are cut to `max_part_chars` characters, then the whole
+/// serialization to `max_attribute_bytes` bytes. Returns the string and
+/// whether anything was cut.
+///
+/// A cut serialization is no longer valid JSON, so use this only for
+/// attributes the backend treats as plain text (tool arguments and results);
+/// message lists and tool definitions have structure-preserving bounds
+/// ([`bound_messages`], [`bound_tool_definitions`]).
+pub fn bounded_json_string(value: &serde_json::Value, limits: &Limits) -> (String, bool) {
+ let mut value = value.clone();
+ let cut_parts = bound_json_strings(&mut value, limits.max_part_chars);
+ let json = match value {
+ // A bare string reads better than its quoted JSON encoding.
+ serde_json::Value::String(s) => s,
+ other => other.to_string(),
+ };
+ match truncate_bytes(&json, limits.max_attribute_bytes) {
+ Cow::Borrowed(_) => (json, cut_parts),
+ Cow::Owned(cut) => (cut, true),
+ }
+}
+
+/// A bounded `gen_ai.*.messages` list.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct BoundedMessages {
+ /// The serialized JSON array.
+ pub json: String,
+ /// How many of the oldest messages were dropped to fit the budget.
+ pub omitted: usize,
+ /// Whether any content was cut (parts shortened or messages dropped).
+ pub truncated: bool,
+}
+
+/// Bound a message list (see [`crate::messages`]) to `limits`: every string
+/// leaf is cut to `max_part_chars`, then the oldest messages are dropped —
+/// never the last one, which is the current turn — until the serialized list
+/// fits `max_attribute_bytes`. A last message that is over budget on its own
+/// is shrunk in place - its leaves cut shorter, then its oldest parts dropped
+/// - so the result is always a valid JSON array.
+pub fn bound_messages(mut messages: Vec<serde_json::Value>, limits: &Limits) -> BoundedMessages {
+ let mut truncated = false;
+ for message in &mut messages {
+ truncated |= bound_json_strings(message, limits.max_part_chars);
+ }
+ let mut omitted = 0;
+ let mut json = serialize(&messages);
+ while json.len() > limits.max_attribute_bytes && messages.len() > 1 {
+ messages.remove(0);
+ omitted += 1;
+ json = serialize(&messages);
+ }
+ if json.len() > limits.max_attribute_bytes
+ && let Some(last) = messages.last_mut()
+ {
+ shrink_message(last, limits);
+ truncated = true;
+ json = serialize(&messages);
+ }
+ BoundedMessages {
+ json,
+ omitted,
+ truncated: truncated || omitted > 0,
+ }
+}
+
+/// Shrink one message until it serializes within the attribute budget: halve
+/// the leaf budget while that helps, then drop its oldest parts (never the
+/// last), and as a last resort keep only a note of what was dropped.
+fn shrink_message(message: &mut serde_json::Value, limits: &Limits) {
+ let over = |message: &serde_json::Value| message.to_string().len() > limits.max_attribute_bytes;
+ let mut part_chars = limits.max_part_chars / 2;
+ while over(message) && part_chars >= 16 {
+ bound_json_strings(message, part_chars);
+ part_chars /= 2;
+ }
+ while over(message) {
+ let Some(parts) = message
+ .get_mut("parts")
+ .and_then(serde_json::Value::as_array_mut)
+ else {
+ break;
+ };
+ if parts.len() <= 1 {
+ break;
+ }
+ parts.remove(0);
+ }
+ if over(message)
+ && let Some(object) = message.as_object_mut()
+ {
+ let dropped = object
+ .get("parts")
+ .map(|parts| parts.to_string().len())
+ .unwrap_or(0);
+ object.insert(
+ "parts".to_owned(),
+ serde_json::json!([{
+ "type": "text",
+ "content": format!("[content omitted: {dropped} bytes]"),
+ }]),
+ );
+ }
+}
+
+/// Bound a `gen_ai.tool.definitions` list to `limits`. Definitions are our
+/// own code, not user content, so nothing is dropped unless the list is over
+/// budget; then the parameter schemas go first (a judge grades a tool choice
+/// on names and descriptions), then descriptions are cut. Returns the
+/// serialized array and whether anything was removed or cut.
+pub fn bound_tool_definitions(
+ mut definitions: Vec<serde_json::Value>,
+ limits: &Limits,
+) -> (String, bool) {
+ let json = serialize(&definitions);
+ if json.len() <= limits.max_attribute_bytes {
+ return (json, false);
+ }
+ for definition in &mut definitions {
+ if let Some(object) = definition.as_object_mut() {
+ object.remove("parameters");
+ }
+ }
+ let json = serialize(&definitions);
+ if json.len() <= limits.max_attribute_bytes {
+ return (json, true);
+ }
+ for definition in &mut definitions {
+ bound_json_strings(definition, limits.max_part_chars);
+ }
+ // Still over: the list itself is too long. Drop definitions from the end
+ // rather than cut the serialization, so what remains is valid JSON.
+ let mut json = serialize(&definitions);
+ while json.len() > limits.max_attribute_bytes && !definitions.is_empty() {
+ definitions.pop();
+ json = serialize(&definitions);
+ }
+ (json, true)
+}
+
+fn serialize(values: &[serde_json::Value]) -> String {
+ serde_json::to_string(values).unwrap_or_else(|_| "[]".to_string())
+}
diff --git a/crates/genai_telemetry/src/content/test.rs b/crates/genai_telemetry/src/content/test.rs
new file mode 100644
index 0000000000..2816e3c1e2
--- /dev/null
+++ b/crates/genai_telemetry/src/content/test.rs
@@ -0,0 +1,222 @@
+use super::*;
+use serde_json::json;
+
+#[test]
+fn truncate_chars_borrows_when_within_budget() {
+ assert!(matches!(truncate_chars("hello", 5), Cow::Borrowed("hello")));
+ assert!(matches!(truncate_chars("", 0), Cow::Borrowed("")));
+}
+
+#[test]
+fn truncate_chars_cuts_on_char_boundaries_and_marks_the_cut() {
+ // Multi-byte characters: the byte length exceeds the budget but the char
+ // count does not, so nothing is cut.
+ assert!(matches!(truncate_chars("héllo", 5), Cow::Borrowed("héllo")));
+ assert_eq!(
+ truncate_chars("héllo wörld", 5),
+ "héllo…[truncated 6 chars]"
+ );
+}
+
+#[test]
+fn policy_parses_the_standard_switch() {
+ assert!(ContentPolicy::parse(None).capture);
+ assert!(ContentPolicy::parse(Some("true")).capture);
+ assert!(ContentPolicy::parse(Some("anything")).capture);
+ for off in ["false", "FALSE", " 0 ", "no", "Off"] {
+ assert!(
+ !ContentPolicy::parse(Some(off)).capture,
+ "{off:?} should disable capture"
+ );
+ }
+ assert_eq!(ContentPolicy::parse(None).limits, Limits::default());
+}
+
+#[test]
+fn bounded_json_string_cuts_leaves_then_the_whole_value() {
+ let limits = Limits {
+ max_part_chars: 4,
+ max_attribute_bytes: 1_000,
+ };
+ let (json, cut) = bounded_json_string(&json!({"a": "abcdefgh", "n": 1}), &limits);
+ assert!(cut);
+ assert_eq!(json, r#"{"a":"abcd…[truncated 4 chars]","n":1}"#);
+
+ let (json, cut) = bounded_json_string(&json!({"a": "ab"}), &limits);
+ assert!(!cut);
+ assert_eq!(json, r#"{"a":"ab"}"#);
+
+ // A bare string is recorded unquoted.
+ let (json, cut) = bounded_json_string(&json!("plain"), &limits);
+ assert!(cut);
+ assert_eq!(json, "plai…[truncated 1 chars]");
+
+ let tight = Limits {
+ max_part_chars: 1_000,
+ max_attribute_bytes: 10,
+ };
+ let (json, cut) = bounded_json_string(&json!({"key": "value"}), &tight);
+ assert!(cut);
+ assert!(json.starts_with(r#"{"key"#), "{json}");
+ assert!(json.ends_with("bytes]"), "{json}");
+}
+
+fn message(role: &str, text: &str) -> serde_json::Value {
+ json!({"role": role, "parts": [{"type": "text", "content": text}]})
+}
+
+#[test]
+fn bound_messages_keeps_everything_within_budget() {
+ let bounded = bound_messages(
+ vec![message("user", "hi"), message("assistant", "hello")],
+ &Limits::default(),
+ );
+ assert_eq!(bounded.omitted, 0);
+ assert!(!bounded.truncated);
+ let parsed: Vec<serde_json::Value> = serde_json::from_str(&bounded.json).unwrap();
+ assert_eq!(parsed.len(), 2);
+}
+
+#[test]
+fn bound_messages_drops_the_oldest_messages_first_and_never_the_last() {
+ let messages = vec![
+ message("user", "first"),
+ message("assistant", "second"),
+ message("user", "third"),
+ ];
+ let one_message_budget = serde_json::to_string(&vec![message("user", "third")])
+ .unwrap()
+ .len();
+ let limits = Limits {
+ max_part_chars: 1_000,
+ max_attribute_bytes: one_message_budget,
+ };
+ let bounded = bound_messages(messages, &limits);
+ assert_eq!(bounded.omitted, 2);
+ assert!(bounded.truncated);
+ let parsed: Vec<serde_json::Value> = serde_json::from_str(&bounded.json).unwrap();
+ assert_eq!(parsed, vec![message("user", "third")]);
+}
+
+#[test]
+fn bound_messages_cuts_long_parts() {
+ let limits = Limits {
+ max_part_chars: 3,
+ max_attribute_bytes: 10_000,
+ };
+ let bounded = bound_messages(vec![message("user", "abcdef")], &limits);
+ assert_eq!(bounded.omitted, 0);
+ assert!(bounded.truncated);
+ let parsed: Vec<serde_json::Value> = serde_json::from_str(&bounded.json).unwrap();
+ assert_eq!(parsed[0]["parts"][0]["content"], "abc…[truncated 3 chars]");
+}
+
+fn definition(name: &str) -> serde_json::Value {
+ json!({
+ "type": "function",
+ "name": name,
+ "description": format!("does {name}"),
+ "parameters": {"type": "object", "properties": {"q": {"type": "string", "description": "a long schema description"}}}
+ })
+}
+
+#[test]
+fn bound_tool_definitions_keeps_schemas_within_budget() {
+ let (json, cut) =
+ bound_tool_definitions(vec![definition("a"), definition("b")], &Limits::default());
+ assert!(!cut);
+ let parsed: Vec<serde_json::Value> = serde_json::from_str(&json).unwrap();
+ assert!(parsed.iter().all(|d| d.get("parameters").is_some()));
+}
+
+#[test]
+fn bound_tool_definitions_drops_parameters_before_names_and_descriptions() {
+ let full = serde_json::to_string(&vec![definition("a"), definition("b")]).unwrap();
+ let limits = Limits {
+ max_part_chars: 1_000,
+ max_attribute_bytes: full.len() - 1,
+ };
+ let (json, cut) = bound_tool_definitions(vec![definition("a"), definition("b")], &limits);
+ assert!(cut);
+ let parsed: Vec<serde_json::Value> = serde_json::from_str(&json).unwrap();
+ assert_eq!(parsed.len(), 2);
+ assert!(parsed.iter().all(|d| d.get("parameters").is_none()));
+ assert_eq!(parsed[0]["name"], "a");
+ assert_eq!(parsed[0]["description"], "does a");
+}
+
+#[test]
+fn truncate_bytes_respects_the_byte_budget_on_char_boundaries() {
+ assert!(matches!(truncate_bytes("héllo", 6), Cow::Borrowed("héllo")));
+ let emoji = "😀".repeat(1_000);
+ let cut = truncate_bytes(&emoji, 200);
+ assert!(cut.len() <= 200, "{} bytes", cut.len());
+ assert!(cut.starts_with("😀"), "cut on a character boundary");
+ assert!(cut.contains("…[truncated"), "{cut}");
+}
+
+#[test]
+fn bounded_json_string_enforces_bytes_not_chars() {
+ let limits = Limits {
+ max_part_chars: 100_000,
+ max_attribute_bytes: 500,
+ };
+ let (json, cut) = bounded_json_string(&json!("😀".repeat(1_000)), &limits);
+ assert!(cut);
+ assert!(json.len() <= 500, "{} bytes", json.len());
+}
+
+#[test]
+fn an_oversized_last_message_is_shrunk_into_valid_json() {
+ let limits = Limits {
+ max_part_chars: 100_000,
+ max_attribute_bytes: 300,
+ };
+ let big = json!({
+ "role": "user",
+ "parts": [
+ {"type": "text", "content": "a".repeat(400)},
+ {"type": "text", "content": "b".repeat(400)},
+ ]
+ });
+ let bounded = bound_messages(vec![message("assistant", "earlier"), big], &limits);
+ assert!(bounded.truncated);
+ assert_eq!(bounded.omitted, 1);
+ assert!(bounded.json.len() <= 300, "{} bytes", bounded.json.len());
+ let parsed: Vec<serde_json::Value> =
+ serde_json::from_str(&bounded.json).expect("valid JSON: {bounded.json}");
+ assert_eq!(parsed.len(), 1);
+ assert_eq!(parsed[0]["role"], "user");
+
+ // Even a budget too small for any of its content leaves a note behind.
+ let bounded = bound_messages(
+ vec![message("user", &"x".repeat(1_000))],
+ &Limits {
+ max_part_chars: 100_000,
+ max_attribute_bytes: 60,
+ },
+ );
+ let parsed: Vec<serde_json::Value> = serde_json::from_str(&bounded.json).expect("valid JSON");
+ assert!(
+ parsed[0]["parts"][0]["content"]
+ .as_str()
+ .is_some_and(|content| content.starts_with("[content omitted")),
+ "{}",
+ bounded.json
+ );
+}
+
+#[test]
+fn an_overlong_tool_list_drops_definitions_but_stays_valid_json() {
+ let definitions: Vec<_> = (0..50).map(|i| definition(&format!("tool_{i}"))).collect();
+ let limits = Limits {
+ max_part_chars: 1_000,
+ max_attribute_bytes: 600,
+ };
+ let (json, cut) = bound_tool_definitions(definitions, &limits);
+ assert!(cut);
+ assert!(json.len() <= 600, "{} bytes", json.len());
+ let parsed: Vec<serde_json::Value> = serde_json::from_str(&json).expect("valid JSON");
+ assert!(!parsed.is_empty());
+ assert_eq!(parsed[0]["name"], "tool_0");
+}
diff --git a/crates/genai_telemetry/src/lib.rs b/crates/genai_telemetry/src/lib.rs
new file mode 100644
index 0000000000..c92a203303
--- /dev/null
+++ b/crates/genai_telemetry/src/lib.rs
@@ -0,0 +1,43 @@
+//! GenAI span telemetry shared by the AI toolset and the agent loop.
+//!
+//! The agent runtime (`rig`) already emits the *structural* OpenTelemetry GenAI
+//! spans — `invoke_agent`, `chat`, `execute_tool` — with model, provider, usage
+//! and tool names. What it does not give us, and what Datadog LLM Observability
+//! evaluations (Tool Selection, Goal Completeness, …) need, is
+//!
+//! - the **content**: prompts, tool arguments, tool results and model output
+//! (`gen_ai.input.messages`, `gen_ai.output.messages`,
+//! `gen_ai.tool.call.arguments`, `gen_ai.tool.call.result`, …),
+//! - the **tool definitions** offered on each model call
+//! (`gen_ai.tool.definitions`), and
+//! - the **session** the trace belongs to (`gen_ai.conversation.id`).
+//!
+//! This crate provides the shared pieces to record them:
+//!
+//! - [`attr`]: the `gen_ai.*` attribute names and well-known values, spelled once.
+//! - [`ContentPolicy`] / [`Limits`]: whether content may be captured (the
+//! OpenTelemetry-standard `OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT`
+//! switch) and how large a single attribute may grow. Content is always
+//! bounded — a 12-turn agent run otherwise re-sends the whole conversation on
+//! every `chat` span and a tool can return a whole document.
+//! - [`messages`]: builders for the semconv JSON shapes (`{"role", "parts"}`
+//! messages, tool definitions, system instructions) that Datadog parses.
+//! - [`GenAiSpanExt`]: attribute setters that write straight to the
+//! OpenTelemetry span. `tracing` only accepts fields declared at span
+//! creation, so this is the only way to add attributes to spans another crate
+//! (rig) opened — and it keeps large content out of the JSON log layer, which
+//! would otherwise repeat every span field on every log line.
+
+#![deny(missing_docs)]
+
+pub mod attr;
+mod content;
+pub mod messages;
+pub mod propagation;
+mod span;
+
+pub use content::{
+ BoundedMessages, ContentPolicy, Limits, bound_messages, bound_tool_definitions,
+ bounded_json_string, truncate_bytes, truncate_chars,
+};
+pub use span::{GenAiSpanExt, current_span_is_named};
diff --git a/crates/genai_telemetry/src/messages.rs b/crates/genai_telemetry/src/messages.rs
new file mode 100644
index 0000000000..e8f78a890e
--- /dev/null
+++ b/crates/genai_telemetry/src/messages.rs
@@ -0,0 +1,156 @@
+//! Builders for the GenAI semantic-convention JSON shapes.
+//!
+//! `gen_ai.input.messages` / `gen_ai.output.messages` are arrays of
+//! `{"role": …, "parts": […]}` objects (output messages add
+//! `"finish_reason"`), `gen_ai.system_instructions` is an array of parts and
+//! `gen_ai.tool.definitions` an array of function definitions. Datadog parses
+//! exactly these shapes; a backend without structured attributes receives them
+//! as JSON strings, which is how this workspace records them (see
+//! [`crate::bound_messages`] for the size bounds applied on the way out).
+//!
+//! The builders take plain values so callers stay independent of any agent
+//! runtime's message types.
+
+#[cfg(test)]
+mod test;
+
+use serde_json::{Value, json};
+
+/// The `user` role.
+pub const ROLE_USER: &str = "user";
+/// The `assistant` role.
+pub const ROLE_ASSISTANT: &str = "assistant";
+/// The `system` role.
+pub const ROLE_SYSTEM: &str = "system";
+/// The `tool` role (tool results).
+pub const ROLE_TOOL: &str = "tool";
+
+/// Where a media part's bytes live. Inline data is never copied onto a span.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum MediaSource {
+ /// A URL.
+ Uri(String),
+ /// A provider-side file id.
+ FileId(String),
+ /// Inline bytes (base64, raw, …) — recorded as a placeholder only.
+ Inline,
+}
+
+/// A text part.
+pub fn text_part(content: impl Into<String>) -> Value {
+ json!({ "type": "text", "content": content.into() })
+}
+
+/// A reasoning (thinking) part.
+pub fn reasoning_part(content: impl Into<String>) -> Value {
+ json!({ "type": "reasoning", "content": content.into() })
+}
+
+/// A tool call requested by the model.
+pub fn tool_call_part(id: Option<&str>, name: &str, arguments: Value) -> Value {
+ let mut part = json!({ "type": "tool_call", "name": name, "arguments": arguments });
+ if let Some(id) = id {
+ part["id"] = Value::String(id.to_string());
+ }
+ part
+}
+
+/// The response to a tool call.
+pub fn tool_call_response_part(id: Option<&str>, response: Value) -> Value {
+ let mut part = json!({ "type": "tool_call_response", "response": response });
+ if let Some(id) = id {
+ part["id"] = Value::String(id.to_string());
+ }
+ part
+}
+
+/// A media part (`image`, `audio`, `video`, `document` modality). Inline
+/// bytes are replaced by a placeholder: a span is not the place for a file.
+/// A URI is recorded without its credentials, query or fragment (see
+/// [`redact_uri`]).
+pub fn media_part(modality: &str, mime_type: Option<&str>, source: MediaSource) -> Value {
+ let mut part = match source {
+ MediaSource::Uri(uri) => {
+ json!({ "type": "uri", "modality": modality, "uri": redact_uri(&uri) })
+ }
+ MediaSource::FileId(file_id) => {
+ json!({ "type": "file", "modality": modality, "file_id": file_id })
+ }
+ MediaSource::Inline => {
+ json!({ "type": "blob", "modality": modality, "content": "[inline data omitted]" })
+ }
+ };
+ if let Some(mime_type) = mime_type {
+ part["mime_type"] = Value::String(mime_type.to_string());
+ }
+ part
+}
+
+/// A URI reduced to what identifies the resource: scheme, host and path.
+/// Userinfo (`user:password@`), the query string and the fragment are
+/// dropped - that is where credentials, signed-URL tokens and session state
+/// travel, none of which belongs on a span. A `data:` URI *is* its payload,
+/// so only its media type survives.
+pub fn redact_uri(uri: &str) -> String {
+ let uri = uri.split(['?', '#']).next().unwrap_or_default();
+ let (scheme, rest) = match uri.split_once(':') {
+ Some((scheme, rest)) if is_scheme(scheme) => (Some(scheme), rest),
+ _ => (None, uri),
+ };
+ if scheme.is_some_and(|scheme| scheme.eq_ignore_ascii_case("data")) {
+ let media_type = rest.split([';', ',']).next().unwrap_or_default();
+ return format!("data:{media_type};[inline data omitted]");
+ }
+ // Hierarchical (`scheme://authority/path`) or protocol-relative
+ // (`//authority/path`): the authority may carry userinfo.
+ if let Some(rest) = rest.strip_prefix("//") {
+ let (authority, path) = match rest.find('/') {
+ Some(slash) => rest.split_at(slash),
+ None => (rest, ""),
+ };
+ let host = authority
+ .rsplit_once('@')
+ .map_or(authority, |(_, host)| host);
+ return match scheme {
+ Some(scheme) => format!("{scheme}://{host}{path}"),
+ None => format!("//{host}{path}"),
+ };
+ }
+ // Opaque (`urn:…`, `mailto:…`): the identifier is the whole thing.
+ uri.to_owned()
+}
+
+/// Whether `candidate` is a URI scheme (RFC 3986: a letter, then letters,
+/// digits, `+`, `-` or `.`).
+fn is_scheme(candidate: &str) -> bool {
+ let mut chars = candidate.chars();
+ chars
+ .next()
+ .is_some_and(|first| first.is_ascii_alphabetic())
+ && chars.all(|c| c.is_ascii_alphanumeric() || matches!(c, '+' | '-' | '.'))
+}
+
+/// An input message.
+pub fn message(role: &str, parts: Vec<Value>) -> Value {
+ json!({ "role": role, "parts": parts })
+}
+
+/// An output message (one choice) with the reason generation stopped.
+pub fn output_message(role: &str, parts: Vec<Value>, finish_reason: &str) -> Value {
+ json!({ "role": role, "parts": parts, "finish_reason": finish_reason })
+}
+
+/// The `gen_ai.system_instructions` value for one block of instructions.
+pub fn system_instructions(text: impl Into<String>) -> Vec<Value> {
+ vec![text_part(text)]
+}
+
+/// One entry of `gen_ai.tool.definitions`.
+pub fn tool_definition(name: &str, description: &str, parameters: Value) -> Value {
+ json!({
+ "type": "function",
+ "name": name,
+ "description": description,
+ "parameters": parameters,
+ })
+}
diff --git a/crates/genai_telemetry/src/messages/test.rs b/crates/genai_telemetry/src/messages/test.rs
new file mode 100644
index 0000000000..f698a343f3
--- /dev/null
+++ b/crates/genai_telemetry/src/messages/test.rs
@@ -0,0 +1,93 @@
+use super::*;
+
+#[test]
+fn parts_follow_the_semconv_shapes() {
+ assert_eq!(text_part("hi"), json!({"type": "text", "content": "hi"}));
+ assert_eq!(
+ reasoning_part("hmm"),
+ json!({"type": "reasoning", "content": "hmm"})
+ );
+ assert_eq!(
+ tool_call_part(Some("call_1"), "search", json!({"q": "x"})),
+ json!({"type": "tool_call", "id": "call_1", "name": "search", "arguments": {"q": "x"}})
+ );
+ assert_eq!(
+ tool_call_part(None, "search", json!({})),
+ json!({"type": "tool_call", "name": "search", "arguments": {}})
+ );
+ assert_eq!(
+ tool_call_response_part(Some("call_1"), json!({"hits": 3})),
+ json!({"type": "tool_call_response", "id": "call_1", "response": {"hits": 3}})
+ );
+}
+
+#[test]
+fn media_parts_never_carry_inline_bytes() {
+ assert_eq!(
+ media_part(
+ "image",
+ Some("image/png"),
+ MediaSource::Uri("https://x/a.png".into())
+ ),
+ json!({"type": "uri", "modality": "image", "uri": "https://x/a.png", "mime_type": "image/png"})
+ );
+ assert_eq!(
+ media_part("document", None, MediaSource::FileId("file_1".into())),
+ json!({"type": "file", "modality": "document", "file_id": "file_1"})
+ );
+ assert_eq!(
+ media_part("image", None, MediaSource::Inline),
+ json!({"type": "blob", "modality": "image", "content": "[inline data omitted]"})
+ );
+}
+
+#[test]
+fn messages_and_definitions_follow_the_semconv_shapes() {
+ assert_eq!(
+ message(ROLE_USER, vec![text_part("hi")]),
+ json!({"role": "user", "parts": [{"type": "text", "content": "hi"}]})
+ );
+ assert_eq!(
+ output_message(ROLE_ASSISTANT, vec![text_part("hello")], "stop"),
+ json!({"role": "assistant", "parts": [{"type": "text", "content": "hello"}], "finish_reason": "stop"})
+ );
+ assert_eq!(
+ system_instructions("be brief"),
+ vec![json!({"type": "text", "content": "be brief"})]
+ );
+ assert_eq!(
+ tool_definition("search", "Search things", json!({"type": "object"})),
+ json!({"type": "function", "name": "search", "description": "Search things", "parameters": {"type": "object"}})
+ );
+}
+
+#[test]
+fn uris_lose_their_credentials_query_and_fragment() {
+ assert_eq!(
+ redact_uri("https://user:[email protected]/a/b.png?sig=abc#frag"),
+ "https://files.example.com/a/b.png"
+ );
+ assert_eq!(redact_uri("https://x/a.png"), "https://x/a.png");
+ assert_eq!(redact_uri("https://host"), "https://host");
+ assert_eq!(redact_uri("file:///tmp/a.txt"), "file:///tmp/a.txt");
+ assert_eq!(redact_uri("urn:doc:123?x=1"), "urn:doc:123");
+ // Forms without `scheme://` still carry secrets or payloads.
+ assert_eq!(redact_uri("//user:secret@host/path?k=v"), "//host/path");
+ assert_eq!(
+ redact_uri("data:image/png;base64,iVBORw0KGgoAAAANSUhEUg"),
+ "data:image/png;[inline data omitted]"
+ );
+ assert_eq!(
+ redact_uri("DATA:text/plain,hello%20world"),
+ "data:text/plain;[inline data omitted]"
+ );
+ assert_eq!(redact_uri("HTTPS://user@Host/x"), "HTTPS://Host/x");
+ assert_eq!(
+ media_part(
+ "image",
+ None,
+ MediaSource::Uri("https://token@x/a.png?expires=1".into())
+ )["uri"],
+ "https://x/a.png"
+ );
+}
diff --git a/crates/genai_telemetry/src/propagation.rs b/crates/genai_telemetry/src/propagation.rs
new file mode 100644
index 0000000000..bc24362348
--- /dev/null
+++ b/crates/genai_telemetry/src/propagation.rs
@@ -0,0 +1,66 @@
+//! W3C trace context carried in ACP's extensible `_meta` object.
+//!
+//! Only context is transported: content capture and GenAI span ownership stay
+//! with the session actor. Runtimes that do not understand this key ignore it.
+
+#[cfg(test)]
+mod test;
+
+use opentelemetry::propagation::{Extractor, Injector};
+use opentelemetry::trace::TraceContextExt as _;
+use serde_json::{Map, Value};
+use tracing_opentelemetry::OpenTelemetrySpanExt as _;
+
+const TRACE_CONTEXT: &str = "macro.dev/trace-context";
+
+struct Carrier<'a>(&'a Map<String, Value>);
+
+impl Extractor for Carrier<'_> {
+ fn get(&self, key: &str) -> Option<&str> {
+ self.0.get(key)?.as_str()
+ }
+
+ fn keys(&self) -> Vec<&str> {
+ self.0.keys().map(String::as_str).collect()
+ }
+}
+
+struct CarrierMut<'a>(&'a mut Map<String, Value>);
+
+impl Injector for CarrierMut<'_> {
+ fn set(&mut self, key: &str, value: String) {
+ self.0.insert(key.to_owned(), Value::String(value));
+ }
+}
+
+/// Replace our trace-context metadata with `span`'s context, preserving all
+/// other ACP metadata. A disabled span clears any stale context in our key.
+pub fn inject(span: &tracing::Span, meta: &mut Map<String, Value>) {
+ meta.remove(TRACE_CONTEXT);
+ let context = span.context();
+ if !context.span().span_context().is_valid() {
+ return;
+ }
+ let mut carrier = Map::new();
+ opentelemetry::global::get_text_map_propagator(|propagator| {
+ propagator.inject_context(&context, &mut CarrierMut(&mut carrier));
+ });
+ if !carrier.is_empty() {
+ meta.insert(TRACE_CONTEXT.to_owned(), Value::Object(carrier));
+ }
+}
+
+/// Parent runtime work under the invocation that sent its ACP prompt. Missing
+/// or malformed context is ignored; unrelated ambient context is never used
+/// as an extraction fallback.
+pub fn set_parent(span: &tracing::Span, meta: Option<&Map<String, Value>>) {
+ let Some(carrier) = meta.and_then(|meta| meta.get(TRACE_CONTEXT)?.as_object()) else {
+ return;
+ };
+ let context = opentelemetry::global::get_text_map_propagator(|propagator| {
+ propagator.extract_with_context(&opentelemetry::Context::new(), &Carrier(carrier))
+ });
+ if context.span().span_context().is_valid() {
+ let _ = span.set_parent(context);
+ }
+}
diff --git a/crates/genai_telemetry/src/propagation/test.rs b/crates/genai_telemetry/src/propagation/test.rs
new file mode 100644
index 0000000000..f1153c2acd
--- /dev/null
+++ b/crates/genai_telemetry/src/propagation/test.rs
@@ -0,0 +1,75 @@
+use super::*;
+use opentelemetry::trace::{SpanId, TraceContextExt as _, TracerProvider as _};
+use opentelemetry_sdk::propagation::TraceContextPropagator;
+use opentelemetry_sdk::trace::{InMemorySpanExporter, SdkTracerProvider};
+use serde_json::json;
+use tracing_subscriber::layer::SubscriberExt as _;
+
+#[test]
+fn context_survives_serialization_and_is_replaced_for_each_prompt() {
+ opentelemetry::global::set_text_map_propagator(TraceContextPropagator::new());
+ let exporter = InMemorySpanExporter::default();
+ let provider = SdkTracerProvider::builder()
+ .with_simple_exporter(exporter.clone())
+ .build();
+ let subscriber = tracing_subscriber::registry()
+ .with(tracing_opentelemetry::layer().with_tracer(provider.tracer("test")));
+ let _guard = tracing::subscriber::set_default(subscriber);
+ let mut meta = Map::from_iter([("other/client".to_owned(), json!({"keep": true}))]);
+ let mut expected = Vec::new();
+ for _ in 0..2 {
+ let parent = tracing::info_span!(parent: None, "invocation");
+ let context = parent.context();
+ let parent_context = context.span().span_context().clone();
+ inject(&parent, &mut meta);
+ let wire = serde_json::to_string(&meta).expect("serialize metadata");
+ let received = serde_json::from_str(&wire).expect("deserialize metadata");
+ let child = tracing::info_span!(parent: None, "runtime");
+ set_parent(&child, Some(&received));
+ assert_eq!(
+ child.context().span().span_context().trace_id(),
+ parent_context.trace_id()
+ );
+ expected.push(parent_context);
+ }
+ assert_eq!(meta["other/client"], json!({"keep": true}));
+ provider.force_flush().expect("flush");
+ let spans = exporter.get_finished_spans().expect("spans");
+ let children: Vec<_> = spans.iter().filter(|s| s.name == "runtime").collect();
+ assert_eq!(children.len(), 2);
+ for (child, parent) in children.iter().zip(&expected) {
+ assert_eq!(child.parent_span_id, parent.span_id());
+ assert_ne!(child.parent_span_id, SpanId::INVALID);
+ }
+ assert_ne!(expected[0].trace_id(), expected[1].trace_id());
+ inject(&tracing::Span::none(), &mut meta);
+ assert!(
+ !meta.contains_key(TRACE_CONTEXT),
+ "clear stale context when tracing is off"
+ );
+ assert!(meta.contains_key("other/client"));
+}
+
+#[test]
+fn invalid_metadata_does_not_attach_an_unrelated_ambient_context() {
+ opentelemetry::global::set_text_map_propagator(TraceContextPropagator::new());
+ let provider = SdkTracerProvider::builder().build();
+ let subscriber = tracing_subscriber::registry()
+ .with(tracing_opentelemetry::layer().with_tracer(provider.tracer("test")));
+ let _guard = tracing::subscriber::set_default(subscriber);
+ let unrelated = tracing::info_span!("unrelated");
+ let _context = unrelated.context().attach();
+ for value in [
+ Value::Null,
+ json!("bad"),
+ json!({"traceparent": "invalid"}),
+ json!({"traceparent": 5}),
+ ] {
+ let meta = Map::from_iter([(TRACE_CONTEXT.to_owned(), value)]);
+ let child = tracing::info_span!(parent: None, "runtime");
+ let before = child.context().span().span_context().trace_id();
+ set_parent(&child, Some(&meta));
+ set_parent(&child, None);
+ assert_eq!(child.context().span().span_context().trace_id(), before);
+ }
+}
diff --git a/crates/genai_telemetry/src/span.rs b/crates/genai_telemetry/src/span.rs
new file mode 100644
index 0000000000..0dccba56cd
--- /dev/null
+++ b/crates/genai_telemetry/src/span.rs
@@ -0,0 +1,97 @@
+//! Attribute helpers that write straight to a span's OpenTelemetry data.
+//!
+//! `tracing` only records fields a span declared when it was created, so a
+//! span opened by another crate (the agent runtime's `chat` and `execute_tool`
+//! spans) cannot take new fields through `Span::record`. `tracing-opentelemetry`
+//! exposes the underlying OpenTelemetry span, which accepts any attribute at
+//! any time before the span closes. Writing there has a second benefit: the
+//! JSON log layer repeats a span's `tracing` fields on every log line emitted
+//! inside it, and content attributes are far too large for that.
+//!
+//! Every setter is a no-op when no OpenTelemetry layer is installed (local
+//! runs without a collector, unit tests without a subscriber).
+
+#[cfg(test)]
+mod test;
+
+use opentelemetry::trace::Status;
+use opentelemetry::{Array, Key, StringValue, Value};
+use tracing_opentelemetry::OpenTelemetrySpanExt as _;
+
+use crate::attr;
+
+/// Attribute setters for GenAI spans; implemented for [`tracing::Span`].
+pub trait GenAiSpanExt {
+ /// Set a string attribute.
+ fn set_str(&self, key: &'static str, value: impl Into<String>);
+ /// Set an integer attribute.
+ fn set_i64(&self, key: &'static str, value: i64);
+ /// Set an integer attribute from a `u64`, saturating at `i64::MAX`.
+ fn set_u64(&self, key: &'static str, value: u64);
+ /// Set a float attribute.
+ fn set_f64(&self, key: &'static str, value: f64);
+ /// Set a boolean attribute.
+ fn set_bool(&self, key: &'static str, value: bool);
+ /// Set a string-array attribute.
+ fn set_str_array<I, S>(&self, key: &'static str, values: I)
+ where
+ I: IntoIterator<Item = S>,
+ S: Into<String>;
+ /// Mark the span failed: records `error.type` and sets the OpenTelemetry
+ /// error status with `description`.
+ fn set_error(&self, error_type: &str, description: impl Into<String>);
+}
+
+impl GenAiSpanExt for tracing::Span {
+ fn set_str(&self, key: &'static str, value: impl Into<String>) {
+ self.set_attribute(
+ Key::from_static_str(key),
+ Value::String(value.into().into()),
+ );
+ }
+
+ fn set_i64(&self, key: &'static str, value: i64) {
+ self.set_attribute(Key::from_static_str(key), Value::I64(value));
+ }
+
+ fn set_u64(&self, key: &'static str, value: u64) {
+ self.set_i64(key, i64::try_from(value).unwrap_or(i64::MAX));
+ }
+
+ fn set_f64(&self, key: &'static str, value: f64) {
+ self.set_attribute(Key::from_static_str(key), Value::F64(value));
+ }
+
+ fn set_bool(&self, key: &'static str, value: bool) {
+ self.set_attribute(Key::from_static_str(key), Value::Bool(value));
+ }
+
+ fn set_str_array<I, S>(&self, key: &'static str, values: I)
+ where
+ I: IntoIterator<Item = S>,
+ S: Into<String>,
+ {
+ let values: Vec<StringValue> = values.into_iter().map(|v| v.into().into()).collect();
+ self.set_attribute(
+ Key::from_static_str(key),
+ Value::Array(Array::String(values)),
+ );
+ }
+
+ fn set_error(&self, error_type: &str, description: impl Into<String>) {
+ self.set_str(attr::ERROR_TYPE, error_type);
+ self.set_status(Status::error(description.into()));
+ }
+}
+
+/// Whether the current span was opened under `name`.
+///
+/// Lets an inner layer tell that an outer runtime already opened the semconv
+/// span for the operation in flight — the agent runtime's `execute_tool` span
+/// around a tool dispatch, say — so it enriches that span instead of nesting a
+/// duplicate. `false` when there is no current span.
+pub fn current_span_is_named(name: &str) -> bool {
+ tracing::Span::current()
+ .metadata()
+ .is_some_and(|metadata| metadata.name() == name)
+}
diff --git a/crates/genai_telemetry/src/span/test.rs b/crates/genai_telemetry/src/span/test.rs
new file mode 100644
index 0000000000..dedb94db7d
--- /dev/null
+++ b/crates/genai_telemetry/src/span/test.rs
@@ -0,0 +1,101 @@
+use super::*;
+use opentelemetry::trace::TracerProvider as _;
+use opentelemetry_sdk::trace::{InMemorySpanExporter, SdkTracerProvider, SpanData};
+use tracing_subscriber::layer::SubscriberExt as _;
+
+/// An OpenTelemetry pipeline that keeps finished spans in memory, installed as
+/// the thread's default subscriber for the returned guard's lifetime.
+fn otel_test_pipeline() -> (
+ InMemorySpanExporter,
+ SdkTracerProvider,
+ tracing::subscriber::DefaultGuard,
+) {
+ let exporter = InMemorySpanExporter::default();
+ let provider = SdkTracerProvider::builder()
+ .with_simple_exporter(exporter.clone())
+ .build();
+ let layer = tracing_opentelemetry::layer().with_tracer(provider.tracer("test"));
+ let guard = tracing::subscriber::set_default(tracing_subscriber::registry().with(layer));
+ // Other tests in this binary run the same code paths with no subscriber
+ // installed, which can cache "never interested" for a span callsite on
+ // another thread; recompute so this subscriber sees every span.
+ tracing::callsite::rebuild_interest_cache();
+ (exporter, provider, guard)
+}
+
+fn finished(exporter: &InMemorySpanExporter, provider: &SdkTracerProvider) -> Vec<SpanData> {
+ provider.force_flush().expect("flush");
+ exporter.get_finished_spans().expect("finished spans")
+}
+
+fn attribute<'a>(span: &'a SpanData, key: &str) -> Option<&'a Value> {
+ span.attributes
+ .iter()
+ .find(|kv| kv.key.as_str() == key)
+ .map(|kv| &kv.value)
+}
+
+#[test]
+fn setters_write_attributes_and_status_to_the_otel_span() {
+ let (exporter, provider, _guard) = otel_test_pipeline();
+
+ {
+ let span = tracing::info_span!("work");
+ span.set_str("s", "text");
+ span.set_i64("i", -3);
+ span.set_u64("u", u64::MAX);
+ span.set_f64("f", 0.5);
+ span.set_bool("b", true);
+ span.set_str_array("a", ["x", "y"]);
+ span.set_error("boom", "it broke");
+ }
+
+ let spans = finished(&exporter, &provider);
+ assert_eq!(spans.len(), 1);
+ let span = &spans[0];
+ assert_eq!(span.name, "work");
+ assert_eq!(attribute(span, "s"), Some(&Value::String("text".into())));
+ assert_eq!(attribute(span, "i"), Some(&Value::I64(-3)));
+ assert_eq!(attribute(span, "u"), Some(&Value::I64(i64::MAX)));
+ assert_eq!(attribute(span, "f"), Some(&Value::F64(0.5)));
+ assert_eq!(attribute(span, "b"), Some(&Value::Bool(true)));
+ assert_eq!(
+ attribute(span, "a"),
+ Some(&Value::Array(Array::String(vec!["x".into(), "y".into()])))
+ );
+ assert_eq!(
+ attribute(span, attr::ERROR_TYPE),
+ Some(&Value::String("boom".into()))
+ );
+ assert!(
+ matches!(&span.status, Status::Error { description } if description == "it broke"),
+ "{:?}",
+ span.status
+ );
+}
+
+#[test]
+fn setters_are_noops_without_an_otel_layer() {
+ // No subscriber at all: nothing to write to, nothing to panic about.
+ let span = tracing::info_span!("quiet");
+ span.set_str("k", "v");
+ span.set_error("e", "d");
+ assert!(span.is_disabled());
+}
+
+#[test]
+fn current_span_is_named_matches_the_entered_span_only() {
+ let (_exporter, _provider, _guard) = otel_test_pipeline();
+ assert!(!current_span_is_named("outer"));
+ let outer = tracing::info_span!("outer");
+ let _entered = outer.enter();
+ assert!(current_span_is_named("outer"));
+ assert!(!current_span_is_named("inner"));
+ {
+ let inner = tracing::info_span!("inner");
+ let _entered = inner.enter();
+ assert!(current_span_is_named("inner"));
+ assert!(!current_span_is_named("outer"));
+ }
+ assert!(current_span_is_named("outer"));
+}
diff --git a/crates/import/src/domain/service.rs b/crates/import/src/domain/service.rs
index e94b419789..9ede8ce3e4 100644
--- a/crates/import/src/domain/service.rs
+++ b/crates/import/src/domain/service.rs
@@ -2732,7 +2732,7 @@ impl<Context> ToolSet<Context> for NativePlusMcp<Context>
where
Context: Clone + Send + Sync + 'static,
{
- fn try_tool_call<'a>(
+ fn dispatch_tool_call<'a>(
&'a self,
context: Context,
request_context: RequestContext,
@@ -2747,10 +2747,10 @@ where
> {
if self.native_names.contains(tool_name) {
self.native
- .try_tool_call(context, request_context, tool_name, json)
+ .dispatch_tool_call(context, request_context, tool_name, json)
} else {
self.mcp
- .try_tool_call(context, request_context, tool_name, json)
+ .dispatch_tool_call(context, request_context, tool_name, json)
}
}
diff --git a/crates/mcp_client/src/domain/service/toolset.rs b/crates/mcp_client/src/domain/service/toolset.rs
index a279c25f62..d70647e9b5 100644
--- a/crates/mcp_client/src/domain/service/toolset.rs
+++ b/crates/mcp_client/src/domain/service/toolset.rs
@@ -337,7 +337,7 @@ impl McpToolSet {
}
impl<Context: Send + Sync + 'static> ToolSet<Context> for McpToolSet {
- fn try_tool_call<'a>(
+ fn dispatch_tool_call<'a>(
&'a self,
_context: Context,
_request_context: RequestContext,
@@ -440,7 +440,7 @@ impl<T> CombinedToolSet<T> {
}
impl<T: Send + Sync + 'static> ToolSet<T> for CombinedToolSet<T> {
- fn try_tool_call<'a>(
+ fn dispatch_tool_call<'a>(
&'a self,
context: T,
request_context: RequestContext,
@@ -451,10 +451,10 @@ impl<T: Send + Sync + 'static> ToolSet<T> for CombinedToolSet<T> {
> {
if tool_name.starts_with(MANGLED_PREFIX) {
self.mcp_tools
- .try_tool_call(context, request_context, tool_name, json)
+ .dispatch_tool_call(context, request_context, tool_name, json)
} else {
self.static_tools
- .try_tool_call(context, request_context, tool_name, json)
+ .dispatch_tool_call(context, request_context, tool_name, json)
}
}
diff --git a/crates/mcp_select/src/lib.rs b/crates/mcp_select/src/lib.rs
index d2934dbf0b..b7a8140acc 100644
--- a/crates/mcp_select/src/lib.rs
+++ b/crates/mcp_select/src/lib.rs
@@ -65,7 +65,7 @@ impl UserMcpTools {
}
impl<Context: Send + Sync + 'static> ToolSet<Context> for UserMcpTools {
- fn try_tool_call<'a>(
+ fn dispatch_tool_call<'a>(
&'a self,
context: Context,
request_context: RequestContext,
@@ -76,10 +76,10 @@ impl<Context: Send + Sync + 'static> ToolSet<Context> for UserMcpTools {
> {
match self {
UserMcpTools::Pipedream(tools) => {
- tools.try_tool_call(context, request_context, tool_name, json)
+ tools.dispatch_tool_call(context, request_context, tool_name, json)
}
UserMcpTools::Native(tools) => {
- tools.try_tool_call(context, request_context, tool_name, json)
+ tools.dispatch_tool_call(context, request_context, tool_name, json)
}
}
}
@@ -298,7 +298,7 @@ impl<T, Mcp> CombinedToolSet<T, Mcp> {
}
impl<T: Send + Sync + 'static, Mcp: ToolSet<T>> ToolSet<T> for CombinedToolSet<T, Mcp> {
- fn try_tool_call<'a>(
+ fn dispatch_tool_call<'a>(
&'a self,
context: T,
request_context: RequestContext,
@@ -309,10 +309,10 @@ impl<T: Send + Sync + 'static, Mcp: ToolSet<T>> ToolSet<T> for CombinedToolSet<T
> {
if tool_name.starts_with(MANGLED_PREFIX) {
self.mcp_tools
- .try_tool_call(context, request_context, tool_name, json)
+ .dispatch_tool_call(context, request_context, tool_name, json)
} else {
self.static_tools
- .try_tool_call(context, request_context, tool_name, json)
+ .dispatch_tool_call(context, request_context, tool_name, json)
}
}
diff --git a/crates/mcp_toolset/src/toolset.rs b/crates/mcp_toolset/src/toolset.rs
index a9bff2128e..26ffe317de 100644
--- a/crates/mcp_toolset/src/toolset.rs
+++ b/crates/mcp_toolset/src/toolset.rs
@@ -194,7 +194,7 @@ impl RemoteMcpToolSet {
}
impl<Context: Send + Sync + 'static> ToolSet<Context> for RemoteMcpToolSet {
- fn try_tool_call<'a>(
+ fn dispatch_tool_call<'a>(
&'a self,
_context: Context,
_request_context: RequestContext,
diff --git a/crates/pipedream_mcp/src/domain/service/toolset.rs b/crates/pipedream_mcp/src/domain/service/toolset.rs
index 3c797d82e7..f6ce63510d 100644
--- a/crates/pipedream_mcp/src/domain/service/toolset.rs
+++ b/crates/pipedream_mcp/src/domain/service/toolset.rs
@@ -72,7 +72,7 @@ impl PipedreamToolSet {
}
impl<Context: Send + Sync + 'static> ToolSet<Context> for PipedreamToolSet {
- fn try_tool_call<'a>(
+ fn dispatch_tool_call<'a>(
&'a self,
context: Context,
request_context: RequestContext,
@@ -82,7 +82,7 @@ impl<Context: Send + Sync + 'static> ToolSet<Context> for PipedreamToolSet {
Box<dyn Future<Output = Result<ToolResult<serde_json::Value>, ToolSetError>> + 'a + Send>,
> {
self.0
- .try_tool_call(context, request_context, tool_name, json)
+ .dispatch_tool_call(context, request_context, tool_name, json)
}
fn request_schemas(&self) -> Option<Vec<RequestSchema>> {
diff --git a/deny.toml b/deny.toml
index fef442eea4..30a7c69b06 100644
--- a/deny.toml
+++ b/deny.toml
@@ -200,11 +200,16 @@ deny = [
"workspace-hack", # cargo-hakari generated crate
], reason = "Use macro_aws_config for AWS configuration in workspace crates" },
{ crate = "tracing-subscriber", wrappers = [
+ "agent", # GenAI span tests
+ "agent_inmem", # GenAI trace propagation tests
+ "agent_session", # GenAI span tests
+ "ai_toolset", # GenAI span tests
"channels", # Dev logging only
"coding_agent_worker", # Standalone daemon logging setup
"cursor_cloud_agents", # Standalone ACP binary logging; stdout is the protocol, so macro_entrypoint cannot own it
"documents", # Dev logging only
"email", # Dev logging only
+ "genai_telemetry", # GenAI span tests
"lambda_runtime_api_client", # External crate allowed to use tracing-subscriber
"loom", # External concurrency testing dependency
"macro_entrypoint",
diff --git a/docker/analytics-proxy.Dockerfile b/docker/analytics-proxy.Dockerfile
index 7457e1b096..4c672ef930 100644
--- a/docker/analytics-proxy.Dockerfile
+++ b/docker/analytics-proxy.Dockerfile
@@ -3,17 +3,24 @@ RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates
&& rm -rf /var/lib/apt/lists/*
RUN npm install -g bun
+# Wrangler includes platform-specific workerd/esbuild binaries. Install the
+# proxy's locked dependencies outside /app so the repository bind mount cannot
+# replace them with the host platform's node_modules at runtime.
+COPY services/analytics-proxy/package.json services/analytics-proxy/bun.lock /opt/analytics-proxy/
+RUN cd /opt/analytics-proxy && bun install --frozen-lockfile
+ENV PATH="/opt/analytics-proxy/node_modules/.bin:${PATH}"
+
ENV SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt
WORKDIR /app/services/analytics-proxy
EXPOSE 8098
-# The repo is bind-mounted at /app (see docker-compose.yml), so wrangler and
-# node_modules come from the host install. Forward OTLP (both signals) to the
+# The repo is bind-mounted at /app (see docker-compose.yml), while Wrangler and
+# its native dependencies stay under /opt. Forward OTLP (both signals) to the
# local collector (--var overrides the wrangler.jsonc host defaults); no
# DD_API_KEY locally, so the worker skips key injection.
CMD ["sh", "-c", "\
- npx wrangler dev \
+ wrangler dev \
--env local \
--ip 0.0.0.0 \
--port 8098 \
diff --git a/docs/RUNNING_LOCALLY.md b/docs/RUNNING_LOCALLY.md
index 186caacfdf..271cf94820 100644
--- a/docs/RUNNING_LOCALLY.md
+++ b/docs/RUNNING_LOCALLY.md
@@ -184,6 +184,17 @@ across instances) debugging containers:
See `.claude/skills/live-debug/SKILL.md` for query recipes (Tempo/Loki HTTP
APIs) and the browser-debugging workflow.
+For agent turns, use Tempo's TraceQL query
+`{span.gen_ai.operation.name="invoke_agent"}`. The session actor records the
+ACP prompt, output and tool activity on that trace and sends its context in
+`params._meta["macro.dev/trace-context"]`. Macro's in-process runtime restores
+that context so its model calls and backend work appear in the same trace.
+Other runtimes must explicitly consume this metadata to correlate their
+internal spans; their ACP activity is still traced by the session actor.
+
+GenAI content is bounded by `genai_telemetry`; check
+`macro.genai.content_truncated` before using a span for evaluations.
+
## Control the Running Stack
While `run_local` is attached:
diff --git a/docs/STYLE_GUIDE.md b/docs/STYLE_GUIDE.md
index 234e9cfce2..7acf9c0e3e 100644
--- a/docs/STYLE_GUIDE.md
+++ b/docs/STYLE_GUIDE.md
@@ -162,6 +162,17 @@ TypeScript · `[ui]` UI / UX conventions
- **CS-54** `[rust]` New crates put `#![deny(missing_docs)]` in `lib.rs` and document
all public items. Do not mark documentation code blocks `ignore` to skip doctests
unless explicitly directed.
+- **CS-55** `[err]` AI spans follow the OpenTelemetry GenAI conventions (`gen_ai.*`,
+ see `crates/genai_telemetry`). Agent sessions of every harness are traced in one
+ place: the session actor projects each session's ACP frames onto `invoke_agent` /
+ `execute_tool` spans (`crates/agent_session/src/domain/session/telemetry.rs`), so
+ do not add GenAI spans to a harness runtime — `agent_inmem` runs its agent loop with
+ `with_genai_telemetry(false)` for that reason. Outside sessions (chat in
+ `document_cognition_service`), the agent loop and `ai_toolset::ToolSet::try_tool_call`
+ emit and enrich the `invoke_agent` / `chat` / `execute_tool` spans; do not open tool
+ spans in tools or toolsets, and never record prompts, tool arguments or tool results
+ as span fields or logs — they are content, recorded on the spans under the
+ `OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT` policy and a size budget.
## Frontend and shared TypeScript (`apps/web`, `packages/`)
diff --git a/services/agent_harness_service/src/main.rs b/services/agent_harness_service/src/main.rs
index 9ee7cd6fdc..bd7b826fca 100644
--- a/services/agent_harness_service/src/main.rs
+++ b/services/agent_harness_service/src/main.rs
@@ -61,6 +61,7 @@ use agent_inmem::outbound::acp_mcp::AcpMcpConnector;
use agent_inmem::outbound::egress_mcp::EgressMcpClient;
use agent_inmem::outbound::log_frames::LogFrameSource;
use agent_inmem::outbound::manager::InMemAgentManager;
+use agent_inmem::outbound::tool_catalog::McpToolCatalog;
use agent_inmem::rig_engine::RigTurnEngine;
use agent_runtime_directory::PgAgentRuntimeDirectory;
use agent_session::domain::model::{AgentMcpServers, ReplicaId};
@@ -369,6 +370,14 @@ async fn run() -> anyhow::Result<()> {
// against it, so the two must be the same string.
let egress_base_url = AgentHarnessEgressUrl::new()?.to_string();
+ // Every session's MCP tools, listed for its telemetry the way the harness
+ // itself lists them: through the egress proxy, in process.
+ let tool_catalog: Arc<dyn agent_session::domain::ports::SessionToolCatalog> =
+ Arc::new(McpToolCatalog::new(Arc::new(AcpMcpConnector::new(
+ EgressMcpClient::new(Arc::clone(&egress), &egress_base_url),
+ ))));
+ let sessions = sessions.with_tool_catalog(Arc::clone(&tool_catalog));
+
let tool_context =
ai_tools::build_tool_service_context_from_env(pool.clone(), event_broker_tracker.clone())
.await
@@ -399,7 +408,8 @@ async fn run() -> anyhow::Result<()> {
turn_observer.clone(),
lifecycle_publisher.clone(),
replica,
- );
+ )
+ .with_tool_catalog(tool_catalog);
let sandbox_and_inmem = RoutedContainers::new(sandbox, Some(inmem), inmem_sessions);
// Cursor sessions run on their owner's own Cursor account, so there is no
diff --git a/services/document_cognition_service/src/api/stream/chat_message.rs b/services/document_cognition_service/src/api/stream/chat_message.rs
index 5f76236dbd..83753c2ad7 100644
--- a/services/document_cognition_service/src/api/stream/chat_message.rs
+++ b/services/document_cognition_service/src/api/stream/chat_message.rs
@@ -538,8 +538,12 @@ fn stream_and_save_message(
};
let toolset: Arc<dyn ai_toolset::ToolSet<_> + Send + Sync> =
Arc::new(mcp_select::CombinedToolSet::new(static_tools, mcp_tools));
- let agent_loop =
- AgentLoop::new(tool_context.recorder.clone()).with_model(&model);
+ // The chat is the conversation every span of this turn belongs to
+ // (`gen_ai.conversation.id`), so the turns of one chat form one session
+ // in the observability backend.
+ let agent_loop = AgentLoop::new(tool_context.recorder.clone())
+ .with_model(&model)
+ .with_conversation_id(&chat_id);
let rig_messages = agent::to_rig_messages(&request);