Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
feat(ai): tracing / observability (#6272)
Eric Hayesa34ebdde96a1
diff --git a/.github/workspace-dep-closures.json b/.github/workspace-dep-closures.jsonindex fb2635c67e..c54d44c1f9 100644--- a/.github/workspace-dep-closures.json+++ b/.github/workspace-dep-closures.json@@ -7,6 +7,7 @@"crates/bot_id","crates/channel_sender","crates/cowlike",+ "crates/genai_telemetry","crates/kafka_util","crates/macro_env","crates/macro_env_var",@@ -24,6 +25,7 @@"crates/bot_id","crates/bot_token","crates/cowlike",+ "crates/genai_telemetry","crates/harness_id","crates/harness_token","crates/macro_auth",@@ -74,6 +76,7 @@"crates/foreign_entity","crates/frecency","crates/fusionauth",+ "crates/genai_telemetry","crates/github","crates/harness_id","crates/harness_token",@@ -176,6 +179,7 @@"crates/foreign_entity","crates/frecency","crates/fusionauth",+ "crates/genai_telemetry","crates/github","crates/harness_id","crates/harness_token",@@ -283,6 +287,7 @@"crates/foreign_entity","crates/frecency","crates/fusionauth",+ "crates/genai_telemetry","crates/generic_email_domains","crates/github","crates/harness_id",@@ -418,6 +423,7 @@"crates/foreign_entity","crates/frecency","crates/fusionauth",+ "crates/genai_telemetry","crates/generic_email_domains","crates/github","crates/harness_id",@@ -532,6 +538,7 @@"crates/entity_mutation","crates/filter_ast","crates/frecency",+ "crates/genai_telemetry","crates/harness_id","crates/harness_token","crates/invite_email",@@ -597,6 +604,7 @@"crates/entity_mutation","crates/filter_ast","crates/frecency",+ "crates/genai_telemetry","crates/harness_id","crates/harness_token","crates/invite_email",@@ -663,6 +671,7 @@"crates/entity_mutation","crates/filter_ast","crates/frecency",+ "crates/genai_telemetry","crates/harness_id","crates/harness_token","crates/invite_email",@@ -748,6 +757,7 @@"crates/filter_ast","crates/foreign_entity","crates/frecency",+ "crates/genai_telemetry","crates/generic_email_domains","crates/harness_id","crates/harness_token",@@ -855,6 +865,7 @@"crates/entity_mutation","crates/filter_ast","crates/frecency",+ "crates/genai_telemetry","crates/harness_id","crates/harness_token","crates/invite_email",@@ -946,6 +957,7 @@"crates/filter_ast","crates/foreign_entity","crates/frecency",+ "crates/genai_telemetry","crates/generic_email_domains","crates/harness_id","crates/harness_token",@@ -1031,6 +1043,8 @@"ai_toolset": ["crates/ai_toolset","crates/cowlike",+ "crates/genai_telemetry",+ "crates/macro_env_var","crates/macro_user_id","crates/workspace-hack"],@@ -1060,6 +1074,7 @@"crates/ai_toolset","crates/anthropic","crates/cowlike",+ "crates/genai_telemetry","crates/macro_env_var","crates/macro_user_id","crates/workspace-hack"@@ -1119,6 +1134,7 @@"crates/foreign_entity","crates/frecency","crates/fusionauth",+ "crates/genai_telemetry","crates/generic_email_domains","crates/github","crates/harness_id",@@ -1208,6 +1224,7 @@"crates/bot_token","crates/cowlike","crates/document_sub_type",+ "crates/genai_telemetry","crates/harness_id","crates/harness_token","crates/macro_auth",@@ -1261,6 +1278,7 @@"crates/entity_mutation","crates/filter_ast","crates/frecency",+ "crates/genai_telemetry","crates/harness_id","crates/harness_token","crates/invite_email",@@ -1351,6 +1369,7 @@"crates/filter_ast","crates/foreign_entity","crates/frecency",+ "crates/genai_telemetry","crates/graphql_common","crates/graphql_soup_filter_input","crates/harness_id",@@ -1421,6 +1440,7 @@"crates/calendar_events","crates/channel_sender","crates/cowlike",+ "crates/genai_telemetry","crates/harness_id","crates/harness_token","crates/invite_email",@@ -1471,6 +1491,7 @@"crates/entity_mutation","crates/filter_ast","crates/frecency",+ "crates/genai_telemetry","crates/harness_id","crates/harness_token","crates/invite_email",@@ -1561,6 +1582,7 @@"crates/filter_ast","crates/foreign_entity","crates/frecency",+ "crates/genai_telemetry","crates/generic_email_domains","crates/harness_id","crates/harness_token",@@ -1672,6 +1694,7 @@"crates/entity_mutation","crates/filter_ast","crates/frecency",+ "crates/genai_telemetry","crates/harness_id","crates/harness_token","crates/invite_email",@@ -1727,6 +1750,7 @@"crates/entity_access_db_utils","crates/entity_access_management","crates/entity_mutation",+ "crates/genai_telemetry","crates/harness_id","crates/harness_token","crates/kafka_util",@@ -1792,6 +1816,7 @@"crates/filter_ast","crates/foreign_entity","crates/frecency",+ "crates/genai_telemetry","crates/harness_id","crates/harness_token","crates/harnesses",@@ -1863,6 +1888,7 @@"crates/database_env_vars","crates/document_sub_type","crates/entity_access",+ "crates/genai_telemetry","crates/harness_id","crates/harness_token","crates/lexical_client",@@ -1913,6 +1939,7 @@"crates/entity_mutation","crates/filter_ast","crates/frecency",+ "crates/genai_telemetry","crates/harness_id","crates/harness_token","crates/invite_email",@@ -1990,6 +2017,7 @@"crates/filter_ast","crates/foreign_entity","crates/frecency",+ "crates/genai_telemetry","crates/graphql_activity","crates/graphql_channel","crates/graphql_common",@@ -2081,6 +2109,7 @@"crates/database_env_vars","crates/document_sub_type","crates/entity_access",+ "crates/genai_telemetry","crates/harness_id","crates/harness_token","crates/macro_auth",@@ -2119,6 +2148,7 @@"crates/ensure_exists","crates/filter_ast","crates/frecency",+ "crates/genai_telemetry","crates/harness_id","crates/harness_token","crates/item_filters",@@ -2195,6 +2225,7 @@"crates/cowlike","crates/database_env_vars","crates/document_sub_type",+ "crates/genai_telemetry","crates/harness_id","crates/harness_token","crates/macro_auth",@@ -2252,6 +2283,7 @@"crates/entity_mutation","crates/filter_ast","crates/frecency",+ "crates/genai_telemetry","crates/harness_id","crates/harness_token","crates/invite_email",@@ -2332,6 +2364,7 @@"crates/document_sub_type","crates/entity_access","crates/entity_access_db_utils",+ "crates/genai_telemetry","crates/harness_id","crates/harness_token","crates/invite_email",@@ -2401,6 +2434,7 @@"crates/entity_mutation","crates/filter_ast","crates/frecency",+ "crates/genai_telemetry","crates/harness_id","crates/harness_token","crates/invite_email",@@ -2507,6 +2541,7 @@"crates/filter_ast","crates/foreign_entity","crates/frecency",+ "crates/genai_telemetry","crates/harness_id","crates/harness_token","crates/invite_email",@@ -2615,6 +2650,7 @@"crates/filter_ast","crates/foreign_entity","crates/frecency",+ "crates/genai_telemetry","crates/generic_email_domains","crates/harness_id","crates/harness_token",@@ -2761,6 +2797,7 @@"crates/foreign_entity","crates/frecency","crates/fusionauth",+ "crates/genai_telemetry","crates/generic_email_domains","crates/github","crates/graphql_activity",@@ -2884,6 +2921,7 @@"crates/cowlike","crates/document_storage_service_client","crates/document_sub_type",+ "crates/genai_telemetry","crates/harness_id","crates/harness_token","crates/macro_auth",@@ -2917,6 +2955,7 @@"crates/bot_token","crates/cowlike","crates/document_sub_type",+ "crates/genai_telemetry","crates/harness_id","crates/harness_token","crates/macro_auth",@@ -2967,6 +3006,7 @@"crates/entity_mutation","crates/filter_ast","crates/foreign_entity",+ "crates/genai_telemetry","crates/harness_id","crates/harness_token","crates/item_filters",@@ -3035,6 +3075,7 @@"crates/entity_mutation","crates/filter_ast","crates/foreign_entity",+ "crates/genai_telemetry","crates/harness_id","crates/harness_token","crates/item_filters",@@ -3102,6 +3143,7 @@"crates/entity_mutation","crates/filter_ast","crates/frecency",+ "crates/genai_telemetry","crates/harness_id","crates/harness_token","crates/invite_email",@@ -3190,6 +3232,7 @@"crates/entity_mutation","crates/filter_ast","crates/frecency",+ "crates/genai_telemetry","crates/harness_id","crates/harness_token","crates/invite_email",@@ -3260,6 +3303,7 @@"crates/channel_sender","crates/cowlike","crates/email_formatting",+ "crates/genai_telemetry","crates/harness_id","crates/harness_token","crates/invite_email",@@ -3311,6 +3355,7 @@"crates/entity_mutation","crates/filter_ast","crates/frecency",+ "crates/genai_telemetry","crates/harness_id","crates/harness_token","crates/invite_email",@@ -3383,6 +3428,7 @@"crates/entity_mutation","crates/filter_ast","crates/frecency",+ "crates/genai_telemetry","crates/harness_id","crates/harness_token","crates/invite_email",@@ -3462,6 +3508,7 @@"crates/entity_mutation","crates/filter_ast","crates/frecency",+ "crates/genai_telemetry","crates/gmail_client","crates/harness_id","crates/harness_token",@@ -3552,6 +3599,7 @@"crates/entity_mutation","crates/filter_ast","crates/frecency",+ "crates/genai_telemetry","crates/harness_id","crates/harness_token","crates/invite_email",@@ -3618,6 +3666,7 @@"crates/entity_mutation","crates/filter_ast","crates/frecency",+ "crates/genai_telemetry","crates/harness_id","crates/harness_token","crates/invite_email",@@ -3675,6 +3724,7 @@"crates/cowlike","crates/document_sub_type","crates/entity_access_db_utils",+ "crates/genai_telemetry","crates/harness_id","crates/harness_token","crates/macro_auth",@@ -3733,6 +3783,7 @@"crates/database_env_vars","crates/document_sub_type","crates/entity_access",+ "crates/genai_telemetry","crates/harness_id","crates/harness_token","crates/macro_auth",@@ -3791,6 +3842,7 @@"crates/document_sub_type","crates/entity_access","crates/entity_mutation",+ "crates/genai_telemetry","crates/harness_id","crates/harness_token","crates/macro_auth",@@ -3826,6 +3878,7 @@"crates/document_sub_type","crates/entity_access","crates/favorites",+ "crates/genai_telemetry","crates/harness_id","crates/harness_token","crates/macro_auth",@@ -3865,6 +3918,7 @@"crates/entity_access","crates/filter_ast","crates/foreign_entity",+ "crates/genai_telemetry","crates/harness_id","crates/harness_token","crates/item_filters",@@ -3907,6 +3961,11 @@"crates/fusionauth","crates/workspace-hack"],+ "genai_telemetry": [+ "crates/genai_telemetry",+ "crates/macro_env_var",+ "crates/workspace-hack"+ ],"generic_email_domains": ["crates/generic_email_domains","crates/workspace-hack"@@ -3938,6 +3997,7 @@"crates/filter_ast","crates/foreign_entity","crates/fusionauth",+ "crates/genai_telemetry","crates/github","crates/harness_id","crates/harness_token",@@ -4006,6 +4066,7 @@"crates/cowlike","crates/document_sub_type","crates/filter_ast",+ "crates/genai_telemetry","crates/graphql_activity","crates/graphql_common","crates/harness_id",@@ -4052,6 +4113,7 @@"crates/entity_mutation","crates/filter_ast","crates/frecency",+ "crates/genai_telemetry","crates/graphql_channel","crates/graphql_common","crates/harness_id",@@ -4143,6 +4205,7 @@"crates/entity_mutation","crates/filter_ast","crates/frecency",+ "crates/genai_telemetry","crates/graphql_common","crates/graphql_email","crates/harness_id",@@ -4221,6 +4284,7 @@"crates/filter_ast","crates/foreign_entity","crates/frecency",+ "crates/genai_telemetry","crates/graphql_common","crates/graphql_email","crates/graphql_entity_mutation",@@ -4330,6 +4394,7 @@"crates/filter_ast","crates/foreign_entity","crates/frecency",+ "crates/genai_telemetry","crates/graphql_common","crates/graphql_email","crates/graphql_entity_mutation",@@ -4412,6 +4477,7 @@"crates/cowlike","crates/document_sub_type","crates/filter_ast",+ "crates/genai_telemetry","crates/graphql_common","crates/graphql_notification","crates/harness_id",@@ -4455,6 +4521,7 @@"crates/database_env_vars","crates/document_sub_type","crates/entity_access",+ "crates/genai_telemetry","crates/graphql_permission","crates/harness_id","crates/harness_token",@@ -4495,6 +4562,7 @@"crates/entity_access","crates/entity_access_db_utils","crates/filter_ast",+ "crates/genai_telemetry","crates/graphql_common","crates/graphql_properties","crates/harness_id",@@ -4571,6 +4639,7 @@"crates/filter_ast","crates/foreign_entity","crates/frecency",+ "crates/genai_telemetry","crates/graphql_common","crates/graphql_email","crates/graphql_permission",@@ -4711,6 +4780,7 @@"crates/cowlike","crates/document_sub_type","crates/entity_access_db_utils",+ "crates/genai_telemetry","crates/harness_id","crates/harness_token","crates/ip_extractor",@@ -4766,6 +4836,7 @@"crates/document_sub_type","crates/entity_access","crates/entity_access_db_utils",+ "crates/genai_telemetry","crates/harness_id","crates/harness_token","crates/import",@@ -4815,6 +4886,7 @@"crates/bot_token","crates/broadcast","crates/cowlike",+ "crates/genai_telemetry","crates/harness_id","crates/harness_token","crates/invite_email",@@ -4915,6 +4987,7 @@"crates/entity_mutation","crates/filter_ast","crates/frecency",+ "crates/genai_telemetry","crates/harness_id","crates/harness_token","crates/invite_email",@@ -5044,6 +5117,7 @@"crates/cowlike","crates/document_sub_type","crates/entity_access_db_utils",+ "crates/genai_telemetry","crates/harness_id","crates/harness_token","crates/macro_auth",@@ -5114,6 +5188,7 @@"crates/cowlike","crates/document_sub_type","crates/entity_access_db_utils",+ "crates/genai_telemetry","crates/harness_id","crates/harness_token","crates/ip_extractor",@@ -5170,6 +5245,7 @@"crates/bot_token","crates/cowlike","crates/document_sub_type",+ "crates/genai_telemetry","crates/harness_id","crates/harness_token","crates/macro_auth",@@ -5228,6 +5304,7 @@"crates/bot_id","crates/bot_token","crates/cowlike",+ "crates/genai_telemetry","crates/harness_id","crates/harness_token","crates/macro_auth",@@ -5254,6 +5331,7 @@"crates/bot_id","crates/bot_token","crates/cowlike",+ "crates/genai_telemetry","crates/harness_id","crates/harness_token","crates/macro_auth",@@ -5318,6 +5396,7 @@"crates/foreign_entity","crates/frecency","crates/fusionauth",+ "crates/genai_telemetry","crates/generic_email_domains","crates/harness_id","crates/harness_token",@@ -5407,6 +5486,8 @@"mcp_toolset": ["crates/ai_toolset","crates/cowlike",+ "crates/genai_telemetry",+ "crates/macro_env_var","crates/macro_user_id","crates/mcp_toolset","crates/workspace-hack"@@ -5453,6 +5534,7 @@"crates/filter_ast","crates/foreign_entity","crates/frecency",+ "crates/genai_telemetry","crates/generic_email_domains","crates/harness_id","crates/harness_token",@@ -5561,6 +5643,7 @@"crates/bot_token","crates/cowlike","crates/document_sub_type",+ "crates/genai_telemetry","crates/harness_id","crates/harness_token","crates/macro_auth",@@ -5601,6 +5684,7 @@"crates/broadcast","crates/channel_sender","crates/cowlike",+ "crates/genai_telemetry","crates/harness_id","crates/harness_token","crates/invite_email",@@ -5719,6 +5803,7 @@"crates/entity_mutation","crates/filter_ast","crates/frecency",+ "crates/genai_telemetry","crates/harness_id","crates/harness_token","crates/invite_email",@@ -5809,6 +5894,7 @@"crates/entity_mutation","crates/filter_ast","crates/frecency",+ "crates/genai_telemetry","crates/harness_id","crates/harness_token","crates/invite_email",@@ -5881,6 +5967,7 @@"crates/entity_mutation","crates/filter_ast","crates/frecency",+ "crates/genai_telemetry","crates/harness_id","crates/harness_token","crates/invite_email",@@ -5942,6 +6029,7 @@"crates/bot_token","crates/broadcast","crates/cowlike",+ "crates/genai_telemetry","crates/harness_id","crates/harness_token","crates/kafka_util",@@ -5972,6 +6060,7 @@"crates/broadcast","crates/channel_sender","crates/cowlike",+ "crates/genai_telemetry","crates/harness_id","crates/harness_token","crates/invite_email",@@ -6007,6 +6096,7 @@"crates/channel_sender","crates/cowlike","crates/email_formatting",+ "crates/genai_telemetry","crates/harness_id","crates/harness_token","crates/invite_email",@@ -6049,6 +6139,7 @@"crates/document_sub_type","crates/email_formatting","crates/entity_access_db_utils",+ "crates/genai_telemetry","crates/harness_id","crates/harness_token","crates/invite_email",@@ -6122,6 +6213,7 @@"crates/document_sub_type","crates/entity_access","crates/entity_access_db_utils",+ "crates/genai_telemetry","crates/generic_email_domains","crates/harness_id","crates/harness_token",@@ -6203,6 +6295,7 @@"crates/entity_mutation","crates/filter_ast","crates/frecency",+ "crates/genai_telemetry","crates/harness_id","crates/harness_token","crates/invite_email",@@ -6275,6 +6368,7 @@"crates/entity_mutation","crates/filter_ast","crates/frecency",+ "crates/genai_telemetry","crates/harness_id","crates/harness_token","crates/invite_email",@@ -6327,6 +6421,7 @@"crates/bot_id","crates/bot_token","crates/cowlike",+ "crates/genai_telemetry","crates/harness_id","crates/harness_token","crates/macro_auth",@@ -6376,6 +6471,7 @@"crates/filter_ast","crates/foreign_entity","crates/frecency",+ "crates/genai_telemetry","crates/harness_id","crates/harness_token","crates/invite_email",@@ -6450,6 +6546,7 @@"crates/document_sub_type","crates/entity_access","crates/entity_access_db_utils",+ "crates/genai_telemetry","crates/harness_id","crates/harness_token","crates/invite_email",@@ -6502,6 +6599,7 @@"crates/document_sub_type","crates/entity_access","crates/entity_access_db_utils",+ "crates/genai_telemetry","crates/harness_id","crates/harness_token","crates/invite_email",@@ -6554,6 +6652,7 @@"crates/bot_token","crates/broadcast","crates/cowlike",+ "crates/genai_telemetry","crates/harness_id","crates/harness_token","crates/invite_email",@@ -6593,6 +6692,7 @@"crates/database_env_vars","crates/document_sub_type","crates/entity_access",+ "crates/genai_telemetry","crates/harness_id","crates/harness_token","crates/invite_email",@@ -6696,6 +6796,7 @@"crates/filter_ast","crates/foreign_entity","crates/frecency",+ "crates/genai_telemetry","crates/generic_email_domains","crates/harness_id","crates/harness_token",@@ -6823,6 +6924,7 @@"crates/filter_ast","crates/foreign_entity","crates/frecency",+ "crates/genai_telemetry","crates/harness_id","crates/harness_token","crates/invite_email",@@ -6934,6 +7036,7 @@"crates/entity_mutation","crates/filter_ast","crates/frecency",+ "crates/genai_telemetry","crates/harness_id","crates/harness_token","crates/invite_email",@@ -7025,6 +7128,7 @@"crates/entity_mutation","crates/filter_ast","crates/frecency",+ "crates/genai_telemetry","crates/harness_id","crates/harness_token","crates/invite_email",@@ -7087,6 +7191,7 @@"crates/cowlike","crates/document_storage_service_client","crates/document_sub_type",+ "crates/genai_telemetry","crates/harness_id","crates/harness_token","crates/macro_auth",@@ -7147,6 +7252,7 @@"crates/filter_ast","crates/frecency","crates/fusionauth",+ "crates/genai_telemetry","crates/harness_id","crates/harness_token","crates/invite_email",@@ -7262,6 +7368,7 @@"crates/filter_ast","crates/foreign_entity","crates/frecency",+ "crates/genai_telemetry","crates/harness_id","crates/harness_token","crates/invite_email",@@ -7355,6 +7462,7 @@"crates/filter_ast","crates/foreign_entity","crates/frecency",+ "crates/genai_telemetry","crates/harness_id","crates/harness_token","crates/invite_email",@@ -7437,6 +7545,7 @@"crates/filter_ast","crates/foreign_entity","crates/frecency",+ "crates/genai_telemetry","crates/graphql_common","crates/graphql_soup_filter_input","crates/harness_id",@@ -7525,6 +7634,7 @@"crates/filter_ast","crates/foreign_entity","crates/frecency",+ "crates/genai_telemetry","crates/harness_id","crates/harness_token","crates/invite_email",@@ -7612,6 +7722,7 @@"crates/filter_ast","crates/foreign_entity","crates/frecency",+ "crates/genai_telemetry","crates/harness_id","crates/harness_token","crates/invite_email",@@ -7695,6 +7806,7 @@"crates/entity_mutation","crates/filter_ast","crates/frecency",+ "crates/genai_telemetry","crates/harness_id","crates/harness_token","crates/invite_email",@@ -7762,6 +7874,7 @@"crates/bot_token","crates/cowlike","crates/document_sub_type",+ "crates/genai_telemetry","crates/harness_id","crates/harness_token","crates/macro_auth",@@ -7823,6 +7936,7 @@"crates/bot_token","crates/cowlike","crates/document_sub_type",+ "crates/genai_telemetry","crates/harness_id","crates/harness_token","crates/macro_auth",@@ -7891,6 +8005,7 @@"crates/cowlike","crates/document_sub_type","crates/embedding",+ "crates/genai_telemetry","crates/harness_id","crates/harness_token","crates/lexical_client",@@ -7947,6 +8062,7 @@"crates/entity_mutation","crates/filter_ast","crates/frecency",+ "crates/genai_telemetry","crates/generic_email_domains","crates/harness_id","crates/harness_token",@@ -8045,6 +8161,7 @@"crates/entity_mutation","crates/filter_ast","crates/frecency",+ "crates/genai_telemetry","crates/harness_id","crates/harness_token","crates/invite_email",@@ -8118,6 +8235,7 @@"crates/entity_mutation","crates/filter_ast","crates/frecency",+ "crates/genai_telemetry","crates/harness_id","crates/harness_token","crates/invite_email",@@ -8192,6 +8310,7 @@"crates/cowlike","crates/document_sub_type","crates/entity_access_db_utils",+ "crates/genai_telemetry","crates/harness_id","crates/harness_token","crates/macro_auth",@@ -8263,6 +8382,7 @@"crates/filter_ast","crates/foreign_entity","crates/frecency",+ "crates/genai_telemetry","crates/harness_id","crates/harness_token","crates/invite_email",diff --git a/Cargo.lock b/Cargo.lockindex 0a981f9bbd..9a7c664ced 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -100,10 +100,13 @@ dependencies = ["async-trait","attachment","futures",+ "genai_telemetry","macro_env_var","macro_user_id","model-entity","non_empty",+ "opentelemetry",+ "opentelemetry_sdk","prompt","regex","rig-agent",@@ -116,6 +119,8 @@ dependencies = ["tokio","tokio-util","tracing",+ "tracing-opentelemetry",+ "tracing-subscriber","utoipa","workspace-hack",]@@ -379,6 +384,7 @@ dependencies = ["chat","dashmap","futures",+ "genai_telemetry","http 1.4.0","http-body-util","macro_user_id",@@ -386,6 +392,8 @@ dependencies = ["mcp_select","mcp_toolset","memory",+ "opentelemetry",+ "opentelemetry_sdk","prompt","rig-agent","rmcp",@@ -398,6 +406,8 @@ dependencies = ["tokio","tokio-util","tracing",+ "tracing-opentelemetry",+ "tracing-subscriber","workspace-hack",]@@ -441,6 +451,7 @@ dependencies = ["dashmap","entity_access","entity_access_db_utils",+ "genai_telemetry","harness_id","macro_authorization","macro_db_migrator",@@ -452,6 +463,8 @@ dependencies = ["model-entity","model_user","non_empty",+ "opentelemetry",+ "opentelemetry_sdk","rootcause","serde","serde_json",@@ -463,6 +476,8 @@ dependencies = ["tokio-util","tower 0.5.3","tracing",+ "tracing-opentelemetry",+ "tracing-subscriber","utoipa","uuid","workspace-hack",@@ -712,13 +727,19 @@ dependencies = ["async-trait","axum","axum-macros",+ "genai_telemetry","macro_user_id",+ "opentelemetry",+ "opentelemetry_sdk","schemars 1.2.1","serde","serde_json","thiserror 2.0.18",+ "tokio","tokio-util","tracing",+ "tracing-opentelemetry",+ "tracing-subscriber","utoipa","workspace-hack",]@@ -6896,6 +6917,20 @@ dependencies = ["byteorder",]+[[package]]+name = "genai_telemetry"+version = "0.1.0"+dependencies = [+ "macro_env_var",+ "opentelemetry",+ "opentelemetry_sdk",+ "serde_json",+ "tracing",+ "tracing-opentelemetry",+ "tracing-subscriber",+ "workspace-hack",+]+[[package]]name = "generator"version = "0.8.8"diff --git a/Cargo.toml b/Cargo.tomlindex 932425af2b..c555f01844 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -32,6 +32,7 @@ members = ["crates/entity_mutation","crates/favorites","crates/foreign_entity",+ "crates/genai_telemetry","crates/generic_email_domains","crates/gmail_client","crates/graphql_activity",diff --git a/crates/agent/Cargo.toml b/crates/agent/Cargo.tomlindex d4e2f529b0..139e5b0aaa 100644--- a/crates/agent/Cargo.toml+++ b/crates/agent/Cargo.toml@@ -27,6 +27,7 @@ tracing = { workspace = true }ai_usage = { path = "../ai_usage" }ai_toolset = { path = "../ai_toolset" }attachment = { path = "../attachment" }+genai_telemetry = { path = "../genai_telemetry" }macro_env_var = { path = "../macro_env_var" }macro_user_id = { path = "../macro_user_id" }prompt = { path = "../prompt" }@@ -36,6 +37,10 @@ tokio-util.workspace = true[dev-dependencies]model-entity = { path = "../model-entity" }non_empty = { path = "../non_empty" }+opentelemetry = { workspace = true, features = ["trace"] }+opentelemetry_sdk = { workspace = true, features = ["testing", "trace"] }+tracing-opentelemetry = { workspace = true }+tracing-subscriber = { workspace = true }# Enable rig's scripted MockCompletionModel for tests (fake AI streams).rig-core = { version = "0.41.0", features = ["test-utils"] }rig-agent = { version = "0.41.0", features = ["test-utils"] }diff --git a/crates/agent/src/agent_loop.rs b/crates/agent/src/agent_loop.rsindex 406c9175b4..243e63e95c 100644--- a/crates/agent/src/agent_loop.rs+++ b/crates/agent/src/agent_loop.rs@@ -4,15 +4,18 @@ use crate::hook::{BridgeInputs, RegisterFn, ToolRouter, UserToolFinisher};use crate::model::PredefinedModel;use crate::model::router::{ModelRouter, ProviderAgent};use crate::stream::ChatCompletionStream;+use crate::telemetry::GenAiContext;use crate::tool_adapter::DynToolSetAdapter;use ai_toolset::{RequestContext, SearchableTool, ToolLoader, ToolSet as AiToolSet};use ai_usage::{UsageContext, UsageRecorder};+use genai_telemetry::ContentPolicy;use rig_agent::tool::server::{ToolServer, ToolServerHandle};use rig_core::message::Message;use std::future::Future;use std::pin::Pin;use std::sync::{Arc, Mutex, RwLock};use tokio_util::sync::CancellationToken;+use tracing::Instrument as _;const DEFAULT_MAX_TURNS: usize = 16;const DEFAULT_MAX_TOKENS: u64 = 16_000;@@ -31,6 +34,13 @@ pub struct AgentLoop {max_tokens: u64,recorder: Arc<dyn UsageRecorder>,user_tool_finisher: Option<UserToolFinisher>,+ /// The conversation (session) sessions belong to, for telemetry.+ conversation_id: Option<String>,+ /// The agent name spans carry; defaults to the usage context's feature.+ agent_name: Option<String>,+ /// Whether this loop enriches the runtime's GenAI spans (see+ /// [`Self::with_genai_telemetry`]).+ genai_telemetry: bool,}impl AgentLoop {@@ -48,6 +58,9 @@ impl AgentLoop {max_tokens: DEFAULT_MAX_TOKENS,recorder,user_tool_finisher: None,+ conversation_id: None,+ agent_name: None,+ genai_telemetry: true,}}@@ -86,6 +99,39 @@ impl AgentLoop {self}+ /// Tag every span of the sessions created from this loop with the+ /// conversation they belong to (`gen_ai.conversation.id`, e.g. the chat+ /// id). Observability backends group the turns of one conversation into a+ /// session by it, which is what session-level evaluations run over.+ pub fn with_conversation_id<S: Into<String>>(mut self, conversation_id: S) -> Self {+ self.conversation_id = Some(conversation_id.into());+ self+ }++ /// Override the agent name spans carry (`gen_ai.agent.name`). Defaults to+ /// the usage context's feature (`chat`, `automation`, …).+ pub fn with_agent_name<S: Into<String>>(mut self, agent_name: S) -> Self {+ self.agent_name = Some(agent_name.into());+ self+ }++ /// Whether this loop records GenAI telemetry on the runtime's spans: the+ /// `invoke_agent` span with the run's input, output and usage, the+ /// content, tool definitions and conversation id on each `chat` span, and+ /// the arguments and result on each `execute_tool` span. On by default.+ ///+ /// Off for a loop whose turns are already traced from outside - Macro's+ /// in-process agent session runtime, whose ACP frames the session actor+ /// projects onto GenAI spans for every harness alike. With it off the run+ /// is still wrapped in a span (so the runtime adopts it rather than+ /// opening an `invoke_agent` of its own), but that span carries no GenAI+ /// fields, nothing is recorded onto the runtime's spans, and the tool+ /// calls run with `RequestContext::genai_telemetry` off.+ pub fn with_genai_telemetry(mut self, enabled: bool) -> Self {+ self.genai_telemetry = enabled;+ self+ }+/// Start a new streaming session.////// `toolset` is the combined tool set (static + MCP) for this request.@@ -111,10 +157,17 @@ impl AgentLoop {context,system_prompt,usage_ctx,- |handle, prompt, max_turns, max_tokens| {+ |handle, prompt, max_turns, max_tokens, telemetry| {ModelRouter::shared().expect("failed to initialize model router")- .agent(&self.model, handle, prompt, max_turns, max_tokens)+ .agent(+ &self.model,+ handle,+ prompt,+ max_turns,+ max_tokens,+ telemetry,+ )},).await@@ -131,7 +184,7 @@ impl AgentLoop {context: Arc<Context>,system_prompt: &str,usage_ctx: UsageContext,- build: impl FnOnce(ToolServerHandle, &str, usize, u64) -> ProviderAgent,+ build: impl FnOnce(ToolServerHandle, &str, usize, u64, GenAiContext) -> ProviderAgent,) -> SessionwhereContext: Clone + Send + Sync + 'static,@@ -153,8 +206,9 @@ impl AgentLoop {buffer.lock().expect("loaded_buffer poisoned").extend(tools)})};- let request_context =- RequestContext::new(usage_ctx.user.clone()).with_tool_search(Arc::new(catalog), loader);+ let request_context = RequestContext::new(usage_ctx.user.clone())+ .with_tool_search(Arc::new(catalog), loader)+ .with_genai_telemetry(self.genai_telemetry);// TODO this is cringe, make request context a RW lock newtypelet request_context_rw = Arc::new(RwLock::new(request_context.clone()));@@ -238,7 +292,24 @@ impl AgentLoop {system_prompt.push_str(§ion);}- let agent = build(handle, &system_prompt, self.max_turns, self.max_tokens);+ // Session-scoped GenAI telemetry: the conversation id ties the spans of+ // every turn of this session together, the agent name labels them and+ // the content policy governs what is recorded (see `crate::telemetry`).+ let telemetry = GenAiContext::new(+ self.conversation_id.clone(),+ self.agent_name+ .clone()+ .unwrap_or_else(|| usage_ctx.feature.to_string()),+ ContentPolicy::from_env(),+ self.genai_telemetry,+ );+ let agent = build(+ handle,+ &system_prompt,+ self.max_turns,+ self.max_tokens,+ telemetry.clone(),+ );Session {agent,@@ -254,6 +325,7 @@ impl AgentLoop {usage_ctx,model: self.model.clone(),request_context,+ telemetry,}}@@ -279,8 +351,8 @@ impl AgentLoop {context,system_prompt,usage_ctx,- move |handle, prompt, max_turns, max_tokens| {- ProviderAgent::test(model, prompt, max_turns, max_tokens, handle)+ move |handle, prompt, max_turns, max_tokens, telemetry| {+ ProviderAgent::test(model, prompt, max_turns, max_tokens, handle, telemetry)},).await@@ -299,6 +371,7 @@ pub struct Session {usage_ctx: UsageContext,model: String,request_context: RequestContext,+ telemetry: GenAiContext,}impl Session {@@ -311,10 +384,50 @@ impl Session {////// The returned stream yields [`StreamPart`] items compatible with the/// existing DCS consumer code.- #[tracing::instrument(name = "invoke_agent", skip_all)]+ ///+ /// The run is wrapped in a span the runtime adopts instead of opening an+ /// `invoke_agent` of its own. With GenAI telemetry on (the default) that+ /// span *is* the run's `invoke_agent` span: the semconv fields are declared+ /// on it and the run's usage and output are recorded onto it by the stream+ /// driver (see `crate::telemetry`). With it off the span is a plain+ /// `agent.turn`, traced from outside instead. Either way it stays open+ /// until the returned stream ends.pub async fn send_message(&mut self,messages: Vec<Message>,+ ) -> Result<ChatCompletionStream<'_>, AgentError> {+ let span = if self.telemetry.enabled() {+ tracing::info_span!(+ "invoke_agent",+ gen_ai.operation.name = "invoke_agent",+ gen_ai.agent.name = %self.telemetry.agent_name(),+ gen_ai.conversation.id = self.telemetry.conversation_id(),+ gen_ai.provider.name = self.telemetry.provider_name(),+ gen_ai.request.model = self.telemetry.model_name(),+ )+ } else {+ tracing::info_span!("agent.turn", agent.name = %self.telemetry.agent_name())+ };+ let telemetry = self.telemetry.clone();+ let result = self+ .send_message_in(messages)+ .instrument(span.clone())+ .await;+ if let Err(error) = &result {+ // The run never started; the span still says why.+ telemetry.record_agent_failure(+ &span,+ "agent_error",+ genai_telemetry::attr::finish_reason::ERROR,+ &error.to_string(),+ );+ }+ result+ }++ async fn send_message_in(+ &mut self,+ messages: Vec<Message>,) -> Result<ChatCompletionStream<'_>, AgentError> {self.history = messages;@@ -335,6 +448,7 @@ impl Session {self.usage_ctx.clone(),self.model.clone(),self.request_context.clone(),+ self.telemetry.clone(),).await;diff --git a/crates/agent/src/completion.rs b/crates/agent/src/completion.rsindex e78ea1e5d5..7c021d59aa 100644--- a/crates/agent/src/completion.rs+++ b/crates/agent/src/completion.rs@@ -2,8 +2,13 @@//!//! This is the non-streaming API layer: routing only resolves a model, and the//! actual prompting lives here.+#[cfg(test)]+mod test;+use crate::model::router::{ModelRouter, RoutedModel};+use crate::telemetry::{ChatSpanHook, GenAiContext, TracedModel};use ai_usage::{UsageContext, UsageRecorder};+use genai_telemetry::ContentPolicy;use rig_agent::agent::{AgentBuilder, PromptResponse};use rig_agent::completion::Prompt;use rig_core::completion::CompletionModel;@@ -28,15 +33,35 @@ pub async fn complete<M: ToString>(ctx: UsageContext,) -> anyhow::Result<String> {let model = model.to_string();- let response = match ModelRouter::shared()?.route_or_default(&model) {+ let routed = ModelRouter::shared()?.route_or_default(&model);+ let telemetry = telemetry_for(&ctx, &routed);+ let response = match routed {RoutedModel::Anthropic(m) => {- prompt_once(m.completion(), system_prompt, user_message).await?+ prompt_once(+ TracedModel::new(m.completion(), telemetry.clone()),+ system_prompt,+ user_message,+ telemetry,+ )+ .await?}RoutedModel::OpenAiChatCompletions(m) => {- prompt_once(m.completion(), system_prompt, user_message).await?+ prompt_once(+ TracedModel::new(m.completion(), telemetry.clone()),+ system_prompt,+ user_message,+ telemetry,+ )+ .await?}RoutedModel::OpenAiResponses(m) => {- prompt_once(m.completion(), system_prompt, user_message).await?+ prompt_once(+ TracedModel::new(m.completion(), telemetry.clone()),+ system_prompt,+ user_message,+ telemetry,+ )+ .await?}};record(recorder, ctx, model, &response);@@ -56,21 +81,54 @@ pub async fn complete_with_history<M: ToString>(ctx: UsageContext,) -> anyhow::Result<String> {let model = model.to_string();- let response = match ModelRouter::shared()?.route_or_default(&model) {+ let routed = ModelRouter::shared()?.route_or_default(&model);+ let telemetry = telemetry_for(&ctx, &routed);+ let response = match routed {RoutedModel::Anthropic(m) => {- prompt_with_history(m.completion(), system_prompt, messages).await?+ prompt_with_history(+ TracedModel::new(m.completion(), telemetry.clone()),+ system_prompt,+ messages,+ telemetry,+ )+ .await?}RoutedModel::OpenAiChatCompletions(m) => {- prompt_with_history(m.completion(), system_prompt, messages).await?+ prompt_with_history(+ TracedModel::new(m.completion(), telemetry.clone()),+ system_prompt,+ messages,+ telemetry,+ )+ .await?}RoutedModel::OpenAiResponses(m) => {- prompt_with_history(m.completion(), system_prompt, messages).await?+ prompt_with_history(+ TracedModel::new(m.completion(), telemetry.clone()),+ system_prompt,+ messages,+ telemetry,+ )+ .await?}};record(recorder, ctx, model, &response);Ok(response.output)}+/// GenAI telemetry for a one-shot completion: named after the feature making+/// it, reporting the routed model, with no conversation (a one-shot has none).+fn telemetry_for(ctx: &UsageContext, routed: &RoutedModel<'_>) -> GenAiContext {+ let telemetry = GenAiContext::new(+ None,+ ctx.feature.to_string(),+ ContentPolicy::from_env(),+ true,+ );+ telemetry.set_model(routed.provider(), routed.model_name());+ telemetry+}+/// Record the usage of a one-shot completion.fn record(recorder: &dyn UsageRecorder,@@ -86,17 +144,27 @@ fn record(}/// Build a toolless agent and prompt it with a single user message.+///+/// The model is a [`TracedModel`]; `telemetry` also drives the hook that+/// records the model's output on the `chat` span.async fn prompt_once<M: CompletionModel + 'static>(completion_model: M,system_prompt: &str,user_message: &str,+ telemetry: GenAiContext,) -> anyhow::Result<PromptResponse> {let agent = AgentBuilder::new(completion_model)+ .name(telemetry.agent_name())+ .record_content_telemetry(false).preamble(system_prompt).max_tokens(ONE_SHOT_MAX_TOKENS).build();- Ok(agent.prompt(user_message).extended_details().await?)+ Ok(agent+ .prompt(user_message)+ .extended_details()+ .add_hook(ChatSpanHook(telemetry))+ .await?)}/// Build a toolless agent and prompt it with the last message of `messages`,@@ -105,8 +173,11 @@ async fn prompt_with_history<M: CompletionModel + 'static>(completion_model: M,system_prompt: &str,messages: Vec<Message>,+ telemetry: GenAiContext,) -> anyhow::Result<PromptResponse> {let agent = AgentBuilder::new(completion_model)+ .name(telemetry.agent_name())+ .record_content_telemetry(false).preamble(system_prompt).max_tokens(ONE_SHOT_MAX_TOKENS).build();@@ -119,5 +190,6 @@ async fn prompt_with_history<M: CompletionModel + 'static>(.prompt(prompt.clone()).extended_details().history(history.to_vec())+ .add_hook(ChatSpanHook(telemetry)).await?)}diff --git a/crates/agent/src/completion/test.rs b/crates/agent/src/completion/test.rsnew file mode 100644index 0000000000..d6d324d811--- /dev/null+++ b/crates/agent/src/completion/test.rs@@ -0,0 +1,107 @@+//! The one-shot paths trace their single model call like the agent loop does.+use super::*;+use genai_telemetry::attr;+use opentelemetry::trace::{Status, TracerProvider as _};+use opentelemetry_sdk::trace::{InMemorySpanExporter, SdkTracerProvider, SpanData};+use rig_core::test_utils::{MockCompletionModel, MockTurn};+use tracing_subscriber::layer::SubscriberExt as _;++fn otel_test_pipeline() -> (+ InMemorySpanExporter,+ SdkTracerProvider,+ tracing::subscriber::DefaultGuard,+) {+ let exporter = InMemorySpanExporter::default();+ let provider = SdkTracerProvider::builder()+ .with_simple_exporter(exporter.clone())+ .build();+ let layer = tracing_opentelemetry::layer().with_tracer(provider.tracer("test"));+ let guard = tracing::subscriber::set_default(tracing_subscriber::registry().with(layer));+ tracing::callsite::rebuild_interest_cache();+ (exporter, provider, guard)+}++fn finished(exporter: &InMemorySpanExporter, provider: &SdkTracerProvider) -> Vec<SpanData> {+ provider.force_flush().expect("flush");+ exporter.get_finished_spans().expect("spans")+}++fn string_attribute(span: &SpanData, key: &str) -> Option<String> {+ span.attributes+ .iter()+ .find(|kv| kv.key.as_str() == key)+ .map(|kv| kv.value.to_string())+}++fn chat_spans(spans: &[SpanData]) -> Vec<&SpanData> {+ spans+ .iter()+ .filter(|span| {+ string_attribute(span, attr::OPERATION_NAME).as_deref() == Some(attr::operation::CHAT)+ })+ .collect()+}++#[tokio::test]+async fn a_one_shot_completion_traces_its_model_call() {+ let (exporter, provider, _guard) = otel_test_pipeline();+ let telemetry = GenAiContext::new(None, "summarize".to_owned(), ContentPolicy::enabled(), true);+ let model = TracedModel::new(+ MockCompletionModel::new([MockTurn::text("a summary")]),+ telemetry.clone(),+ );++ let response = prompt_once(model, "be brief", "summarize this", telemetry)+ .await+ .expect("the completion succeeds");+ assert_eq!(response.output, "a summary");++ let spans = finished(&exporter, &provider);+ let chats = chat_spans(&spans);+ assert_eq!(chats.len(), 1, "{spans:#?}");+ let chat = chats[0];+ assert_eq!(chat.status, Status::Unset);+ assert!(string_attribute(chat, attr::INPUT_MESSAGES).is_some());+ assert!(+ string_attribute(chat, attr::OUTPUT_MESSAGES)+ .is_some_and(|output| output.contains("a summary"))+ );+}++/// A provider that refuses the call leaves no turn for the hook to record, so+/// the traced model itself marks the call as failed.+#[tokio::test]+async fn a_failed_one_shot_completion_marks_its_chat_span() {+ let (exporter, provider, _guard) = otel_test_pipeline();+ let telemetry = GenAiContext::new(None, "summarize".to_owned(), ContentPolicy::enabled(), true);+ let model = TracedModel::new(+ MockCompletionModel::new([MockTurn::error("upstream is down")]),+ telemetry.clone(),+ );++ let error = prompt_once(model, "be brief", "summarize this", telemetry)+ .await+ .expect_err("the completion fails");+ assert!(error.to_string().contains("upstream is down"), "{error}");++ let spans = finished(&exporter, &provider);+ let chats = chat_spans(&spans);+ assert_eq!(chats.len(), 1, "{spans:#?}");+ let chat = chats[0];+ match &chat.status {+ Status::Error { description } => {+ assert!(description.contains("upstream is down"), "{description}");+ }+ other => panic!("expected an error status, got {other:?}"),+ }+ assert_eq!(+ string_attribute(chat, attr::ERROR_TYPE).as_deref(),+ Some("provider_error")+ );+ assert!(+ string_attribute(chat, attr::RESPONSE_FINISH_REASONS)+ .is_some_and(|reasons| reasons.contains("error"))+ );+ assert!(string_attribute(chat, attr::INPUT_MESSAGES).is_some());+ assert_eq!(string_attribute(chat, attr::OUTPUT_MESSAGES), None);+}diff --git a/crates/agent/src/lib.rs b/crates/agent/src/lib.rsindex 99154451b2..6599d821ad 100644--- a/crates/agent/src/lib.rs+++ b/crates/agent/src/lib.rs@@ -12,6 +12,7 @@ mod model;mod stream;/// Structured output via prompted JSON generation.pub mod structured_output;+mod telemetry;mod tool_adapter;#[cfg(test)]diff --git a/crates/agent/src/model/anthropic.rs b/crates/agent/src/model/anthropic.rsindex 4de920a80d..53a45b529a 100644--- a/crates/agent/src/model/anthropic.rs+++ b/crates/agent/src/model/anthropic.rs@@ -18,6 +18,11 @@ impl<'a> AnthropicModel<'a> {Self { model, client }}+ /// The routed id this model was bound to.+ pub fn model(&self) -> &Model<'a> {+ &self.model+ }+/// The rig completion model for this id. The id is passed verbatim to the/// Anthropic API.pub fn completion(&self) -> anthropic::completion::CompletionModel {diff --git a/crates/agent/src/model/openai.rs b/crates/agent/src/model/openai.rsindex 3ed0c22eef..0ac99b6bad 100644--- a/crates/agent/src/model/openai.rs+++ b/crates/agent/src/model/openai.rs@@ -21,6 +21,11 @@ impl<'a> OpenAiChatCompletionsModel<'a> {Self { model, client }}+ /// The routed id this model was bound to.+ pub fn model(&self) -> &Model<'a> {+ &self.model+ }+/// The rig completion model for this id.////// Unlike the Responses API, the Chat Completions API does not coerce tools@@ -74,6 +79,11 @@ impl<'a> OpenAiResponsesModel<'a> {Self { model, client }}+ /// The routed id this model was bound to.+ pub fn model(&self) -> &Model<'a> {+ &self.model+ }+/// The rig Responses API completion model for this id.////// Rig maps the generic `max_tokens` agent setting to the Responses API'sdiff --git a/crates/agent/src/model/router.rs b/crates/agent/src/model/router.rsindex ddc856fbe7..fad7b8daa1 100644--- a/crates/agent/src/model/router.rs+++ b/crates/agent/src/model/router.rs@@ -29,6 +29,7 @@ use rig_core::completion::{CompletionModel, GetTokenUsage};use rig_core::message::Message;use rig_core::providers::{anthropic, openai};use rig_core::streaming::StreamedAssistantContent;+use tracing::Instrument as _;use super::PredefinedModel;use super::anthropic::AnthropicModel;@@ -37,6 +38,7 @@ use super::types::Model;use crate::error::AgentError;use crate::hook::{BridgeInputs, StreamBridge};use crate::stream::{ChatCompletionStream, StreamPart};+use crate::telemetry::{ChatSpanHook, GenAiContext, TracedModel};env_var! {struct ApiKeys {@@ -67,6 +69,24 @@ pub(crate) enum RoutedModel<'a> {}impl<'a> RoutedModel<'a> {+ /// The provider segment of the routed id (`anthropic`, `openai`, …).+ pub(crate) fn provider(&self) -> &str {+ match self {+ RoutedModel::Anthropic(m) => m.model().provider(),+ RoutedModel::OpenAiChatCompletions(m) => m.model().provider(),+ RoutedModel::OpenAiResponses(m) => m.model().provider(),+ }+ }++ /// The bare model name sent to the provider.+ pub(crate) fn model_name(&self) -> &str {+ match self {+ RoutedModel::Anthropic(m) => m.model().name(),+ RoutedModel::OpenAiChatCompletions(m) => m.model().name(),+ RoutedModel::OpenAiResponses(m) => m.model().name(),+ }+ }+/// Build the rig agent for this model, applying provider-specific thinking/// config. Pure construction — no model call is made here.pub(crate) fn into_agent(@@ -75,6 +95,7 @@ impl<'a> RoutedModel<'a> {system_prompt: &str,max_turns: usize,max_tokens: u64,+ telemetry: &GenAiContext,) -> ProviderAgent {match self {RoutedModel::Anthropic(m) => {@@ -86,6 +107,7 @@ impl<'a> RoutedModel<'a> {system_prompt,max_turns,max_tokens,+ telemetry,))}RoutedModel::OpenAiChatCompletions(m) => {@@ -97,6 +119,7 @@ impl<'a> RoutedModel<'a> {system_prompt,max_turns,max_tokens,+ telemetry,))}RoutedModel::OpenAiResponses(m) => {@@ -108,6 +131,7 @@ impl<'a> RoutedModel<'a> {system_prompt,max_turns,max_tokens,+ telemetry,))}}@@ -122,11 +146,11 @@ impl<'a> RoutedModel<'a> {/// [`run_stream`]: ProviderAgent::run_streampub(crate) enum ProviderAgent {/// An agent over Anthropic's native completion model.- Anthropic(Agent<anthropic::completion::CompletionModel>),+ Anthropic(Agent<TracedModel<anthropic::completion::CompletionModel>>),/// An agent over the OpenAI Chat Completions model.- OpenAiChatCompletions(Agent<openai::completion::CompletionModel>),+ OpenAiChatCompletions(Agent<TracedModel<openai::completion::CompletionModel>>),/// An agent over the OpenAI Responses model.- OpenAiResponses(Agent<openai::responses_api::ResponsesCompletionModel>),+ OpenAiResponses(Agent<TracedModel<openai::responses_api::ResponsesCompletionModel>>),/// A test-only agent over an arbitrary completion model (e.g. a scripted/// fake), type-erased so the enum itself stays non-generic.#[cfg(test)]@@ -147,6 +171,7 @@ impl ProviderAgent {usage_ctx: UsageContext,model: String,request_context: RequestContext,+ telemetry: GenAiContext,) -> ChatCompletionStream<'static> {match self {ProviderAgent::Anthropic(agent) => {@@ -160,6 +185,7 @@ impl ProviderAgent {usage_ctx,model,request_context.clone(),+ telemetry,).await}@@ -174,6 +200,7 @@ impl ProviderAgent {usage_ctx,model,request_context.clone(),+ telemetry,).await}@@ -188,6 +215,7 @@ impl ProviderAgent {usage_ctx,model,request_context.clone(),+ telemetry,).await}@@ -203,6 +231,7 @@ impl ProviderAgent {usage_ctx,model,request_context.clone(),+ telemetry,).await}@@ -306,7 +335,9 @@ impl ModelRouter {}/// Route + build the agent in one step, falling back to the default model on- /// an unroutable id.+ /// an unroutable id. Tells `telemetry` which provider and model the session+ /// actually runs on, so its spans report the routed model, not the+ /// requested id.pub(crate) fn agent(&self,model: &str,@@ -314,9 +345,11 @@ impl ModelRouter {system_prompt: &str,max_turns: usize,max_tokens: u64,+ telemetry: GenAiContext,) -> ProviderAgent {- self.route_or_default(model)- .into_agent(handle, system_prompt, max_turns, max_tokens)+ let routed = self.route_or_default(model);+ telemetry.set_model(routed.provider(), routed.model_name());+ routed.into_agent(handle, system_prompt, max_turns, max_tokens, &telemetry)}/// Route a `provider/model` id to the provider that serves it.@@ -366,6 +399,10 @@ impl ModelRouter {}/// Build a rig agent from a completion model and per-session config.+///+/// The model is wrapped in [`TracedModel`] so every model call records its+/// request on the `chat` span. rig's own content recording stays off — it is+/// unbounded; `crate::telemetry` records bounded content instead.fn build_agent<M: CompletionModel>(model: M,thinking: Option<serde_json::Value>,@@ -373,8 +410,11 @@ fn build_agent<M: CompletionModel>(system_prompt: &str,max_turns: usize,max_tokens: u64,-) -> Agent<M> {- let mut builder = AgentBuilder::new(model)+ telemetry: &GenAiContext,+) -> Agent<TracedModel<M>> {+ let mut builder = AgentBuilder::new(TracedModel::new(model, telemetry.clone()))+ .name(telemetry.agent_name())+ .record_content_telemetry(false).tool_server_handle(handle).default_max_turns(max_turns).max_tokens(max_tokens)@@ -398,11 +438,18 @@ async fn drive_stream<M>(usage_ctx: UsageContext,model: String,request_context: RequestContext,+ telemetry: GenAiContext,) -> ChatCompletionStream<'static>whereM: CompletionModel + 'static,M::StreamingResponse: GetTokenUsage + Send + Sync,{+ // The caller's `invoke_agent` span (see `Session::send_message`): rig adopts+ // it as the run's agent span but never records onto a span it did not+ // open, so the run's input, output and usage are recorded here.+ let agent_span = tracing::Span::current();+ telemetry.record_agent_input(&agent_span, &prompt);+let (bridge, mut rx) = StreamBridge::channel(inputs,request_context.searchable_tools.clone(),@@ -419,6 +466,7 @@ where.max_turns(max_turns).max_invalid_tool_call_retries(crate::hook::MAX_INVALID_TOOL_CALL_RETRIES).add_hook(bridge)+ .add_hook(ChatSpanHook(telemetry.clone())).await;// Drive the rig stream on its own task. The hook emits a tool call the@@ -430,49 +478,113 @@ where// to the client as soon as it is produced — so a tool call renders in its// pending state immediately and its response renders when execution// finishes.- let driver = tokio::spawn(async move {- let mut thinking_buf = String::new();-- while let Some(item) = rig_stream.next().await {- match item {- Ok(MultiTurnStreamItem::StreamAssistantItem(- StreamedAssistantContent::ReasoningDelta { reasoning, .. },- )) => {- thinking_buf.push_str(&reasoning);- }- other => {- if !thinking_buf.is_empty() {- let _ = driver_tx- .send(Ok(StreamPart::Thinking(std::mem::take(&mut thinking_buf))));+ // Whatever ends the driver - the stream running dry, a provider error, an+ // abort when the consumer drops the stream - the model call's parked+ // `chat` span is released with it (see `GenAiContext::finish_run`), and a+ // run that never reached its final response or an error is recorded as+ // cancelled: the consumer went away, and the run went with it.+ struct FinishRun {+ telemetry: GenAiContext,+ agent_span: tracing::Span,+ concluded: bool,+ }+ impl Drop for FinishRun {+ fn drop(&mut self) {+ if !self.concluded {+ self.telemetry.record_agent_failure(+ &self.agent_span,+ "cancelled",+ genai_telemetry::attr::finish_reason::CANCELLED,+ "the run was cancelled before the agent answered",+ );+ }+ self.telemetry.finish_run();+ }+ }+ let mut finish_run = FinishRun {+ telemetry: telemetry.clone(),+ agent_span: agent_span.clone(),+ concluded: false,+ };+ let driver_span = agent_span.clone();+ let driver = tokio::spawn(+ async move {+ let mut thinking_buf = String::new();++ while let Some(item) = rig_stream.next().await {+ match item {+ Ok(MultiTurnStreamItem::StreamAssistantItem(+ StreamedAssistantContent::ReasoningDelta { reasoning, .. },+ )) => {+ thinking_buf.push_str(&reasoning);}- match other {- Ok(MultiTurnStreamItem::FinalResponse(final_resp)) => {- let usage = final_resp.usage;- // Best-effort cost logging; never fails the stream.- recorder.record(usage_ctx.clone().into_event(- model.clone(),- usage.input_tokens,- usage.output_tokens,- ));- let _ = driver_tx.send(Ok(StreamPart::Usage(crate::stream::Usage {- input_tokens: usage.input_tokens,- output_tokens: usage.output_tokens,- })));+ other => {+ if !thinking_buf.is_empty() {+ let _ = driver_tx+ .send(Ok(StreamPart::Thinking(std::mem::take(&mut thinking_buf))));}- Err(e) => {- let _ = driver_tx.send(Err(AgentError::Streaming(e)));+ match other {+ Ok(MultiTurnStreamItem::FinalResponse(final_resp)) => {+ let usage = final_resp.usage;+ finish_run.concluded = true;+ telemetry.record_agent_output(+ &agent_span,+ &final_resp.output,+ &usage,+ );+ // Best-effort cost logging; never fails the stream.+ recorder.record(usage_ctx.clone().into_event(+ model.clone(),+ usage.input_tokens,+ usage.output_tokens,+ ));+ let _ =+ driver_tx.send(Ok(StreamPart::Usage(crate::stream::Usage {+ input_tokens: usage.input_tokens,+ output_tokens: usage.output_tokens,+ })));+ }+ Err(e) => {+ finish_run.concluded = true;+ let error = AgentError::Streaming(e);+ if error.was_cancelled() {+ // The caller stopped the run through its+ // cancellation token: a stop, not a fault.+ telemetry.record_agent_failure(+ &agent_span,+ "cancelled",+ genai_telemetry::attr::finish_reason::CANCELLED,+ "the run was cancelled",+ );+ } else {+ // A provider error, or the runtime giving+ // up (retries exhausted): the run failed.+ telemetry.record_agent_failure(+ &agent_span,+ "streaming_error",+ genai_telemetry::attr::finish_reason::ERROR,+ &error.to_string(),+ );+ }+ let _ = driver_tx.send(Err(error));+ }+ _ => {}}- _ => {}}}}+ if !thinking_buf.is_empty() {+ let _ = driver_tx.send(Ok(StreamPart::Thinking(std::mem::take(&mut thinking_buf))));+ }+ // Dropping `rig_stream` (and with it the hook's sender) plus `driver_tx`+ // here closes the channel, ending the consumer stream below.+ drop(finish_run);}- if !thinking_buf.is_empty() {- let _ = driver_tx.send(Ok(StreamPart::Thinking(std::mem::take(&mut thinking_buf))));- }- // Dropping `rig_stream` (and with it the hook's sender) plus `driver_tx`- // here closes the channel, ending the consumer stream below.- });+ // Entered into the agent span: the runtime opens its `chat` and+ // `execute_tool` spans from inside this task, and they belong under+ // the run, not at the root of a trace of their own.+ .instrument(driver_span),+ );// Abort the driver when the consumer drops the returned stream (e.g. on// cancellation), which drops `rig_stream` and cancels any in-flight tool —@@ -511,6 +623,7 @@ pub(crate) trait DynStreamAgent: Send + Sync {usage_ctx: UsageContext,model: String,request_context: RequestContext,+ telemetry: GenAiContext,) -> std::pin::Pin<Box<dyn std::future::Future<Output = ChatCompletionStream<'static>> + Send + 'a>,>;@@ -532,6 +645,7 @@ whereusage_ctx: UsageContext,model: String,request_context: RequestContext,+ telemetry: GenAiContext,) -> std::pin::Pin<Box<dyn std::future::Future<Output = ChatCompletionStream<'static>> + Send + 'a>,> {@@ -545,6 +659,7 @@ whereusage_ctx,model,request_context,+ telemetry,))}}@@ -559,11 +674,13 @@ impl ProviderAgent {max_turns: usize,max_tokens: u64,handle: ToolServerHandle,+ telemetry: GenAiContext,) -> SelfwhereM: CompletionModel + 'static,M::StreamingResponse: GetTokenUsage + Send + Sync,{+ telemetry.set_model("test", "fake-model");ProviderAgent::Test(Box::new(build_agent(model,None,@@ -571,6 +688,7 @@ impl ProviderAgent {system_prompt,max_turns,max_tokens,+ &telemetry,)))}}diff --git a/crates/agent/src/telemetry.rs b/crates/agent/src/telemetry.rsnew file mode 100644index 0000000000..d79042fcc3--- /dev/null+++ b/crates/agent/src/telemetry.rs@@ -0,0 +1,558 @@+//! GenAI span enrichment for the agent loop.+//!+//! rig opens the OpenTelemetry GenAI spans — `invoke_agent` (adopting ours, see+//! [`crate::Session::send_message`]), one `chat` span per model call and one+//! `execute_tool` span per tool call — and records the structural fields:+//! provider, model, response id, token usage, tool name and call id. Its own+//! content recording (`record_content_telemetry`) stays **off**: it is+//! unbounded, so every `chat` span would carry the whole conversation so far.+//! This module records what Datadog's evaluations need on top, under the+//! content policy and size bounds of [`genai_telemetry`]:+//!+//! - on each `chat` span: the tool definitions offered (`gen_ai.tool.definitions`,+//! what a tool-selection judge grades against), the input messages, the+//! system instructions, request parameters, the conversation id and — once+//! the turn finishes — the output messages and finish reason;+//! - on the `invoke_agent` span: the run's token usage, its input (the prompt)+//! and its output (the final answer), which a session-level judge reads.+//!+//! [`TracedModel`] sits between rig and the provider. rig instruments the+//! model call with the `chat` span, so inside `completion` / `stream` the+//! current span *is* that span: the request side is recorded there and the+//! span is parked in the shared [`GenAiContext`]. The response side arrives+//! through rig's per-turn hook ([`ChatSpanHook`]) after the provider stream+//! ends, when the current span is the agent span again.++#[cfg(test)]+mod test;++use std::sync::{Arc, Mutex, OnceLock};++use genai_telemetry::messages::{self, MediaSource};+use genai_telemetry::{+ ContentPolicy, GenAiSpanExt as _, attr, bound_messages, bound_tool_definitions,+};+use rig_agent::agent::hook::{AgentHook, HookContext, ModelTurnAction, ModelTurnFinished};+use rig_core::OneOrMany;+use rig_core::completion::{+ CompletionError, CompletionModel, CompletionRequest, CompletionResponse, ToolDefinition, Usage,+};+use rig_core::message::{+ AssistantContent, DocumentSourceKind, Message, MimeType, Reasoning, ReasoningContent,+ ToolResultContent, UserContent,+};+use rig_core::streaming::StreamingCompletionResponse;++/// Session-scoped telemetry state shared by the traced model, the hooks and+/// the agent span. Cheap to clone; all clones share one state.+#[derive(Clone, Default)]+pub(crate) struct GenAiContext(Arc<Inner>);++#[derive(Default)]+struct Inner {+ /// The session (chat) every span of this run belongs to.+ conversation_id: Option<String>,+ /// Label for the agent, e.g. the AI feature driving it.+ agent_name: String,+ policy: ContentPolicy,+ /// Whether anything is recorded at all. Off, every `record_*` is a no-op+ /// and the model call's span is never parked.+ enabled: bool,+ /// The routed `(provider, model)`, set once routing has decided.+ model: OnceLock<(String, String)>,+ /// The `chat` span of the model call in flight: parked by+ /// [`TracedModel`] on request, consumed by [`ChatSpanHook`] on response.+ /// Holding the handle keeps the span open, so it is always taken out again.+ chat_span: Mutex<Option<tracing::Span>>,+}++impl GenAiContext {+ pub(crate) fn new(+ conversation_id: Option<String>,+ agent_name: String,+ policy: ContentPolicy,+ enabled: bool,+ ) -> Self {+ Self(Arc::new(Inner {+ conversation_id,+ agent_name,+ policy,+ enabled,+ model: OnceLock::new(),+ chat_span: Mutex::new(None),+ }))+ }++ /// Whether this context records anything (see+ /// [`crate::AgentLoop::with_genai_telemetry`]).+ pub(crate) fn enabled(&self) -> bool {+ self.0.enabled+ }++ pub(crate) fn conversation_id(&self) -> Option<&str> {+ self.0.conversation_id.as_deref()+ }++ pub(crate) fn agent_name(&self) -> &str {+ &self.0.agent_name+ }++ /// Record the provider and model routing settled on. First call wins.+ pub(crate) fn set_model(&self, provider: &str, model: &str) {+ let _ = self.0.model.set((provider.to_owned(), model.to_owned()));+ }++ pub(crate) fn provider_name(&self) -> Option<&str> {+ self.0.model.get().map(|(provider, _)| provider.as_str())+ }++ pub(crate) fn model_name(&self) -> Option<&str> {+ self.0.model.get().map(|(_, model)| model.as_str())+ }++ /// Record the request side of a model call on the current span (rig's+ /// `chat` span when called from [`TracedModel`]) and park that span for+ /// [`Self::record_response`].+ fn record_request(&self, request: &CompletionRequest) {+ if !self.0.enabled {+ return;+ }+ let span = tracing::Span::current();+ let policy = &self.0.policy;++ if let Some(conversation_id) = self.conversation_id() {+ span.set_str(attr::CONVERSATION_ID, conversation_id);+ }+ if let Some(max_tokens) = request.max_tokens {+ span.set_u64(attr::REQUEST_MAX_TOKENS, max_tokens);+ }+ if let Some(temperature) = request.temperature {+ span.set_f64(attr::REQUEST_TEMPERATURE, temperature);+ }+ // Tool definitions are our own code, not user content: always recorded.+ if !request.tools.is_empty() {+ let definitions = request.tools.iter().map(tool_definition_json).collect();+ let (json, truncated) = bound_tool_definitions(definitions, &policy.limits);+ span.set_str(attr::TOOL_DEFINITIONS, json);+ if truncated {+ span.set_bool(attr::MACRO_CONTENT_TRUNCATED, true);+ }+ }+ if policy.capture {+ let (system, input) = split_system(request);+ if let Some(system) = system {+ let bounded = bound_messages(messages::system_instructions(system), &policy.limits);+ span.set_str(attr::SYSTEM_INSTRUCTIONS, bounded.json);+ if bounded.truncated {+ span.set_bool(attr::MACRO_CONTENT_TRUNCATED, true);+ }+ }+ let bounded = bound_messages(input, &policy.limits);+ span.set_str(attr::INPUT_MESSAGES, bounded.json);+ if bounded.omitted > 0 {+ span.set_u64(attr::MACRO_INPUT_MESSAGES_OMITTED, bounded.omitted as u64);+ }+ if bounded.truncated {+ span.set_bool(attr::MACRO_CONTENT_TRUNCATED, true);+ }+ }++ *self.0.chat_span.lock().expect("chat span slot poisoned") = Some(span);+ }++ /// Record the response side of the model call whose request was recorded+ /// last, then release the parked span so it can close.+ fn record_response(&self, content: &OneOrMany<AssistantContent>) {+ let Some(span) = self+ .0+ .chat_span+ .lock()+ .expect("chat span slot poisoned")+ .take()+ else {+ return;+ };+ let finish_reason = finish_reason(content);+ span.set_str_array(attr::RESPONSE_FINISH_REASONS, [finish_reason]);+ if self.0.policy.capture {+ let message = messages::output_message(+ messages::ROLE_ASSISTANT,+ assistant_parts(content),+ finish_reason,+ );+ let bounded = bound_messages(vec![message], &self.0.policy.limits);+ span.set_str(attr::OUTPUT_MESSAGES, bounded.json);+ if bounded.truncated {+ span.set_bool(attr::MACRO_CONTENT_TRUNCATED, true);+ }+ }+ }++ /// Record that an agent run failed: `error_type` names the class of+ /// failure, `finish_reason` how the run stopped (`error`, `cancelled`) and+ /// `detail` the message, which is recorded bounded when content may be+ /// captured and left out otherwise - a provider's error text can quote the+ /// request.+ ///+ /// A model call still in flight - its `chat` span parked, its turn hook+ /// never to fire - failed the same way: the span is marked and released,+ /// so a failed call never reads as a successful one with no response.+ pub(crate) fn record_agent_failure(+ &self,+ agent_span: &tracing::Span,+ error_type: &str,+ finish_reason: &'static str,+ detail: &str,+ ) {+ if !self.0.enabled {+ return;+ }+ let description = self.failure_description(error_type, detail);+ agent_span.set_str_array(attr::RESPONSE_FINISH_REASONS, [finish_reason]);+ agent_span.set_error(error_type, description.clone());+ self.fail_parked_chat(error_type, finish_reason, description);+ }++ /// Record that the model call whose request was recorded last failed+ /// before it produced a response: the provider refused or errored. The+ /// parked `chat` span is marked and released. Called by [`TracedModel`]+ /// on every path - a one-shot completion has no run driver to do it.+ fn record_chat_failure(&self, error: &CompletionError) {+ if !self.0.enabled {+ return;+ }+ let description = self.failure_description("provider_error", &error.to_string());+ self.fail_parked_chat("provider_error", attr::finish_reason::ERROR, description);+ }++ /// The description a failure's span status carries: the detail, bounded,+ /// when content may be recorded - an error message can quote the request+ /// - and a fixed summary otherwise.+ fn failure_description(&self, error_type: &str, detail: &str) -> String {+ if self.0.policy.capture {+ genai_telemetry::truncate_chars(detail, self.0.policy.limits.max_part_chars)+ .into_owned()+ } else {+ format!("the call ended with {error_type}")+ }+ }++ /// Mark the parked `chat` span, if any, as ended by `error_type` and+ /// release it. A no-op when no model call is in flight.+ fn fail_parked_chat(&self, error_type: &str, finish_reason: &'static str, description: String) {+ if let Some(chat_span) = self+ .0+ .chat_span+ .lock()+ .expect("chat span slot poisoned")+ .take()+ {+ chat_span.set_str_array(attr::RESPONSE_FINISH_REASONS, [finish_reason]);+ chat_span.set_error(error_type, description);+ }+ }++ /// Release the parked `chat` span, if any, so it closes now. Called when+ /// a run ends without its turn hook firing and without a recorded failure;+ /// a no-op when the hook (or `record_agent_failure`) already released it.+ pub(crate) fn finish_run(&self) {+ self.0+ .chat_span+ .lock()+ .expect("chat span slot poisoned")+ .take();+ }++ /// Record the prompt of an agent run as the agent span's input.+ pub(crate) fn record_agent_input(&self, agent_span: &tracing::Span, prompt: &Message) {+ if !self.0.enabled || !self.0.policy.capture {+ return;+ }+ let bounded = bound_messages(message_json(prompt), &self.0.policy.limits);+ agent_span.set_str(attr::INPUT_MESSAGES, bounded.json);+ if bounded.truncated {+ agent_span.set_bool(attr::MACRO_CONTENT_TRUNCATED, true);+ }+ }++ /// Record the final answer and aggregate usage of an agent run on the+ /// agent span. Usage is structure and always recorded; the answer follows+ /// the content policy.+ pub(crate) fn record_agent_output(+ &self,+ agent_span: &tracing::Span,+ output: &str,+ usage: &Usage,+ ) {+ if !self.0.enabled {+ return;+ }+ agent_span.set_u64(attr::USAGE_INPUT_TOKENS, usage.input_tokens);+ agent_span.set_u64(attr::USAGE_OUTPUT_TOKENS, usage.output_tokens);+ agent_span.set_str_array(attr::RESPONSE_FINISH_REASONS, [attr::finish_reason::STOP]);+ if !self.0.policy.capture {+ return;+ }+ let message = messages::output_message(+ messages::ROLE_ASSISTANT,+ vec![messages::text_part(output)],+ attr::finish_reason::STOP,+ );+ let bounded = bound_messages(vec![message], &self.0.policy.limits);+ agent_span.set_str(attr::OUTPUT_MESSAGES, bounded.json);+ if bounded.truncated {+ agent_span.set_bool(attr::MACRO_CONTENT_TRUNCATED, true);+ }+ }+}++/// A [`CompletionModel`] that records the request side of every model call+/// on the current (`chat`) span before delegating to the provider model.+#[derive(Clone)]+pub(crate) struct TracedModel<M> {+ inner: M,+ telemetry: GenAiContext,+}++impl<M> TracedModel<M> {+ pub(crate) fn new(inner: M, telemetry: GenAiContext) -> Self {+ Self { inner, telemetry }+ }+}++impl<M: CompletionModel> CompletionModel for TracedModel<M> {+ type Response = M::Response;+ type StreamingResponse = M::StreamingResponse;+ type Client = M::Client;++ fn make(client: &Self::Client, model: impl Into<String>) -> Self {+ Self {+ inner: M::make(client, model),+ telemetry: GenAiContext::default(),+ }+ }++ async fn completion(+ &self,+ request: CompletionRequest,+ ) -> Result<CompletionResponse<Self::Response>, CompletionError> {+ self.telemetry.record_request(&request);+ let result = self.inner.completion(request).await;+ if let Err(error) = &result {+ self.telemetry.record_chat_failure(error);+ }+ result+ }++ async fn stream(+ &self,+ request: CompletionRequest,+ ) -> Result<StreamingCompletionResponse<Self::StreamingResponse>, CompletionError> {+ self.telemetry.record_request(&request);+ let result = self.inner.stream(request).await;+ if let Err(error) = &result {+ self.telemetry.record_chat_failure(error);+ }+ result+ }++ fn composes_native_output_with_tools(&self) -> bool {+ self.inner.composes_native_output_with_tools()+ }+}++/// rig hook completing the `chat` span [`TracedModel`] parked: records the+/// model's output and finish reason once the turn's response is in.+///+/// `ModelTurnFinished` is the one per-turn event both of rig's surfaces fire+/// for every accepted turn, tool-only turns included (`StreamResponseFinish`+/// skips those). A turn rig recovers from an invalid tool call fires nothing;+/// its span is released, without output, when the next call parks its own.+#[derive(Clone)]+pub(crate) struct ChatSpanHook(pub(crate) GenAiContext);++impl AgentHook for ChatSpanHook {+ async fn on_model_turn_finished(+ &self,+ _ctx: &HookContext,+ event: ModelTurnFinished<'_>,+ ) -> ModelTurnAction {+ self.0.record_response(event.content);+ ModelTurnAction::Continue+ }+}++/// Why the model stopped, as far as rig's normalized content tells: a turn+/// with tool calls stopped to call them, anything else is a natural stop+/// (rig does not retain a provider's `length` / `content_filter` reasons).+fn finish_reason(content: &OneOrMany<AssistantContent>) -> &'static str {+ if content+ .iter()+ .any(|content| matches!(content, AssistantContent::ToolCall(_)))+ {+ attr::finish_reason::TOOL_CALL+ } else {+ attr::finish_reason::STOP+ }+}++/// Separate the system instructions (the preamble and any `System` messages,+/// which rig prepends to the history) from the chat messages.+fn split_system(request: &CompletionRequest) -> (Option<String>, Vec<serde_json::Value>) {+ let mut system: Vec<String> = request.preamble.iter().cloned().collect();+ let mut input = Vec::new();+ for message in request.chat_history.iter() {+ match message {+ Message::System { content } => system.push(content.clone()),+ other => input.extend(message_json(other)),+ }+ }+ let system = (!system.is_empty()).then(|| system.join("\n\n"));+ (system, input)+}++fn tool_definition_json(definition: &ToolDefinition) -> serde_json::Value {+ messages::tool_definition(+ &definition.name,+ &definition.description,+ definition.parameters.clone(),+ )+}++/// One rig message as semconv messages. A user message carrying tool results+/// yields a `tool` message for them (and a `user` message for anything else),+/// matching the semconv role split.+fn message_json(message: &Message) -> Vec<serde_json::Value> {+ match message {+ Message::System { content } => vec![messages::message(+ messages::ROLE_SYSTEM,+ vec![messages::text_part(content.clone())],+ )],+ Message::User { content } => {+ let (tool_parts, user_parts): (Vec<_>, Vec<_>) = content+ .iter()+ .partition(|content| matches!(content, UserContent::ToolResult(_)));+ let mut out = Vec::new();+ if !tool_parts.is_empty() {+ out.push(messages::message(+ messages::ROLE_TOOL,+ tool_parts.into_iter().map(user_part).collect(),+ ));+ }+ if !user_parts.is_empty() {+ out.push(messages::message(+ messages::ROLE_USER,+ user_parts.into_iter().map(user_part).collect(),+ ));+ }+ out+ }+ Message::Assistant { content, .. } => vec![messages::message(+ messages::ROLE_ASSISTANT,+ assistant_parts(content),+ )],+ }+}++fn user_part(content: &UserContent) -> serde_json::Value {+ match content {+ UserContent::Text(text) => messages::text_part(text.text.clone()),+ UserContent::ToolResult(result) => {+ messages::tool_call_response_part(Some(&result.id), tool_result_json(&result.content))+ }+ UserContent::Image(image) => media_part(+ "image",+ image.media_type.as_ref().map(MimeType::to_mime_type),+ &image.data,+ ),+ UserContent::Audio(audio) => media_part(+ "audio",+ audio.media_type.as_ref().map(MimeType::to_mime_type),+ &audio.data,+ ),+ UserContent::Video(video) => media_part(+ "video",+ video.media_type.as_ref().map(MimeType::to_mime_type),+ &video.data,+ ),+ UserContent::Document(document) => media_part(+ "document",+ document.media_type.as_ref().map(MimeType::to_mime_type),+ &document.data,+ ),+ }+}++/// A tool result as the `response` of a `tool_call_response` part: a lone+/// text or JSON item as itself, several items as an array.+fn tool_result_json(content: &OneOrMany<ToolResultContent>) -> serde_json::Value {+ let mut items: Vec<serde_json::Value> = content+ .iter()+ .map(|item| match item {+ ToolResultContent::Text(text) => serde_json::Value::String(text.text.clone()),+ ToolResultContent::Json { value } => value.clone(),+ ToolResultContent::Image(image) => media_part(+ "image",+ image.media_type.as_ref().map(MimeType::to_mime_type),+ &image.data,+ ),+ })+ .collect();+ if items.len() == 1 {+ items.remove(0)+ } else {+ serde_json::Value::Array(items)+ }+}++fn assistant_parts(content: &OneOrMany<AssistantContent>) -> Vec<serde_json::Value> {+ content+ .iter()+ .filter_map(|content| match content {+ AssistantContent::Text(text) => Some(messages::text_part(text.text.clone())),+ AssistantContent::ToolCall(call) => Some(messages::tool_call_part(+ Some(&call.id),+ &call.function.name,+ call.function.arguments.clone(),+ )),+ AssistantContent::Reasoning(reasoning) => {+ reasoning_text(reasoning).map(messages::reasoning_part)+ }+ AssistantContent::Image(image) => Some(media_part(+ "image",+ image.media_type.as_ref().map(MimeType::to_mime_type),+ &image.data,+ )),+ })+ .collect()+}++/// The readable text of a reasoning block; `None` when it is entirely+/// encrypted or redacted.+fn reasoning_text(reasoning: &Reasoning) -> Option<String> {+ let text: Vec<&str> = reasoning+ .content+ .iter()+ .filter_map(|content| match content {+ ReasoningContent::Text { text, .. } => Some(text.as_str()),+ ReasoningContent::Summary(summary) => Some(summary.as_str()),+ // Encrypted, redacted, or a variant added later: nothing readable.+ _ => None,+ })+ .collect();+ (!text.is_empty()).then(|| text.join("\n"))+}++fn media_part(+ modality: &str,+ mime_type: Option<&str>,+ data: &DocumentSourceKind,+) -> serde_json::Value {+ let source = match data {+ DocumentSourceKind::Url(uri) => MediaSource::Uri(uri.clone()),+ DocumentSourceKind::FileId(file_id) => MediaSource::FileId(file_id.clone()),+ // Base64, raw bytes, inline strings: never copied onto a span.+ _ => MediaSource::Inline,+ };+ messages::media_part(modality, mime_type, source)+}diff --git a/crates/agent/src/telemetry/test.rs b/crates/agent/src/telemetry/test.rsnew file mode 100644index 0000000000..f5b5153fdd--- /dev/null+++ b/crates/agent/src/telemetry/test.rs@@ -0,0 +1,126 @@+//! Unit tests for the rig → semconv message encoding.+use super::*;+use rig_core::message::{Text, ToolCall, ToolFunction, ToolResult};+use serde_json::json;++#[test]+fn user_text_becomes_a_user_message_with_a_text_part() {+ let encoded = message_json(&Message::user("hello"));+ assert_eq!(+ encoded,+ vec![json!({"role": "user", "parts": [{"type": "text", "content": "hello"}]})]+ );+}++#[test]+fn tool_results_become_a_tool_message() {+ let message = Message::User {+ content: OneOrMany::many([+ UserContent::ToolResult(ToolResult {+ id: "call-1".to_string(),+ call_id: None,+ content: OneOrMany::one(ToolResultContent::Json {+ value: json!({ "echo": "a" }),+ }),+ }),+ UserContent::Text(Text {+ text: "and more".to_string(),+ additional_params: None,+ }),+ ])+ .expect("two items"),+ };+ let encoded = message_json(&message);+ assert_eq!(+ encoded,+ vec![+ json!({"role": "tool", "parts": [{"type": "tool_call_response", "id": "call-1", "response": {"echo": "a"}}]}),+ json!({"role": "user", "parts": [{"type": "text", "content": "and more"}]}),+ ]+ );+}++#[test]+fn assistant_tool_calls_and_text_are_parts_and_decide_the_finish_reason() {+ let content = OneOrMany::many([+ AssistantContent::Text(Text {+ text: "let me check".to_string(),+ additional_params: None,+ }),+ AssistantContent::ToolCall(ToolCall {+ id: "call-1".to_string(),+ call_id: None,+ function: ToolFunction {+ name: "echo_tool".to_string(),+ arguments: json!({ "value": "a" }),+ },+ signature: None,+ additional_params: None,+ }),+ ])+ .expect("two items");+ assert_eq!(+ assistant_parts(&content),+ vec![+ json!({"type": "text", "content": "let me check"}),+ json!({"type": "tool_call", "id": "call-1", "name": "echo_tool", "arguments": {"value": "a"}}),+ ]+ );+ assert_eq!(finish_reason(&content), attr::finish_reason::TOOL_CALL);+ assert_eq!(+ finish_reason(&OneOrMany::one(AssistantContent::text("done"))),+ attr::finish_reason::STOP+ );+}++#[test]+fn a_lone_text_tool_result_is_a_plain_string_and_several_items_an_array() {+ assert_eq!(+ tool_result_json(&OneOrMany::one(ToolResultContent::Text(Text {+ text: "ok".to_string(),+ additional_params: None,+ }))),+ json!("ok")+ );+ assert_eq!(+ tool_result_json(+ &OneOrMany::many([+ ToolResultContent::Text(Text {+ text: "ok".to_string(),+ additional_params: None,+ }),+ ToolResultContent::Json { value: json!(1) },+ ])+ .expect("two items")+ ),+ json!(["ok", 1])+ );+}++#[test]+fn finishing_a_run_releases_the_parked_chat_span() {+ use opentelemetry::trace::TracerProvider as _;+ use opentelemetry_sdk::trace::{InMemorySpanExporter, SdkTracerProvider};+ use tracing_subscriber::layer::SubscriberExt as _;++ let exporter = InMemorySpanExporter::default();+ let provider = SdkTracerProvider::builder()+ .with_simple_exporter(exporter.clone())+ .build();+ let layer = tracing_opentelemetry::layer().with_tracer(provider.tracer("test"));+ let _guard = tracing::subscriber::set_default(tracing_subscriber::registry().with(layer));+ tracing::callsite::rebuild_interest_cache();++ let telemetry = GenAiContext::new(None, "chat".to_owned(), ContentPolicy::enabled(), true);+ // A model call whose turn never finished: its span sits parked.+ *telemetry.0.chat_span.lock().unwrap() = Some(tracing::info_span!("chat"));+ provider.force_flush().unwrap();+ assert!(exporter.get_finished_spans().unwrap().is_empty());++ telemetry.finish_run();+ provider.force_flush().unwrap();+ assert_eq!(exporter.get_finished_spans().unwrap().len(), 1);+ assert!(telemetry.0.chat_span.lock().unwrap().is_none());+ // Idempotent.+ telemetry.finish_run();+}diff --git a/crates/agent/src/test/agent_loop/mod.rs b/crates/agent/src/test/agent_loop/mod.rsindex cf5a1931d4..6816d89245 100644--- a/crates/agent/src/test/agent_loop/mod.rs+++ b/crates/agent/src/test/agent_loop/mod.rs@@ -7,6 +7,7 @@ mod util;mod test_cooperative_cancellation;mod test_eager_tools;+mod test_telemetry;mod test_tool;mod test_tool_search;mod test_user_tools;diff --git a/crates/agent/src/test/agent_loop/test_telemetry.rs b/crates/agent/src/test/agent_loop/test_telemetry.rsnew file mode 100644index 0000000000..f14a7f93ae--- /dev/null+++ b/crates/agent/src/test/agent_loop/test_telemetry.rs@@ -0,0 +1,588 @@+//! GenAI telemetry: what the agent loop adds to rig's `invoke_agent`, `chat`+//! and `execute_tool` spans so an observability backend can evaluate a run+//! (tool selection against the offered tools, goal completion across a+//! conversation).+use super::util;+use ai_toolset::{+ AsyncTool, RequestContext, ServiceContext, ToolAnnotated, ToolAnnotations, ToolResult,+};+use async_trait::async_trait;+use genai_telemetry::attr;+use opentelemetry::trace::TracerProvider as _;+use opentelemetry::{Array, Value};+use opentelemetry_sdk::trace::{InMemorySpanExporter, SdkTracerProvider, SpanData};+use rig_core::test_utils::{MockCompletionModel, MockStreamEvent};+use schemars::JsonSchema;+use serde::Deserialize;+use serde_json::json;+use std::sync::Arc;+use tracing_subscriber::layer::SubscriberExt as _;++/// A tool that echoes its input.+#[derive(Deserialize, JsonSchema)]+#[schemars(title = "echo_tool", description = "Echoes its input back.")]+struct EchoTool {+ value: String,+}++impl ToolAnnotated for EchoTool {+ const ANNOTATIONS: ToolAnnotations = ToolAnnotations::read_only("Echo");+}++#[async_trait]+impl AsyncTool<()> for EchoTool {+ type Output = serde_json::Value;++ async fn call(+ &self,+ _service_context: ServiceContext<()>,+ _request_context: RequestContext,+ ) -> ToolResult<Self::Output> {+ Ok(json!({ "echo": self.value }))+ }+}++/// An OpenTelemetry pipeline that keeps finished spans in memory, installed as+/// the thread's default subscriber for the returned guard's lifetime. The+/// tests run on tokio's current-thread runtime, so the driver task the agent+/// loop spawns sees the same subscriber.+fn otel_test_pipeline() -> (+ InMemorySpanExporter,+ SdkTracerProvider,+ tracing::subscriber::DefaultGuard,+) {+ let exporter = InMemorySpanExporter::default();+ let provider = SdkTracerProvider::builder()+ .with_simple_exporter(exporter.clone())+ .build();+ let layer = tracing_opentelemetry::layer().with_tracer(provider.tracer("test"));+ let guard = tracing::subscriber::set_default(tracing_subscriber::registry().with(layer));+ // Other tests in this binary run the same code paths with no subscriber+ // installed, which can cache "never interested" for a span callsite on+ // another thread; recompute so this subscriber sees every span.+ tracing::callsite::rebuild_interest_cache();+ (exporter, provider, guard)+}++fn finished(exporter: &InMemorySpanExporter, provider: &SdkTracerProvider) -> Vec<SpanData> {+ provider.force_flush().expect("flush");+ exporter.get_finished_spans().expect("finished spans")+}++fn attribute<'a>(span: &'a SpanData, key: &str) -> Option<&'a Value> {+ span.attributes+ .iter()+ .find(|kv| kv.key.as_str() == key)+ .map(|kv| &kv.value)+}++fn string_attribute(span: &SpanData, key: &str) -> Option<String> {+ match attribute(span, key)? {+ Value::String(s) => Some(s.as_str().to_string()),+ _ => None,+ }+}++fn int_attribute(span: &SpanData, key: &str) -> Option<i64> {+ match attribute(span, key)? {+ Value::I64(i) => Some(*i),+ _ => None,+ }+}++fn string_array_attribute(span: &SpanData, key: &str) -> Option<Vec<String>> {+ match attribute(span, key)? {+ Value::Array(Array::String(values)) => {+ Some(values.iter().map(|v| v.as_str().to_string()).collect())+ }+ _ => None,+ }+}++fn json_attribute(span: &SpanData, key: &str) -> Vec<serde_json::Value> {+ let raw = string_attribute(span, key).unwrap_or_else(|| panic!("{key} on {}", span.name));+ serde_json::from_str(&raw).unwrap_or_else(|e| panic!("{key} is not a JSON array: {e}: {raw}"))+}++/// Spans with the given `gen_ai.operation.name`, in the order they finished.+fn spans_with_operation<'a>(spans: &'a [SpanData], operation: &str) -> Vec<&'a SpanData> {+ spans+ .iter()+ .filter(|span| string_attribute(span, attr::OPERATION_NAME).as_deref() == Some(operation))+ .collect()+}++fn names(spans: &[SpanData]) -> Vec<&str> {+ spans.iter().map(|span| span.name.as_ref()).collect()+}++/// Every exported span with its attribute keys, for assertion messages.+fn describe(spans: &[SpanData]) -> String {+ spans+ .iter()+ .map(|span| {+ let keys: Vec<&str> = span.attributes.iter().map(|kv| kv.key.as_str()).collect();+ format!("{} {:?}", span.name, keys)+ })+ .collect::<Vec<_>>()+ .join("\n")+}++/// One run: the model calls `echo_tool`, then answers "done".+#[tokio::test]+async fn a_run_records_the_attributes_evaluations_read() {+ let (exporter, provider, _guard) = otel_test_pipeline();+ let model = MockCompletionModel::from_stream_turns([+ vec![+ MockStreamEvent::tool_call("call-1", "echo_tool", json!({ "value": "a" })),+ MockStreamEvent::final_response_with_default_usage(),+ ],+ vec![+ MockStreamEvent::text("done"),+ MockStreamEvent::final_response_with_default_usage(),+ ],+ ]);+ let toolset = util::single_tool_set::<EchoTool, ()>();+ let mut session = util::test_loop()+ .with_conversation_id("chat-123")+ .test_session(+ toolset,+ Arc::new(()),+ "test preamble",+ util::usage_ctx(),+ model,+ )+ .await;++ let result = util::drive(&mut session, "call echo").await;+ assert_eq!(result.content(), "done");+ drop(session);++ let spans = finished(&exporter, &provider);++ // The agent span: ours, adopted by rig, carrying the session and the run's+ // input, output and usage.+ let agents = spans_with_operation(&spans, attr::operation::INVOKE_AGENT);+ assert_eq!(agents.len(), 1, "one agent span, got {:?}", names(&spans));+ let agent = agents[0];+ assert_eq!(agent.name, "invoke_agent");+ assert_eq!(+ string_attribute(agent, attr::CONVERSATION_ID).as_deref(),+ Some("chat-123")+ );+ assert_eq!(+ string_attribute(agent, attr::AGENT_NAME).as_deref(),+ Some("chat"),+ "the agent is named after the usage context's feature"+ );+ assert_eq!(+ string_attribute(agent, attr::PROVIDER_NAME).as_deref(),+ Some("test")+ );+ assert_eq!(+ string_attribute(agent, attr::REQUEST_MODEL).as_deref(),+ Some("fake-model")+ );+ assert!(int_attribute(agent, attr::USAGE_INPUT_TOKENS).is_some());+ assert!(int_attribute(agent, attr::USAGE_OUTPUT_TOKENS).is_some());+ let input = json_attribute(agent, attr::INPUT_MESSAGES);+ assert_eq!(input[0]["role"], "user");+ assert_eq!(input[0]["parts"][0]["content"], "call echo");+ let output = json_attribute(agent, attr::OUTPUT_MESSAGES);+ assert_eq!(output[0]["parts"][0]["content"], "done");+ assert_eq!(output[0]["finish_reason"], "stop");++ // One chat span per model call, each with the tool set it was offered, the+ // history it saw, the session, and what it produced.+ let chats = spans_with_operation(&spans, attr::operation::CHAT);+ assert_eq!(+ chats.len(),+ 2,+ "two model calls, got:\n{}",+ describe(&spans)+ );+ let (first, second) = (chats[0], chats[1]);+ for chat in &chats {+ assert_eq!(+ string_attribute(chat, attr::CONVERSATION_ID).as_deref(),+ Some("chat-123")+ );+ assert!(int_attribute(chat, attr::REQUEST_MAX_TOKENS).is_some());+ let definitions = json_attribute(chat, attr::TOOL_DEFINITIONS);+ assert!(+ definitions+ .iter()+ .any(|d| d["type"] == "function" && d["name"] == "echo_tool"),+ "{definitions:?}"+ );+ let system = json_attribute(chat, attr::SYSTEM_INSTRUCTIONS);+ assert!(+ system[0]["content"]+ .as_str()+ .is_some_and(|s| s.contains("test preamble")),+ "{system:?}"+ );+ }++ let input = json_attribute(first, attr::INPUT_MESSAGES);+ assert!(+ input.iter().all(|m| m["role"] != "system"),+ "system instructions stay out of the history: {input:?}"+ );+ assert_eq!(input.last().unwrap()["parts"][0]["content"], "call echo");+ assert_eq!(+ string_array_attribute(first, attr::RESPONSE_FINISH_REASONS),+ Some(vec!["tool_call".to_string()])+ );+ let output = json_attribute(first, attr::OUTPUT_MESSAGES);+ assert_eq!(output[0]["role"], "assistant");+ assert_eq!(output[0]["finish_reason"], "tool_call");+ assert_eq!(output[0]["parts"][0]["type"], "tool_call");+ assert_eq!(output[0]["parts"][0]["name"], "echo_tool");+ assert_eq!(output[0]["parts"][0]["arguments"], json!({ "value": "a" }));++ let input = json_attribute(second, attr::INPUT_MESSAGES);+ let tool_message = input+ .iter()+ .find(|m| m["role"] == "tool")+ .expect("the second call sees the tool result as a tool message");+ assert_eq!(tool_message["parts"][0]["type"], "tool_call_response");+ assert_eq!(tool_message["parts"][0]["response"], json!({ "echo": "a" }));+ assert_eq!(+ string_array_attribute(second, attr::RESPONSE_FINISH_REASONS),+ Some(vec!["stop".to_string()])+ );+ let output = json_attribute(second, attr::OUTPUT_MESSAGES);+ assert_eq!(output[0]["parts"][0]["content"], "done");++ // The tool span: rig's, with the model's call id, plus our arguments and+ // result — and only one of them per call.+ let tools = spans_with_operation(&spans, attr::operation::EXECUTE_TOOL);+ assert_eq!(+ tools.len(),+ 1,+ "one tool span per call, got {:?}",+ names(&spans)+ );+ let tool = tools[0];+ assert_eq!(+ string_attribute(tool, attr::TOOL_NAME).as_deref(),+ Some("echo_tool")+ );+ assert_eq!(+ string_attribute(tool, attr::TOOL_CALL_ID).as_deref(),+ Some("call-1")+ );+ assert_eq!(+ string_attribute(tool, attr::TOOL_CALL_ARGUMENTS).as_deref(),+ Some(r#"{"value":"a"}"#)+ );+ assert_eq!(+ string_attribute(tool, attr::TOOL_CALL_RESULT).as_deref(),+ Some(r#"{"echo":"a"}"#)+ );++ // Everything is one trace under the agent span: the model calls hang off+ // it directly, and the tool call off the run too (rig opens it from the+ // driver, which runs inside the agent span).+ let trace_id = agent.span_context.trace_id();+ assert!(+ spans+ .iter()+ .all(|span| span.span_context.trace_id() == trace_id),+ "{:?}",+ names(&spans)+ );+ let agent_id = agent.span_context.span_id();+ for chat in &chats {+ assert_eq!(chat.parent_span_id, agent_id, "{}", describe(&spans));+ }+ assert_eq!(tool.parent_span_id, agent_id, "{}", describe(&spans));+}++/// Without a conversation id nothing is invented, and the run still traces.+#[tokio::test]+async fn a_run_without_a_conversation_id_has_no_session_attribute() {+ let (exporter, provider, _guard) = otel_test_pipeline();+ let model = MockCompletionModel::from_stream_turns([vec![+ MockStreamEvent::text("hi"),+ MockStreamEvent::final_response_with_default_usage(),+ ]]);+ let toolset = util::single_tool_set::<EchoTool, ()>();+ let mut session = util::session(toolset, Arc::new(()), model).await;++ let result = util::drive(&mut session, "hello").await;+ assert_eq!(result.content(), "hi");+ drop(session);++ let spans = finished(&exporter, &provider);+ let agent = spans_with_operation(&spans, attr::operation::INVOKE_AGENT)[0];+ assert_eq!(string_attribute(agent, attr::CONVERSATION_ID), None);+ let chat = spans_with_operation(&spans, attr::operation::CHAT)[0];+ assert_eq!(string_attribute(chat, attr::CONVERSATION_ID), None);+ assert_eq!(+ string_array_attribute(chat, attr::RESPONSE_FINISH_REASONS),+ Some(vec!["stop".to_string()])+ );+}++/// A run the provider fails is a failed run on the agent span, not a quiet+/// success.+#[tokio::test]+async fn a_provider_error_marks_the_agent_span_failed() {+ let (exporter, provider, _guard) = otel_test_pipeline();+ let model = MockCompletionModel::from_stream_turns([vec![+ MockStreamEvent::text("partial"),+ MockStreamEvent::error("upstream exploded"),+ ]]);+ let toolset = util::single_tool_set::<EchoTool, ()>();+ let mut session = util::session(toolset, Arc::new(()), model).await;++ let stream = session+ .send_message(vec![rig_core::message::Message::user("hello")])+ .await+ .expect("the stream starts");+ let collected = util::collect(stream).await;+ assert!(+ collected.error.is_some(),+ "the failure reaches the consumer"+ );+ drop(session);++ let spans = finished(&exporter, &provider);+ let agent = spans_with_operation(&spans, attr::operation::INVOKE_AGENT)[0];+ assert!(+ matches!(agent.status, opentelemetry::trace::Status::Error { .. }),+ "{agent:#?}"+ );+ assert_eq!(+ string_attribute(agent, attr::ERROR_TYPE).as_deref(),+ Some("streaming_error")+ );+ assert_eq!(+ string_array_attribute(agent, attr::RESPONSE_FINISH_REASONS),+ Some(vec!["error".to_string()])+ );++ // The model call that was in flight failed the same way: its span carries+ // the request, the failure, and no output.+ let chats = spans_with_operation(&spans, attr::operation::CHAT);+ assert_eq!(chats.len(), 1, "{}", describe(&spans));+ let chat = chats[0];+ assert!(+ matches!(chat.status, opentelemetry::trace::Status::Error { .. }),+ "{chat:#?}"+ );+ assert_eq!(+ string_attribute(chat, attr::ERROR_TYPE).as_deref(),+ Some("streaming_error")+ );+ assert_eq!(+ string_array_attribute(chat, attr::RESPONSE_FINISH_REASONS),+ Some(vec!["error".to_string()])+ );+ assert!(string_attribute(chat, attr::INPUT_MESSAGES).is_some());+ assert_eq!(string_attribute(chat, attr::OUTPUT_MESSAGES), None);+}++/// A tool that never returns, so a run can be abandoned mid-flight.+#[derive(Deserialize, JsonSchema)]+#[schemars(title = "hang_tool", description = "Never finishes.")]+struct HangTool {}++impl ToolAnnotated for HangTool {+ const ANNOTATIONS: ToolAnnotations = ToolAnnotations::read_only("Hang");+}++#[async_trait]+impl AsyncTool<()> for HangTool {+ type Output = serde_json::Value;++ async fn call(+ &self,+ _service_context: ServiceContext<()>,+ _request_context: RequestContext,+ ) -> ToolResult<Self::Output> {+ std::future::pending().await+ }+}++/// A consumer that walks away from the stream cancels the run, and the agent+/// span says so rather than closing as if the agent had answered.+#[tokio::test]+async fn dropping_the_stream_marks_the_agent_span_cancelled() {+ let (exporter, provider, _guard) = otel_test_pipeline();+ let model = MockCompletionModel::from_stream_turns([vec![+ MockStreamEvent::tool_call("call-1", "hang_tool", json!({})),+ MockStreamEvent::final_response_with_default_usage(),+ ]]);+ let toolset = util::single_tool_set::<HangTool, ()>();+ let mut session = util::session(toolset, Arc::new(()), model).await;++ let mut stream = session+ .send_message(vec![rig_core::message::Message::user("hang")])+ .await+ .expect("the stream starts");+ // The tool call is announced the moment the model emits it, while the+ // tool itself hangs; walking away here abandons the run mid-tool.+ let first = util::next_within(&mut stream, std::time::Duration::from_secs(5)).await;+ assert!(first.is_some(), "the pending tool call is streamed");+ drop(stream);+ drop(session);+ // The aborted driver's destructors run on the runtime's next turns.+ for _ in 0..20 {+ tokio::task::yield_now().await;+ }++ let spans = finished(&exporter, &provider);+ let agent = spans_with_operation(&spans, attr::operation::INVOKE_AGENT)[0];+ assert_eq!(+ string_attribute(agent, attr::ERROR_TYPE).as_deref(),+ Some("cancelled"),+ "{}",+ describe(&spans)+ );+ assert_eq!(+ string_array_attribute(agent, attr::RESPONSE_FINISH_REASONS),+ Some(vec!["cancelled".to_string()])+ );+}++/// A tool that cancels the request it runs in, then returns: the runtime+/// notices the cancellation before its next model call and ends the run with+/// a cancellation error.+#[derive(Deserialize, JsonSchema)]+#[schemars(title = "cancel_tool", description = "Cancels the request.")]+struct CancelTool {}++impl ToolAnnotated for CancelTool {+ const ANNOTATIONS: ToolAnnotations = ToolAnnotations::read_only("Cancel");+}++#[async_trait]+impl AsyncTool<()> for CancelTool {+ type Output = serde_json::Value;++ async fn call(+ &self,+ _service_context: ServiceContext<()>,+ request_context: RequestContext,+ ) -> ToolResult<Self::Output> {+ request_context.cancel.cancel();+ Ok(json!({ "status": "cancelling" }))+ }+}++/// A user stopping the run is a cancelled run, not a failed one, even though+/// the runtime reports it as an error item on the stream.+#[tokio::test]+async fn a_cancelled_run_is_recorded_as_cancelled_not_failed() {+ let (exporter, provider, _guard) = otel_test_pipeline();+ let model = MockCompletionModel::from_stream_turns([+ vec![+ MockStreamEvent::tool_call("call-1", "cancel_tool", json!({})),+ MockStreamEvent::final_response_with_default_usage(),+ ],+ vec![+ MockStreamEvent::text("never reached"),+ MockStreamEvent::final_response_with_default_usage(),+ ],+ ]);+ let toolset = util::single_tool_set::<CancelTool, ()>();+ let mut session = util::session(toolset, Arc::new(()), model).await;++ let stream = session+ .send_message(vec![rig_core::message::Message::user("stop soon")])+ .await+ .expect("the stream starts");+ let collected = util::collect(stream).await;+ assert!(+ collected+ .error+ .as_ref()+ .is_some_and(|error| error.was_cancelled()),+ "the run ends with a cancellation: {:?}",+ collected.error+ );+ drop(session);++ let spans = finished(&exporter, &provider);+ let agent = spans_with_operation(&spans, attr::operation::INVOKE_AGENT)[0];+ assert_eq!(+ string_attribute(agent, attr::ERROR_TYPE).as_deref(),+ Some("cancelled"),+ "{}",+ describe(&spans)+ );+ assert_eq!(+ string_array_attribute(agent, attr::RESPONSE_FINISH_REASONS),+ Some(vec!["cancelled".to_string()])+ );+}++/// With GenAI telemetry off - a loop whose turns another layer reports - the+/// run records nothing GenAI onto any span, tool calls included; the runtime's+/// structural spans still nest under a plain `agent.turn`.+#[tokio::test]+async fn a_loop_with_telemetry_off_records_no_genai_content() {+ let (exporter, provider, _guard) = otel_test_pipeline();+ let model = MockCompletionModel::from_stream_turns([+ vec![+ MockStreamEvent::tool_call("call-1", "echo_tool", json!({ "value": "a" })),+ MockStreamEvent::final_response_with_default_usage(),+ ],+ vec![+ MockStreamEvent::text("done"),+ MockStreamEvent::final_response_with_default_usage(),+ ],+ ]);+ let toolset = util::single_tool_set::<EchoTool, ()>();+ let mut session = util::test_loop()+ .with_genai_telemetry(false)+ .test_session(+ toolset,+ Arc::new(()),+ "test preamble",+ util::usage_ctx(),+ model,+ )+ .await;++ let result = util::drive(&mut session, "call echo").await;+ assert_eq!(result.content(), "done");+ drop(session);++ let spans = finished(&exporter, &provider);+ assert!(+ spans.iter().any(|span| span.name == "agent.turn"),+ "{:?}",+ names(&spans)+ );+ for span in &spans {+ assert_eq!(+ string_attribute(span, attr::INPUT_MESSAGES),+ None,+ "{span:#?}"+ );+ assert_eq!(+ string_attribute(span, attr::OUTPUT_MESSAGES),+ None,+ "{span:#?}"+ );+ assert_eq!(+ string_attribute(span, attr::TOOL_CALL_ARGUMENTS),+ None,+ "{span:#?}"+ );+ assert_eq!(+ string_attribute(span, attr::TOOL_CALL_RESULT),+ None,+ "{span:#?}"+ );+ assert_eq!(+ string_attribute(span, attr::TOOL_DEFINITIONS),+ None,+ "{span:#?}"+ );+ }+}diff --git a/crates/agent/src/test/agent_loop/test_tool_search.rs b/crates/agent/src/test/agent_loop/test_tool_search.rsindex ab68910144..3ddfdb1558 100644--- a/crates/agent/src/test/agent_loop/test_tool_search.rs+++ b/crates/agent/src/test/agent_loop/test_tool_search.rs@@ -108,7 +108,7 @@ impl FakeCombined {}impl AiToolSet<()> for FakeCombined {- fn try_tool_call<'a>(+ fn dispatch_tool_call<'a>(&'a self,context: (),request_context: RequestContext,@@ -129,7 +129,7 @@ impl AiToolSet<()> for FakeCombined {});}self.static_tools- .try_tool_call(context, request_context, tool_name, json)+ .dispatch_tool_call(context, request_context, tool_name, json)}fn request_schemas(&self) -> Option<Vec<RequestSchema>> {diff --git a/crates/agent/src/tool_adapter.rs b/crates/agent/src/tool_adapter.rsindex e8936034fd..e4133a5b75 100644--- a/crates/agent/src/tool_adapter.rs+++ b/crates/agent/src/tool_adapter.rs@@ -1,8 +1,13 @@/// Adapts `ai_toolset` tool types into RIG [`DynamicTool`] objects.+use ai_toolset::telemetry::ToolCallSpan;use ai_toolset::tool_object::ToolSetCallable;-use ai_toolset::{AsyncToolCollection, RequestContext, RequestSchema, ToolSet as AiToolSet};+use ai_toolset::{+ AsyncToolCollection, RequestContext, RequestSchema, ToolResult, ToolSet as AiToolSet,+ ToolSetError,+};use rig_agent::tool::{DynamicTool, ToolExecutionError, ToolOutput};use std::sync::{Arc, RwLock};+use tracing::Instrument as _;/// Ensure every object schema carries an explicit `properties` map.///@@ -90,6 +95,7 @@ impl ToolsetToolAdapter {);let context = context.clone();let request_context = request_context.clone();+ let tool_name = name.clone();DynamicTool::new(name,@@ -99,20 +105,41 @@ impl ToolsetToolAdapter {let deserializer = deserializer.clone();let context = context.clone();let request_context = request_context.clone();+ let tool_name = tool_name.clone();Box::pin(async move {- let callable =- (deserializer)(&args).map_err(invalid_args)?;- let ctx = (*context).clone();let req_ctx = request_context.read().expect("request_context lock poisoned").clone();- match callable.call(ctx, req_ctx).await {- Ok(value) => Ok(ToolOutput::json(value)),- Err(e) => {+ // Bypasses `ToolSet::try_tool_call`, so it carries+ // the same `execute_tool` telemetry itself - under+ // the same request-level opt-out.+ let telemetry = req_ctx+ .genai_telemetry+ .then(|| ToolCallSpan::begin(&tool_name, &args));+ let span = telemetry.as_ref().map_or_else(tracing::Span::none, |t| {+ t.span().clone()+ });+ let result: Result<ToolResult<serde_json::Value>, ToolSetError> =+ async {+ let callable = (deserializer)(&args)+ .map_err(ToolSetError::Deserialization)?;+ let ctx = (*context).clone();+ Ok(callable.call(ctx, req_ctx).await)+ }+ .instrument(span)+ .await;+ if let Some(telemetry) = &telemetry {+ telemetry.finish(&result);+ }+ match result {+ Ok(Ok(value)) => Ok(ToolOutput::json(value)),+ Ok(Err(e)) => {tracing::error!(error = ?e.internal_error, "toolset tool error");Err(ToolExecutionError::other(e.description))}+ Err(ToolSetError::Deserialization(e)) => Err(invalid_args(e)),+ Err(e) => Err(ToolExecutionError::other(e.to_string())),}})},diff --git a/crates/agent_inmem/Cargo.toml b/crates/agent_inmem/Cargo.tomlindex 02eb2bb4ac..df4c13052d 100644--- a/crates/agent_inmem/Cargo.toml+++ b/crates/agent_inmem/Cargo.toml@@ -22,6 +22,7 @@ axum = { workspace = true }chat = { path = "../chat", default-features = false }dashmap = { workspace = true }futures = { workspace = true }+genai_telemetry = { path = "../genai_telemetry" }macro_user_id = { path = "../macro_user_id" }http = { workspace = true }http-body-util = { workspace = true }@@ -48,4 +49,8 @@ agent_session = { path = "../agent_session", features = ["test-utils"] }bot_id = { path = "../bot_id", features = ["test-utils"] }futures = { workspace = true }rig-agent = "0.41.0"+opentelemetry = { workspace = true, features = ["trace"] }+opentelemetry_sdk = { workspace = true, features = ["testing", "trace"] }+tracing-opentelemetry = { workspace = true }+tracing-subscriber = { workspace = true }tokio = { workspace = true, features = ["macros", "rt-multi-thread", "time"] }diff --git a/crates/agent_inmem/src/domain/agent.rs b/crates/agent_inmem/src/domain/agent.rsindex ddc973dd7f..8d7342095a 100644--- a/crates/agent_inmem/src/domain/agent.rs+++ b/crates/agent_inmem/src/domain/agent.rs@@ -38,6 +38,7 @@ use ai_tools::user_tool_review::{use async_trait::async_trait;use macro_user_id::user_id::MacroUserIdStr;use tokio_util::sync::CancellationToken;+use tracing::Instrument as _;use crate::domain::engine::{AgentIdentity, TurnEngine, TurnRequest};use crate::domain::mcp::{DynMcpToolConnector, dialable_servers};@@ -552,6 +553,13 @@ pub async fn serve(state: Arc<AgentState>, acp: AcpChannel) -> Result<(), AcpErrif let Err(error) = state.expect_session(&request.session_id) {return responder.respond_with_error(error);}+ let span = tracing::info_span!(+ parent: None,+ "agent.acp.prompt",+ agent.session.id = %state.session_id,+ gen_ai.conversation.id = %state.session_id,+ );+ genai_telemetry::propagation::set_parent(&span, request.meta.as_ref());let prompt = prompt_text(&request);if prompt.trim() == COMPACT_COMMAND {state.clear_history();@@ -585,6 +593,7 @@ pub async fn serve(state: Arc<AgentState>, acp: AcpChannel) -> Result<(), AcpErrlet _ = responder.respond(PromptResponse::new(stop));Ok(())}+ .instrument(span)})?;return Ok(());}@@ -602,6 +611,7 @@ pub async fn serve(state: Arc<AgentState>, acp: AcpChannel) -> Result<(), AcpErrlet _ = responder.respond(PromptResponse::new(stop));Ok(())}+ .instrument(span)})?;Ok(())}diff --git a/crates/agent_inmem/src/domain/agent/test.rs b/crates/agent_inmem/src/domain/agent/test.rsindex 05c0508453..103a877d8b 100644--- a/crates/agent_inmem/src/domain/agent/test.rs+++ b/crates/agent_inmem/src/domain/agent/test.rs@@ -1024,3 +1024,5 @@ async fn ask_without_form_support_explains_instead_of_asking() {spoken(¬ifications));}++mod telemetry;diff --git a/crates/agent_inmem/src/domain/agent/test/telemetry.rs b/crates/agent_inmem/src/domain/agent/test/telemetry.rsnew file mode 100644index 0000000000..16bdbf4b55--- /dev/null+++ b/crates/agent_inmem/src/domain/agent/test/telemetry.rs@@ -0,0 +1,83 @@+use super::*;++/// The ACP dispatcher and spawned prompt task must restore context from each+/// request, rather than inheriting the connection's context or the last turn.+#[tokio::test]+async fn prompt_trace_context_reaches_the_engine_for_each_turn() {+ use opentelemetry::trace::{TraceContextExt as _, TracerProvider as _};+ use opentelemetry_sdk::trace::{InMemorySpanExporter, SdkTracerProvider};+ use tracing_opentelemetry::OpenTelemetrySpanExt as _;+ use tracing_subscriber::layer::SubscriberExt as _;++ struct TracedEngine;+ impl TurnEngine for TracedEngine {+ fn supported_models(&self) -> &[&str] {+ crate::testing::TEST_MODELS+ }++ fn run_turn(+ &self,+ _request: TurnRequest,+ ) -> tokio::sync::mpsc::Receiver<Result<StreamPart, agent::AgentError>> {+ let _work = tracing::info_span!("engine.work");+ let (_sender, receiver) = tokio::sync::mpsc::channel(1);+ receiver+ }+ }++ opentelemetry::global::set_text_map_propagator(+ opentelemetry_sdk::propagation::TraceContextPropagator::new(),+ );+ let exporter = InMemorySpanExporter::default();+ let provider = SdkTracerProvider::builder()+ .with_simple_exporter(exporter.clone())+ .build();+ let subscriber = tracing_subscriber::registry()+ .with(tracing_opentelemetry::layer().with_tracer(provider.tracer("test")));+ let _guard = tracing::subscriber::set_default(subscriber);+ // While exactly one dispatcher is registered, tracing-core caches each+ // new span callsite's interest from the registering thread's default+ // subscriber. Other tests in this binary hit the prompt callsite with no+ // subscriber, which would cache "never interested" and disable the span+ // on this thread too. A second registered dispatcher makes tracing-core+ // combine every registered subscriber's interest instead, so the cache+ // stays "sometimes" and each thread's own default subscriber decides.+ let _pin_interest = tracing::Dispatch::new(tracing::subscriber::NoSubscriber::default());+ let (_, _, parents) = with_agent(Arc::new(TracedEngine), async |connection, session| {+ let mut parents = Vec::new();+ for _ in 0..2 {+ let parent = tracing::info_span!(parent: None, "invocation");+ parents.push(parent.context().span().span_context().clone());+ let mut request = text_prompt(&session, "test prompt");+ let mut meta = Meta::new();+ genai_telemetry::propagation::inject(&parent, &mut meta);+ request.meta = Some(meta);+ connection+ .send_request(request)+ .block_task()+ .await+ .expect("prompt response");+ }+ parents+ })+ .await;+ provider.force_flush().expect("flush");+ let spans = exporter.get_finished_spans().expect("spans");+ assert_ne!(parents[0].trace_id(), parents[1].trace_id());+ for parent in parents {+ let prompt = spans+ .iter()+ .find(|span| {+ span.name == "agent.acp.prompt" && span.span_context.trace_id() == parent.trace_id()+ })+ .expect("ACP prompt stays in its caller's trace");+ assert_eq!(prompt.parent_span_id, parent.span_id());+ let work = spans+ .iter()+ .find(|span| {+ span.name == "engine.work" && span.span_context.trace_id() == parent.trace_id()+ })+ .expect("engine runs in the prompt trace");+ assert_eq!(work.parent_span_id, prompt.span_context.span_id());+ }+}diff --git a/crates/agent_inmem/src/outbound/mod.rs b/crates/agent_inmem/src/outbound/mod.rsindex 2449f6e45c..3eaa8e7833 100644--- a/crates/agent_inmem/src/outbound/mod.rs+++ b/crates/agent_inmem/src/outbound/mod.rs@@ -4,3 +4,4 @@ pub mod acp_mcp;pub mod egress_mcp;pub mod log_frames;pub mod manager;+pub mod tool_catalog;diff --git a/crates/agent_inmem/src/outbound/tool_catalog.rs b/crates/agent_inmem/src/outbound/tool_catalog.rsnew file mode 100644index 0000000000..247e63eec3--- /dev/null+++ b/crates/agent_inmem/src/outbound/tool_catalog.rs@@ -0,0 +1,71 @@+//! Listing a session's MCP tools for its telemetry.+//!+//! The session actor records, on each turn's span, the tools the agent had to+//! choose from (`gen_ai.tool.definitions`). A sandboxed harness learns those+//! by dialing the MCP servers it was handed in `session/new`; this lists the+//! same servers the same way, in process through the egress proxy, so the+//! definitions on the span are the ones the agent saw - Macro's own tools and+//! the owner's connected apps alike, under the names the harness calls them+//! by. Which is also why Macro's server is *not* skipped here the way+//! [`dialable_servers`](crate::domain::mcp::dialable_servers) skips it for the+//! in-process runtime's own toolset: every other harness reaches Macro's tools+//! over that server, and the listing has to say so.++#[cfg(test)]+mod test;++use std::pin::Pin;+use std::sync::Arc;++use agent_client_protocol::schema::v1::McpServer;+use agent_session::domain::ports::{SessionToolCatalog, SessionToolDefinition};++use crate::domain::mcp::McpToolConnector;++/// A [`SessionToolCatalog`] over the connector the in-process runtime dials+/// its MCP servers with.+pub struct McpToolCatalog<Connector> {+ connector: Arc<Connector>,+}++impl<Connector> McpToolCatalog<Connector> {+ /// A catalog dialing through `connector`.+ pub fn new(connector: Arc<Connector>) -> Self {+ Self { connector }+ }+}++impl<Connector> SessionToolCatalog for McpToolCatalog<Connector>+where+ Connector: McpToolConnector,+{+ fn tool_definitions(+ &self,+ servers: Vec<McpServer>,+ ) -> Pin<Box<dyn Future<Output = Vec<SessionToolDefinition>> + Send + '_>> {+ Box::pin(async move {+ // HTTP entries only: the egress proxy serves nothing else, and a+ // stdio or SSE entry is not something this process can dial.+ let http = servers+ .into_iter()+ .filter_map(|server| match server {+ McpServer::Http(http) => Some(http),+ _ => None,+ })+ .collect();+ let Some(tools) = self.connector.connect(http).await else {+ return Vec::new();+ };+ tools+ .searchable_catalog()+ .into_iter()+ .map(|tool| SessionToolDefinition {+ name: tool.name,+ description: tool.description,+ parameters: serde_json::to_value(tool.schema)+ .unwrap_or(serde_json::Value::Null),+ })+ .collect()+ })+ }+}diff --git a/crates/agent_inmem/src/outbound/tool_catalog/test.rs b/crates/agent_inmem/src/outbound/tool_catalog/test.rsnew file mode 100644index 0000000000..2361fd53df--- /dev/null+++ b/crates/agent_inmem/src/outbound/tool_catalog/test.rs@@ -0,0 +1,50 @@+use super::*;+use agent_client_protocol::schema::v1::{HttpHeader, McpServerHttp, McpServerStdio};+use mcp_toolset::RemoteMcpToolSet;+use std::sync::Mutex;++/// A connector that records which servers it was asked to dial and finds no+/// tools on them.+#[derive(Default)]+struct RecordingConnector {+ dialed: Mutex<Vec<String>>,+}++impl McpToolConnector for RecordingConnector {+ async fn connect(&self, servers: Vec<McpServerHttp>) -> Option<RemoteMcpToolSet> {+ self.dialed+ .lock()+ .expect("not poisoned")+ .extend(servers.into_iter().map(|server| server.name));+ None+ }+}++fn http(name: &str) -> McpServer {+ McpServer::Http(+ McpServerHttp::new(name, format!("https://egress.test/mcp/{name}")).headers(vec![+ HttpHeader::new("Authorization", "Bearer session-token"),+ ]),+ )+}++#[tokio::test]+async fn every_http_server_is_listed_including_macros_own() {+ let connector = Arc::new(RecordingConnector::default());+ let catalog = McpToolCatalog::new(Arc::clone(&connector));++ let definitions = catalog+ .tool_definitions(vec![+ http("macro"),+ McpServer::Stdio(McpServerStdio::new("local", "some-binary")),+ http("linear"),+ ])+ .await;++ assert!(definitions.is_empty(), "no server yielded a tool");+ assert_eq!(+ connector.dialed.lock().expect("not poisoned").as_slice(),+ ["macro", "linear"],+ "HTTP servers are dialed, Macro's included; stdio entries are not"+ );+}diff --git a/crates/agent_inmem/src/rig_engine.rs b/crates/agent_inmem/src/rig_engine.rsindex 80f3bf034d..55b9ffccf7 100644--- a/crates/agent_inmem/src/rig_engine.rs+++ b/crates/agent_inmem/src/rig_engine.rs@@ -35,6 +35,7 @@ use memory::domain::service::MemoryServiceImpl;use memory::outbound::pg_memory_repo::PgMemoryRepo;use sqlx::PgPool;use tokio::sync::mpsc;+use tracing::Instrument as _;use crate::domain::engine::{AgentIdentity, TurnEngine, TurnRequest};use crate::inbound::ask_user::{AskUser, AskUserContext};@@ -102,11 +103,14 @@ impl TurnEngine for RigTurnEngine {let (parts, receiver) = mpsc::channel(PART_BUFFER);let db = self.db.clone();let tool_context = self.tool_context.clone();- tokio::spawn(async move {- if let Err(error) = drive_turn(db, tool_context, request, &parts).await {- let _ = parts.send(Err(error)).await;+ tokio::spawn(+ async move {+ if let Err(error) = drive_turn(db, tool_context, request, &parts).await {+ let _ = parts.send(Err(error)).await;+ }}- });+ .in_current_span(),+ );receiver}}@@ -158,7 +162,13 @@ async fn drive_turn(},};- let mut agent_loop = AgentLoop::new(base_context.recorder.clone()).with_model(&model);+ // GenAI telemetry stays off here: this runtime's turns and tool calls+ // reach the session actor as ACP frames, and the actor projects those onto+ // `invoke_agent` / `execute_tool` spans for every harness alike. Enriching+ // rig's spans too would report each turn twice.+ let mut agent_loop = AgentLoop::new(base_context.recorder.clone())+ .with_model(&model)+ .with_genai_telemetry(false);if let Some(reviewer) = reviewer {agent_loop = agent_loop.with_user_tool_finisher(user_tool_finisher(Arc::clone(&toolset),diff --git a/crates/agent_session/Cargo.toml b/crates/agent_session/Cargo.tomlindex 1ab79578a9..fb6dba7743 100644--- a/crates/agent_session/Cargo.toml+++ b/crates/agent_session/Cargo.toml@@ -56,6 +56,7 @@ entity_access = { path = "../entity_access", default-features = false, features"ports",] }entity_access_db_utils = { path = "../entity_access_db_utils", optional = true }+genai_telemetry = { path = "../genai_telemetry" }model-entity = { path = "../model-entity", optional = true }clap = { workspace = true, optional = true }macro_authorization = { path = "../macro_authorization", default-features = false, optional = true }@@ -89,6 +90,10 @@ non_empty = { path = "../non_empty" }agent_fold = { path = "../agent_fold", features = ["test-utils"] }axum = { workspace = true, features = ["ws"] }macro_db_migrator = { path = "../macro_db_migrator" }+opentelemetry = { workspace = true, features = ["trace"] }+opentelemetry_sdk = { workspace = true, features = ["testing", "trace"] }+tracing-opentelemetry = { workspace = true }+tracing-subscriber = { workspace = true }tokio = { workspace = true, features = ["fs", "io-std", "io-util", "macros", "net", "rt-multi-thread", "time"] }tokio-tungstenite = { workspace = true }tower = { workspace = true, features = ["util"] }diff --git a/crates/agent_session/src/domain/ports.rs b/crates/agent_session/src/domain/ports.rsindex ccddeab4be..1a4027d75b 100644--- a/crates/agent_session/src/domain/ports.rs+++ b/crates/agent_session/src/domain/ports.rs@@ -4,7 +4,7 @@ use super::error::{AgentSessionError, Result};use super::model::*;use super::session::StopReason;use crate::domain::events::AgentSessionLifecycleEvent;-use agent_client_protocol::schema::v1::SessionId;+use agent_client_protocol::schema::v1::{McpServer, SessionId};use agent_fold::domain::model::TurnSignal;use agent_runtime_protocol::domain::action::{AgentAction, AgentActionId};use agent_runtime_protocol::domain::ports::Transport;@@ -509,6 +509,57 @@ pub trait AgentSessionLogRepo: Send + Sync + 'static {) -> impl Future<Output = Result<Vec<MacroUserIdStr<'static>>>> + Send;}+/// One tool a session's agent may call, as the MCP server offering it+/// describes it - the same name, description and parameter schema the agent+/// itself is shown.+#[derive(Debug, Clone, PartialEq)]+pub struct SessionToolDefinition {+ /// The name the agent calls the tool by, exactly as the server registers+ /// it for this session (an MCP tool arrives as `mcp__<server>__<tool>`).+ pub name: String,+ /// The description the agent chooses the tool on.+ pub description: String,+ /// The JSON schema of the tool's arguments.+ pub parameters: serde_json::Value,+}++/// Lists the tools a session's MCP servers advertise, so the session's+/// telemetry can say what the agent had to choose from+/// (`gen_ai.tool.definitions`).+///+/// Best-effort by contract: a listing that fails, or takes too long, costs+/// a span its tool definitions and nothing else. The one production+/// implementation dials the same egress-proxy URLs the agent is handed, in+/// process, so what it lists is what the agent sees - Macro's own tools and+/// the owner's connected apps alike. A harness's built-in tools (its shell,+/// its file editor) are not on any server and are not listed; the+/// convention does not require them.+///+/// Object-safe on purpose: the session service stores it erased so wiring it+/// is not another type parameter.+pub trait SessionToolCatalog: Send + Sync + 'static {+ /// Every tool the given servers offer. Empty when none do, or when the+ /// listing failed.+ fn tool_definitions(+ &self,+ servers: Vec<McpServer>,+ ) -> std::pin::Pin<Box<dyn Future<Output = Vec<SessionToolDefinition>> + Send + '_>>;+}++/// A [`SessionToolCatalog`] that lists nothing: tests, offline tooling, and+/// deployments with no in-process MCP client.+#[derive(Debug, Clone, Copy, Default)]+pub struct NoOpToolCatalog;++impl SessionToolCatalog for NoOpToolCatalog {+ fn tool_definitions(+ &self,+ _servers: Vec<McpServer>,+ ) -> std::pin::Pin<Box<dyn Future<Output = Vec<SessionToolDefinition>> + Send + '_>> {+ Box::pin(async { Vec::new() })+ }+}+/// One frame appended: its durable identity, and what the fold made of it.#[derive(Debug, Clone, PartialEq, Eq)]pub struct Appended {diff --git a/crates/agent_session/src/domain/service.rs b/crates/agent_session/src/domain/service.rsindex 7ca89426d3..1b6ba31089 100644--- a/crates/agent_session/src/domain/service.rs+++ b/crates/agent_session/src/domain/service.rs@@ -58,7 +58,8 @@ use super::model::{use super::ports::{AgentConnector, AgentSessionLifecyclePublisher, AgentSessionLogRepo, AgentSessionLogWriter,AgentSessionNameGenerator, AgentSessionQueueChanged, AgentSessionRealtime, AgentSessionRepo,- Appended, NoOpAgentSessionNameGenerator, SessionOwnership, SessionTurnObserver,+ Appended, NoOpAgentSessionNameGenerator, NoOpToolCatalog, SessionOwnership, SessionToolCatalog,+ SessionTurnObserver,};use super::session::actors::{SessionActor, SessionCommand, Stepped};use super::session::{CloseReason, Input};@@ -285,6 +286,9 @@ pub struct AgentSessionServiceImpl<R, Folds, Rt, Namer = NoOpAgentSessionNameGen/// Told when a session's turn ends or its actor stops - the harness's/// prompt-queue gate. Erased so wiring it is not another type parameter.turn_observer: Arc<dyn SessionTurnObserver>,+ /// Lists a session's MCP tools for its telemetry. Erased like the+ /// observer, for the same reason.+ tool_catalog: Arc<dyn SessionToolCatalog>,/// Where lifecycle facts go - renames, from here; everything else from/// the harness. Erased for the same reason as the observer.lifecycle_publisher: Arc<dyn AgentSessionLifecyclePublisher>,@@ -306,7 +310,10 @@ impl<R, Folds, Rt, Namer> AgentSessionServiceImpl<R, Folds, Rt, Namer> {/// it passes [`NoOpTurnObserver`], one with nothing downstream passes/// [`NoopLifecyclePublisher`], and one that is the only service instance/// in its process mints its own [`ReplicaId`] - each choice visible at the- /// call site rather than hidden in a builder's default.+ /// call site rather than hidden in a builder's default. The one+ /// exception is the tool catalog: it starts as [`NoOpToolCatalog`] and+ /// [`Self::with_tool_catalog`] swaps in a real one, since only a process+ /// with an in-process MCP client can list anything.////// `replica` is this service's identity in the session-management lease./// A restarted process is a new replica whose claims are recovered by@@ -329,6 +336,7 @@ impl<R, Folds, Rt, Namer> AgentSessionServiceImpl<R, Folds, Rt, Namer> {name_generator,turn_observer,lifecycle_publisher,+ tool_catalog: Arc::new(NoOpToolCatalog),active: Arc::new(DashMap::new()),replica,tasks: TaskTracker::new(),@@ -337,6 +345,14 @@ impl<R, Folds, Rt, Namer> AgentSessionServiceImpl<R, Folds, Rt, Namer> {}}+ /// Replace the no-op tool catalog with one that lists a session's MCP+ /// tools, so its turns' spans carry the tools the agent could choose from.+ #[must_use]+ pub fn with_tool_catalog(mut self, tool_catalog: Arc<dyn SessionToolCatalog>) -> Self {+ self.tool_catalog = tool_catalog;+ self+ }+/// This service's identity in the session-management lease, for the/// composition root to heartbeat while the process lives.#[must_use]@@ -437,6 +453,7 @@ impl<R, Folds, Rt, Namer> AgentSessionServiceImpl<R, Folds, Rt, Namer> {command_rx,attachment.handshake,Arc::clone(&self.turn_observer),+ Arc::clone(&self.tool_catalog),);self.tasks.spawn(run_session(diff --git a/crates/agent_session/src/domain/service/test.rs b/crates/agent_session/src/domain/service/test.rsindex e02d650d9b..42ade3811b 100644--- a/crates/agent_session/src/domain/service/test.rs+++ b/crates/agent_session/src/domain/service/test.rs@@ -842,6 +842,7 @@ async fn cancellation_does_not_drop_an_effect_batch_after_machine_mutation() {command_rx,handshake,Arc::new(crate::domain::ports::NoOpTurnObserver),+ Arc::new(crate::domain::ports::NoOpToolCatalog),);let active = Arc::new(ActiveSessions::new());let cancellation = CancellationToken::new();@@ -922,6 +923,7 @@ async fn live_inbound_logs_do_not_reuse_the_expired_handshake_deadline() {command_rx,handshake,Arc::new(crate::domain::ports::NoOpTurnObserver),+ Arc::new(crate::domain::ports::NoOpToolCatalog),);let active = Arc::new(ActiveSessions::new());let cancellation = CancellationToken::new();@@ -1384,6 +1386,7 @@ async fn shared_transport_copies_durable_initialization_before_load() {commands,handshake.clone(),Arc::new(crate::domain::ports::NoOpTurnObserver),+ Arc::new(crate::domain::ports::NoOpToolCatalog),);actor.dispatch(Input::Inbound(ToServerMessage::Event {@@ -1436,6 +1439,7 @@ async fn shared_transport_copies_durable_initialization_before_load() {commands,handshake,Arc::new(crate::domain::ports::NoOpTurnObserver),+ Arc::new(crate::domain::ports::NoOpToolCatalog),);let ready = actor.next_input().await;assert!(matches!(ready, Input::SharedReady { .. }));@@ -1540,6 +1544,7 @@ async fn assert_restore_persistence_failure_does_not_send_prompt(failure: Restorcommand_rx,handshake.clone(),Arc::new(crate::domain::ports::NoOpTurnObserver),+ Arc::new(crate::domain::ports::NoOpToolCatalog),);let (completed, completion) = oneshot::channel();commands@@ -1647,6 +1652,156 @@ async fn assert_restore_persistence_failure_does_not_send_prompt(failure: Restor));}+/// The actor's GenAI projection, end to end: a prompt delivered through the+/// actor opens an `invoke_agent` span under the command that carried it, and+/// the runtime's answer closes it with what the agent said.+#[tokio::test]+async fn a_prompt_turn_is_traced_as_an_agent_span_under_its_command() {+ use genai_telemetry::attr;+ use opentelemetry::trace::{TraceContextExt as _, TracerProvider as _};+ use opentelemetry_sdk::trace::{InMemorySpanExporter, SdkTracerProvider};+ use tracing_opentelemetry::OpenTelemetrySpanExt as _;+ use tracing_subscriber::layer::SubscriberExt as _;++ let exporter = InMemorySpanExporter::default();+ let provider = SdkTracerProvider::builder()+ .with_simple_exporter(exporter.clone())+ .build();+ let layer = tracing_opentelemetry::layer().with_tracer(provider.tracer("test"));+ let _guard = tracing::subscriber::set_default(tracing_subscriber::registry().with(layer));+ tracing::callsite::rebuild_interest_cache();++ let repo = InMemoryAgentSessionRepo::new();+ let session = test_session();+ repo.insert_session(test_agent_session(session));+ let release = Arc::new(Notify::new());+ let logs = BlockingPromptLogs {+ repo: repo.clone(),+ entered: Arc::new(Notify::new()),+ release: release.clone(),+ hang_disconnect: false,+ fail_restore_log: None,+ };+ let (outbound_tx, mut outbound_rx) = mpsc::channel(8);+ let (inbound_tx, inbound_rx) = mpsc::channel(8);+ let (commands, command_rx) = mpsc::channel(8);+ let (handshake, _) = watch::channel(HandshakeStatus::Pending);+ let actor = SessionActor::new(+ session,+ None,+ "/workspace".to_owned(),+ Vec::new(),+ RecordingTransport {+ outbound: outbound_tx,+ inbound: inbound_rx,+ },+ logs,+ command_rx,+ handshake,+ Arc::new(crate::domain::ports::NoOpTurnObserver),+ Arc::new(crate::domain::ports::NoOpToolCatalog),+ );+ let active = Arc::new(ActiveSessions::new());+ let cancellation = CancellationToken::new();+ let (stopped_tx, _) = watch::channel(false);+ let claim = claim_for_test(&repo, session).await;+ let task = tokio::spawn(+ run_session(+ actor,+ Arc::downgrade(&active),+ Arc::new(()),+ stopped_tx,+ cancellation.clone(),+ repo.clone(),+ claim,+ Arc::new(crate::domain::ports::NoOpTurnObserver),+ )+ .with_current_subscriber(),+ );+ open_test_session(&inbound_tx, &mut outbound_rx, session).await;++ release.notify_one();+ let command_span = tracing::info_span!("agent.session.command");+ let command_id = command_span.context().span().span_context().span_id();+ let action_id = AgentActionId::mint();+ let (completed, result) = oneshot::channel();+ commands+ .send(SessionCommand {+ user_id: None,+ action: AgentAction::prompt("what time is it?"),+ action_id,+ completed,+ span: command_span,+ enqueued_at: tokio::time::Instant::now(),+ })+ .await+ .unwrap();+ let prompt = outbound_rx.recv().await.expect("prompt is dispatched");+ assert!(matches!(+ &prompt,+ ToRuntimeMessage::Acp(AcpMessage(+ agent_client_protocol::RawJsonRpcMessage::Request(request)+ )) if request.method.as_ref() == "session/prompt"+ ));+ result.await.unwrap().expect("delivery completes");++ let update = agent_client_protocol::RawJsonRpcMessage::notification(+ "session/update".to_owned(),+ serde_json::json!({+ "sessionId": "acp-1",+ "update": {+ "sessionUpdate": "agent_message_chunk",+ "content": { "type": "text", "text": "It is noon." }+ }+ }),+ )+ .unwrap();+ inbound_tx+ .send(ToServerMessage::Acp(AcpMessage(update)))+ .await+ .unwrap();+ inbound_tx+ .send(ToServerMessage::Acp(AcpMessage(+ agent_client_protocol::RawJsonRpcMessage::response(+ action_id.to_request_id(),+ Ok(serde_json::json!({ "stopReason": "end_turn" })),+ ),+ )))+ .await+ .unwrap();+ // The transport closing stops the actor, which flushes anything open.+ drop(inbound_tx);+ task.await.unwrap();++ provider.force_flush().expect("flush");+ let spans = exporter.get_finished_spans().expect("finished spans");+ let agent: Vec<_> = spans+ .iter()+ .filter(|span| span.name == "invoke_agent")+ .collect();+ assert_eq!(agent.len(), 1, "one prompt, one agent span: {spans:#?}");+ assert_eq!(agent[0].parent_span_id, command_id);+ let attribute = |key: &str| {+ agent[0]+ .attributes+ .iter()+ .find(|kv| kv.key.as_str() == key)+ .map(|kv| kv.value.to_string())+ };+ assert_eq!(+ attribute(attr::CONVERSATION_ID).as_deref(),+ Some(session.to_string().as_str())+ );+ assert_eq!(+ attribute(attr::RESPONSE_FINISH_REASONS).as_deref(),+ Some(r#"["stop"]"#)+ );+ let output = attribute(attr::OUTPUT_MESSAGES).expect("output recorded");+ assert!(output.contains("It is noon."), "{output}");+ let input = attribute(attr::INPUT_MESSAGES).expect("input recorded");+ assert!(input.contains("what time is it?"), "{input}");+}+mod owner_binding;/// The live writer's fold says what each appended frame meant for the turn;diff --git a/crates/agent_session/src/domain/session/actors.rs b/crates/agent_session/src/domain/session/actors.rsindex ee43461719..272b14f4a1 100644--- a/crates/agent_session/src/domain/session/actors.rs+++ b/crates/agent_session/src/domain/session/actors.rs@@ -12,7 +12,7 @@//! loop (see [`super::super::service`]), while tests exercise its input and//! dispatch halves directly.-use std::{collections::VecDeque, time::Duration};+use std::{collections::VecDeque, sync::Arc, time::Duration};use agent_client_protocol::RawJsonRpcMessage;use agent_client_protocol::schema::v1::{McpServer, SessionId};@@ -30,10 +30,12 @@ use crate::domain::model::{AgentSessionId, AgentSessionLog, Message};use agent_runtime_protocol::domain::ports::{TransportReceiver, TransportSender};use crate::domain::ports::{- AgentConnector, AgentSessionLogWriter, AgentSessionRepo, SessionTurnObserver,+ AgentConnector, AgentSessionLogWriter, AgentSessionRepo, SessionToolCatalog,+ SessionTurnObserver,};use agent_fold::domain::model::TurnSignal;+use super::telemetry::GenAiProjector;use super::{CloseReason, Effect, HandshakeStatus, Input, RuntimeStatus, SessionMachine, StopReason,};@@ -42,6 +44,9 @@ use super::{const HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(60);/// How long one caller's action has to reach the runtime.const COMMAND_DELIVERY_TIMEOUT: Duration = Duration::from_secs(60);+/// How long the session's MCP servers have to list their tools for telemetry+/// before the listing is given up on. Off the actor's own path, so generous.+const TOOL_LISTING_TIMEOUT: Duration = Duration::from_secs(60);/// A caller's request to deliver one action, and the wire back to them.pub(crate) struct SessionCommand {@@ -90,7 +95,17 @@ pub(crate) struct SessionActor<Connector: AgentConnector, Logs> {handshake_deadline: Instant,/// Told when the machine reports a turn over, so the harness can dispatch/// its next queued prompt.- turn_observer: std::sync::Arc<dyn SessionTurnObserver>,+ turn_observer: Arc<dyn SessionTurnObserver>,+ /// GenAI spans for the turns and tool calls this connection carries, fed+ /// every frame the effects below send or log. See [`super::telemetry`].+ telemetry: GenAiProjector,+ /// Lists the session's MCP tools once the connection is live, for the+ /// projector's `gen_ai.tool.definitions`.+ tool_catalog: Arc<dyn SessionToolCatalog>,+ /// The servers to list, as the machine sends them on `session/new`.+ mcp_servers: Vec<McpServer>,+ /// Whether the listing has been started; it runs once per connection.+ tools_listed: bool,}impl<Connector, Logs> SessionActor<Connector, Logs>@@ -110,7 +125,8 @@ wherelogs: Logs,commands: mpsc::Receiver<SessionCommand>,handshake: watch::Sender<HandshakeStatus>,- turn_observer: std::sync::Arc<dyn SessionTurnObserver>,+ turn_observer: Arc<dyn SessionTurnObserver>,+ tool_catalog: Arc<dyn SessionToolCatalog>,) -> Self {// Marked unseen so the first wait reports the *current* state rather// than only later changes: a session binding after the handshake@@ -134,8 +150,10 @@ wherehandshake_seen,handshake,machine: match acp_session_id {- None => SessionMachine::new(id, workspace, mcp_servers),- Some(session_id) => SessionMachine::resume(id, session_id, workspace, mcp_servers),+ None => SessionMachine::new(id, workspace, mcp_servers.clone()),+ Some(session_id) => {+ SessionMachine::resume(id, session_id, workspace, mcp_servers.clone())+ }}.with_connection_context(macro_uuid::generate_uuid_v7()),logs,@@ -143,6 +161,14 @@ wherehandshake_span: Some(handshake_span),handshake_deadline: Instant::now() + HANDSHAKE_TIMEOUT,turn_observer,+ telemetry: GenAiProjector::new(+ id,+ genai_telemetry::ContentPolicy::from_env(),+ Default::default(),+ ),+ tool_catalog,+ mcp_servers,+ tools_listed: false,}}@@ -240,11 +266,16 @@ wherewhile let Some(effect) = effects.pop_front() {match effect {- Effect::Send { from, message } => {+ Effect::Send { from, mut message } => {let command_span = effects.front().and_then(|effect| match effect {Effect::Complete { token, .. } => Some(token.span.clone()),_ => None,});+ // Before delivery, so the turn's span starts when the+ // prompt leaves. A delivery that fails closes the+ // connection, which ends the turn in error below.+ self.telemetry+ .on_outbound(&mut message, command_span.as_ref());let delivery = self.deliver(from, message);let (result, close_reason) =match (command_span.as_ref(), handshake_span.as_ref()) {@@ -302,6 +333,7 @@ where}}Effect::Log { message, boundary } => {+ self.telemetry.on_inbound(&message);if let Some(ToRuntimeMessage::Acp(AcpMessage(RawJsonRpcMessage::Request(request,)))) = &self.initialization_request@@ -424,6 +456,7 @@ wherelet _ = token.completed.send(result);}Effect::Stop { reason } => {+ self.telemetry.on_stopped(&reason);// The one place a session's `disconnected` event is// written. The frame itself carries no cause, so a// routine idle teardown and a runtime lost mid-turn are@@ -585,6 +618,7 @@ wherematch self.machine.status() {RuntimeStatus::Live { .. } => {self.handshake_span.take();+ self.list_tools_once();}RuntimeStatus::Dead => {if let Some(span) = self.handshake_span.take() {@@ -596,6 +630,40 @@ where}}+ /// List the session's MCP tools for its telemetry, once, off this actor's+ /// path: the listing dials the same servers the agent was just handed and+ /// must never hold up a prompt. Whatever it finds is published for the+ /// projector to pick up on the next turn it records.+ fn list_tools_once(&mut self) {+ if self.tools_listed {+ return;+ }+ self.tools_listed = true;+ if self.mcp_servers.is_empty() {+ return;+ }+ let catalog = Arc::clone(&self.tool_catalog);+ let servers = self.mcp_servers.clone();+ let definitions = self.telemetry.tool_definitions();+ let id = self.machine.id();+ tokio::spawn(+ async move {+ match tokio::time::timeout(TOOL_LISTING_TIMEOUT, catalog.tool_definitions(servers))+ .await+ {+ Ok(listed) => {+ tracing::debug!(%id, tools = listed.len(), "listed an agent session's tools");+ definitions.publish(listed);+ }+ Err(_) => {+ tracing::warn!(%id, "timed out listing an agent session's tools");+ }+ }+ }+ .in_current_span(),+ );+ }+/// The failing effect's own caller gets `cause` itself; anything still/// queued behind it never reached the runtime, so it gets `Disconnected`.fn fail_remaining_completions(diff --git a/crates/agent_session/src/domain/session/mod.rs b/crates/agent_session/src/domain/session/mod.rsindex 5eb2e3d175..4418597fee 100644--- a/crates/agent_session/src/domain/session/mod.rs+++ b/crates/agent_session/src/domain/session/mod.rs@@ -5,6 +5,7 @@ pub(crate) mod actors;// choice: logic in `session`, vocabulary in `types`.#[allow(clippy::module_inception)]mod session;+pub(crate) mod telemetry;mod types;#[cfg(test)]diff --git a/crates/agent_session/src/domain/session/telemetry.rs b/crates/agent_session/src/domain/session/telemetry.rsnew file mode 100644index 0000000000..ba6797f994--- /dev/null+++ b/crates/agent_session/src/domain/session/telemetry.rs@@ -0,0 +1,870 @@+//! GenAI spans projected from a live session's ACP frames.+//!+//! Every harness a session can run on - a sandboxed OpenCode, a Cursor cloud+//! agent, Macro's own in-process agent, an external `macrod` - speaks ACP+//! through the [`SessionActor`](super::actors::SessionActor), and the actor+//! sees every frame in both directions exactly once. So this is the one place+//! a session's turns and tool calls can be traced the same way for all of+//! them, in the OpenTelemetry GenAI vocabulary an evaluation backend reads:+//!+//! - one `invoke_agent` span per prompt turn, from the `session/prompt` going+//! out to its response coming back: the prompt as `gen_ai.input.messages`,+//! the agent's streamed prose, reasoning and tool calls as+//! `gen_ai.output.messages`, the stop reason as the finish reason, the+//! turn's token usage, the tools the session's MCP servers offered as+//! `gen_ai.tool.definitions`, the harness as the agent name and the session+//! as the conversation id;+//! - one `execute_tool` span per tool call, from its `tool_call` to the+//! `tool_call_update` that finishes it, with the tool's name, arguments and+//! result read through the harness conventions `agent_fold` knows, and an+//! error status when the call failed.+//!+//! What ACP does not carry is not invented: there is no `chat` span, because+//! the model calls happen inside the harness and never cross the wire.+//!+//! Content follows the [`ContentPolicy`] and size limits of+//! [`genai_telemetry`]; structure (names, ids, status, usage) is always+//! recorded. Everything here is best-effort and total: a frame the projector+//! cannot read changes nothing, and never the session.+//!+//! Only live frames are projected. The actor feeds this from its effect loop,+//! not from the log writer, so a reconnecting actor catching its fold up on+//! the stored log re-emits nothing.++#[cfg(test)]+mod test;++use std::collections::{HashMap, HashSet};+use std::sync::{Arc, OnceLock};++use agent_client_protocol::schema::v1::{+ ContentBlock, EmbeddedResourceResource, InitializeRequest, InitializeResponse,+ LoadSessionRequest, NewSessionRequest, PromptRequest, PromptResponse, RequestId, Response,+ ResumeSessionRequest, SessionConfigKind, SessionConfigOption, SessionNotification,+ SessionUpdate, SetSessionConfigOptionRequest, StopReason as AcpStopReason, ToolCall,+ ToolCallUpdate, ToolKind,+};+use agent_client_protocol::{JsonRpcMessage, RawJsonRpcMessage, RawJsonRpcParams};+use agent_fold::domain::harness::{ToolFrame, tool_name};+use agent_fold::domain::model::{Harness, ToolName, ToolStatus};+use agent_runtime_protocol::domain::action::MODEL_CONFIG_ID;+use agent_runtime_protocol::domain::schema::v0::{AcpMessage, ToRuntimeMessage, ToServerMessage};+use genai_telemetry::messages::{self, MediaSource};+use genai_telemetry::{+ ContentPolicy, GenAiSpanExt as _, attr, bound_messages, bound_tool_definitions,+ bounded_json_string, truncate_chars,+};+use serde::Deserialize;+use serde_json::Value;++use crate::domain::model::AgentSessionId;+use crate::domain::ports::SessionToolDefinition;++use super::StopReason;++/// The tools a session's MCP servers advertise, listed once per connection by+/// a task the actor spawns and read by the projector when it records a turn.+///+/// A write-once cell rather than a channel: the listing is one fact, whoever+/// reads it wants the latest and only value, and a turn that opens before the+/// listing lands picks it up when it closes instead.+#[derive(Clone, Default)]+pub(crate) struct SharedToolDefinitions(Arc<OnceLock<Vec<SessionToolDefinition>>>);++impl SharedToolDefinitions {+ /// Publish the listing. A second publish is dropped; the first stands.+ pub(crate) fn publish(&self, definitions: Vec<SessionToolDefinition>) {+ let _ = self.0.set(definitions);+ }++ fn get(&self) -> Option<&[SessionToolDefinition]> {+ self.0.get().map(Vec::as_slice)+ }+}++/// Projects one session connection's ACP frames onto GenAI spans.+pub(crate) struct GenAiProjector {+ session: AgentSessionId,+ policy: ContentPolicy,+ /// The harness producing the frames: announced by `initialize`, or+ /// recognized from the first tool frame of a log with no `initialize`.+ harness: Harness,+ /// The model the session runs on, per the runtime's own config options.+ model: Option<String>,+ tool_definitions: SharedToolDefinitions,+ /// The `initialize` request whose response names the harness.+ initialize: Option<RequestId>,+ /// Requests whose responses carry the session's config options (and so+ /// its model): `session/new`, `session/load`, `session/resume`, and every+ /// `session/set_config_option`.+ config_requests: HashSet<RequestId>,+ /// The turn in flight, if a `session/prompt` is awaiting its answer.+ turn: Option<Turn>,+ /// Tool calls opened and not yet finished, by ACP tool call id. Under the+ /// turn in flight when there is one; a call streamed outside any turn (a+ /// resumed session whose prompt this connection never sent) gets a root+ /// span of its own.+ tools: HashMap<String, OpenTool>,+ /// The cumulative token totals the last prompt response reported, so+ /// each turn records its own share rather than the running total.+ usage_totals: Option<(u64, u64)>,+}++/// One prompt turn's span and the output it has streamed so far.+struct Turn {+ request_id: RequestId,+ span: tracing::Span,+ /// The agent's output, in order: prose, reasoning and tool calls.+ parts: Vec<OutputPart>,+ /// Facts recorded at open so they are not recorded twice at close. Each+ /// is recorded as soon as it is known, which may be at open or at close.+ agent_name_recorded: bool,+ model_recorded: bool,+ definitions_recorded: bool,+}++enum OutputPart {+ Text(String),+ Reasoning(String),+ ToolCall {+ id: String,+ name: String,+ arguments: Value,+ },+}++impl OutputPart {+ fn into_json(self) -> Value {+ match self {+ Self::Text(text) => messages::text_part(text),+ Self::Reasoning(text) => messages::reasoning_part(text),+ Self::ToolCall {+ id,+ name,+ arguments,+ } => messages::tool_call_part(Some(&id), &name, arguments),+ }+ }+}++/// One tool call's span and what its frames have said so far.+struct OpenTool {+ span: tracing::Span,+ name: ToolName,+ status: ToolStatus,+ /// The latest `rawInput` seen: a harness may stream the arguments over+ /// several patches, and the last complete value is the call's.+ arguments: Option<Value>,+ /// Index of this call's part in the turn's output, when it has one.+ part: Option<usize>,+}++/// How a turn ended.+enum Outcome<'outcome> {+ /// The runtime answered the prompt; the result carries the stop reason.+ Answered(&'outcome Value),+ /// The runtime refused the prompt with a JSON-RPC error.+ Refused(&'outcome str),+ /// Another prompt went out before this one was answered.+ Superseded,+ /// The connection ended with the turn in flight.+ Stopped(&'outcome StopReason),+}++impl GenAiProjector {+ /// A projector for `session`, recording content under `policy` and tool+ /// definitions from `tool_definitions` once they are published.+ pub(crate) fn new(+ session: AgentSessionId,+ policy: ContentPolicy,+ tool_definitions: SharedToolDefinitions,+ ) -> Self {+ Self {+ session,+ policy,+ harness: Harness::Unknown,+ model: None,+ tool_definitions,+ initialize: None,+ config_requests: HashSet::new(),+ turn: None,+ tools: HashMap::new(),+ usage_totals: None,+ }+ }++ /// The cell the connection's tool listing is published into.+ pub(crate) fn tool_definitions(&self) -> SharedToolDefinitions {+ self.tool_definitions.clone()+ }++ /// A frame this side is sending to the runtime. A `session/prompt` opens a+ /// turn, as a child of `parent` - the span of the command that carried+ /// the prompt - when there is one.+ pub(crate) fn on_outbound(+ &mut self,+ message: &mut ToRuntimeMessage,+ parent: Option<&tracing::Span>,+ ) {+ let ToRuntimeMessage::Acp(AcpMessage(RawJsonRpcMessage::Request(request))) = message else {+ return;+ };+ let method = &request.method;+ if InitializeRequest::matches_method(method) {+ self.initialize = Some(request.id.clone());+ } else if NewSessionRequest::matches_method(method)+ || LoadSessionRequest::matches_method(method)+ || ResumeSessionRequest::matches_method(method)+ || SetSessionConfigOptionRequest::matches_method(method)+ {+ self.config_requests.insert(request.id.clone());+ } else if PromptRequest::matches_method(method) {+ let prompt: Option<PromptRequest> = deserialize_params(request.params.as_ref());+ self.begin_turn(request.id.clone(), prompt, parent);+ if let Some(turn) = &self.turn+ && let Some(RawJsonRpcParams::Object(params)) = &mut request.params+ && let Some(meta) = params+ .entry("_meta")+ .or_insert_with(|| Value::Object(Default::default()))+ .as_object_mut()+ {+ genai_telemetry::propagation::inject(&turn.span, meta);+ }+ }+ }++ /// A frame the runtime sent.+ pub(crate) fn on_inbound(&mut self, message: &ToServerMessage) {+ let ToServerMessage::Acp(AcpMessage(frame)) = message else {+ return;+ };+ match frame {+ RawJsonRpcMessage::Response(Response::Result { id, result }) => {+ self.on_result(id, result);+ }+ RawJsonRpcMessage::Response(Response::Error { id, error }) => {+ if self+ .turn+ .as_ref()+ .is_some_and(|turn| &turn.request_id == id)+ && let Some(turn) = self.turn.take()+ {+ self.end_turn(turn, Outcome::Refused(&error.message));+ }+ }+ RawJsonRpcMessage::Notification(notification)+ if SessionNotification::matches_method(¬ification.method) =>+ {+ if let Some(notification) =+ deserialize_params::<SessionNotification>(notification.params.as_ref())+ {+ self.on_update(notification.update);+ }+ }+ RawJsonRpcMessage::Request(_) | RawJsonRpcMessage::Notification(_) => {}+ }+ }++ /// The connection is over. A turn in flight ends in error; tool calls+ /// still open are marked abandoned.+ pub(crate) fn on_stopped(&mut self, reason: &StopReason) {+ if let Some(turn) = self.turn.take() {+ self.end_turn(turn, Outcome::Stopped(reason));+ }+ for (_, tool) in self.tools.drain() {+ tool.span.set_error("session_stopped", reason.to_string());+ }+ }++ fn on_result(&mut self, id: &RequestId, result: &Value) {+ if self.initialize.as_ref() == Some(id) {+ self.initialize = None;+ if let Ok(response) = serde_json::from_value::<InitializeResponse>(result.clone())+ && let Some(info) = response.agent_info+ {+ self.harness = Harness::from_agent_info(&info.name);+ }+ return;+ }+ if self.config_requests.remove(id) {+ #[derive(Deserialize)]+ #[serde(rename_all = "camelCase")]+ struct ConfigCarrier {+ #[serde(default)]+ config_options: Vec<SessionConfigOption>,+ }+ if let Ok(carrier) = serde_json::from_value::<ConfigCarrier>(result.clone()) {+ self.apply_config_options(carrier.config_options);+ }+ return;+ }+ if self+ .turn+ .as_ref()+ .is_some_and(|turn| &turn.request_id == id)+ && let Some(turn) = self.turn.take()+ {+ self.end_turn(turn, Outcome::Answered(result));+ }+ }++ fn on_update(&mut self, update: SessionUpdate) {+ match update {+ SessionUpdate::AgentMessageChunk(chunk) => self.append_output(&chunk.content, false),+ SessionUpdate::AgentThoughtChunk(chunk) => self.append_output(&chunk.content, true),+ SessionUpdate::ToolCall(call) => self.open_tool(call),+ SessionUpdate::ToolCallUpdate(update) => self.patch_tool(update),+ SessionUpdate::ConfigOptionUpdate(update) => {+ self.apply_config_options(update.config_options);+ }+ SessionUpdate::UsageUpdate(usage) => {+ if let Some(turn) = &self.turn {+ turn.span+ .set_u64(attr::MACRO_CONTEXT_USED_TOKENS, usage.used);+ turn.span+ .set_u64(attr::MACRO_CONTEXT_SIZE_TOKENS, usage.size);+ }+ }+ // Plans, modes, commands, titles and the user's own echoed+ // chunks: nothing a judge of the turn needs.+ _ => {}+ }+ }++ fn apply_config_options(&mut self, options: Vec<SessionConfigOption>) {+ let model = options+ .into_iter()+ .find(|option| option.id.to_string() == MODEL_CONFIG_ID);+ if let Some(SessionConfigOption {+ kind: SessionConfigKind::Select(select),+ ..+ }) = model+ {+ self.model = Some(select.current_value.to_string());+ }+ }++ fn begin_turn(+ &mut self,+ request_id: RequestId,+ prompt: Option<PromptRequest>,+ parent: Option<&tracing::Span>,+ ) {+ if let Some(previous) = self.turn.take() {+ self.end_turn(previous, Outcome::Superseded);+ }+ // The literal stays in step with `attr::span_name::INVOKE_AGENT`;+ // `tracing` bakes the name into static metadata.+ let span = match parent {+ Some(parent) => tracing::info_span!(+ parent: parent,+ "invoke_agent",+ gen_ai.operation.name = attr::operation::INVOKE_AGENT,+ gen_ai.conversation.id = %self.session,+ agent.session.id = %self.session,+ ),+ None => tracing::info_span!(+ "invoke_agent",+ gen_ai.operation.name = attr::operation::INVOKE_AGENT,+ gen_ai.conversation.id = %self.session,+ agent.session.id = %self.session,+ ),+ };+ let mut turn = Turn {+ request_id,+ span,+ parts: Vec::new(),+ agent_name_recorded: false,+ model_recorded: false,+ definitions_recorded: false,+ };+ self.record_turn_facts(&mut turn);+ if self.policy.capture+ && let Some(prompt) = prompt+ {+ let parts: Vec<Value> = prompt.prompt.iter().filter_map(prompt_part).collect();+ let bounded = bound_messages(+ vec![messages::message(messages::ROLE_USER, parts)],+ &self.policy.limits,+ );+ turn.span.set_str(attr::INPUT_MESSAGES, bounded.json);+ if bounded.truncated {+ turn.span.set_bool(attr::MACRO_CONTENT_TRUNCATED, true);+ }+ }+ self.turn = Some(turn);+ }++ /// Record what is known about the session onto a turn's span: the+ /// harness, the model and the offered tools, each once, as soon as known.+ fn record_turn_facts(&self, turn: &mut Turn) {+ if !turn.agent_name_recorded && self.harness != Harness::Unknown {+ turn.span+ .set_str(attr::AGENT_NAME, harness_name(self.harness));+ turn.agent_name_recorded = true;+ }+ if !turn.model_recorded+ && let Some(model) = &self.model+ {+ turn.span.set_str(attr::REQUEST_MODEL, model.clone());+ turn.model_recorded = true;+ }+ if !turn.definitions_recorded+ && let Some(definitions) = self.tool_definitions.get()+ {+ turn.definitions_recorded = true;+ if !definitions.is_empty() {+ let definitions = definitions+ .iter()+ .map(|definition| {+ messages::tool_definition(+ &self.definition_name(&definition.name),+ &definition.description,+ definition.parameters.clone(),+ )+ })+ .collect();+ let (json, truncated) = bound_tool_definitions(definitions, &self.policy.limits);+ turn.span.set_str(attr::TOOL_DEFINITIONS, json);+ if truncated {+ turn.span.set_bool(attr::MACRO_CONTENT_TRUNCATED, true);+ }+ }+ }+ }++ /// A listed tool's name as this harness calls it. Macro's in-process agent+ /// calls Macro's tools natively, by their bare names, while the listing+ /// sees them through Macro's MCP server as `mcp__macro__<tool>`; every+ /// other harness calls them by the listed name.+ fn definition_name(&self, listed: &str) -> String {+ let name: ToolName = listed.parse().unwrap_or_else(|never| match never {});+ if self.harness == Harness::Macro+ && let ToolName::Mcp { .. } = &name+ && let Some(tool) = self.harness.reader().macro_tool(&name)+ {+ return tool.to_owned();+ }+ listed.to_owned()+ }++ /// The listed description of the tool behind `name`, if it was listed.+ fn tool_description(&self, name: &ToolName) -> Option<String> {+ let display = name.display();+ let mangled = match name {+ ToolName::Mcp { server, tool } => Some(format!("mcp__{server}__{tool}")),+ ToolName::Native { .. } => None,+ };+ self.tool_definitions.get()?.iter().find_map(|definition| {+ (definition.name == display+ || mangled.as_deref() == Some(definition.name.as_str())+ || self.definition_name(&definition.name) == display)+ .then(|| definition.description.clone())+ })+ }++ fn end_turn(&mut self, mut turn: Turn, outcome: Outcome<'_>) {+ let finish_reason = match outcome {+ Outcome::Answered(result) => {+ self.record_usage(&turn.span, result);+ let stop_reason = serde_json::from_value::<PromptResponse>(result.clone())+ .ok()+ .map(|response| response.stop_reason);+ finish_reason(stop_reason)+ }+ Outcome::Refused(message) => {+ turn.span.set_error(+ "prompt_refused",+ self.error_description(message, "the runtime refused the prompt"),+ );+ attr::finish_reason::ERROR+ }+ Outcome::Superseded => {+ turn.span.set_error(+ "superseded",+ "another prompt was sent before this turn was answered",+ );+ attr::finish_reason::ERROR+ }+ Outcome::Stopped(reason) => {+ turn.span.set_error("session_stopped", reason.to_string());+ attr::finish_reason::ERROR+ }+ };++ // A call this turn opened and the turn ended on is over for the+ // judge's purposes, whatever the harness would have said next: mark+ // it rather than leak its span. A call opened outside any turn (its+ // `part` is `None`) belongs to no turn and stays open for its own+ // finish.+ let abandoned = if finish_reason == attr::finish_reason::CANCELLED {+ "the turn was cancelled before the tool call finished"+ } else {+ "the turn ended before the tool call finished"+ };+ self.tools.retain(|_, tool| {+ if tool.part.is_some() {+ tool.span.set_error("abandoned", abandoned);+ false+ } else {+ true+ }+ });++ self.record_turn_facts(&mut turn);+ if !turn.agent_name_recorded {+ turn.span+ .set_str(attr::AGENT_NAME, harness_name(self.harness));+ }+ turn.span+ .set_str_array(attr::RESPONSE_FINISH_REASONS, [finish_reason]);+ if self.policy.capture {+ let parts = turn.parts.into_iter().map(OutputPart::into_json).collect();+ let bounded = bound_messages(+ vec![messages::output_message(+ messages::ROLE_ASSISTANT,+ parts,+ finish_reason,+ )],+ &self.policy.limits,+ );+ turn.span.set_str(attr::OUTPUT_MESSAGES, bounded.json);+ if bounded.truncated {+ turn.span.set_bool(attr::MACRO_CONTENT_TRUNCATED, true);+ }+ }+ // Dropping the last handle closes the span.+ }++ /// The turn's token usage from the prompt response. ACP reports running+ /// session totals, so each turn records the increase over the previous+ /// response; the first response a connection sees records the total,+ /// which for a resumed session includes earlier turns.+ fn record_usage(&mut self, span: &tracing::Span, result: &Value) {+ let input = result.pointer("/usage/inputTokens").and_then(Value::as_u64);+ let output = result+ .pointer("/usage/outputTokens")+ .and_then(Value::as_u64);+ let (Some(input), Some(output)) = (input, output) else {+ return;+ };+ let (turn_input, turn_output) = match self.usage_totals {+ Some((previous_input, previous_output))+ if input >= previous_input && output >= previous_output =>+ {+ (input - previous_input, output - previous_output)+ }+ _ => (input, output),+ };+ self.usage_totals = Some((input, output));+ span.set_u64(attr::USAGE_INPUT_TOKENS, turn_input);+ span.set_u64(attr::USAGE_OUTPUT_TOKENS, turn_output);+ }++ fn append_output(&mut self, content: &ContentBlock, reasoning: bool) {+ let ContentBlock::Text(text) = content else {+ return;+ };+ let Some(turn) = &mut self.turn else {+ return;+ };+ match (turn.parts.last_mut(), reasoning) {+ (Some(OutputPart::Text(existing)), false)+ | (Some(OutputPart::Reasoning(existing)), true) => existing.push_str(&text.text),+ (_, false) => turn.parts.push(OutputPart::Text(text.text.clone())),+ (_, true) => turn.parts.push(OutputPart::Reasoning(text.text.clone())),+ }+ }++ fn open_tool(&mut self, call: ToolCall) {+ let frame = ToolFrame::of_call(&call);+ self.sniff_harness(&frame);+ let reader = self.harness.reader();+ let name = tool_name(reader, &frame);+ let id = call.tool_call_id.0.to_string();+ let tool_display = name.display().to_owned();++ // The literal stays in step with `attr::span_name::EXECUTE_TOOL`;+ // `otel.name` gives the exported span the semconv `execute_tool+ // {name}` name.+ let otel_name = format!("{} {tool_display}", attr::operation::EXECUTE_TOOL);+ let span = match &self.turn {+ Some(turn) => tracing::info_span!(+ parent: &turn.span,+ "execute_tool",+ otel.name = %otel_name,+ gen_ai.operation.name = attr::operation::EXECUTE_TOOL,+ gen_ai.tool.type = attr::tool_type::FUNCTION,+ gen_ai.tool.name = %tool_display,+ gen_ai.tool.call.id = %id,+ gen_ai.conversation.id = %self.session,+ agent.session.id = %self.session,+ ),+ None => tracing::info_span!(+ "execute_tool",+ otel.name = %otel_name,+ gen_ai.operation.name = attr::operation::EXECUTE_TOOL,+ gen_ai.tool.type = attr::tool_type::FUNCTION,+ gen_ai.tool.name = %tool_display,+ gen_ai.tool.call.id = %id,+ gen_ai.conversation.id = %self.session,+ agent.session.id = %self.session,+ ),+ };+ span.set_str(attr::MACRO_TOOL_KIND, tool_kind_name(call.kind));+ if !call.title.is_empty() {+ span.set_str(attr::MACRO_TOOL_TITLE, call.title.clone());+ }+ if let ToolName::Mcp { server, .. } = &name {+ span.set_str(attr::MACRO_MCP_SERVER, server.clone());+ }+ if let Some(description) = self.tool_description(&name) {+ span.set_str(attr::TOOL_DESCRIPTION, description);+ }++ let arguments = call.raw_input.clone().filter(|input| !input.is_null());+ let part = self.turn.as_mut().map(|turn| {+ turn.parts.push(OutputPart::ToolCall {+ id: id.clone(),+ name: tool_display,+ arguments: arguments.clone().unwrap_or(Value::Null),+ });+ turn.parts.len() - 1+ });+ let status = frame.status.unwrap_or_default();+ if let Some(replaced) = self.tools.insert(+ id.clone(),+ OpenTool {+ span,+ name,+ status,+ arguments,+ part,+ },+ ) {+ replaced+ .span+ .set_error("superseded", "the harness reopened this tool call id");+ }+ if status.is_finished() {+ self.finish_tool(&id, &frame);+ }+ }++ fn patch_tool(&mut self, update: ToolCallUpdate) {+ let id = update.tool_call_id.0.to_string();+ let mut frame = ToolFrame::of_update(&update);+ self.sniff_harness(&frame);+ let Some(tool) = self.tools.get_mut(&id) else {+ // A call this connection never saw open: a resumed session+ // mid-call. Nothing to attach it to.+ return;+ };+ if frame.status.is_none() {+ frame = frame.with_status(tool.status);+ }+ if let Some(status) = frame.status {+ tool.status = status;+ }+ if let Some(input) = frame.raw_input+ && !input.is_null()+ {+ tool.arguments = Some(input.clone());+ }+ // A harness may name the tool only once it knows what it is running.+ if tool.name.is_empty() {+ let named = tool_name(self.harness.reader(), &frame);+ if !named.is_empty() {+ let display = named.display().to_owned();+ tool.span.record("gen_ai.tool.name", display.as_str());+ tool.span.record(+ "otel.name",+ format!("{} {display}", attr::operation::EXECUTE_TOOL).as_str(),+ );+ tool.name = named;+ }+ }+ if frame.finished() {+ self.finish_tool(&id, &frame);+ }+ }++ fn finish_tool(&mut self, id: &str, frame: &ToolFrame<'_>) {+ let Some(tool) = self.tools.remove(id) else {+ return;+ };+ let reader = self.harness.reader();+ let (result, mut error) = match frame.raw_output {+ Some(raw) => {+ let (value, error) = reader.unwrap_tool_output(raw);+ (Some(value), error)+ }+ None => (+ frame+ .content_text()+ .or_else(|| reader.terminal_output(frame))+ .map(Value::String),+ None,+ ),+ };+ if error.is_none()+ && let Some(code) = reader.terminal_exit_code(frame)+ && code != 0+ {+ error = Some(format!("the command exited with code {code}"));+ }++ if self.policy.capture {+ if let Some(arguments) = &tool.arguments {+ let (json, truncated) = bounded_json_string(arguments, &self.policy.limits);+ tool.span.set_str(attr::TOOL_CALL_ARGUMENTS, json);+ if truncated {+ tool.span.set_bool(attr::MACRO_CONTENT_TRUNCATED, true);+ }+ }+ if let Some(result) = &result {+ let (json, truncated) = bounded_json_string(result, &self.policy.limits);+ tool.span.set_str(attr::TOOL_CALL_RESULT, json);+ if truncated {+ tool.span.set_bool(attr::MACRO_CONTENT_TRUNCATED, true);+ }+ }+ }+ // The error text is the tool's own output and so content: recorded+ // under the policy, bounded.+ match (tool.status, error) {+ (ToolStatus::Failed, error) => tool.span.set_error(+ "tool_error",+ error.map_or_else(+ || "the tool call failed".to_owned(),+ |error| self.error_description(&error, "the tool call failed"),+ ),+ ),+ (_, Some(error)) => tool.span.set_error(+ "tool_error",+ self.error_description(&error, "the tool reported an error"),+ ),+ (_, None) => {}+ }++ // The turn's record of the call gets the final arguments and name.+ if let (Some(turn), Some(index)) = (&mut self.turn, tool.part)+ && let Some(OutputPart::ToolCall {+ name, arguments, ..+ }) = turn.parts.get_mut(index)+ {+ if let Some(final_arguments) = tool.arguments {+ *arguments = final_arguments;+ }+ if name.is_empty() {+ *name = tool.name.display().to_owned();+ }+ }+ // Dropping the last handle closes the span.+ }++ /// An error description fit for a span's status: the detail, bounded,+ /// when content may be recorded; a fixed summary otherwise.+ fn error_description(&self, detail: &str, summary: &'static str) -> String {+ if self.policy.capture {+ truncate_chars(detail, self.policy.limits.max_part_chars).into_owned()+ } else {+ summary.to_owned()+ }+ }++ /// Recognize the harness from a tool frame when no `initialize` named it.+ fn sniff_harness(&mut self, frame: &ToolFrame<'_>) {+ if self.harness == Harness::Unknown+ && let Some(harness) = Harness::sniff(frame)+ {+ self.harness = harness;+ }+ }+}++/// Deserialize a request's or notification's params as a specific ACP type.+/// `None` for positional params or a shape mismatch: less is derived from the+/// frame, nothing fails.+fn deserialize_params<T: serde::de::DeserializeOwned>(+ params: Option<&RawJsonRpcParams>,+) -> Option<T> {+ match params? {+ RawJsonRpcParams::Object(map) => serde_json::from_value(Value::Object(map.clone())).ok(),+ RawJsonRpcParams::Array(_) => None,+ }+}++/// A prompt content block as a semconv message part. Only the user's own text+/// is copied onto a span; attached resources - inline bytes and embedded text+/// alike - are named by their URI, since a document the user attached is not+/// the prompt and can be arbitrarily large.+fn prompt_part(block: &ContentBlock) -> Option<Value> {+ Some(match block {+ ContentBlock::Text(text) => messages::text_part(text.text.clone()),+ ContentBlock::Image(image) => messages::media_part(+ "image",+ Some(&image.mime_type),+ image+ .uri+ .clone()+ .map_or(MediaSource::Inline, MediaSource::Uri),+ ),+ ContentBlock::Audio(audio) => {+ messages::media_part("audio", Some(&audio.mime_type), MediaSource::Inline)+ }+ ContentBlock::ResourceLink(link) => messages::media_part(+ "document",+ link.mime_type.as_deref(),+ MediaSource::Uri(link.uri.clone()),+ ),+ ContentBlock::Resource(embedded) => match &embedded.resource {+ EmbeddedResourceResource::TextResourceContents(text) => messages::media_part(+ "document",+ text.mime_type.as_deref(),+ MediaSource::Uri(text.uri.clone()),+ ),+ EmbeddedResourceResource::BlobResourceContents(blob) => messages::media_part(+ "document",+ blob.mime_type.as_deref(),+ MediaSource::Uri(blob.uri.clone()),+ ),+ // `#[non_exhaustive]`: a resource shape this build does not know.+ _ => return None,+ },+ // `#[non_exhaustive]`: a block this build does not know.+ _ => return None,+ })+}++/// The semconv finish reason for how a turn stopped.+fn finish_reason(stop_reason: Option<AcpStopReason>) -> &'static str {+ match stop_reason {+ Some(AcpStopReason::EndTurn) => attr::finish_reason::STOP,+ Some(AcpStopReason::MaxTokens | AcpStopReason::MaxTurnRequests) => {+ attr::finish_reason::LENGTH+ }+ Some(AcpStopReason::Refusal) => attr::finish_reason::CONTENT_FILTER,+ Some(AcpStopReason::Cancelled) => attr::finish_reason::CANCELLED,+ // `#[non_exhaustive]`, and a response this build cannot read: the+ // turn was answered, which is all that is known.+ Some(_) | None => attr::finish_reason::STOP,+ }+}++/// The harness as the agent name spans carry, in the fold's own snake_case+/// spelling so it matches the `harness` field a session's readers show.+fn harness_name(harness: Harness) -> String {+ serde_json::to_value(harness)+ .ok()+ .and_then(|value| value.as_str().map(str::to_owned))+ .unwrap_or_else(|| "unknown".to_owned())+}++/// ACP's coarse tool kind, in its wire spelling.+fn tool_kind_name(kind: ToolKind) -> String {+ serde_json::to_value(kind)+ .ok()+ .and_then(|value| value.as_str().map(str::to_owned))+ .unwrap_or_else(|| "unknown".to_owned())+}diff --git a/crates/agent_session/src/domain/session/telemetry/test.rs b/crates/agent_session/src/domain/session/telemetry/test.rsnew file mode 100644index 0000000000..f048f2c3ec--- /dev/null+++ b/crates/agent_session/src/domain/session/telemetry/test.rs@@ -0,0 +1,776 @@+//! The projector over recorded sessions: which spans come out, with which+//! attributes, from the frames a live actor would feed it.++use super::*;+use crate::domain::model::Message;+use crate::domain::ports::SessionToolDefinition;+use crate::domain::session::CloseReason;+use agent_fold::testing::{fixtures, parse_log};+use opentelemetry::trace::{SpanId, Status, TraceContextExt as _, TracerProvider as _};+use opentelemetry::{Array, Value as OtelValue};+use opentelemetry_sdk::trace::{InMemorySpanExporter, SdkTracerProvider, SpanData};+use serde_json::json;+use tracing_opentelemetry::OpenTelemetrySpanExt as _;+use tracing_subscriber::layer::SubscriberExt as _;++/// An OpenTelemetry pipeline that keeps finished spans in memory, installed as+/// the thread's default subscriber for the returned guard's lifetime.+fn otel_test_pipeline() -> (+ InMemorySpanExporter,+ SdkTracerProvider,+ tracing::subscriber::DefaultGuard,+) {+ let exporter = InMemorySpanExporter::default();+ let provider = SdkTracerProvider::builder()+ .with_simple_exporter(exporter.clone())+ .build();+ let layer = tracing_opentelemetry::layer().with_tracer(provider.tracer("test"));+ let guard = tracing::subscriber::set_default(tracing_subscriber::registry().with(layer));+ // Other tests in this binary run the same code paths with no subscriber+ // installed, which can cache "never interested" for a span callsite on+ // another thread; recompute so this subscriber sees every span.+ tracing::callsite::rebuild_interest_cache();+ (exporter, provider, guard)+}++fn finished(exporter: &InMemorySpanExporter, provider: &SdkTracerProvider) -> Vec<SpanData> {+ provider.force_flush().expect("flush");+ exporter.get_finished_spans().expect("finished spans")+}++fn attribute<'a>(span: &'a SpanData, key: &str) -> Option<&'a OtelValue> {+ span.attributes+ .iter()+ .find(|kv| kv.key.as_str() == key)+ .map(|kv| &kv.value)+}++fn string_attribute(span: &SpanData, key: &str) -> Option<String> {+ match attribute(span, key)? {+ OtelValue::String(s) => Some(s.to_string()),+ other => panic!("{key} is not a string: {other:?}"),+ }+}++fn int_attribute(span: &SpanData, key: &str) -> Option<i64> {+ match attribute(span, key)? {+ OtelValue::I64(i) => Some(*i),+ other => panic!("{key} is not an integer: {other:?}"),+ }+}++fn string_array_attribute(span: &SpanData, key: &str) -> Option<Vec<String>> {+ match attribute(span, key)? {+ OtelValue::Array(Array::String(values)) => {+ Some(values.iter().map(ToString::to_string).collect())+ }+ other => panic!("{key} is not a string array: {other:?}"),+ }+}++fn json_attribute(span: &SpanData, key: &str) -> Option<Value> {+ string_attribute(span, key).map(|json| serde_json::from_str(&json).expect("valid JSON"))+}++fn spans_named<'a>(spans: &'a [SpanData], name: &str) -> Vec<&'a SpanData> {+ spans.iter().filter(|span| span.name == name).collect()+}++fn projector(policy: ContentPolicy) -> GenAiProjector {+ GenAiProjector::new(+ AgentSessionId::TEST_A,+ policy,+ SharedToolDefinitions::default(),+ )+}++/// Feed a recorded log through the projector the way the actor does: frames+/// to the runtime through `on_outbound`, frames from it through `on_inbound`.+fn replay(projector: &mut GenAiProjector, jsonl: &str) {+ for mut entry in parse_log(jsonl) {+ match &mut entry.content {+ Message::ToRuntime(message) => projector.on_outbound(message, None),+ Message::ToServer(message) => projector.on_inbound(message),+ }+ }+}++#[test]+fn a_turn_becomes_an_agent_span_with_a_tool_span_per_call() {+ let (exporter, provider, _guard) = otel_test_pipeline();+ let mut projector = projector(ContentPolicy::enabled());+ replay(&mut projector, fixtures::TURN);+ let spans = finished(&exporter, &provider);++ let agent = spans_named(&spans, "invoke_agent");+ assert_eq!(agent.len(), 1, "one prompt, one agent span: {spans:#?}");+ let agent = agent[0];+ assert_eq!(+ string_attribute(agent, attr::OPERATION_NAME).as_deref(),+ Some("invoke_agent")+ );+ assert_eq!(+ string_attribute(agent, attr::CONVERSATION_ID).as_deref(),+ Some(AgentSessionId::TEST_A.to_string().as_str())+ );+ // The fixture's `initialize` names no agent; the harness is recognized+ // from the `_meta.claudeCode` namespace on its tool frames.+ assert_eq!(+ string_attribute(agent, attr::AGENT_NAME).as_deref(),+ Some("claude_code")+ );+ assert_eq!(+ string_array_attribute(agent, attr::RESPONSE_FINISH_REASONS),+ Some(vec!["stop".to_owned()])+ );+ assert_eq!(int_attribute(agent, attr::USAGE_INPUT_TOKENS), Some(10));+ assert_eq!(int_attribute(agent, attr::USAGE_OUTPUT_TOKENS), Some(8));+ assert_eq!(+ int_attribute(agent, attr::MACRO_CONTEXT_USED_TOKENS),+ Some(10)+ );+ assert_eq!(+ int_attribute(agent, attr::MACRO_CONTEXT_SIZE_TOKENS),+ Some(1_000_000)+ );++ let input = json_attribute(agent, attr::INPUT_MESSAGES).expect("input recorded");+ assert_eq!(+ input,+ json!([{ "role": "user", "parts": [{ "type": "text", "content": "list the examples and write a file" }] }])+ );+ let output = json_attribute(agent, attr::OUTPUT_MESSAGES).expect("output recorded");+ assert_eq!(output[0]["role"], "assistant");+ assert_eq!(output[0]["finish_reason"], "stop");+ let parts = output[0]["parts"].as_array().expect("parts");+ assert_eq!(+ parts[0],+ json!({ "type": "text", "content": "Sure, one moment." })+ );+ assert_eq!(parts[1]["type"], "tool_call");+ assert_eq!(parts[1]["id"], "tc-run");+ assert_eq!(parts[1]["name"], "Bash");+ assert_eq!(parts[1]["arguments"], json!({ "command": "ls examples" }));+ assert_eq!(parts[2]["type"], "tool_call");+ assert_eq!(parts[2]["name"], "Write");+ assert_eq!(parts[3], json!({ "type": "text", "content": "Done." }));++ let tools = spans_named(&spans, "execute_tool Bash");+ assert_eq!(tools.len(), 1, "one Bash call: {spans:#?}");+ let bash = tools[0];+ assert_eq!(+ bash.parent_span_id,+ agent.span_context.span_id(),+ "tool spans hang off the turn"+ );+ assert_eq!(+ string_attribute(bash, attr::TOOL_NAME).as_deref(),+ Some("Bash")+ );+ assert_eq!(+ string_attribute(bash, attr::TOOL_CALL_ID).as_deref(),+ Some("tc-run")+ );+ assert_eq!(+ string_attribute(bash, attr::MACRO_TOOL_KIND).as_deref(),+ Some("execute")+ );+ assert_eq!(+ json_attribute(bash, attr::TOOL_CALL_ARGUMENTS),+ Some(json!({ "command": "ls examples" }))+ );+ // No `rawOutput`; the terminal output in `_meta` is the result.+ let result = string_attribute(bash, attr::TOOL_CALL_RESULT).expect("result recorded");+ assert!(+ result.contains("events.rs"),+ "terminal output recorded: {result}"+ );+ assert_eq!(bash.status, Status::Unset);++ let write = spans_named(&spans, "execute_tool Write");+ assert_eq!(write.len(), 1);+ assert_eq!(write[0].parent_span_id, agent.span_context.span_id());+ // The tool spans close before the turn does.+ assert!(bash.end_time <= agent.end_time);+}++#[test]+fn the_harness_announced_by_initialize_names_the_agent() {+ let (exporter, provider, _guard) = otel_test_pipeline();+ let mut projector = projector(ContentPolicy::enabled());+ replay(+ &mut projector,+ include_str!("../../../../../agent_fold/fixtures/real/real_single_turn.jsonl"),+ );+ let spans = finished(&exporter, &provider);++ let agent = spans_named(&spans, "invoke_agent");+ assert_eq!(agent.len(), 1);+ assert_eq!(+ string_attribute(agent[0], attr::AGENT_NAME).as_deref(),+ Some("claude_code")+ );+ assert_eq!(int_attribute(agent[0], attr::USAGE_INPUT_TOKENS), Some(2));+ assert_eq!(int_attribute(agent[0], attr::USAGE_OUTPUT_TOKENS), Some(29));+ let output = json_attribute(agent[0], attr::OUTPUT_MESSAGES).expect("output");+ assert_eq!(+ output[0]["parts"][0]["content"],+ "Hey! What are we working on today?"+ );+}++#[test]+fn usage_is_recorded_per_turn_from_the_running_totals() {+ let (exporter, provider, _guard) = otel_test_pipeline();+ let mut projector = projector(ContentPolicy::enabled());+ replay(+ &mut projector,+ include_str!("../../../../../agent_fold/fixtures/real/real_multi_turn.jsonl"),+ );+ let spans = finished(&exporter, &provider);++ let agents = spans_named(&spans, "invoke_agent");+ assert!(agents.len() > 1, "a multi-turn recording: {}", agents.len());+ let first = agents+ .iter()+ .min_by_key(|span| span.start_time)+ .expect("a first turn");+ assert_eq!(int_attribute(first, attr::USAGE_OUTPUT_TOKENS), Some(2161));+ // Every later turn reports its increase, never the total so far.+ let total: i64 = agents+ .iter()+ .filter_map(|span| int_attribute(span, attr::USAGE_OUTPUT_TOKENS))+ .sum();+ assert_eq!(total, 48946, "the turns' shares add up to the final total");+}++#[test]+fn a_failed_mcp_call_marks_its_span_and_unwraps_the_envelope() {+ let (exporter, provider, _guard) = otel_test_pipeline();+ let mut projector = projector(ContentPolicy::enabled());+ replay(&mut projector, fixtures::MACRO_MCP);+ let spans = finished(&exporter, &provider);++ let tools: Vec<_> = spans+ .iter()+ .filter(|span| span.name.starts_with("execute_tool "))+ .collect();+ assert!(!tools.is_empty(), "tool spans: {spans:#?}");+ for tool in &tools {+ assert_eq!(+ string_attribute(tool, attr::MACRO_MCP_SERVER).as_deref(),+ Some("macro"),+ "Macro's tools are reached over its MCP server: {tool:#?}"+ );+ // The MCP `CallToolResult` envelope is not what the tool returned:+ // a text result is recorded as its bare text, a structured one as+ // its own JSON, never `{"content": [...], "isError": ...}`.+ if let Some(result) = string_attribute(tool, attr::TOOL_CALL_RESULT) {+ assert!(+ !result.contains("\"isError\""),+ "envelope unwrapped: {result}"+ );+ }+ }+ let failed: Vec<_> = tools+ .iter()+ .filter(|span| matches!(span.status, Status::Error { .. }))+ .collect();+ assert_eq!(failed.len(), 1, "one failed call: {tools:#?}");+ assert_eq!(+ string_attribute(failed[0], attr::ERROR_TYPE).as_deref(),+ Some("tool_error")+ );+}++#[test]+fn a_stopped_connection_ends_the_turn_and_its_tools_in_error() {+ let (exporter, provider, _guard) = otel_test_pipeline();+ let mut projector = projector(ContentPolicy::enabled());+ // Everything up to, not including, the prompt's answer.+ let frames: Vec<&str> = fixtures::TURN.lines().collect();+ let truncated = frames[..frames.len() - 3].join("\n");+ replay(&mut projector, &truncated);+ projector.on_stopped(&StopReason::Closed(CloseReason::TransportClosed));+ let spans = finished(&exporter, &provider);++ let agent = spans_named(&spans, "invoke_agent");+ assert_eq!(agent.len(), 1);+ assert!(matches!(agent[0].status, Status::Error { .. }));+ assert_eq!(+ string_attribute(agent[0], attr::ERROR_TYPE).as_deref(),+ Some("session_stopped")+ );+ assert_eq!(+ string_array_attribute(agent[0], attr::RESPONSE_FINISH_REASONS),+ Some(vec!["error".to_owned()])+ );+ // The Write call had not finished when the connection dropped.+ let write = spans_named(&spans, "execute_tool Write");+ assert_eq!(write.len(), 1);+ assert!(matches!(write[0].status, Status::Error { .. }));+ assert_eq!(+ string_attribute(write[0], attr::ERROR_TYPE).as_deref(),+ Some("abandoned")+ );+}++#[test]+fn a_refused_prompt_ends_the_turn_in_error() {+ let (exporter, provider, _guard) = otel_test_pipeline();+ let mut projector = projector(ContentPolicy::enabled());+ let prompt = RawJsonRpcMessage::request(+ "session/prompt".to_owned(),+ json!({ "sessionId": "s1", "prompt": [{ "type": "text", "text": "hi" }] }),+ RequestId::Str("p1".to_owned()),+ )+ .expect("a request");+ projector.on_outbound(&mut ToRuntimeMessage::Acp(AcpMessage(prompt)), None);+ projector.on_inbound(&ToServerMessage::Acp(AcpMessage(+ RawJsonRpcMessage::response(+ RequestId::Str("p1".to_owned()),+ Err(agent_client_protocol::Error::internal_error()),+ ),+ )));+ let spans = finished(&exporter, &provider);++ let agent = spans_named(&spans, "invoke_agent");+ assert_eq!(agent.len(), 1);+ assert_eq!(+ string_attribute(agent[0], attr::ERROR_TYPE).as_deref(),+ Some("prompt_refused")+ );+ assert_eq!(+ string_array_attribute(agent[0], attr::RESPONSE_FINISH_REASONS),+ Some(vec!["error".to_owned()])+ );+}++#[test]+fn content_is_not_recorded_when_capture_is_off() {+ let (exporter, provider, _guard) = otel_test_pipeline();+ let mut projector = projector(ContentPolicy::disabled());+ replay(&mut projector, fixtures::TURN);+ let spans = finished(&exporter, &provider);++ let agent = spans_named(&spans, "invoke_agent")[0];+ assert!(attribute(agent, attr::INPUT_MESSAGES).is_none());+ assert!(attribute(agent, attr::OUTPUT_MESSAGES).is_none());+ // Structure stays.+ assert_eq!(+ string_array_attribute(agent, attr::RESPONSE_FINISH_REASONS),+ Some(vec!["stop".to_owned()])+ );+ assert_eq!(int_attribute(agent, attr::USAGE_OUTPUT_TOKENS), Some(8));++ let bash = spans_named(&spans, "execute_tool Bash")[0];+ assert!(attribute(bash, attr::TOOL_CALL_ARGUMENTS).is_none());+ assert!(attribute(bash, attr::TOOL_CALL_RESULT).is_none());+ assert_eq!(+ string_attribute(bash, attr::TOOL_NAME).as_deref(),+ Some("Bash")+ );+}++#[test]+fn published_tool_definitions_reach_the_turn_and_describe_its_calls() {+ let (exporter, provider, _guard) = otel_test_pipeline();+ let definitions = SharedToolDefinitions::default();+ let mut projector = GenAiProjector::new(+ AgentSessionId::TEST_A,+ ContentPolicy::enabled(),+ definitions.clone(),+ );+ definitions.publish(vec![+ SessionToolDefinition {+ name: "mcp__macro__ReadContent".to_owned(),+ description: "Read a document.".to_owned(),+ parameters: json!({ "type": "object", "properties": { "id": { "type": "string" } } }),+ },+ SessionToolDefinition {+ name: "mcp__linear__create_issue".to_owned(),+ description: "Create a Linear issue.".to_owned(),+ parameters: json!({ "type": "object" }),+ },+ ]);+ replay(&mut projector, fixtures::MACRO_MCP);+ let spans = finished(&exporter, &provider);++ let agent = spans_named(&spans, "invoke_agent");+ assert!(!agent.is_empty());+ let listed = json_attribute(agent[0], attr::TOOL_DEFINITIONS).expect("definitions recorded");+ assert_eq!(listed[0]["name"], "mcp__macro__ReadContent");+ assert_eq!(listed[0]["description"], "Read a document.");+ assert_eq!(listed[1]["name"], "mcp__linear__create_issue");++ let read = spans+ .iter()+ .find(|span| string_attribute(span, attr::TOOL_NAME).as_deref() == Some("ReadContent"))+ .expect("a ReadContent call in the fixture");+ assert_eq!(+ string_attribute(read, attr::TOOL_DESCRIPTION).as_deref(),+ Some("Read a document.")+ );+}++#[test]+fn definitions_published_after_the_prompt_are_recorded_when_the_turn_closes() {+ let (exporter, provider, _guard) = otel_test_pipeline();+ let definitions = SharedToolDefinitions::default();+ let mut projector = GenAiProjector::new(+ AgentSessionId::TEST_A,+ ContentPolicy::enabled(),+ definitions.clone(),+ );+ let frames: Vec<&str> = fixtures::TURN.lines().collect();+ // Up to and including the prompt.+ replay(&mut projector, &frames[..4].join("\n"));+ definitions.publish(vec![SessionToolDefinition {+ name: "mcp__linear__create_issue".to_owned(),+ description: "Create a Linear issue.".to_owned(),+ parameters: json!({ "type": "object" }),+ }]);+ replay(&mut projector, &frames[4..].join("\n"));+ let spans = finished(&exporter, &provider);++ let agent = spans_named(&spans, "invoke_agent")[0];+ let listed = json_attribute(agent, attr::TOOL_DEFINITIONS).expect("definitions recorded");+ assert_eq!(listed.as_array().map(Vec::len), Some(1));+}++#[test]+fn macro_tools_are_listed_by_their_bare_names_for_the_in_process_agent() {+ let (exporter, provider, _guard) = otel_test_pipeline();+ let definitions = SharedToolDefinitions::default();+ let mut projector = GenAiProjector::new(+ AgentSessionId::TEST_A,+ ContentPolicy::enabled(),+ definitions.clone(),+ );+ definitions.publish(vec![+ SessionToolDefinition {+ name: "mcp__macro__ReadContent".to_owned(),+ description: "Read a document.".to_owned(),+ parameters: json!({ "type": "object" }),+ },+ SessionToolDefinition {+ name: "mcp__linear__create_issue".to_owned(),+ description: "Create a Linear issue.".to_owned(),+ parameters: json!({ "type": "object" }),+ },+ ]);+ replay(&mut projector, fixtures::SUBAGENT_MACRO_INMEM);+ let spans = finished(&exporter, &provider);++ let agent = spans_named(&spans, "invoke_agent");+ assert!(!agent.is_empty(), "{spans:#?}");+ assert_eq!(+ string_attribute(agent[0], attr::AGENT_NAME).as_deref(),+ Some("macro")+ );+ let listed = json_attribute(agent[0], attr::TOOL_DEFINITIONS).expect("definitions recorded");+ assert_eq!(listed[0]["name"], "ReadContent");+ assert_eq!(listed[1]["name"], "mcp__linear__create_issue");+}++#[test]+fn the_turn_hangs_off_the_command_that_carried_the_prompt() {+ let (exporter, provider, _guard) = otel_test_pipeline();+ opentelemetry::global::set_text_map_propagator(+ opentelemetry_sdk::propagation::TraceContextPropagator::new(),+ );+ let mut projector = projector(ContentPolicy::enabled());+ let command = tracing::info_span!("agent.session.command");+ let command_id = command.context().span().span_context().span_id();+ assert_ne!(command_id, SpanId::INVALID);+ let prompt = RawJsonRpcMessage::request(+ "session/prompt".to_owned(),+ json!({ "sessionId": "s1", "prompt": [{ "type": "text", "text": "hi" }] }),+ RequestId::Str("p1".to_owned()),+ )+ .expect("a request");+ let mut message = ToRuntimeMessage::Acp(AcpMessage(prompt));+ projector.on_outbound(&mut message, Some(&command));+ let wire = serde_json::to_value(&message).expect("serialize prompt");+ let request: PromptRequest = serde_json::from_value(wire["params"].clone()).expect("prompt");+ let runtime = tracing::info_span!(parent: None, "runtime");+ genai_telemetry::propagation::set_parent(&runtime, request.meta.as_ref());+ drop(runtime);+ projector.on_inbound(&ToServerMessage::Acp(AcpMessage(+ RawJsonRpcMessage::response(+ RequestId::Str("p1".to_owned()),+ Ok(json!({ "stopReason": "cancelled" })),+ ),+ )));+ drop(command);+ let spans = finished(&exporter, &provider);++ let agent = spans_named(&spans, "invoke_agent");+ assert_eq!(agent.len(), 1);+ assert_eq!(agent[0].parent_span_id, command_id);+ let runtime = spans_named(&spans, "runtime");+ assert_eq!(runtime.len(), 1);+ assert_eq!(runtime[0].parent_span_id, agent[0].span_context.span_id());+ assert_eq!(+ runtime[0].span_context.trace_id(),+ agent[0].span_context.trace_id()+ );+ assert_eq!(+ string_array_attribute(agent[0], attr::RESPONSE_FINISH_REASONS),+ Some(vec!["cancelled".to_owned()])+ );+ // No harness ever announced itself and no tool frame gave it away.+ assert_eq!(+ string_attribute(agent[0], attr::AGENT_NAME).as_deref(),+ Some("unknown")+ );+}++#[test]+fn the_model_comes_from_the_runtime_config_options() {+ let (exporter, provider, _guard) = otel_test_pipeline();+ let mut projector = projector(ContentPolicy::enabled());+ let open = RawJsonRpcMessage::request(+ "session/new".to_owned(),+ json!({ "cwd": "/w", "mcpServers": [] }),+ RequestId::Str("n1".to_owned()),+ )+ .expect("a request");+ projector.on_outbound(&mut ToRuntimeMessage::Acp(AcpMessage(open)), None);+ projector.on_inbound(&ToServerMessage::Acp(AcpMessage(+ RawJsonRpcMessage::response(+ RequestId::Str("n1".to_owned()),+ Ok(json!({+ "sessionId": "s1",+ "configOptions": [{+ "id": "model",+ "name": "Model",+ "category": "model",+ "type": "select",+ "currentValue": "claude-opus-5",+ "options": [{ "value": "claude-opus-5", "name": "Opus" }]+ }]+ })),+ ),+ )));+ let prompt = RawJsonRpcMessage::request(+ "session/prompt".to_owned(),+ json!({ "sessionId": "s1", "prompt": [{ "type": "text", "text": "hi" }] }),+ RequestId::Str("p1".to_owned()),+ )+ .expect("a request");+ projector.on_outbound(&mut ToRuntimeMessage::Acp(AcpMessage(prompt)), None);+ projector.on_inbound(&ToServerMessage::Acp(AcpMessage(+ RawJsonRpcMessage::response(+ RequestId::Str("p1".to_owned()),+ Ok(json!({ "stopReason": "max_tokens" })),+ ),+ )));+ let spans = finished(&exporter, &provider);++ let agent = spans_named(&spans, "invoke_agent")[0];+ assert_eq!(+ string_attribute(agent, attr::REQUEST_MODEL).as_deref(),+ Some("claude-opus-5")+ );+ assert_eq!(+ string_array_attribute(agent, attr::RESPONSE_FINISH_REASONS),+ Some(vec!["length".to_owned()])+ );+}++#[test]+fn oversized_tool_results_are_cut_and_marked() {+ let (exporter, provider, _guard) = otel_test_pipeline();+ let mut projector = projector(ContentPolicy {+ capture: true,+ limits: genai_telemetry::Limits {+ max_part_chars: 16,+ max_attribute_bytes: 64,+ },+ });+ let call = RawJsonRpcMessage::notification(+ "session/update".to_owned(),+ json!({+ "sessionId": "s1",+ "update": {+ "sessionUpdate": "tool_call",+ "toolCallId": "t1",+ "title": "Read",+ "kind": "read",+ "status": "completed",+ "rawInput": { "path": "/a" },+ "rawOutput": { "text": "x".repeat(500) }+ }+ }),+ )+ .expect("a notification");+ projector.on_inbound(&ToServerMessage::Acp(AcpMessage(call)));+ let spans = finished(&exporter, &provider);++ let read = spans_named(&spans, "execute_tool Read");+ assert_eq!(read.len(), 1, "a call outside any turn still gets a span");+ assert_eq!(read[0].parent_span_id, SpanId::INVALID);+ let result = string_attribute(read[0], attr::TOOL_CALL_RESULT).expect("result");+ assert!(result.len() <= 64, "cut to the budget: {}", result.len());+ assert_eq!(+ attribute(read[0], attr::MACRO_CONTENT_TRUNCATED),+ Some(&OtelValue::Bool(true))+ );+}++#[test]+fn a_tool_call_opened_outside_any_turn_survives_a_later_turn_ending() {+ let (exporter, provider, _guard) = otel_test_pipeline();+ let mut projector = projector(ContentPolicy::enabled());+ // A call streamed by a resumed session, before this connection prompts.+ let open = RawJsonRpcMessage::notification(+ "session/update".to_owned(),+ json!({+ "sessionId": "s1",+ "update": {+ "sessionUpdate": "tool_call",+ "toolCallId": "bg-1",+ "title": "Bash",+ "kind": "execute",+ "status": "in_progress",+ "rawInput": { "command": "sleep 60" }+ }+ }),+ )+ .expect("a notification");+ projector.on_inbound(&ToServerMessage::Acp(AcpMessage(open)));++ // A whole turn passes.+ let prompt = RawJsonRpcMessage::request(+ "session/prompt".to_owned(),+ json!({ "sessionId": "s1", "prompt": [{ "type": "text", "text": "hi" }] }),+ RequestId::Str("p1".to_owned()),+ )+ .expect("a request");+ projector.on_outbound(&mut ToRuntimeMessage::Acp(AcpMessage(prompt)), None);+ projector.on_inbound(&ToServerMessage::Acp(AcpMessage(+ RawJsonRpcMessage::response(+ RequestId::Str("p1".to_owned()),+ Ok(json!({ "stopReason": "end_turn" })),+ ),+ )));++ // Then the background call finishes, and its span records that.+ let done = RawJsonRpcMessage::notification(+ "session/update".to_owned(),+ json!({+ "sessionId": "s1",+ "update": {+ "sessionUpdate": "tool_call_update",+ "toolCallId": "bg-1",+ "status": "completed",+ "rawOutput": { "exit": 0 }+ }+ }),+ )+ .expect("a notification");+ projector.on_inbound(&ToServerMessage::Acp(AcpMessage(done)));+ let spans = finished(&exporter, &provider);++ let bash = spans_named(&spans, "execute_tool Bash");+ assert_eq!(bash.len(), 1, "{spans:#?}");+ assert_eq!(bash[0].status, Status::Unset, "not abandoned by the turn");+ assert_eq!(+ json_attribute(bash[0], attr::TOOL_CALL_RESULT),+ Some(json!({ "exit": 0 }))+ );+ assert!(bash[0].end_time >= spans_named(&spans, "invoke_agent")[0].end_time);+}++#[test]+fn attached_resources_are_named_not_copied() {+ let (exporter, provider, _guard) = otel_test_pipeline();+ let mut projector = projector(ContentPolicy::enabled());+ let prompt = RawJsonRpcMessage::request(+ "session/prompt".to_owned(),+ json!({+ "sessionId": "s1",+ "prompt": [+ { "type": "text", "text": "summarize this" },+ { "type": "resource", "resource": {+ "uri": "https://user:[email protected]/doc.md?sig=abc",+ "mimeType": "text/markdown",+ "text": "CONFIDENTIAL BODY"+ } },+ { "type": "resource_link", "name": "notes", "uri": "macro://doc/42#p3" }+ ]+ }),+ RequestId::Str("p1".to_owned()),+ )+ .expect("a request");+ projector.on_outbound(&mut ToRuntimeMessage::Acp(AcpMessage(prompt)), None);+ projector.on_inbound(&ToServerMessage::Acp(AcpMessage(+ RawJsonRpcMessage::response(+ RequestId::Str("p1".to_owned()),+ Ok(json!({ "stopReason": "end_turn" })),+ ),+ )));+ let spans = finished(&exporter, &provider);++ let agent = spans_named(&spans, "invoke_agent")[0];+ let input = string_attribute(agent, attr::INPUT_MESSAGES).expect("input recorded");+ assert!(input.contains("summarize this"));+ assert!(!input.contains("CONFIDENTIAL BODY"), "{input}");+ assert!(!input.contains("secret"), "{input}");+ assert!(!input.contains("sig=abc"), "{input}");+ let parsed: Value = serde_json::from_str(&input).expect("valid JSON");+ assert_eq!(parsed[0]["parts"][1]["type"], "uri");+ assert_eq!(+ parsed[0]["parts"][1]["uri"],+ "https://files.example.com/doc.md"+ );+ assert_eq!(parsed[0]["parts"][2]["uri"], "macro://doc/42");+}++#[test]+fn tool_error_text_follows_the_content_policy() {+ let failing_call = |id: &str| {+ RawJsonRpcMessage::notification(+ "session/update".to_owned(),+ json!({+ "sessionId": "s1",+ "update": {+ "sessionUpdate": "tool_call",+ "toolCallId": id,+ "title": "Read",+ "kind": "read",+ "status": "failed",+ "rawOutput": {+ "content": [{ "type": "text", "text": "permission denied: /home/eric/.ssh/id_rsa" }],+ "isError": true+ }+ }+ }),+ )+ .expect("a notification")+ };++ let (exporter, provider, guard) = otel_test_pipeline();+ let mut quiet = projector(ContentPolicy::disabled());+ quiet.on_inbound(&ToServerMessage::Acp(AcpMessage(failing_call("t1"))));+ let spans = finished(&exporter, &provider);+ match &spans_named(&spans, "execute_tool Read")[0].status {+ Status::Error { description } => {+ assert!(!description.contains("id_rsa"), "{description}");+ }+ other => panic!("expected an error status, got {other:?}"),+ }+ drop(guard);++ let (exporter, provider, _guard) = otel_test_pipeline();+ let mut verbose = projector(ContentPolicy::enabled());+ verbose.on_inbound(&ToServerMessage::Acp(AcpMessage(failing_call("t2"))));+ let spans = finished(&exporter, &provider);+ match &spans_named(&spans, "execute_tool Read")[0].status {+ Status::Error { description } => {+ assert!(description.contains("permission denied"), "{description}");+ }+ other => panic!("expected an error status, got {other:?}"),+ }+}diff --git a/crates/ai_toolset/Cargo.toml b/crates/ai_toolset/Cargo.tomlindex 1eb7115baf..f8bb8d2787 100644--- a/crates/ai_toolset/Cargo.toml+++ b/crates/ai_toolset/Cargo.toml@@ -13,8 +13,16 @@ async-trait = { workspace = true }thiserror = { workspace = true }anyhow = { workspace = true }macro_user_id = { path = "../macro_user_id" }+genai_telemetry = { path = "../genai_telemetry" }axum = { workspace = true }axum-macros = { workspace = true }tracing = { workspace = true }workspace-hack = { version = "0.1", path = "../workspace-hack" }tokio-util.workspace = true++[dev-dependencies]+opentelemetry = { workspace = true, features = ["trace"] }+opentelemetry_sdk = { workspace = true, features = ["testing", "trace"] }+tokio = { workspace = true }+tracing-opentelemetry = { workspace = true }+tracing-subscriber = { workspace = true }diff --git a/crates/ai_toolset/TOOL_DESIGN.md b/crates/ai_toolset/TOOL_DESIGN.mdindex 90ff1dbce5..7ae4d20bf7 100644--- a/crates/ai_toolset/TOOL_DESIGN.md+++ b/crates/ai_toolset/TOOL_DESIGN.md@@ -131,3 +131,21 @@ Before adding a tool, verify:- Anthropic's guide: https://www.anthropic.com/engineering/writing-tools-for-agents- Crate documentation: `src/lib.rs`- Schema requirements: `src/schema/`++## Telemetry++Every tool call dispatched through `ToolSet::try_tool_call` runs inside an+OpenTelemetry GenAI `execute_tool` span (see `ai_toolset::telemetry`). The span+carries the tool name, the arguments as input and the result as output — the+attributes an LLM-as-a-judge evaluation of tool selection reads — and an error+status when the call fails. Two rules follow:++- **Tools and toolsets do not open tool telemetry themselves.** Implement+ `ToolSet::dispatch_tool_call` (routing only) and let `try_tool_call` wrap it;+ a toolset that delegates to children calls *their* `dispatch_tool_call`, so+ one call yields exactly one tool span. Inside a tool, ordinary+ `#[tracing::instrument]` spans for its own work are fine and nest under it.+- **Arguments and results are content**, recorded under the content policy+ (`OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT`, on unless set to+ `false`) and cut to a size budget. Do not record them as span fields or log+ them in tools: that bypasses the policy and repeats them on every log line.diff --git a/crates/ai_toolset/src/context.rs b/crates/ai_toolset/src/context.rsindex 15102cf8e3..94398f705b 100644--- a/crates/ai_toolset/src/context.rs+++ b/crates/ai_toolset/src/context.rs@@ -40,6 +40,12 @@ pub struct RequestContext {/// user has requested stream stop/// long-running tools should comsume thispub cancel: CancellationToken,+ /// Whether tool calls in this request record `execute_tool` GenAI+ /// telemetry (arguments, result, failure) on their span. On by default;+ /// off when another layer already reports the request's tool calls - the+ /// agent session actor does, for the in-process runtime - so each call is+ /// counted once.+ pub genai_telemetry: bool,}impl RequestContext {@@ -51,9 +57,17 @@ impl RequestContext {searchable_tools: Arc::new(Vec::new()),tool_loader: None,cancel: CancellationToken::new(),+ genai_telemetry: true,}}+ /// Turn `execute_tool` GenAI telemetry for this request's tool calls on or+ /// off (see [`Self::genai_telemetry`]).+ pub fn with_genai_telemetry(mut self, enabled: bool) -> Self {+ self.genai_telemetry = enabled;+ self+ }+/// Attach the searchable-tool catalog and loader that power `SearchTools`.pub fn with_tool_search(mut self,diff --git a/crates/ai_toolset/src/lib.rs b/crates/ai_toolset/src/lib.rsindex 5602b11277..0856c30719 100644--- a/crates/ai_toolset/src/lib.rs+++ b/crates/ai_toolset/src/lib.rs@@ -144,12 +144,20 @@//!//! assert!(parent_toolset.tools.contains_key("SubTool"));//! ```+//!+//! # Telemetry+//!+//! Every dispatch through [`ToolSet::try_tool_call`] runs inside an+//! OpenTelemetry GenAI `execute_tool` span carrying the tool name, arguments+//! and result (see [`telemetry`]). Implementors provide+//! [`ToolSet::dispatch_tool_call`] and never open tool telemetry themselves.#![deny(missing_docs)]mod annotations;mod context;pub mod schema;+pub mod telemetry;mod tool;pub mod tool_search;mod toolset;@@ -159,6 +167,6 @@ pub use context::{RequestContext, ServiceContext};pub use tool::{AsyncTool, NoContext, ToolCallError, ToolResult};pub use tool_search::{SearchableTool, ToolLoader};pub use toolset::{- AsyncToolCollection, RequestSchema, ToolCollection, ToolInfo, ToolSchema, ToolSet,- ToolSetCreationError, ToolSetError, tool_object,+ AsyncToolCollection, RequestSchema, ToolCallFuture, ToolCollection, ToolInfo, ToolSchema,+ ToolSet, ToolSetCreationError, ToolSetError, tool_object,};diff --git a/crates/ai_toolset/src/telemetry.rs b/crates/ai_toolset/src/telemetry.rsnew file mode 100644index 0000000000..bd6c629da1--- /dev/null+++ b/crates/ai_toolset/src/telemetry.rs@@ -0,0 +1,153 @@+//! Enforced `execute_tool` telemetry for every tool dispatch.+//!+//! [`ToolSet::try_tool_call`](crate::ToolSet::try_tool_call) is the one door a+//! tool call goes through — the agent loop, user-executed tools and pipelines+//! all dispatch via the trait — and it wraps every call in a [`ToolCallSpan`].+//! Two situations:+//!+//! - The agent runtime already opened the semconv `execute_tool` span around+//! the dispatch (it carries the tool name and the provider's call id). The+//! guard **enriches** that span rather than nesting a duplicate.+//! - Nothing did — a user confirming a deferred tool, a pipeline calling a+//! tool directly, a test. The guard **opens** `execute_tool {name}` itself.+//!+//! Either way the span ends up with what Datadog's tool evaluations read:+//! `gen_ai.operation.name`, `gen_ai.tool.name`, the arguments as input and the+//! result as output (both subject to the content policy and size bounds in+//! [`genai_telemetry`]), and an error status when the call failed.++#[cfg(test)]+mod test;++use crate::{ToolResult, ToolSetError};+use genai_telemetry::{+ ContentPolicy, GenAiSpanExt as _, attr, bounded_json_string, current_span_is_named,+ truncate_chars,+};+use serde::Serialize;++/// Telemetry guard for one tool call.+///+/// Create it with [`ToolCallSpan::begin`], run the call inside+/// [`ToolCallSpan::span`] (`fut.instrument(guard.span().clone())`), then hand+/// the outcome to [`ToolCallSpan::finish`].+#[derive(Debug)]+pub struct ToolCallSpan {+ span: tracing::Span,+ policy: ContentPolicy,+ tool_name: String,+}++impl ToolCallSpan {+ /// Begin telemetry for a call of `tool_name` with `arguments`, under the+ /// process-wide [`ContentPolicy`].+ pub fn begin(tool_name: &str, arguments: &serde_json::Value) -> Self {+ Self::begin_with_policy(tool_name, arguments, ContentPolicy::from_env())+ }++ /// [`Self::begin`] with an explicit content policy.+ pub fn begin_with_policy(+ tool_name: &str,+ arguments: &serde_json::Value,+ policy: ContentPolicy,+ ) -> Self {+ let span = if current_span_is_named(attr::span_name::EXECUTE_TOOL) {+ // The runtime's span already names the tool and carries the call+ // id; only the content and outcome are missing.+ tracing::Span::current()+ } else {+ // The literal must stay in step with `attr::span_name::EXECUTE_TOOL`+ // (`tracing` bakes the name into static metadata); `otel.name`+ // gives the exported span the semconv `execute_tool {name}` name.+ let otel_name = format!("{} {tool_name}", attr::operation::EXECUTE_TOOL);+ tracing::info_span!(+ "execute_tool",+ otel.name = %otel_name,+ gen_ai.operation.name = attr::operation::EXECUTE_TOOL,+ gen_ai.tool.type = attr::tool_type::FUNCTION,+ gen_ai.tool.name = %tool_name,+ )+ };+ if policy.capture {+ let (arguments, truncated) = bounded_json_string(arguments, &policy.limits);+ span.set_str(attr::TOOL_CALL_ARGUMENTS, arguments);+ if truncated {+ span.set_bool(attr::MACRO_CONTENT_TRUNCATED, true);+ }+ }+ Self {+ span,+ policy,+ tool_name: tool_name.to_owned(),+ }+ }++ /// The span the call must run inside.+ pub fn span(&self) -> &tracing::Span {+ &self.span+ }++ /// Record the outcome: the result as the span's output on success, an+ /// error status (and the model-facing description as output) otherwise.+ pub fn finish<T: Serialize>(&self, result: &Result<ToolResult<T>, ToolSetError>) {+ match result {+ Ok(Ok(output)) => {+ if self.policy.capture {+ match serde_json::to_value(output) {+ Ok(value) => self.record_result(&value),+ Err(error) => tracing::debug!(+ error = ?error,+ tool_name = %self.tool_name,+ "tool result is not serializable; not recorded on the span"+ ),+ }+ }+ }+ Ok(Err(error)) => {+ // The tool ran and failed; `description` is what the model+ // sees, so it doubles as the span's output - and is content,+ // recorded under the same policy.+ self.span.set_error(+ "tool_error",+ self.error_description(&error.description, "the tool reported an error"),+ );+ if self.policy.capture {+ self.record_result(&serde_json::Value::String(error.description.clone()));+ }+ }+ Err(error @ ToolSetError::NotFound(_)) => {+ tracing::warn!(tool_name = %self.tool_name, error = %error, "tool call could not be dispatched");+ self.span.set_error("tool_not_found", "no such tool");+ }+ Err(error @ ToolSetError::Deserialization(_)) => {+ tracing::warn!(tool_name = %self.tool_name, error = %error, "tool call could not be dispatched");+ // The message quotes the arguments it could not read.+ self.span.set_error(+ "invalid_arguments",+ self.error_description(+ &error.to_string(),+ "the arguments did not match the tool's schema",+ ),+ );+ }+ }+ }++ /// An error description fit for the span's status: the detail, bounded,+ /// when content may be recorded; a fixed summary otherwise.+ fn error_description(&self, detail: &str, summary: &'static str) -> String {+ if self.policy.capture {+ truncate_chars(detail, self.policy.limits.max_part_chars).into_owned()+ } else {+ summary.to_owned()+ }+ }++ fn record_result(&self, value: &serde_json::Value) {+ let (result, truncated) = bounded_json_string(value, &self.policy.limits);+ self.span.set_str(attr::TOOL_CALL_RESULT, result);+ if truncated {+ self.span.set_bool(attr::MACRO_CONTENT_TRUNCATED, true);+ }+ }+}diff --git a/crates/ai_toolset/src/telemetry/test.rs b/crates/ai_toolset/src/telemetry/test.rsnew file mode 100644index 0000000000..bf1d8fdff6--- /dev/null+++ b/crates/ai_toolset/src/telemetry/test.rs@@ -0,0 +1,369 @@+use super::*;+use crate::{+ AsyncTool, AsyncToolCollection, RequestContext, ServiceContext, ToolAnnotated, ToolAnnotations,+ ToolCallError, ToolSet as _,+};+use async_trait::async_trait;+use macro_user_id::user_id::MacroUserIdStr;+use opentelemetry::Value;+use opentelemetry::trace::{Status, TracerProvider as _};+use opentelemetry_sdk::trace::{InMemorySpanExporter, SdkTracerProvider, SpanData};+use schemars::JsonSchema;+use serde::Deserialize;+use serde_json::json;+use tracing::Instrument as _;+use tracing_subscriber::layer::SubscriberExt as _;++/// A tool that echoes its input.+#[derive(Deserialize, JsonSchema)]+#[schemars(title = "echo_tool", description = "Echoes its input back.")]+struct EchoTool {+ value: String,+}++impl ToolAnnotated for EchoTool {+ const ANNOTATIONS: ToolAnnotations = ToolAnnotations::read_only("Echo");+}++#[async_trait]+impl AsyncTool<()> for EchoTool {+ type Output = serde_json::Value;++ async fn call(+ &self,+ _service_context: ServiceContext<()>,+ _request_context: RequestContext,+ ) -> ToolResult<Self::Output> {+ Ok(json!({ "echo": self.value }))+ }+}++/// A tool that always fails.+#[derive(Deserialize, JsonSchema)]+#[schemars(title = "boom_tool", description = "Always fails.")]+struct BoomTool {}++impl ToolAnnotated for BoomTool {+ const ANNOTATIONS: ToolAnnotations = ToolAnnotations::read_only("Boom");+}++#[async_trait]+impl AsyncTool<()> for BoomTool {+ type Output = serde_json::Value;++ async fn call(+ &self,+ _service_context: ServiceContext<()>,+ _request_context: RequestContext,+ ) -> ToolResult<Self::Output> {+ Err(ToolCallError {+ description: "boom failed".to_string(),+ internal_error: anyhow::anyhow!("boom"),+ })+ }+}++fn toolset() -> AsyncToolCollection<()> {+ AsyncToolCollection::<()>::new()+ .add_tool::<EchoTool, ()>()+ .add_tool::<BoomTool, ()>()+}++/// `echo_tool` registered as a user-executed (deferred) tool instead.+fn user_toolset() -> AsyncToolCollection<()> {+ AsyncToolCollection::<()>::new().add_user_tool::<EchoTool, ()>()+}++fn request_context() -> RequestContext {+ RequestContext::new(MacroUserIdStr::try_from_email("[email protected]").expect("valid user id"))+}++/// An OpenTelemetry pipeline that keeps finished spans in memory, installed as+/// the thread's default subscriber for the returned guard's lifetime.+fn otel_test_pipeline() -> (+ InMemorySpanExporter,+ SdkTracerProvider,+ tracing::subscriber::DefaultGuard,+) {+ let exporter = InMemorySpanExporter::default();+ let provider = SdkTracerProvider::builder()+ .with_simple_exporter(exporter.clone())+ .build();+ let layer = tracing_opentelemetry::layer().with_tracer(provider.tracer("test"));+ let guard = tracing::subscriber::set_default(tracing_subscriber::registry().with(layer));+ // Other tests in this binary run the same code paths with no subscriber+ // installed, which can cache "never interested" for a span callsite on+ // another thread; recompute so this subscriber sees every span.+ tracing::callsite::rebuild_interest_cache();+ (exporter, provider, guard)+}++fn finished(exporter: &InMemorySpanExporter, provider: &SdkTracerProvider) -> Vec<SpanData> {+ provider.force_flush().expect("flush");+ exporter.get_finished_spans().expect("finished spans")+}++fn string_attribute(span: &SpanData, key: &str) -> Option<String> {+ span.attributes+ .iter()+ .find(|kv| kv.key.as_str() == key)+ .and_then(|kv| match &kv.value {+ Value::String(s) => Some(s.as_str().to_string()),+ _ => None,+ })+}++fn tool_spans(spans: &[SpanData]) -> Vec<&SpanData> {+ spans+ .iter()+ .filter(|span| span.name.starts_with("execute_tool"))+ .collect()+}++#[tokio::test]+async fn try_tool_call_opens_an_execute_tool_span_with_arguments_and_result() {+ let (exporter, provider, _guard) = otel_test_pipeline();+ let toolset = toolset();++ let result = toolset+ .try_tool_call((), request_context(), "echo_tool", &json!({ "value": "a" }))+ .await+ .expect("dispatch")+ .expect("tool success");+ assert_eq!(result, json!({ "echo": "a" }));++ let spans = finished(&exporter, &provider);+ let tool_spans = tool_spans(&spans);+ assert_eq!(tool_spans.len(), 1, "exactly one tool span, got {spans:#?}");+ let span = tool_spans[0];+ assert_eq!(span.name, "execute_tool echo_tool");+ assert_eq!(+ string_attribute(span, attr::OPERATION_NAME).as_deref(),+ Some("execute_tool")+ );+ assert_eq!(+ string_attribute(span, attr::TOOL_TYPE).as_deref(),+ Some("function")+ );+ assert_eq!(+ string_attribute(span, attr::TOOL_NAME).as_deref(),+ Some("echo_tool")+ );+ assert_eq!(+ string_attribute(span, attr::TOOL_CALL_ARGUMENTS).as_deref(),+ Some(r#"{"value":"a"}"#)+ );+ assert_eq!(+ string_attribute(span, attr::TOOL_CALL_RESULT).as_deref(),+ Some(r#"{"echo":"a"}"#)+ );+ assert_eq!(span.status, Status::Unset);+}++#[tokio::test]+async fn try_tool_call_enriches_an_open_execute_tool_span_instead_of_nesting() {+ let (exporter, provider, _guard) = otel_test_pipeline();+ let toolset = toolset();++ // The agent runtime opens `execute_tool` around the dispatch; the toolset+ // must add to it, not open a second tool span underneath.+ let runtime_span = tracing::info_span!("execute_tool");+ toolset+ .try_tool_call((), request_context(), "echo_tool", &json!({ "value": "b" }))+ .instrument(runtime_span)+ .await+ .expect("dispatch")+ .expect("tool success");++ let spans = finished(&exporter, &provider);+ let tool_spans = tool_spans(&spans);+ assert_eq!(tool_spans.len(), 1, "no nested tool span, got {spans:#?}");+ let span = tool_spans[0];+ assert_eq!(+ span.name, "execute_tool",+ "the runtime's span is kept as-is"+ );+ assert_eq!(+ string_attribute(span, attr::TOOL_CALL_ARGUMENTS).as_deref(),+ Some(r#"{"value":"b"}"#)+ );+ assert_eq!(+ string_attribute(span, attr::TOOL_CALL_RESULT).as_deref(),+ Some(r#"{"echo":"b"}"#)+ );+}++#[tokio::test]+async fn a_failing_tool_marks_the_span_failed_with_its_description_as_output() {+ let (exporter, provider, _guard) = otel_test_pipeline();+ let toolset = toolset();++ let result = toolset+ .try_tool_call((), request_context(), "boom_tool", &json!({}))+ .await+ .expect("dispatch");+ assert!(result.is_err());++ let spans = finished(&exporter, &provider);+ let span = tool_spans(&spans)[0];+ assert!(+ matches!(&span.status, Status::Error { description } if description == "boom failed"),+ "{:?}",+ span.status+ );+ assert_eq!(+ string_attribute(span, attr::ERROR_TYPE).as_deref(),+ Some("tool_error")+ );+ assert_eq!(+ string_attribute(span, attr::TOOL_CALL_RESULT).as_deref(),+ Some("boom failed")+ );+}++#[tokio::test]+async fn dispatch_errors_are_classified() {+ let (exporter, provider, _guard) = otel_test_pipeline();+ let toolset = toolset();++ assert!(matches!(+ toolset+ .try_tool_call((), request_context(), "missing_tool", &json!({}))+ .await,+ Err(ToolSetError::NotFound(_))+ ));+ assert!(matches!(+ toolset+ .try_tool_call((), request_context(), "echo_tool", &json!({ "wrong": 1 }))+ .await,+ Err(ToolSetError::Deserialization(_))+ ));++ let spans = finished(&exporter, &provider);+ let tool_spans = tool_spans(&spans);+ assert_eq!(tool_spans.len(), 2);+ assert_eq!(tool_spans[0].name, "execute_tool missing_tool");+ assert_eq!(+ string_attribute(tool_spans[0], attr::ERROR_TYPE).as_deref(),+ Some("tool_not_found")+ );+ assert_eq!(+ string_attribute(tool_spans[1], attr::ERROR_TYPE).as_deref(),+ Some("invalid_arguments")+ );+ assert!(+ tool_spans+ .iter()+ .all(|span| matches!(span.status, Status::Error { .. }))+ );+}++#[tokio::test]+async fn user_tool_calls_are_traced_too() {+ let (exporter, provider, _guard) = otel_test_pipeline();+ let toolset = user_toolset();++ toolset+ .try_user_tool_call((), request_context(), "echo_tool", &json!({ "value": "c" }))+ .await+ .expect("dispatch")+ .expect("tool success");++ let spans = finished(&exporter, &provider);+ let span = tool_spans(&spans)[0];+ assert_eq!(span.name, "execute_tool echo_tool");+ assert_eq!(+ string_attribute(span, attr::TOOL_CALL_ARGUMENTS).as_deref(),+ Some(r#"{"value":"c"}"#)+ );+ // The tool's own output, not the `UserToolResponse` API wrapper.+ assert_eq!(+ string_attribute(span, attr::TOOL_CALL_RESULT).as_deref(),+ Some(r#"{"echo":"c"}"#)+ );+}++#[tokio::test]+async fn content_capture_off_records_structure_only() {+ let (exporter, provider, _guard) = otel_test_pipeline();++ let guard = ToolCallSpan::begin_with_policy(+ "echo_tool",+ &json!({ "value": "secret" }),+ ContentPolicy::disabled(),+ );+ let result: Result<ToolResult<serde_json::Value>, ToolSetError> = Ok(Err(ToolCallError {+ description: "private failure".to_string(),+ internal_error: anyhow::anyhow!("boom"),+ }));+ guard.finish(&result);+ drop(guard);++ let spans = finished(&exporter, &provider);+ let span = tool_spans(&spans)[0];+ assert_eq!(+ string_attribute(span, attr::TOOL_NAME).as_deref(),+ Some("echo_tool")+ );+ assert_eq!(string_attribute(span, attr::TOOL_CALL_ARGUMENTS), None);+ assert_eq!(string_attribute(span, attr::TOOL_CALL_RESULT), None);+ // The failure itself is not content, but its description is.+ match &span.status {+ Status::Error { description } => {+ assert!(!description.contains("private failure"), "{description}");+ }+ other => panic!("expected an error status, got {other:?}"),+ }+}++#[tokio::test]+async fn oversized_content_is_bounded_and_flagged() {+ let (exporter, provider, _guard) = otel_test_pipeline();+ let toolset = toolset();++ let big = "x".repeat(20_000);+ toolset+ .try_tool_call((), request_context(), "echo_tool", &json!({ "value": big }))+ .await+ .expect("dispatch")+ .expect("tool success");++ let spans = finished(&exporter, &provider);+ let span = tool_spans(&spans)[0];+ let arguments = string_attribute(span, attr::TOOL_CALL_ARGUMENTS).expect("arguments");+ assert!(+ arguments.len() < 10_000,+ "arguments should be cut, got {} bytes",+ arguments.len()+ );+ assert!(arguments.contains("[truncated"), "{arguments}");+ assert_eq!(+ span.attributes+ .iter()+ .find(|kv| kv.key.as_str() == attr::MACRO_CONTENT_TRUNCATED)+ .map(|kv| kv.value.clone()),+ Some(Value::Bool(true))+ );+}++/// A request that opted out - another layer reports its tool calls - runs+/// its tools without `execute_tool` telemetry of their own.+#[tokio::test]+async fn a_request_that_opted_out_records_no_tool_span() {+ let (exporter, provider, _guard) = otel_test_pipeline();+ let toolset = toolset();++ toolset+ .try_tool_call(+ (),+ request_context().with_genai_telemetry(false),+ "echo_tool",+ &json!({ "value": "quiet" }),+ )+ .await+ .expect("dispatch")+ .expect("tool success");++ let spans = finished(&exporter, &provider);+ assert!(tool_spans(&spans).is_empty(), "{spans:#?}");+}diff --git a/crates/ai_toolset/src/toolset/mod.rs b/crates/ai_toolset/src/toolset/mod.rsindex 65c7c6c591..c27b132e46 100644--- a/crates/ai_toolset/src/toolset/mod.rs+++ b/crates/ai_toolset/src/toolset/mod.rs@@ -4,7 +4,7 @@ pub mod tool_object;mod traits;mod types;-pub use traits::ToolSet;+pub use traits::{ToolCallFuture, ToolSet};pub use types::{AsyncToolCollection, RequestSchema, ToolCollection, ToolInfo, ToolSchema, ToolSetCreationError,ToolSetError,diff --git a/crates/ai_toolset/src/toolset/tool_object/tool_async.rs b/crates/ai_toolset/src/toolset/tool_object/tool_async.rsindex e51bb7dab7..1a67ffef0e 100644--- a/crates/ai_toolset/src/toolset/tool_object/tool_async.rs+++ b/crates/ai_toolset/src/toolset/tool_object/tool_async.rs@@ -123,7 +123,10 @@ pub type AsyncToolObject<ToolSetContext> = ToolObject<AsyncDeserializer<ToolSetCimpl<ToolSetContext> ToolObject<AsyncDeserializer<ToolSetContext>> {/// Attempts to deserialize JSON input into a callable async tool instance.- #[tracing::instrument(err, skip(self))]+ ///+ /// `data` is the model's arguments: content, recorded on the tool span+ /// under the content policy rather than as a field here.+ #[tracing::instrument(err, skip_all)]pub fn try_deserialize(&self,data: &serde_json::Value,diff --git a/crates/ai_toolset/src/toolset/traits.rs b/crates/ai_toolset/src/toolset/traits.rsindex fb84991cb2..345538a121 100644--- a/crates/ai_toolset/src/toolset/traits.rs+++ b/crates/ai_toolset/src/toolset/traits.rs@@ -1,20 +1,64 @@//! The core ToolSet trait implemented by the ToolSet and AsyncToolset+use crate::telemetry::ToolCallSpan;use crate::toolset::types::{RequestSchema, ToolInfo};use crate::{AsyncToolCollection, RequestContext, SearchableTool, ToolResult, ToolSetError};use std::pin::Pin;+use tracing::Instrument as _;++/// The future a tool dispatch resolves to.+pub type ToolCallFuture<'a> =+ Pin<Box<dyn Future<Output = Result<ToolResult<serde_json::Value>, ToolSetError>> + 'a + Send>>;/// An object with a set of toolspub trait ToolSet<Context>: Send + Sync {- /// Try to call a tool indexed by name with the provided raw json+ /// Route a tool call by name to its implementation and return the result.+ ///+ /// This is the implementor's hook and carries **no telemetry**:+ /// [`Self::try_tool_call`] wraps it in the `execute_tool` span (see+ /// [`crate::telemetry`]). A toolset that delegates to child toolsets calls+ /// the children's `dispatch_tool_call`, so one tool call yields one span.+ fn dispatch_tool_call<'a>(+ &'a self,+ context: Context,+ request_context: RequestContext,+ tool_name: &'a str,+ json: &'a serde_json::Value,+ ) -> ToolCallFuture<'a>;++ /// Try to call a tool indexed by name with the provided raw json.+ ///+ /// This is what callers use — the agent loop, user-executed tools,+ /// pipelines. Every call runs inside `execute_tool` telemetry: the span is+ /// enriched when the agent runtime already opened it, opened otherwise, and+ /// records the arguments, the result and any failure (see+ /// [`crate::telemetry::ToolCallSpan`]) - unless the request opted out+ /// ([`RequestContext::genai_telemetry`]) because another layer reports its+ /// tool calls.fn try_tool_call<'a>(&'a self,context: Context,request_context: RequestContext,tool_name: &'a str,json: &'a serde_json::Value,- ) -> Pin<- Box<dyn Future<Output = Result<ToolResult<serde_json::Value>, ToolSetError>> + 'a + Send>,- >;+ ) -> ToolCallFuture<'a>+ where+ Context: Send + 'a,+ {+ Box::pin(async move {+ if !request_context.genai_telemetry {+ return self+ .dispatch_tool_call(context, request_context, tool_name, json)+ .await;+ }+ let telemetry = ToolCallSpan::begin(tool_name, json);+ let result = self+ .dispatch_tool_call(context, request_context, tool_name, json)+ .instrument(telemetry.span().clone())+ .await;+ telemetry.finish(&result);+ result+ })+ }/// Returns the input schemas for the tools sent to the provider on every/// request. Tools loaded on demand via tool search are excluded — they are@@ -45,15 +89,13 @@ impl<Context> ToolSet<Context> for AsyncToolCollection<Context>whereContext: Send + Sync + 'static,{- fn try_tool_call<'a>(+ fn dispatch_tool_call<'a>(&'a self,context: Context,request_context: RequestContext,tool_name: &'a str,json: &'a serde_json::Value,- ) -> Pin<- Box<dyn Future<Output = Result<ToolResult<serde_json::Value>, ToolSetError>> + 'a + Send>,- > {+ ) -> ToolCallFuture<'a> {Box::pin(self.try_tool_call_internal(context, request_context, tool_name, json))}diff --git a/crates/ai_toolset/src/toolset/types.rs b/crates/ai_toolset/src/toolset/types.rsindex f1ec7fbc51..bb0be177d7 100644--- a/crates/ai_toolset/src/toolset/types.rs+++ b/crates/ai_toolset/src/toolset/types.rs@@ -3,6 +3,7 @@ use super::tool_object::{AsyncToolObject, UserTool, UserToolResponse};use crate::RequestContext;use crate::annotations::ToolAnnotated;use crate::schema::ValidationError;+use crate::telemetry::ToolCallSpan;use crate::{AsyncTool, ToolResult};use axum::extract::FromRef;use schemars::{JsonSchema, Schema};@@ -10,6 +11,7 @@ use serde::Serialize;use serde::de::Deserialize;use std::collections::BTreeMap;use thiserror::Error;+use tracing::Instrument as _;#[cfg(test)]mod test;@@ -200,7 +202,10 @@ where/// was added.////// Returns an error if the tool is not found or if deserialization fails.- #[tracing::instrument(err, skip(self, context, request_context))]+ ///+ /// No span of its own: it runs inside the `execute_tool` span that+ /// [`ToolSet::try_tool_call`](crate::ToolSet::try_tool_call) opens, which+ /// records the arguments and result under the content policy.pub(crate) async fn try_tool_call_internal(&self,context: ToolSetContext,@@ -220,7 +225,9 @@ where}/// this isn't called in the tool loop it's called by the user-facing API- #[tracing::instrument(err, skip(self, context, request_context))]+ ///+ /// A user-executed tool is still a tool execution, so it gets the same+ /// `execute_tool` telemetry as a model-initiated call.pub async fn try_user_tool_call(&self,context: ToolSetContext,@@ -228,22 +235,35 @@ wheretool_name: &str,json: &serde_json::Value,) -> Result<ToolResult<UserToolResponse<serde_json::Value>>, ToolSetError> {- let tool = self- .user_tools- .get(tool_name)- .ok_or_else(|| ToolSetError::NotFound(tool_name.to_owned()))- .and_then(|tool| {- tool.try_deserialize(json)- .map_err(ToolSetError::Deserialization)- })?;- Ok(tool- .call(context, request_context)- .await- .map(UserToolResponse::UserAction))+ let telemetry = request_context+ .genai_telemetry+ .then(|| ToolCallSpan::begin(tool_name, json));+ let span = telemetry+ .as_ref()+ .map_or_else(tracing::Span::none, |telemetry| telemetry.span().clone());+ let result: Result<ToolResult<serde_json::Value>, ToolSetError> = async {+ let tool = self+ .user_tools+ .get(tool_name)+ .ok_or_else(|| ToolSetError::NotFound(tool_name.to_owned()))+ .and_then(|tool| {+ tool.try_deserialize(json)+ .map_err(ToolSetError::Deserialization)+ })?;+ Ok(tool.call(context, request_context).await)+ }+ .instrument(span)+ .await;+ // Recorded before the API wrapper goes on, so the span carries the+ // tool's own output rather than `{"UserAction": ...}`.+ if let Some(telemetry) = &telemetry {+ telemetry.finish(&result);+ }+ result.map(|result| result.map(UserToolResponse::UserAction))}/// check if json + name matches a known tool in the toolset- #[tracing::instrument(skip(self))]+ #[tracing::instrument(skip_all, fields(tool_name = %tool_name))]pub fn is_valid_tool(&self, tool_name: &str, json: &serde_json::Value) -> bool {let Some(tool) = self.user_toolsdiff --git a/crates/genai_telemetry/Cargo.toml b/crates/genai_telemetry/Cargo.tomlnew file mode 100644index 0000000000..fe1588158c--- /dev/null+++ b/crates/genai_telemetry/Cargo.toml@@ -0,0 +1,17 @@+[package]+edition = "2024"+name = "genai_telemetry"+publish = false+version = "0.1.0"++[dependencies]+macro_env_var = { path = "../macro_env_var" }+opentelemetry = { workspace = true, features = ["trace"] }+serde_json = { workspace = true }+tracing = { workspace = true }+tracing-opentelemetry = { workspace = true }+workspace-hack = { version = "0.1", path = "../workspace-hack" }++[dev-dependencies]+opentelemetry_sdk = { workspace = true, features = ["testing", "trace"] }+tracing-subscriber = { workspace = true }diff --git a/crates/genai_telemetry/src/attr.rs b/crates/genai_telemetry/src/attr.rsnew file mode 100644index 0000000000..45e5de5afa--- /dev/null+++ b/crates/genai_telemetry/src/attr.rs@@ -0,0 +1,122 @@+//! OpenTelemetry GenAI semantic-convention attribute names and well-known values.+//!+//! Mirrors the `gen_ai.*` registry of the OpenTelemetry GenAI semantic+//! conventions, plus the two `macro.genai.*` attributes this workspace adds to+//! describe its own truncation. Datadog maps these onto its LLM Observability+//! schema: `gen_ai.operation.name` picks the span kind (`chat` → llm,+//! `execute_tool` → tool, `invoke_agent` → agent), `gen_ai.conversation.id` is+//! the session, `gen_ai.tool.definitions` the tool set an evaluation judges a+//! tool choice against, and the `*.messages` / `tool.call.*` attributes are the+//! span's input and output.++/// The operation performed: see [`operation`].+pub const OPERATION_NAME: &str = "gen_ai.operation.name";+/// The GenAI provider serving the request (`anthropic`, `openai`, …).+pub const PROVIDER_NAME: &str = "gen_ai.provider.name";+/// The model requested.+pub const REQUEST_MODEL: &str = "gen_ai.request.model";+/// The `max_tokens` request parameter.+pub const REQUEST_MAX_TOKENS: &str = "gen_ai.request.max_tokens";+/// The `temperature` request parameter.+pub const REQUEST_TEMPERATURE: &str = "gen_ai.request.temperature";+/// The model that actually answered.+pub const RESPONSE_MODEL: &str = "gen_ai.response.model";+/// The provider-assigned response id.+pub const RESPONSE_ID: &str = "gen_ai.response.id";+/// Why generation stopped, one entry per choice: see [`finish_reason`].+pub const RESPONSE_FINISH_REASONS: &str = "gen_ai.response.finish_reasons";+/// Input tokens consumed.+pub const USAGE_INPUT_TOKENS: &str = "gen_ai.usage.input_tokens";+/// Output tokens produced.+pub const USAGE_OUTPUT_TOKENS: &str = "gen_ai.usage.output_tokens";+/// The messages sent to the model / agent, as a JSON array of+/// `{"role", "parts"}` objects (see [`crate::messages`]).+pub const INPUT_MESSAGES: &str = "gen_ai.input.messages";+/// The messages produced by the model / agent, as a JSON array of+/// `{"role", "parts", "finish_reason"}` objects (see [`crate::messages`]).+pub const OUTPUT_MESSAGES: &str = "gen_ai.output.messages";+/// The system instructions, as a JSON array of parts, kept separate from the+/// chat history.+pub const SYSTEM_INSTRUCTIONS: &str = "gen_ai.system_instructions";+/// The tools offered to the model, as a JSON array of+/// `{"type": "function", "name", "description", "parameters"}` objects.+pub const TOOL_DEFINITIONS: &str = "gen_ai.tool.definitions";+/// The name of the tool executed.+pub const TOOL_NAME: &str = "gen_ai.tool.name";+/// The description of the tool executed.+pub const TOOL_DESCRIPTION: &str = "gen_ai.tool.description";+/// The kind of tool executed: see [`tool_type`].+pub const TOOL_TYPE: &str = "gen_ai.tool.type";+/// The provider-assigned tool call id.+pub const TOOL_CALL_ID: &str = "gen_ai.tool.call.id";+/// The arguments the tool was called with (opt-in content).+pub const TOOL_CALL_ARGUMENTS: &str = "gen_ai.tool.call.arguments";+/// The result the tool returned (opt-in content).+pub const TOOL_CALL_RESULT: &str = "gen_ai.tool.call.result";+/// The name of the agent.+pub const AGENT_NAME: &str = "gen_ai.agent.name";+/// The conversation (session) this operation belongs to.+pub const CONVERSATION_ID: &str = "gen_ai.conversation.id";+/// Low-cardinality error class when the operation failed.+pub const ERROR_TYPE: &str = "error.type";++/// Macro-specific: how many of the oldest messages were dropped from+/// [`INPUT_MESSAGES`] to fit the size budget (see [`crate::Limits`]).+pub const MACRO_INPUT_MESSAGES_OMITTED: &str = "macro.genai.input_messages_omitted";+/// Macro-specific: `true` when any content attribute on the span was cut to fit+/// the size budget (see [`crate::Limits`]).+pub const MACRO_CONTENT_TRUNCATED: &str = "macro.genai.content_truncated";+/// Macro-specific: tokens in the agent's context window after a turn, as the+/// harness reported it (ACP `usage_update.used`).+pub const MACRO_CONTEXT_USED_TOKENS: &str = "macro.genai.context.used_tokens";+/// Macro-specific: the agent's context window size in tokens (ACP+/// `usage_update.size`).+pub const MACRO_CONTEXT_SIZE_TOKENS: &str = "macro.genai.context.size_tokens";+/// Macro-specific: the coarse ACP tool kind (`execute`, `edit`, `read`, …).+pub const MACRO_TOOL_KIND: &str = "macro.genai.tool.kind";+/// Macro-specific: the human-readable title the harness gave a tool call.+pub const MACRO_TOOL_TITLE: &str = "macro.genai.tool.title";+/// Macro-specific: the MCP server a tool was reached through, when it was.+pub const MACRO_MCP_SERVER: &str = "macro.genai.mcp.server";++/// Well-known [`OPERATION_NAME`] values.+pub mod operation {+ /// An agent run (a turn of the agent loop).+ pub const INVOKE_AGENT: &str = "invoke_agent";+ /// A model completion.+ pub const CHAT: &str = "chat";+ /// A tool execution.+ pub const EXECUTE_TOOL: &str = "execute_tool";+}++/// Well-known [`TOOL_TYPE`] values.+pub mod tool_type {+ /// A client-side function the agent executes.+ pub const FUNCTION: &str = "function";+}++/// Well-known [`RESPONSE_FINISH_REASONS`] values.+pub mod finish_reason {+ /// The model finished naturally.+ pub const STOP: &str = "stop";+ /// The model stopped to call one or more tools.+ pub const TOOL_CALL: &str = "tool_call";+ /// The model hit a token or request limit.+ pub const LENGTH: &str = "length";+ /// The model refused to continue.+ pub const CONTENT_FILTER: &str = "content_filter";+ /// The run failed before the model finished.+ pub const ERROR: &str = "error";+ /// The run was cancelled by the caller.+ pub const CANCELLED: &str = "cancelled";+}++/// Span names used for the semconv spans this workspace opens or adopts. The+/// agent runtime names its spans identically, which is how the toolset detects+/// an already-open tool span (see [`crate::current_span_is_named`]).+pub mod span_name {+ /// The per-tool-call span.+ pub const EXECUTE_TOOL: &str = "execute_tool";+ /// The per-agent-run span.+ pub const INVOKE_AGENT: &str = "invoke_agent";+}diff --git a/crates/genai_telemetry/src/content.rs b/crates/genai_telemetry/src/content.rsnew file mode 100644index 0000000000..92fc82d021--- /dev/null+++ b/crates/genai_telemetry/src/content.rs@@ -0,0 +1,317 @@+//! Content-capture policy and size bounding for GenAI span attributes.+//!+//! Prompts, tool arguments, tool results and model output are the most useful+//! and the most dangerous span attributes: an LLM-as-a-judge evaluation needs+//! them, but they carry user data and can be arbitrarily large. Two controls+//! apply to every content attribute this workspace records:+//!+//! - **Capture** — [`ContentPolicy::capture`], read once from the+//! OpenTelemetry-standard `OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT`+//! variable. It defaults to **on** here (the spec's default is off): the point+//! of the instrumentation is evaluation, which is impossible without content,+//! and traces already reach the same backend as the service logs. Set it to+//! `false` to export structure (spans, names, usage, ids) only.+//! - **Size** — [`Limits`]. Every string leaf is cut to `max_part_chars`, and+//! a whole attribute to `max_attribute_bytes`; message lists shed their+//! oldest entries first so the most recent turn survives. Unbounded content+//! would blow the OTLP batch past the agent's receive limit (dropping the+//! whole batch) and pay for the same conversation once per model call.++#[cfg(test)]+mod test;++use macro_env_var::maybe_env_vars;+use std::borrow::Cow;+use std::sync::OnceLock;++maybe_env_vars! {+ pub struct OtelInstrumentationGenaiCaptureMessageContent;+}++/// Size budgets for content attributes.+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub struct Limits {+ /// Maximum characters of a single string leaf — a text part, a tool+ /// argument value, a tool response. Longer values are cut and marked.+ pub max_part_chars: usize,+ /// Maximum bytes of one serialized attribute value. Message lists drop+ /// their oldest messages to fit; tool definitions drop their parameter+ /// schemas; anything else is cut and marked.+ pub max_attribute_bytes: usize,+}++impl Default for Limits {+ fn default() -> Self {+ Self {+ max_part_chars: 8_000,+ max_attribute_bytes: 64_000,+ }+ }+}++/// Whether, and how much, content may be recorded on GenAI spans.+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub struct ContentPolicy {+ /// Record prompts, tool arguments/results and model output.+ pub capture: bool,+ /// Size budgets applied to everything recorded.+ pub limits: Limits,+}++impl Default for ContentPolicy {+ /// Capture on, default [`Limits`] — the same as an unset environment.+ fn default() -> Self {+ Self::enabled()+ }+}++impl ContentPolicy {+ /// Capture content with the default [`Limits`].+ pub const fn enabled() -> Self {+ Self {+ capture: true,+ limits: Limits {+ max_part_chars: 8_000,+ max_attribute_bytes: 64_000,+ },+ }+ }++ /// Record structure only.+ pub const fn disabled() -> Self {+ Self {+ capture: false,+ ..Self::enabled()+ }+ }++ /// The process-wide policy, read from+ /// `OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT` once and cached.+ /// Unset means capture is on; see [`Self::parse`].+ pub fn from_env() -> Self {+ static POLICY: OnceLock<ContentPolicy> = OnceLock::new();+ *POLICY.get_or_init(|| {+ Self::parse(OtelInstrumentationGenaiCaptureMessageContent::new().as_deref())+ })+ }++ /// Interpret the capture switch: `false`, `0`, `no` and `off`+ /// (case-insensitive) disable capture; anything else, including unset,+ /// enables it.+ pub fn parse(value: Option<&str>) -> Self {+ let off = value.is_some_and(|value| {+ matches!(+ value.trim().to_ascii_lowercase().as_str(),+ "false" | "0" | "no" | "off"+ )+ });+ if off {+ Self::disabled()+ } else {+ Self::enabled()+ }+ }+}++/// Cut `s` to at most `max_chars` characters, appending a marker naming how+/// many characters were dropped. Borrows when nothing needs cutting.+pub fn truncate_chars(s: &str, max_chars: usize) -> Cow<'_, str> {+ // Bytes are never fewer than chars, so this cheap check settles the common case.+ if s.len() <= max_chars {+ return Cow::Borrowed(s);+ }+ let total = s.chars().count();+ if total <= max_chars {+ return Cow::Borrowed(s);+ }+ let mut cut: String = s.chars().take(max_chars).collect();+ cut.push_str(&format!("…[truncated {} chars]", total - max_chars));+ Cow::Owned(cut)+}++/// Room kept for the marker [`truncate_bytes`] appends, so a cut value never+/// overshoots its byte budget.+const TRUNCATION_MARKER_RESERVE: usize = 40;++/// Cut `s` to at most `max_bytes` bytes of UTF-8, on a character boundary,+/// appending a marker naming how many bytes were dropped. The marker is+/// counted against the budget. Borrows when nothing needs cutting.+pub fn truncate_bytes(s: &str, max_bytes: usize) -> Cow<'_, str> {+ if s.len() <= max_bytes {+ return Cow::Borrowed(s);+ }+ // A budget too small to hold the marker keeps half of itself; the marker+ // then overshoots, which beats recording nothing at all.+ let mut keep = max_bytes.saturating_sub(TRUNCATION_MARKER_RESERVE.min(max_bytes / 2));+ while keep > 0 && !s.is_char_boundary(keep) {+ keep -= 1;+ }+ let mut cut = s[..keep].to_owned();+ cut.push_str(&format!("…[truncated {} bytes]", s.len() - keep));+ Cow::Owned(cut)+}++/// Cut every string leaf in `value` to `max_part_chars` characters, in place.+/// Returns whether anything was cut.+fn bound_json_strings(value: &mut serde_json::Value, max_part_chars: usize) -> bool {+ match value {+ serde_json::Value::String(s) => match truncate_chars(s, max_part_chars) {+ Cow::Borrowed(_) => false,+ Cow::Owned(cut) => {+ *s = cut;+ true+ }+ },+ serde_json::Value::Array(items) => items.iter_mut().fold(false, |cut, item| {+ bound_json_strings(item, max_part_chars) || cut+ }),+ serde_json::Value::Object(map) => map.values_mut().fold(false, |cut, item| {+ bound_json_strings(item, max_part_chars) || cut+ }),+ _ => false,+ }+}++/// Serialize `value` as an opaque string attribute within `limits`: string+/// leaves are cut to `max_part_chars` characters, then the whole+/// serialization to `max_attribute_bytes` bytes. Returns the string and+/// whether anything was cut.+///+/// A cut serialization is no longer valid JSON, so use this only for+/// attributes the backend treats as plain text (tool arguments and results);+/// message lists and tool definitions have structure-preserving bounds+/// ([`bound_messages`], [`bound_tool_definitions`]).+pub fn bounded_json_string(value: &serde_json::Value, limits: &Limits) -> (String, bool) {+ let mut value = value.clone();+ let cut_parts = bound_json_strings(&mut value, limits.max_part_chars);+ let json = match value {+ // A bare string reads better than its quoted JSON encoding.+ serde_json::Value::String(s) => s,+ other => other.to_string(),+ };+ match truncate_bytes(&json, limits.max_attribute_bytes) {+ Cow::Borrowed(_) => (json, cut_parts),+ Cow::Owned(cut) => (cut, true),+ }+}++/// A bounded `gen_ai.*.messages` list.+#[derive(Debug, Clone, PartialEq, Eq)]+pub struct BoundedMessages {+ /// The serialized JSON array.+ pub json: String,+ /// How many of the oldest messages were dropped to fit the budget.+ pub omitted: usize,+ /// Whether any content was cut (parts shortened or messages dropped).+ pub truncated: bool,+}++/// Bound a message list (see [`crate::messages`]) to `limits`: every string+/// leaf is cut to `max_part_chars`, then the oldest messages are dropped —+/// never the last one, which is the current turn — until the serialized list+/// fits `max_attribute_bytes`. A last message that is over budget on its own+/// is shrunk in place - its leaves cut shorter, then its oldest parts dropped+/// - so the result is always a valid JSON array.+pub fn bound_messages(mut messages: Vec<serde_json::Value>, limits: &Limits) -> BoundedMessages {+ let mut truncated = false;+ for message in &mut messages {+ truncated |= bound_json_strings(message, limits.max_part_chars);+ }+ let mut omitted = 0;+ let mut json = serialize(&messages);+ while json.len() > limits.max_attribute_bytes && messages.len() > 1 {+ messages.remove(0);+ omitted += 1;+ json = serialize(&messages);+ }+ if json.len() > limits.max_attribute_bytes+ && let Some(last) = messages.last_mut()+ {+ shrink_message(last, limits);+ truncated = true;+ json = serialize(&messages);+ }+ BoundedMessages {+ json,+ omitted,+ truncated: truncated || omitted > 0,+ }+}++/// Shrink one message until it serializes within the attribute budget: halve+/// the leaf budget while that helps, then drop its oldest parts (never the+/// last), and as a last resort keep only a note of what was dropped.+fn shrink_message(message: &mut serde_json::Value, limits: &Limits) {+ let over = |message: &serde_json::Value| message.to_string().len() > limits.max_attribute_bytes;+ let mut part_chars = limits.max_part_chars / 2;+ while over(message) && part_chars >= 16 {+ bound_json_strings(message, part_chars);+ part_chars /= 2;+ }+ while over(message) {+ let Some(parts) = message+ .get_mut("parts")+ .and_then(serde_json::Value::as_array_mut)+ else {+ break;+ };+ if parts.len() <= 1 {+ break;+ }+ parts.remove(0);+ }+ if over(message)+ && let Some(object) = message.as_object_mut()+ {+ let dropped = object+ .get("parts")+ .map(|parts| parts.to_string().len())+ .unwrap_or(0);+ object.insert(+ "parts".to_owned(),+ serde_json::json!([{+ "type": "text",+ "content": format!("[content omitted: {dropped} bytes]"),+ }]),+ );+ }+}++/// Bound a `gen_ai.tool.definitions` list to `limits`. Definitions are our+/// own code, not user content, so nothing is dropped unless the list is over+/// budget; then the parameter schemas go first (a judge grades a tool choice+/// on names and descriptions), then descriptions are cut. Returns the+/// serialized array and whether anything was removed or cut.+pub fn bound_tool_definitions(+ mut definitions: Vec<serde_json::Value>,+ limits: &Limits,+) -> (String, bool) {+ let json = serialize(&definitions);+ if json.len() <= limits.max_attribute_bytes {+ return (json, false);+ }+ for definition in &mut definitions {+ if let Some(object) = definition.as_object_mut() {+ object.remove("parameters");+ }+ }+ let json = serialize(&definitions);+ if json.len() <= limits.max_attribute_bytes {+ return (json, true);+ }+ for definition in &mut definitions {+ bound_json_strings(definition, limits.max_part_chars);+ }+ // Still over: the list itself is too long. Drop definitions from the end+ // rather than cut the serialization, so what remains is valid JSON.+ let mut json = serialize(&definitions);+ while json.len() > limits.max_attribute_bytes && !definitions.is_empty() {+ definitions.pop();+ json = serialize(&definitions);+ }+ (json, true)+}++fn serialize(values: &[serde_json::Value]) -> String {+ serde_json::to_string(values).unwrap_or_else(|_| "[]".to_string())+}diff --git a/crates/genai_telemetry/src/content/test.rs b/crates/genai_telemetry/src/content/test.rsnew file mode 100644index 0000000000..2816e3c1e2--- /dev/null+++ b/crates/genai_telemetry/src/content/test.rs@@ -0,0 +1,222 @@+use super::*;+use serde_json::json;++#[test]+fn truncate_chars_borrows_when_within_budget() {+ assert!(matches!(truncate_chars("hello", 5), Cow::Borrowed("hello")));+ assert!(matches!(truncate_chars("", 0), Cow::Borrowed("")));+}++#[test]+fn truncate_chars_cuts_on_char_boundaries_and_marks_the_cut() {+ // Multi-byte characters: the byte length exceeds the budget but the char+ // count does not, so nothing is cut.+ assert!(matches!(truncate_chars("héllo", 5), Cow::Borrowed("héllo")));+ assert_eq!(+ truncate_chars("héllo wörld", 5),+ "héllo…[truncated 6 chars]"+ );+}++#[test]+fn policy_parses_the_standard_switch() {+ assert!(ContentPolicy::parse(None).capture);+ assert!(ContentPolicy::parse(Some("true")).capture);+ assert!(ContentPolicy::parse(Some("anything")).capture);+ for off in ["false", "FALSE", " 0 ", "no", "Off"] {+ assert!(+ !ContentPolicy::parse(Some(off)).capture,+ "{off:?} should disable capture"+ );+ }+ assert_eq!(ContentPolicy::parse(None).limits, Limits::default());+}++#[test]+fn bounded_json_string_cuts_leaves_then_the_whole_value() {+ let limits = Limits {+ max_part_chars: 4,+ max_attribute_bytes: 1_000,+ };+ let (json, cut) = bounded_json_string(&json!({"a": "abcdefgh", "n": 1}), &limits);+ assert!(cut);+ assert_eq!(json, r#"{"a":"abcd…[truncated 4 chars]","n":1}"#);++ let (json, cut) = bounded_json_string(&json!({"a": "ab"}), &limits);+ assert!(!cut);+ assert_eq!(json, r#"{"a":"ab"}"#);++ // A bare string is recorded unquoted.+ let (json, cut) = bounded_json_string(&json!("plain"), &limits);+ assert!(cut);+ assert_eq!(json, "plai…[truncated 1 chars]");++ let tight = Limits {+ max_part_chars: 1_000,+ max_attribute_bytes: 10,+ };+ let (json, cut) = bounded_json_string(&json!({"key": "value"}), &tight);+ assert!(cut);+ assert!(json.starts_with(r#"{"key"#), "{json}");+ assert!(json.ends_with("bytes]"), "{json}");+}++fn message(role: &str, text: &str) -> serde_json::Value {+ json!({"role": role, "parts": [{"type": "text", "content": text}]})+}++#[test]+fn bound_messages_keeps_everything_within_budget() {+ let bounded = bound_messages(+ vec![message("user", "hi"), message("assistant", "hello")],+ &Limits::default(),+ );+ assert_eq!(bounded.omitted, 0);+ assert!(!bounded.truncated);+ let parsed: Vec<serde_json::Value> = serde_json::from_str(&bounded.json).unwrap();+ assert_eq!(parsed.len(), 2);+}++#[test]+fn bound_messages_drops_the_oldest_messages_first_and_never_the_last() {+ let messages = vec![+ message("user", "first"),+ message("assistant", "second"),+ message("user", "third"),+ ];+ let one_message_budget = serde_json::to_string(&vec![message("user", "third")])+ .unwrap()+ .len();+ let limits = Limits {+ max_part_chars: 1_000,+ max_attribute_bytes: one_message_budget,+ };+ let bounded = bound_messages(messages, &limits);+ assert_eq!(bounded.omitted, 2);+ assert!(bounded.truncated);+ let parsed: Vec<serde_json::Value> = serde_json::from_str(&bounded.json).unwrap();+ assert_eq!(parsed, vec![message("user", "third")]);+}++#[test]+fn bound_messages_cuts_long_parts() {+ let limits = Limits {+ max_part_chars: 3,+ max_attribute_bytes: 10_000,+ };+ let bounded = bound_messages(vec![message("user", "abcdef")], &limits);+ assert_eq!(bounded.omitted, 0);+ assert!(bounded.truncated);+ let parsed: Vec<serde_json::Value> = serde_json::from_str(&bounded.json).unwrap();+ assert_eq!(parsed[0]["parts"][0]["content"], "abc…[truncated 3 chars]");+}++fn definition(name: &str) -> serde_json::Value {+ json!({+ "type": "function",+ "name": name,+ "description": format!("does {name}"),+ "parameters": {"type": "object", "properties": {"q": {"type": "string", "description": "a long schema description"}}}+ })+}++#[test]+fn bound_tool_definitions_keeps_schemas_within_budget() {+ let (json, cut) =+ bound_tool_definitions(vec![definition("a"), definition("b")], &Limits::default());+ assert!(!cut);+ let parsed: Vec<serde_json::Value> = serde_json::from_str(&json).unwrap();+ assert!(parsed.iter().all(|d| d.get("parameters").is_some()));+}++#[test]+fn bound_tool_definitions_drops_parameters_before_names_and_descriptions() {+ let full = serde_json::to_string(&vec![definition("a"), definition("b")]).unwrap();+ let limits = Limits {+ max_part_chars: 1_000,+ max_attribute_bytes: full.len() - 1,+ };+ let (json, cut) = bound_tool_definitions(vec![definition("a"), definition("b")], &limits);+ assert!(cut);+ let parsed: Vec<serde_json::Value> = serde_json::from_str(&json).unwrap();+ assert_eq!(parsed.len(), 2);+ assert!(parsed.iter().all(|d| d.get("parameters").is_none()));+ assert_eq!(parsed[0]["name"], "a");+ assert_eq!(parsed[0]["description"], "does a");+}++#[test]+fn truncate_bytes_respects_the_byte_budget_on_char_boundaries() {+ assert!(matches!(truncate_bytes("héllo", 6), Cow::Borrowed("héllo")));+ let emoji = "😀".repeat(1_000);+ let cut = truncate_bytes(&emoji, 200);+ assert!(cut.len() <= 200, "{} bytes", cut.len());+ assert!(cut.starts_with("😀"), "cut on a character boundary");+ assert!(cut.contains("…[truncated"), "{cut}");+}++#[test]+fn bounded_json_string_enforces_bytes_not_chars() {+ let limits = Limits {+ max_part_chars: 100_000,+ max_attribute_bytes: 500,+ };+ let (json, cut) = bounded_json_string(&json!("😀".repeat(1_000)), &limits);+ assert!(cut);+ assert!(json.len() <= 500, "{} bytes", json.len());+}++#[test]+fn an_oversized_last_message_is_shrunk_into_valid_json() {+ let limits = Limits {+ max_part_chars: 100_000,+ max_attribute_bytes: 300,+ };+ let big = json!({+ "role": "user",+ "parts": [+ {"type": "text", "content": "a".repeat(400)},+ {"type": "text", "content": "b".repeat(400)},+ ]+ });+ let bounded = bound_messages(vec![message("assistant", "earlier"), big], &limits);+ assert!(bounded.truncated);+ assert_eq!(bounded.omitted, 1);+ assert!(bounded.json.len() <= 300, "{} bytes", bounded.json.len());+ let parsed: Vec<serde_json::Value> =+ serde_json::from_str(&bounded.json).expect("valid JSON: {bounded.json}");+ assert_eq!(parsed.len(), 1);+ assert_eq!(parsed[0]["role"], "user");++ // Even a budget too small for any of its content leaves a note behind.+ let bounded = bound_messages(+ vec![message("user", &"x".repeat(1_000))],+ &Limits {+ max_part_chars: 100_000,+ max_attribute_bytes: 60,+ },+ );+ let parsed: Vec<serde_json::Value> = serde_json::from_str(&bounded.json).expect("valid JSON");+ assert!(+ parsed[0]["parts"][0]["content"]+ .as_str()+ .is_some_and(|content| content.starts_with("[content omitted")),+ "{}",+ bounded.json+ );+}++#[test]+fn an_overlong_tool_list_drops_definitions_but_stays_valid_json() {+ let definitions: Vec<_> = (0..50).map(|i| definition(&format!("tool_{i}"))).collect();+ let limits = Limits {+ max_part_chars: 1_000,+ max_attribute_bytes: 600,+ };+ let (json, cut) = bound_tool_definitions(definitions, &limits);+ assert!(cut);+ assert!(json.len() <= 600, "{} bytes", json.len());+ let parsed: Vec<serde_json::Value> = serde_json::from_str(&json).expect("valid JSON");+ assert!(!parsed.is_empty());+ assert_eq!(parsed[0]["name"], "tool_0");+}diff --git a/crates/genai_telemetry/src/lib.rs b/crates/genai_telemetry/src/lib.rsnew file mode 100644index 0000000000..c92a203303--- /dev/null+++ b/crates/genai_telemetry/src/lib.rs@@ -0,0 +1,43 @@+//! GenAI span telemetry shared by the AI toolset and the agent loop.+//!+//! The agent runtime (`rig`) already emits the *structural* OpenTelemetry GenAI+//! spans — `invoke_agent`, `chat`, `execute_tool` — with model, provider, usage+//! and tool names. What it does not give us, and what Datadog LLM Observability+//! evaluations (Tool Selection, Goal Completeness, …) need, is+//!+//! - the **content**: prompts, tool arguments, tool results and model output+//! (`gen_ai.input.messages`, `gen_ai.output.messages`,+//! `gen_ai.tool.call.arguments`, `gen_ai.tool.call.result`, …),+//! - the **tool definitions** offered on each model call+//! (`gen_ai.tool.definitions`), and+//! - the **session** the trace belongs to (`gen_ai.conversation.id`).+//!+//! This crate provides the shared pieces to record them:+//!+//! - [`attr`]: the `gen_ai.*` attribute names and well-known values, spelled once.+//! - [`ContentPolicy`] / [`Limits`]: whether content may be captured (the+//! OpenTelemetry-standard `OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT`+//! switch) and how large a single attribute may grow. Content is always+//! bounded — a 12-turn agent run otherwise re-sends the whole conversation on+//! every `chat` span and a tool can return a whole document.+//! - [`messages`]: builders for the semconv JSON shapes (`{"role", "parts"}`+//! messages, tool definitions, system instructions) that Datadog parses.+//! - [`GenAiSpanExt`]: attribute setters that write straight to the+//! OpenTelemetry span. `tracing` only accepts fields declared at span+//! creation, so this is the only way to add attributes to spans another crate+//! (rig) opened — and it keeps large content out of the JSON log layer, which+//! would otherwise repeat every span field on every log line.++#![deny(missing_docs)]++pub mod attr;+mod content;+pub mod messages;+pub mod propagation;+mod span;++pub use content::{+ BoundedMessages, ContentPolicy, Limits, bound_messages, bound_tool_definitions,+ bounded_json_string, truncate_bytes, truncate_chars,+};+pub use span::{GenAiSpanExt, current_span_is_named};diff --git a/crates/genai_telemetry/src/messages.rs b/crates/genai_telemetry/src/messages.rsnew file mode 100644index 0000000000..e8f78a890e--- /dev/null+++ b/crates/genai_telemetry/src/messages.rs@@ -0,0 +1,156 @@+//! Builders for the GenAI semantic-convention JSON shapes.+//!+//! `gen_ai.input.messages` / `gen_ai.output.messages` are arrays of+//! `{"role": …, "parts": […]}` objects (output messages add+//! `"finish_reason"`), `gen_ai.system_instructions` is an array of parts and+//! `gen_ai.tool.definitions` an array of function definitions. Datadog parses+//! exactly these shapes; a backend without structured attributes receives them+//! as JSON strings, which is how this workspace records them (see+//! [`crate::bound_messages`] for the size bounds applied on the way out).+//!+//! The builders take plain values so callers stay independent of any agent+//! runtime's message types.++#[cfg(test)]+mod test;++use serde_json::{Value, json};++/// The `user` role.+pub const ROLE_USER: &str = "user";+/// The `assistant` role.+pub const ROLE_ASSISTANT: &str = "assistant";+/// The `system` role.+pub const ROLE_SYSTEM: &str = "system";+/// The `tool` role (tool results).+pub const ROLE_TOOL: &str = "tool";++/// Where a media part's bytes live. Inline data is never copied onto a span.+#[derive(Debug, Clone, PartialEq, Eq)]+pub enum MediaSource {+ /// A URL.+ Uri(String),+ /// A provider-side file id.+ FileId(String),+ /// Inline bytes (base64, raw, …) — recorded as a placeholder only.+ Inline,+}++/// A text part.+pub fn text_part(content: impl Into<String>) -> Value {+ json!({ "type": "text", "content": content.into() })+}++/// A reasoning (thinking) part.+pub fn reasoning_part(content: impl Into<String>) -> Value {+ json!({ "type": "reasoning", "content": content.into() })+}++/// A tool call requested by the model.+pub fn tool_call_part(id: Option<&str>, name: &str, arguments: Value) -> Value {+ let mut part = json!({ "type": "tool_call", "name": name, "arguments": arguments });+ if let Some(id) = id {+ part["id"] = Value::String(id.to_string());+ }+ part+}++/// The response to a tool call.+pub fn tool_call_response_part(id: Option<&str>, response: Value) -> Value {+ let mut part = json!({ "type": "tool_call_response", "response": response });+ if let Some(id) = id {+ part["id"] = Value::String(id.to_string());+ }+ part+}++/// A media part (`image`, `audio`, `video`, `document` modality). Inline+/// bytes are replaced by a placeholder: a span is not the place for a file.+/// A URI is recorded without its credentials, query or fragment (see+/// [`redact_uri`]).+pub fn media_part(modality: &str, mime_type: Option<&str>, source: MediaSource) -> Value {+ let mut part = match source {+ MediaSource::Uri(uri) => {+ json!({ "type": "uri", "modality": modality, "uri": redact_uri(&uri) })+ }+ MediaSource::FileId(file_id) => {+ json!({ "type": "file", "modality": modality, "file_id": file_id })+ }+ MediaSource::Inline => {+ json!({ "type": "blob", "modality": modality, "content": "[inline data omitted]" })+ }+ };+ if let Some(mime_type) = mime_type {+ part["mime_type"] = Value::String(mime_type.to_string());+ }+ part+}++/// A URI reduced to what identifies the resource: scheme, host and path.+/// Userinfo (`user:password@`), the query string and the fragment are+/// dropped - that is where credentials, signed-URL tokens and session state+/// travel, none of which belongs on a span. A `data:` URI *is* its payload,+/// so only its media type survives.+pub fn redact_uri(uri: &str) -> String {+ let uri = uri.split(['?', '#']).next().unwrap_or_default();+ let (scheme, rest) = match uri.split_once(':') {+ Some((scheme, rest)) if is_scheme(scheme) => (Some(scheme), rest),+ _ => (None, uri),+ };+ if scheme.is_some_and(|scheme| scheme.eq_ignore_ascii_case("data")) {+ let media_type = rest.split([';', ',']).next().unwrap_or_default();+ return format!("data:{media_type};[inline data omitted]");+ }+ // Hierarchical (`scheme://authority/path`) or protocol-relative+ // (`//authority/path`): the authority may carry userinfo.+ if let Some(rest) = rest.strip_prefix("//") {+ let (authority, path) = match rest.find('/') {+ Some(slash) => rest.split_at(slash),+ None => (rest, ""),+ };+ let host = authority+ .rsplit_once('@')+ .map_or(authority, |(_, host)| host);+ return match scheme {+ Some(scheme) => format!("{scheme}://{host}{path}"),+ None => format!("//{host}{path}"),+ };+ }+ // Opaque (`urn:…`, `mailto:…`): the identifier is the whole thing.+ uri.to_owned()+}++/// Whether `candidate` is a URI scheme (RFC 3986: a letter, then letters,+/// digits, `+`, `-` or `.`).+fn is_scheme(candidate: &str) -> bool {+ let mut chars = candidate.chars();+ chars+ .next()+ .is_some_and(|first| first.is_ascii_alphabetic())+ && chars.all(|c| c.is_ascii_alphanumeric() || matches!(c, '+' | '-' | '.'))+}++/// An input message.+pub fn message(role: &str, parts: Vec<Value>) -> Value {+ json!({ "role": role, "parts": parts })+}++/// An output message (one choice) with the reason generation stopped.+pub fn output_message(role: &str, parts: Vec<Value>, finish_reason: &str) -> Value {+ json!({ "role": role, "parts": parts, "finish_reason": finish_reason })+}++/// The `gen_ai.system_instructions` value for one block of instructions.+pub fn system_instructions(text: impl Into<String>) -> Vec<Value> {+ vec![text_part(text)]+}++/// One entry of `gen_ai.tool.definitions`.+pub fn tool_definition(name: &str, description: &str, parameters: Value) -> Value {+ json!({+ "type": "function",+ "name": name,+ "description": description,+ "parameters": parameters,+ })+}diff --git a/crates/genai_telemetry/src/messages/test.rs b/crates/genai_telemetry/src/messages/test.rsnew file mode 100644index 0000000000..f698a343f3--- /dev/null+++ b/crates/genai_telemetry/src/messages/test.rs@@ -0,0 +1,93 @@+use super::*;++#[test]+fn parts_follow_the_semconv_shapes() {+ assert_eq!(text_part("hi"), json!({"type": "text", "content": "hi"}));+ assert_eq!(+ reasoning_part("hmm"),+ json!({"type": "reasoning", "content": "hmm"})+ );+ assert_eq!(+ tool_call_part(Some("call_1"), "search", json!({"q": "x"})),+ json!({"type": "tool_call", "id": "call_1", "name": "search", "arguments": {"q": "x"}})+ );+ assert_eq!(+ tool_call_part(None, "search", json!({})),+ json!({"type": "tool_call", "name": "search", "arguments": {}})+ );+ assert_eq!(+ tool_call_response_part(Some("call_1"), json!({"hits": 3})),+ json!({"type": "tool_call_response", "id": "call_1", "response": {"hits": 3}})+ );+}++#[test]+fn media_parts_never_carry_inline_bytes() {+ assert_eq!(+ media_part(+ "image",+ Some("image/png"),+ MediaSource::Uri("https://x/a.png".into())+ ),+ json!({"type": "uri", "modality": "image", "uri": "https://x/a.png", "mime_type": "image/png"})+ );+ assert_eq!(+ media_part("document", None, MediaSource::FileId("file_1".into())),+ json!({"type": "file", "modality": "document", "file_id": "file_1"})+ );+ assert_eq!(+ media_part("image", None, MediaSource::Inline),+ json!({"type": "blob", "modality": "image", "content": "[inline data omitted]"})+ );+}++#[test]+fn messages_and_definitions_follow_the_semconv_shapes() {+ assert_eq!(+ message(ROLE_USER, vec![text_part("hi")]),+ json!({"role": "user", "parts": [{"type": "text", "content": "hi"}]})+ );+ assert_eq!(+ output_message(ROLE_ASSISTANT, vec![text_part("hello")], "stop"),+ json!({"role": "assistant", "parts": [{"type": "text", "content": "hello"}], "finish_reason": "stop"})+ );+ assert_eq!(+ system_instructions("be brief"),+ vec![json!({"type": "text", "content": "be brief"})]+ );+ assert_eq!(+ tool_definition("search", "Search things", json!({"type": "object"})),+ json!({"type": "function", "name": "search", "description": "Search things", "parameters": {"type": "object"}})+ );+}++#[test]+fn uris_lose_their_credentials_query_and_fragment() {+ assert_eq!(+ redact_uri("https://user:[email protected]/a/b.png?sig=abc#frag"),+ "https://files.example.com/a/b.png"+ );+ assert_eq!(redact_uri("https://x/a.png"), "https://x/a.png");+ assert_eq!(redact_uri("https://host"), "https://host");+ assert_eq!(redact_uri("file:///tmp/a.txt"), "file:///tmp/a.txt");+ assert_eq!(redact_uri("urn:doc:123?x=1"), "urn:doc:123");+ // Forms without `scheme://` still carry secrets or payloads.+ assert_eq!(redact_uri("//user:secret@host/path?k=v"), "//host/path");+ assert_eq!(+ redact_uri("data:image/png;base64,iVBORw0KGgoAAAANSUhEUg"),+ "data:image/png;[inline data omitted]"+ );+ assert_eq!(+ redact_uri("DATA:text/plain,hello%20world"),+ "data:text/plain;[inline data omitted]"+ );+ assert_eq!(redact_uri("HTTPS://user@Host/x"), "HTTPS://Host/x");+ assert_eq!(+ media_part(+ "image",+ None,+ MediaSource::Uri("https://token@x/a.png?expires=1".into())+ )["uri"],+ "https://x/a.png"+ );+}diff --git a/crates/genai_telemetry/src/propagation.rs b/crates/genai_telemetry/src/propagation.rsnew file mode 100644index 0000000000..bc24362348--- /dev/null+++ b/crates/genai_telemetry/src/propagation.rs@@ -0,0 +1,66 @@+//! W3C trace context carried in ACP's extensible `_meta` object.+//!+//! Only context is transported: content capture and GenAI span ownership stay+//! with the session actor. Runtimes that do not understand this key ignore it.++#[cfg(test)]+mod test;++use opentelemetry::propagation::{Extractor, Injector};+use opentelemetry::trace::TraceContextExt as _;+use serde_json::{Map, Value};+use tracing_opentelemetry::OpenTelemetrySpanExt as _;++const TRACE_CONTEXT: &str = "macro.dev/trace-context";++struct Carrier<'a>(&'a Map<String, Value>);++impl Extractor for Carrier<'_> {+ fn get(&self, key: &str) -> Option<&str> {+ self.0.get(key)?.as_str()+ }++ fn keys(&self) -> Vec<&str> {+ self.0.keys().map(String::as_str).collect()+ }+}++struct CarrierMut<'a>(&'a mut Map<String, Value>);++impl Injector for CarrierMut<'_> {+ fn set(&mut self, key: &str, value: String) {+ self.0.insert(key.to_owned(), Value::String(value));+ }+}++/// Replace our trace-context metadata with `span`'s context, preserving all+/// other ACP metadata. A disabled span clears any stale context in our key.+pub fn inject(span: &tracing::Span, meta: &mut Map<String, Value>) {+ meta.remove(TRACE_CONTEXT);+ let context = span.context();+ if !context.span().span_context().is_valid() {+ return;+ }+ let mut carrier = Map::new();+ opentelemetry::global::get_text_map_propagator(|propagator| {+ propagator.inject_context(&context, &mut CarrierMut(&mut carrier));+ });+ if !carrier.is_empty() {+ meta.insert(TRACE_CONTEXT.to_owned(), Value::Object(carrier));+ }+}++/// Parent runtime work under the invocation that sent its ACP prompt. Missing+/// or malformed context is ignored; unrelated ambient context is never used+/// as an extraction fallback.+pub fn set_parent(span: &tracing::Span, meta: Option<&Map<String, Value>>) {+ let Some(carrier) = meta.and_then(|meta| meta.get(TRACE_CONTEXT)?.as_object()) else {+ return;+ };+ let context = opentelemetry::global::get_text_map_propagator(|propagator| {+ propagator.extract_with_context(&opentelemetry::Context::new(), &Carrier(carrier))+ });+ if context.span().span_context().is_valid() {+ let _ = span.set_parent(context);+ }+}diff --git a/crates/genai_telemetry/src/propagation/test.rs b/crates/genai_telemetry/src/propagation/test.rsnew file mode 100644index 0000000000..f1153c2acd--- /dev/null+++ b/crates/genai_telemetry/src/propagation/test.rs@@ -0,0 +1,75 @@+use super::*;+use opentelemetry::trace::{SpanId, TraceContextExt as _, TracerProvider as _};+use opentelemetry_sdk::propagation::TraceContextPropagator;+use opentelemetry_sdk::trace::{InMemorySpanExporter, SdkTracerProvider};+use serde_json::json;+use tracing_subscriber::layer::SubscriberExt as _;++#[test]+fn context_survives_serialization_and_is_replaced_for_each_prompt() {+ opentelemetry::global::set_text_map_propagator(TraceContextPropagator::new());+ let exporter = InMemorySpanExporter::default();+ let provider = SdkTracerProvider::builder()+ .with_simple_exporter(exporter.clone())+ .build();+ let subscriber = tracing_subscriber::registry()+ .with(tracing_opentelemetry::layer().with_tracer(provider.tracer("test")));+ let _guard = tracing::subscriber::set_default(subscriber);+ let mut meta = Map::from_iter([("other/client".to_owned(), json!({"keep": true}))]);+ let mut expected = Vec::new();+ for _ in 0..2 {+ let parent = tracing::info_span!(parent: None, "invocation");+ let context = parent.context();+ let parent_context = context.span().span_context().clone();+ inject(&parent, &mut meta);+ let wire = serde_json::to_string(&meta).expect("serialize metadata");+ let received = serde_json::from_str(&wire).expect("deserialize metadata");+ let child = tracing::info_span!(parent: None, "runtime");+ set_parent(&child, Some(&received));+ assert_eq!(+ child.context().span().span_context().trace_id(),+ parent_context.trace_id()+ );+ expected.push(parent_context);+ }+ assert_eq!(meta["other/client"], json!({"keep": true}));+ provider.force_flush().expect("flush");+ let spans = exporter.get_finished_spans().expect("spans");+ let children: Vec<_> = spans.iter().filter(|s| s.name == "runtime").collect();+ assert_eq!(children.len(), 2);+ for (child, parent) in children.iter().zip(&expected) {+ assert_eq!(child.parent_span_id, parent.span_id());+ assert_ne!(child.parent_span_id, SpanId::INVALID);+ }+ assert_ne!(expected[0].trace_id(), expected[1].trace_id());+ inject(&tracing::Span::none(), &mut meta);+ assert!(+ !meta.contains_key(TRACE_CONTEXT),+ "clear stale context when tracing is off"+ );+ assert!(meta.contains_key("other/client"));+}++#[test]+fn invalid_metadata_does_not_attach_an_unrelated_ambient_context() {+ opentelemetry::global::set_text_map_propagator(TraceContextPropagator::new());+ let provider = SdkTracerProvider::builder().build();+ let subscriber = tracing_subscriber::registry()+ .with(tracing_opentelemetry::layer().with_tracer(provider.tracer("test")));+ let _guard = tracing::subscriber::set_default(subscriber);+ let unrelated = tracing::info_span!("unrelated");+ let _context = unrelated.context().attach();+ for value in [+ Value::Null,+ json!("bad"),+ json!({"traceparent": "invalid"}),+ json!({"traceparent": 5}),+ ] {+ let meta = Map::from_iter([(TRACE_CONTEXT.to_owned(), value)]);+ let child = tracing::info_span!(parent: None, "runtime");+ let before = child.context().span().span_context().trace_id();+ set_parent(&child, Some(&meta));+ set_parent(&child, None);+ assert_eq!(child.context().span().span_context().trace_id(), before);+ }+}diff --git a/crates/genai_telemetry/src/span.rs b/crates/genai_telemetry/src/span.rsnew file mode 100644index 0000000000..0dccba56cd--- /dev/null+++ b/crates/genai_telemetry/src/span.rs@@ -0,0 +1,97 @@+//! Attribute helpers that write straight to a span's OpenTelemetry data.+//!+//! `tracing` only records fields a span declared when it was created, so a+//! span opened by another crate (the agent runtime's `chat` and `execute_tool`+//! spans) cannot take new fields through `Span::record`. `tracing-opentelemetry`+//! exposes the underlying OpenTelemetry span, which accepts any attribute at+//! any time before the span closes. Writing there has a second benefit: the+//! JSON log layer repeats a span's `tracing` fields on every log line emitted+//! inside it, and content attributes are far too large for that.+//!+//! Every setter is a no-op when no OpenTelemetry layer is installed (local+//! runs without a collector, unit tests without a subscriber).++#[cfg(test)]+mod test;++use opentelemetry::trace::Status;+use opentelemetry::{Array, Key, StringValue, Value};+use tracing_opentelemetry::OpenTelemetrySpanExt as _;++use crate::attr;++/// Attribute setters for GenAI spans; implemented for [`tracing::Span`].+pub trait GenAiSpanExt {+ /// Set a string attribute.+ fn set_str(&self, key: &'static str, value: impl Into<String>);+ /// Set an integer attribute.+ fn set_i64(&self, key: &'static str, value: i64);+ /// Set an integer attribute from a `u64`, saturating at `i64::MAX`.+ fn set_u64(&self, key: &'static str, value: u64);+ /// Set a float attribute.+ fn set_f64(&self, key: &'static str, value: f64);+ /// Set a boolean attribute.+ fn set_bool(&self, key: &'static str, value: bool);+ /// Set a string-array attribute.+ fn set_str_array<I, S>(&self, key: &'static str, values: I)+ where+ I: IntoIterator<Item = S>,+ S: Into<String>;+ /// Mark the span failed: records `error.type` and sets the OpenTelemetry+ /// error status with `description`.+ fn set_error(&self, error_type: &str, description: impl Into<String>);+}++impl GenAiSpanExt for tracing::Span {+ fn set_str(&self, key: &'static str, value: impl Into<String>) {+ self.set_attribute(+ Key::from_static_str(key),+ Value::String(value.into().into()),+ );+ }++ fn set_i64(&self, key: &'static str, value: i64) {+ self.set_attribute(Key::from_static_str(key), Value::I64(value));+ }++ fn set_u64(&self, key: &'static str, value: u64) {+ self.set_i64(key, i64::try_from(value).unwrap_or(i64::MAX));+ }++ fn set_f64(&self, key: &'static str, value: f64) {+ self.set_attribute(Key::from_static_str(key), Value::F64(value));+ }++ fn set_bool(&self, key: &'static str, value: bool) {+ self.set_attribute(Key::from_static_str(key), Value::Bool(value));+ }++ fn set_str_array<I, S>(&self, key: &'static str, values: I)+ where+ I: IntoIterator<Item = S>,+ S: Into<String>,+ {+ let values: Vec<StringValue> = values.into_iter().map(|v| v.into().into()).collect();+ self.set_attribute(+ Key::from_static_str(key),+ Value::Array(Array::String(values)),+ );+ }++ fn set_error(&self, error_type: &str, description: impl Into<String>) {+ self.set_str(attr::ERROR_TYPE, error_type);+ self.set_status(Status::error(description.into()));+ }+}++/// Whether the current span was opened under `name`.+///+/// Lets an inner layer tell that an outer runtime already opened the semconv+/// span for the operation in flight — the agent runtime's `execute_tool` span+/// around a tool dispatch, say — so it enriches that span instead of nesting a+/// duplicate. `false` when there is no current span.+pub fn current_span_is_named(name: &str) -> bool {+ tracing::Span::current()+ .metadata()+ .is_some_and(|metadata| metadata.name() == name)+}diff --git a/crates/genai_telemetry/src/span/test.rs b/crates/genai_telemetry/src/span/test.rsnew file mode 100644index 0000000000..dedb94db7d--- /dev/null+++ b/crates/genai_telemetry/src/span/test.rs@@ -0,0 +1,101 @@+use super::*;+use opentelemetry::trace::TracerProvider as _;+use opentelemetry_sdk::trace::{InMemorySpanExporter, SdkTracerProvider, SpanData};+use tracing_subscriber::layer::SubscriberExt as _;++/// An OpenTelemetry pipeline that keeps finished spans in memory, installed as+/// the thread's default subscriber for the returned guard's lifetime.+fn otel_test_pipeline() -> (+ InMemorySpanExporter,+ SdkTracerProvider,+ tracing::subscriber::DefaultGuard,+) {+ let exporter = InMemorySpanExporter::default();+ let provider = SdkTracerProvider::builder()+ .with_simple_exporter(exporter.clone())+ .build();+ let layer = tracing_opentelemetry::layer().with_tracer(provider.tracer("test"));+ let guard = tracing::subscriber::set_default(tracing_subscriber::registry().with(layer));+ // Other tests in this binary run the same code paths with no subscriber+ // installed, which can cache "never interested" for a span callsite on+ // another thread; recompute so this subscriber sees every span.+ tracing::callsite::rebuild_interest_cache();+ (exporter, provider, guard)+}++fn finished(exporter: &InMemorySpanExporter, provider: &SdkTracerProvider) -> Vec<SpanData> {+ provider.force_flush().expect("flush");+ exporter.get_finished_spans().expect("finished spans")+}++fn attribute<'a>(span: &'a SpanData, key: &str) -> Option<&'a Value> {+ span.attributes+ .iter()+ .find(|kv| kv.key.as_str() == key)+ .map(|kv| &kv.value)+}++#[test]+fn setters_write_attributes_and_status_to_the_otel_span() {+ let (exporter, provider, _guard) = otel_test_pipeline();++ {+ let span = tracing::info_span!("work");+ span.set_str("s", "text");+ span.set_i64("i", -3);+ span.set_u64("u", u64::MAX);+ span.set_f64("f", 0.5);+ span.set_bool("b", true);+ span.set_str_array("a", ["x", "y"]);+ span.set_error("boom", "it broke");+ }++ let spans = finished(&exporter, &provider);+ assert_eq!(spans.len(), 1);+ let span = &spans[0];+ assert_eq!(span.name, "work");+ assert_eq!(attribute(span, "s"), Some(&Value::String("text".into())));+ assert_eq!(attribute(span, "i"), Some(&Value::I64(-3)));+ assert_eq!(attribute(span, "u"), Some(&Value::I64(i64::MAX)));+ assert_eq!(attribute(span, "f"), Some(&Value::F64(0.5)));+ assert_eq!(attribute(span, "b"), Some(&Value::Bool(true)));+ assert_eq!(+ attribute(span, "a"),+ Some(&Value::Array(Array::String(vec!["x".into(), "y".into()])))+ );+ assert_eq!(+ attribute(span, attr::ERROR_TYPE),+ Some(&Value::String("boom".into()))+ );+ assert!(+ matches!(&span.status, Status::Error { description } if description == "it broke"),+ "{:?}",+ span.status+ );+}++#[test]+fn setters_are_noops_without_an_otel_layer() {+ // No subscriber at all: nothing to write to, nothing to panic about.+ let span = tracing::info_span!("quiet");+ span.set_str("k", "v");+ span.set_error("e", "d");+ assert!(span.is_disabled());+}++#[test]+fn current_span_is_named_matches_the_entered_span_only() {+ let (_exporter, _provider, _guard) = otel_test_pipeline();+ assert!(!current_span_is_named("outer"));+ let outer = tracing::info_span!("outer");+ let _entered = outer.enter();+ assert!(current_span_is_named("outer"));+ assert!(!current_span_is_named("inner"));+ {+ let inner = tracing::info_span!("inner");+ let _entered = inner.enter();+ assert!(current_span_is_named("inner"));+ assert!(!current_span_is_named("outer"));+ }+ assert!(current_span_is_named("outer"));+}diff --git a/crates/import/src/domain/service.rs b/crates/import/src/domain/service.rsindex e94b419789..9ede8ce3e4 100644--- a/crates/import/src/domain/service.rs+++ b/crates/import/src/domain/service.rs@@ -2732,7 +2732,7 @@ impl<Context> ToolSet<Context> for NativePlusMcp<Context>whereContext: Clone + Send + Sync + 'static,{- fn try_tool_call<'a>(+ fn dispatch_tool_call<'a>(&'a self,context: Context,request_context: RequestContext,@@ -2747,10 +2747,10 @@ where> {if self.native_names.contains(tool_name) {self.native- .try_tool_call(context, request_context, tool_name, json)+ .dispatch_tool_call(context, request_context, tool_name, json)} else {self.mcp- .try_tool_call(context, request_context, tool_name, json)+ .dispatch_tool_call(context, request_context, tool_name, json)}}diff --git a/crates/mcp_client/src/domain/service/toolset.rs b/crates/mcp_client/src/domain/service/toolset.rsindex a279c25f62..d70647e9b5 100644--- a/crates/mcp_client/src/domain/service/toolset.rs+++ b/crates/mcp_client/src/domain/service/toolset.rs@@ -337,7 +337,7 @@ impl McpToolSet {}impl<Context: Send + Sync + 'static> ToolSet<Context> for McpToolSet {- fn try_tool_call<'a>(+ fn dispatch_tool_call<'a>(&'a self,_context: Context,_request_context: RequestContext,@@ -440,7 +440,7 @@ impl<T> CombinedToolSet<T> {}impl<T: Send + Sync + 'static> ToolSet<T> for CombinedToolSet<T> {- fn try_tool_call<'a>(+ fn dispatch_tool_call<'a>(&'a self,context: T,request_context: RequestContext,@@ -451,10 +451,10 @@ impl<T: Send + Sync + 'static> ToolSet<T> for CombinedToolSet<T> {> {if tool_name.starts_with(MANGLED_PREFIX) {self.mcp_tools- .try_tool_call(context, request_context, tool_name, json)+ .dispatch_tool_call(context, request_context, tool_name, json)} else {self.static_tools- .try_tool_call(context, request_context, tool_name, json)+ .dispatch_tool_call(context, request_context, tool_name, json)}}diff --git a/crates/mcp_select/src/lib.rs b/crates/mcp_select/src/lib.rsindex d2934dbf0b..b7a8140acc 100644--- a/crates/mcp_select/src/lib.rs+++ b/crates/mcp_select/src/lib.rs@@ -65,7 +65,7 @@ impl UserMcpTools {}impl<Context: Send + Sync + 'static> ToolSet<Context> for UserMcpTools {- fn try_tool_call<'a>(+ fn dispatch_tool_call<'a>(&'a self,context: Context,request_context: RequestContext,@@ -76,10 +76,10 @@ impl<Context: Send + Sync + 'static> ToolSet<Context> for UserMcpTools {> {match self {UserMcpTools::Pipedream(tools) => {- tools.try_tool_call(context, request_context, tool_name, json)+ tools.dispatch_tool_call(context, request_context, tool_name, json)}UserMcpTools::Native(tools) => {- tools.try_tool_call(context, request_context, tool_name, json)+ tools.dispatch_tool_call(context, request_context, tool_name, json)}}}@@ -298,7 +298,7 @@ impl<T, Mcp> CombinedToolSet<T, Mcp> {}impl<T: Send + Sync + 'static, Mcp: ToolSet<T>> ToolSet<T> for CombinedToolSet<T, Mcp> {- fn try_tool_call<'a>(+ fn dispatch_tool_call<'a>(&'a self,context: T,request_context: RequestContext,@@ -309,10 +309,10 @@ impl<T: Send + Sync + 'static, Mcp: ToolSet<T>> ToolSet<T> for CombinedToolSet<T> {if tool_name.starts_with(MANGLED_PREFIX) {self.mcp_tools- .try_tool_call(context, request_context, tool_name, json)+ .dispatch_tool_call(context, request_context, tool_name, json)} else {self.static_tools- .try_tool_call(context, request_context, tool_name, json)+ .dispatch_tool_call(context, request_context, tool_name, json)}}diff --git a/crates/mcp_toolset/src/toolset.rs b/crates/mcp_toolset/src/toolset.rsindex a9bff2128e..26ffe317de 100644--- a/crates/mcp_toolset/src/toolset.rs+++ b/crates/mcp_toolset/src/toolset.rs@@ -194,7 +194,7 @@ impl RemoteMcpToolSet {}impl<Context: Send + Sync + 'static> ToolSet<Context> for RemoteMcpToolSet {- fn try_tool_call<'a>(+ fn dispatch_tool_call<'a>(&'a self,_context: Context,_request_context: RequestContext,diff --git a/crates/pipedream_mcp/src/domain/service/toolset.rs b/crates/pipedream_mcp/src/domain/service/toolset.rsindex 3c797d82e7..f6ce63510d 100644--- a/crates/pipedream_mcp/src/domain/service/toolset.rs+++ b/crates/pipedream_mcp/src/domain/service/toolset.rs@@ -72,7 +72,7 @@ impl PipedreamToolSet {}impl<Context: Send + Sync + 'static> ToolSet<Context> for PipedreamToolSet {- fn try_tool_call<'a>(+ fn dispatch_tool_call<'a>(&'a self,context: Context,request_context: RequestContext,@@ -82,7 +82,7 @@ impl<Context: Send + Sync + 'static> ToolSet<Context> for PipedreamToolSet {Box<dyn Future<Output = Result<ToolResult<serde_json::Value>, ToolSetError>> + 'a + Send>,> {self.0- .try_tool_call(context, request_context, tool_name, json)+ .dispatch_tool_call(context, request_context, tool_name, json)}fn request_schemas(&self) -> Option<Vec<RequestSchema>> {diff --git a/deny.toml b/deny.tomlindex fef442eea4..30a7c69b06 100644--- a/deny.toml+++ b/deny.toml@@ -200,11 +200,16 @@ deny = ["workspace-hack", # cargo-hakari generated crate], reason = "Use macro_aws_config for AWS configuration in workspace crates" },{ crate = "tracing-subscriber", wrappers = [+ "agent", # GenAI span tests+ "agent_inmem", # GenAI trace propagation tests+ "agent_session", # GenAI span tests+ "ai_toolset", # GenAI span tests"channels", # Dev logging only"coding_agent_worker", # Standalone daemon logging setup"cursor_cloud_agents", # Standalone ACP binary logging; stdout is the protocol, so macro_entrypoint cannot own it"documents", # Dev logging only"email", # Dev logging only+ "genai_telemetry", # GenAI span tests"lambda_runtime_api_client", # External crate allowed to use tracing-subscriber"loom", # External concurrency testing dependency"macro_entrypoint",diff --git a/docker/analytics-proxy.Dockerfile b/docker/analytics-proxy.Dockerfileindex 7457e1b096..4c672ef930 100644--- a/docker/analytics-proxy.Dockerfile+++ b/docker/analytics-proxy.Dockerfile@@ -3,17 +3,24 @@ RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates&& rm -rf /var/lib/apt/lists/*RUN npm install -g bun+# Wrangler includes platform-specific workerd/esbuild binaries. Install the+# proxy's locked dependencies outside /app so the repository bind mount cannot+# replace them with the host platform's node_modules at runtime.+COPY services/analytics-proxy/package.json services/analytics-proxy/bun.lock /opt/analytics-proxy/+RUN cd /opt/analytics-proxy && bun install --frozen-lockfile+ENV PATH="/opt/analytics-proxy/node_modules/.bin:${PATH}"+ENV SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crtWORKDIR /app/services/analytics-proxyEXPOSE 8098-# The repo is bind-mounted at /app (see docker-compose.yml), so wrangler and-# node_modules come from the host install. Forward OTLP (both signals) to the+# The repo is bind-mounted at /app (see docker-compose.yml), while Wrangler and+# its native dependencies stay under /opt. Forward OTLP (both signals) to the# local collector (--var overrides the wrangler.jsonc host defaults); no# DD_API_KEY locally, so the worker skips key injection.CMD ["sh", "-c", "\- npx wrangler dev \+ wrangler dev \--env local \--ip 0.0.0.0 \--port 8098 \diff --git a/docs/RUNNING_LOCALLY.md b/docs/RUNNING_LOCALLY.mdindex 186caacfdf..271cf94820 100644--- a/docs/RUNNING_LOCALLY.md+++ b/docs/RUNNING_LOCALLY.md@@ -184,6 +184,17 @@ across instances) debugging containers:See `.claude/skills/live-debug/SKILL.md` for query recipes (Tempo/Loki HTTPAPIs) and the browser-debugging workflow.+For agent turns, use Tempo's TraceQL query+`{span.gen_ai.operation.name="invoke_agent"}`. The session actor records the+ACP prompt, output and tool activity on that trace and sends its context in+`params._meta["macro.dev/trace-context"]`. Macro's in-process runtime restores+that context so its model calls and backend work appear in the same trace.+Other runtimes must explicitly consume this metadata to correlate their+internal spans; their ACP activity is still traced by the session actor.++GenAI content is bounded by `genai_telemetry`; check+`macro.genai.content_truncated` before using a span for evaluations.+## Control the Running StackWhile `run_local` is attached:diff --git a/docs/STYLE_GUIDE.md b/docs/STYLE_GUIDE.mdindex 234e9cfce2..7acf9c0e3e 100644--- a/docs/STYLE_GUIDE.md+++ b/docs/STYLE_GUIDE.md@@ -162,6 +162,17 @@ TypeScript · `[ui]` UI / UX conventions- **CS-54** `[rust]` New crates put `#![deny(missing_docs)]` in `lib.rs` and documentall public items. Do not mark documentation code blocks `ignore` to skip doctestsunless explicitly directed.+- **CS-55** `[err]` AI spans follow the OpenTelemetry GenAI conventions (`gen_ai.*`,+ see `crates/genai_telemetry`). Agent sessions of every harness are traced in one+ place: the session actor projects each session's ACP frames onto `invoke_agent` /+ `execute_tool` spans (`crates/agent_session/src/domain/session/telemetry.rs`), so+ do not add GenAI spans to a harness runtime — `agent_inmem` runs its agent loop with+ `with_genai_telemetry(false)` for that reason. Outside sessions (chat in+ `document_cognition_service`), the agent loop and `ai_toolset::ToolSet::try_tool_call`+ emit and enrich the `invoke_agent` / `chat` / `execute_tool` spans; do not open tool+ spans in tools or toolsets, and never record prompts, tool arguments or tool results+ as span fields or logs — they are content, recorded on the spans under the+ `OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT` policy and a size budget.## Frontend and shared TypeScript (`apps/web`, `packages/`)diff --git a/services/agent_harness_service/src/main.rs b/services/agent_harness_service/src/main.rsindex 9ee7cd6fdc..bd7b826fca 100644--- a/services/agent_harness_service/src/main.rs+++ b/services/agent_harness_service/src/main.rs@@ -61,6 +61,7 @@ use agent_inmem::outbound::acp_mcp::AcpMcpConnector;use agent_inmem::outbound::egress_mcp::EgressMcpClient;use agent_inmem::outbound::log_frames::LogFrameSource;use agent_inmem::outbound::manager::InMemAgentManager;+use agent_inmem::outbound::tool_catalog::McpToolCatalog;use agent_inmem::rig_engine::RigTurnEngine;use agent_runtime_directory::PgAgentRuntimeDirectory;use agent_session::domain::model::{AgentMcpServers, ReplicaId};@@ -369,6 +370,14 @@ async fn run() -> anyhow::Result<()> {// against it, so the two must be the same string.let egress_base_url = AgentHarnessEgressUrl::new()?.to_string();+ // Every session's MCP tools, listed for its telemetry the way the harness+ // itself lists them: through the egress proxy, in process.+ let tool_catalog: Arc<dyn agent_session::domain::ports::SessionToolCatalog> =+ Arc::new(McpToolCatalog::new(Arc::new(AcpMcpConnector::new(+ EgressMcpClient::new(Arc::clone(&egress), &egress_base_url),+ ))));+ let sessions = sessions.with_tool_catalog(Arc::clone(&tool_catalog));+let tool_context =ai_tools::build_tool_service_context_from_env(pool.clone(), event_broker_tracker.clone()).await@@ -399,7 +408,8 @@ async fn run() -> anyhow::Result<()> {turn_observer.clone(),lifecycle_publisher.clone(),replica,- );+ )+ .with_tool_catalog(tool_catalog);let sandbox_and_inmem = RoutedContainers::new(sandbox, Some(inmem), inmem_sessions);// Cursor sessions run on their owner's own Cursor account, so there is nodiff --git a/services/document_cognition_service/src/api/stream/chat_message.rs b/services/document_cognition_service/src/api/stream/chat_message.rsindex 5f76236dbd..83753c2ad7 100644--- a/services/document_cognition_service/src/api/stream/chat_message.rs+++ b/services/document_cognition_service/src/api/stream/chat_message.rs@@ -538,8 +538,12 @@ fn stream_and_save_message(};let toolset: Arc<dyn ai_toolset::ToolSet<_> + Send + Sync> =Arc::new(mcp_select::CombinedToolSet::new(static_tools, mcp_tools));- let agent_loop =- AgentLoop::new(tool_context.recorder.clone()).with_model(&model);+ // The chat is the conversation every span of this turn belongs to+ // (`gen_ai.conversation.id`), so the turns of one chat form one session+ // in the observability backend.+ let agent_loop = AgentLoop::new(tool_context.recorder.clone())+ .with_model(&model)+ .with_conversation_id(&chat_id);let rig_messages = agent::to_rig_messages(&request);