Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
feat: add tool_used field to run_metrics [LET-5419] (#5540)
Christina Tong79f49c33d866
diff --git a/alembic/versions/6756d04c3ddb_add_tools_used_field_to_run_metrics_.py b/alembic/versions/6756d04c3ddb_add_tools_used_field_to_run_metrics_.pynew file mode 100644index 000000000..0a948ab0e--- /dev/null+++ b/alembic/versions/6756d04c3ddb_add_tools_used_field_to_run_metrics_.py@@ -0,0 +1,31 @@+"""Add tools_used field to run_metrics table++Revision ID: 6756d04c3ddb+Revises: e67961ed7c32+Create Date: 2025-10-17 14:52:53.601368++"""++from typing import Sequence, Union++import sqlalchemy as sa++from alembic import op++# revision identifiers, used by Alembic.+revision: str = "6756d04c3ddb"+down_revision: Union[str, None] = "e67961ed7c32"+branch_labels: Union[str, Sequence[str], None] = None+depends_on: Union[str, Sequence[str], None] = None+++def upgrade() -> None:+ # ### commands auto generated by Alembic - please adjust! ###+ op.add_column("run_metrics", sa.Column("tools_used", sa.JSON(), nullable=True))+ # ### end Alembic commands ###+++def downgrade() -> None:+ # ### commands auto generated by Alembic - please adjust! ###+ op.drop_column("run_metrics", "tools_used")+ # ### end Alembic commands ###diff --git a/fern/openapi.json b/fern/openapi.jsonindex 912bd408c..0e2258eda 100644--- a/fern/openapi.json+++ b/fern/openapi.json@@ -33932,6 +33932,21 @@"title": "Num Steps","description": "The number of steps in the run."},+ "tools_used": {+ "anyOf": [+ {+ "items": {+ "type": "string"+ },+ "type": "array"+ },+ {+ "type": "null"+ }+ ],+ "title": "Tools Used",+ "description": "List of tool IDs that were used in this run."+ },"template_id": {"anyOf": [{diff --git a/letta/orm/run_metrics.py b/letta/orm/run_metrics.pyindex 71e9aa84c..22c5d8e7a 100644--- a/letta/orm/run_metrics.py+++ b/letta/orm/run_metrics.py@@ -1,7 +1,7 @@from datetime import datetime, timezone-from typing import TYPE_CHECKING, Optional+from typing import TYPE_CHECKING, List, Optional-from sqlalchemy import BigInteger, ForeignKey, Integer, String+from sqlalchemy import JSON, BigInteger, ForeignKey, Integer, Stringfrom sqlalchemy.ext.asyncio import AsyncSessionfrom sqlalchemy.orm import Mapped, Session, mapped_column, relationship@@ -43,6 +43,11 @@ class RunMetrics(SqlalchemyBase, ProjectMixin, AgentMixin, OrganizationMixin, Tenullable=True,doc="The number of steps in the run",)+ tools_used: Mapped[Optional[List[str]]] = mapped_column(+ JSON,+ nullable=True,+ doc="List of tool IDs that were used in this run",+ )run: Mapped[Optional["Run"]] = relationship("Run", foreign_keys=[id])agent: Mapped[Optional["Agent"]] = relationship("Agent")diff --git a/letta/schemas/run_metrics.py b/letta/schemas/run_metrics.pyindex 40971ab6c..458a1cad7 100644--- a/letta/schemas/run_metrics.py+++ b/letta/schemas/run_metrics.py@@ -1,4 +1,4 @@-from typing import Optional+from typing import List, Optionalfrom pydantic import Field@@ -17,5 +17,6 @@ class RunMetrics(RunMetricsBase):run_start_ns: Optional[int] = Field(None, description="The timestamp of the start of the run in nanoseconds.")run_ns: Optional[int] = Field(None, description="Total time for the run in nanoseconds.")num_steps: Optional[int] = Field(None, description="The number of steps in the run.")+ tools_used: Optional[List[str]] = Field(None, description="List of tool IDs that were used in this run.")template_id: Optional[str] = Field(None, description="The template ID that the run belongs to (cloud only).")base_template_id: Optional[str] = Field(None, description="The base template ID that the run belongs to (cloud only).")diff --git a/letta/services/run_manager.py b/letta/services/run_manager.pyindex 75871d8e1..c390462f5 100644--- a/letta/services/run_manager.py+++ b/letta/services/run_manager.py@@ -203,6 +203,22 @@ class RunManager:# update run metrics tablenum_steps = len(await self.step_manager.list_steps_async(run_id=run_id, actor=actor))++ # Collect tools used from run messages+ tools_used = set()+ messages = await self.message_manager.list_messages(actor=actor, run_id=run_id)+ for message in messages:+ if message.tool_calls:+ for tool_call in message.tool_calls:+ if hasattr(tool_call, "function") and hasattr(tool_call.function, "name"):+ # Get tool ID from tool name+ from letta.services.tool_manager import ToolManager++ tool_manager = ToolManager()+ tool_id = await tool_manager.get_tool_id_by_name_async(tool_call.function.name, actor)+ if tool_id:+ tools_used.add(tool_id)+async with db_registry.async_session() as session:metrics = await RunMetricsModel.read_async(db_session=session, identifier=run_id, actor=actor)# Calculate runtime if run is completing@@ -217,6 +233,7 @@ class RunManager:current_ns = int(time.time() * 1e9)metrics.run_ns = current_ns - metrics.run_start_nsmetrics.num_steps = num_steps+ metrics.tools_used = list(tools_used) if tools_used else Noneawait metrics.update_async(db_session=session, actor=actor, no_commit=True, no_refresh=True)await session.commit()