Skip to content

armgit / letta

Public Imported from GitHub
0

Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.

feat: add tool_used field to run_metrics [LET-5419] (#5540)

Christina Tong79f49c33d866
diff --git a/alembic/versions/6756d04c3ddb_add_tools_used_field_to_run_metrics_.py b/alembic/versions/6756d04c3ddb_add_tools_used_field_to_run_metrics_.py
new file mode 100644
index 000000000..0a948ab0e
--- /dev/null
+++ b/alembic/versions/6756d04c3ddb_add_tools_used_field_to_run_metrics_.py
@@ -0,0 +1,31 @@
+"""Add tools_used field to run_metrics table
+
+Revision ID: 6756d04c3ddb
+Revises: e67961ed7c32
+Create Date: 2025-10-17 14:52:53.601368
+
+"""
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "6756d04c3ddb"
+down_revision: Union[str, None] = "e67961ed7c32"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+ # ### commands auto generated by Alembic - please adjust! ###
+ op.add_column("run_metrics", sa.Column("tools_used", sa.JSON(), nullable=True))
+ # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+ # ### commands auto generated by Alembic - please adjust! ###
+ op.drop_column("run_metrics", "tools_used")
+ # ### end Alembic commands ###
diff --git a/fern/openapi.json b/fern/openapi.json
index 912bd408c..0e2258eda 100644
--- a/fern/openapi.json
+++ b/fern/openapi.json
@@ -33932,6 +33932,21 @@
"title": "Num Steps",
"description": "The number of steps in the run."
},
+ "tools_used": {
+ "anyOf": [
+ {
+ "items": {
+ "type": "string"
+ },
+ "type": "array"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Tools Used",
+ "description": "List of tool IDs that were used in this run."
+ },
"template_id": {
"anyOf": [
{
diff --git a/letta/orm/run_metrics.py b/letta/orm/run_metrics.py
index 71e9aa84c..22c5d8e7a 100644
--- a/letta/orm/run_metrics.py
+++ b/letta/orm/run_metrics.py
@@ -1,7 +1,7 @@
from datetime import datetime, timezone
-from typing import TYPE_CHECKING, Optional
+from typing import TYPE_CHECKING, List, Optional
-from sqlalchemy import BigInteger, ForeignKey, Integer, String
+from sqlalchemy import JSON, BigInteger, ForeignKey, Integer, String
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.orm import Mapped, Session, mapped_column, relationship
@@ -43,6 +43,11 @@ class RunMetrics(SqlalchemyBase, ProjectMixin, AgentMixin, OrganizationMixin, Te
nullable=True,
doc="The number of steps in the run",
)
+ tools_used: Mapped[Optional[List[str]]] = mapped_column(
+ JSON,
+ nullable=True,
+ doc="List of tool IDs that were used in this run",
+ )
run: Mapped[Optional["Run"]] = relationship("Run", foreign_keys=[id])
agent: Mapped[Optional["Agent"]] = relationship("Agent")
diff --git a/letta/schemas/run_metrics.py b/letta/schemas/run_metrics.py
index 40971ab6c..458a1cad7 100644
--- a/letta/schemas/run_metrics.py
+++ b/letta/schemas/run_metrics.py
@@ -1,4 +1,4 @@
-from typing import Optional
+from typing import List, Optional
from pydantic import Field
@@ -17,5 +17,6 @@ class RunMetrics(RunMetricsBase):
run_start_ns: Optional[int] = Field(None, description="The timestamp of the start of the run in nanoseconds.")
run_ns: Optional[int] = Field(None, description="Total time for the run in nanoseconds.")
num_steps: Optional[int] = Field(None, description="The number of steps in the run.")
+ tools_used: Optional[List[str]] = Field(None, description="List of tool IDs that were used in this run.")
template_id: Optional[str] = Field(None, description="The template ID that the run belongs to (cloud only).")
base_template_id: Optional[str] = Field(None, description="The base template ID that the run belongs to (cloud only).")
diff --git a/letta/services/run_manager.py b/letta/services/run_manager.py
index 75871d8e1..c390462f5 100644
--- a/letta/services/run_manager.py
+++ b/letta/services/run_manager.py
@@ -203,6 +203,22 @@ class RunManager:
# update run metrics table
num_steps = len(await self.step_manager.list_steps_async(run_id=run_id, actor=actor))
+
+ # Collect tools used from run messages
+ tools_used = set()
+ messages = await self.message_manager.list_messages(actor=actor, run_id=run_id)
+ for message in messages:
+ if message.tool_calls:
+ for tool_call in message.tool_calls:
+ if hasattr(tool_call, "function") and hasattr(tool_call.function, "name"):
+ # Get tool ID from tool name
+ from letta.services.tool_manager import ToolManager
+
+ tool_manager = ToolManager()
+ tool_id = await tool_manager.get_tool_id_by_name_async(tool_call.function.name, actor)
+ if tool_id:
+ tools_used.add(tool_id)
+
async with db_registry.async_session() as session:
metrics = await RunMetricsModel.read_async(db_session=session, identifier=run_id, actor=actor)
# Calculate runtime if run is completing
@@ -217,6 +233,7 @@ class RunManager:
current_ns = int(time.time() * 1e9)
metrics.run_ns = current_ns - metrics.run_start_ns
metrics.num_steps = num_steps
+ metrics.tools_used = list(tools_used) if tools_used else None
await metrics.update_async(db_session=session, actor=actor, no_commit=True, no_refresh=True)
await session.commit()