Skip to content

πŸ™Œ OpenHands: AI-Driven Development

fix(app-server): persist combined LLM usage metrics across all usage buckets (#15354)

Alona6b04532541bf
diff --git a/enterprise/migrations/versions/139_add_cost_event_model_attribution.py b/enterprise/migrations/versions/139_add_cost_event_model_attribution.py
new file mode 100644
index 0000000000..e391dcf85f
--- /dev/null
+++ b/enterprise/migrations/versions/139_add_cost_event_model_attribution.py
@@ -0,0 +1,40 @@
+"""Add model/usage attribution columns to conversation cost events.
+
+Revision ID: 139
+Revises: 138
+Create Date: 2026-07-22
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = '139'
+down_revision = '138'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+ op.add_column(
+ 'conversation_cost_events', sa.Column('usage_id', sa.String(), nullable=True)
+ )
+ op.add_column(
+ 'conversation_cost_events', sa.Column('llm_model', sa.String(), nullable=True)
+ )
+ op.add_column(
+ 'conversation_cost_events',
+ sa.Column('prompt_tokens', sa.Integer(), nullable=True),
+ )
+ op.add_column(
+ 'conversation_cost_events',
+ sa.Column('completion_tokens', sa.Integer(), nullable=True),
+ )
+
+
+def downgrade():
+ op.drop_column('conversation_cost_events', 'completion_tokens')
+ op.drop_column('conversation_cost_events', 'prompt_tokens')
+ op.drop_column('conversation_cost_events', 'llm_model')
+ op.drop_column('conversation_cost_events', 'usage_id')
diff --git a/enterprise/server/routes/org_models.py b/enterprise/server/routes/org_models.py
index f76548d41f..e9f3b28662 100644
--- a/enterprise/server/routes/org_models.py
+++ b/enterprise/server/routes/org_models.py
@@ -913,7 +913,7 @@ class DailyUsageData(BaseModel):
date: str # ISO date string (YYYY-MM-DD)
tokens: int = 0
- conversations: int = 0
+ conversations: int = 0 # Conversations started
class TeamUsageData(BaseModel):
@@ -975,7 +975,8 @@ class OrgUsageStats(BaseModel):
# Top-level metrics
active_users: int = 0 # Users with activity in last 7 days
- agent_runs: int = 0 # Total conversations in last 7 days
+ agent_runs: int = 0 # Conversations started in last 7 days
+ usage_conversation_count: int = 0 # Conversations with usage in last 7 days
total_tokens: int = 0 # Total tokens in last 7 days
estimated_spend: float = 0.0 # Estimated cost in last 7 days
diff --git a/enterprise/server/services/org_conversation_service.py b/enterprise/server/services/org_conversation_service.py
index 1c8b099953..d1f5d17db0 100644
--- a/enterprise/server/services/org_conversation_service.py
+++ b/enterprise/server/services/org_conversation_service.py
@@ -9,7 +9,7 @@ from __future__ import annotations
import math
from dataclasses import dataclass
from datetime import UTC, datetime, timedelta
-from typing import AsyncGenerator
+from typing import Any, AsyncGenerator
from uuid import UUID
from fastapi import Request
@@ -63,27 +63,27 @@ AGENT_LABELS = {
}
-def _format_acp_agent_label(llm_model: str | None) -> str:
- if not llm_model:
- return AGENT_LABELS['acp']
- llm_model_lower = llm_model.lower()
- if 'claude' in llm_model_lower:
- return 'Claude'
- if 'codex' in llm_model_lower:
- return 'Codex'
- if 'gpt' in llm_model_lower or 'openai' in llm_model_lower:
- return 'OpenAI'
- if 'gemini' in llm_model_lower:
- return 'Gemini'
- return llm_model
-
-
-def _format_agent_label(agent_kind: str | None, llm_model: str | None) -> str:
- if agent_kind == 'acp':
- return _format_acp_agent_label(llm_model)
- if not agent_kind:
- return AGENT_LABELS['openhands']
- return AGENT_LABELS.get(agent_kind, agent_kind)
+def _agent_label_expression(llm_model: Any) -> Any:
+ model_lower = func.lower(llm_model)
+ acp_label = case(
+ (llm_model.is_(None), AGENT_LABELS['acp']),
+ (model_lower.contains('claude'), 'Claude'),
+ (model_lower.contains('codex'), 'Codex'),
+ (or_(model_lower.contains('gpt'), model_lower.contains('openai')), 'OpenAI'),
+ (model_lower.contains('gemini'), 'Gemini'),
+ else_=llm_model,
+ )
+ return case(
+ (StoredConversationMetadata.agent_kind == 'acp', acp_label),
+ (
+ or_(
+ StoredConversationMetadata.agent_kind.is_(None),
+ StoredConversationMetadata.agent_kind == 'openhands',
+ ),
+ AGENT_LABELS['openhands'],
+ ),
+ else_=StoredConversationMetadata.agent_kind,
+ )
MAX_SANDBOX_STATUS_FILTER_ROWS = 5000
@@ -573,32 +573,83 @@ class OrgConversationService:
total_tokens=int((total_prompt or 0) + (total_completion or 0)),
)
- async def get_usage_stats(
+ async def _get_model_usage(
self,
- org_id: UUID,
- days: int = 7,
- ) -> OrgUsageStats:
- """Get detailed usage statistics for organization dashboard.
-
- Args:
- org_id: The organization ID
- days: Number of days to look back (default 7)
-
- Returns:
- OrgUsageStats with detailed usage data
- """
- now = datetime.now(UTC)
- cutoff = now - timedelta(days=days)
-
- # Base filter for org conversations
- base_filter = [
- StoredConversationMetadata.conversation_version == 'V1',
- StoredConversationMetadataSaas.org_id == org_id,
- ]
+ base_filter: list,
+ cutoff: datetime,
+ ) -> list[ModelUsageData]:
+ """Model-usage rows: cost-event ledger + legacy no-ledger fallback."""
+ # 6. Model usage (by model) β€” aggregated from the cost-event ledger so
+ # spend is attributed to the model that incurred it (a conversation can
+ # span several LLMs and its llm_model column only holds the latest).
+ # Pre-attribution rows (NULL llm_model) fall back to the conversation's
+ # current model; windowing is on spend time, not conversation creation.
+ model_label = func.coalesce(
+ StoredConversationCostEvent.llm_model,
+ StoredConversationMetadata.llm_model,
+ )
+ model_query = (
+ select(
+ model_label.label('llm_model'),
+ func.count(
+ func.distinct(StoredConversationCostEvent.conversation_id)
+ ).label('conv_count'),
+ func.coalesce(
+ func.sum(
+ func.coalesce(StoredConversationCostEvent.prompt_tokens, 0)
+ + func.coalesce(
+ StoredConversationCostEvent.completion_tokens, 0
+ )
+ ),
+ 0,
+ ).label('token_count'),
+ func.coalesce(
+ func.sum(StoredConversationCostEvent.cost_delta), 0
+ ).label('total_cost'),
+ )
+ .select_from(StoredConversationCostEvent)
+ .join(
+ StoredConversationMetadata,
+ StoredConversationMetadata.conversation_id
+ == StoredConversationCostEvent.conversation_id,
+ )
+ .join(
+ StoredConversationMetadataSaas,
+ StoredConversationMetadata.conversation_id
+ == StoredConversationMetadataSaas.conversation_id,
+ )
+ .where(*base_filter)
+ .where(StoredConversationCostEvent.occurred_at >= cutoff)
+ .group_by(model_label)
+ .order_by(
+ func.coalesce(
+ func.sum(StoredConversationCostEvent.cost_delta), 0
+ ).desc()
+ )
+ )
+ result = await self.db_session.execute(model_query)
+ model_rows = list(result.all())
- # 1. Active users (users with activity in the time window)
- active_users_query = (
- select(func.count(func.distinct(StoredConversationMetadataSaas.user_id)))
+ # Conversations with no ledger rows at all (spend predates the cost
+ # events table) would otherwise vanish; keep them via the legacy
+ # created_at/llm_model aggregation.
+ legacy_model_query = (
+ select(
+ StoredConversationMetadata.llm_model,
+ func.count(StoredConversationMetadata.conversation_id).label(
+ 'conv_count'
+ ),
+ func.coalesce(
+ func.sum(
+ StoredConversationMetadata.prompt_tokens
+ + StoredConversationMetadata.completion_tokens
+ ),
+ 0,
+ ).label('token_count'),
+ func.coalesce(
+ func.sum(StoredConversationMetadata.accumulated_cost), 0
+ ).label('total_cost'),
+ )
.select_from(StoredConversationMetadata)
.join(
StoredConversationMetadataSaas,
@@ -607,33 +658,91 @@ class OrgConversationService:
)
.where(*base_filter)
.where(StoredConversationMetadata.created_at >= cutoff)
+ .where(
+ ~select(StoredConversationCostEvent.id)
+ .where(
+ StoredConversationCostEvent.conversation_id
+ == StoredConversationMetadata.conversation_id
+ )
+ .exists()
+ )
+ .group_by(StoredConversationMetadata.llm_model)
)
- result = await self.db_session.execute(active_users_query)
- active_users = result.scalar() or 0
+ result = await self.db_session.execute(legacy_model_query)
+ model_rows.extend(result.all())
- # 2. Total agent runs (conversations) in time window
- agent_runs_query = (
- select(func.count(StoredConversationMetadata.conversation_id))
- .select_from(StoredConversationMetadata)
+ merged: dict[str, dict[str, float]] = {}
+ for row in model_rows:
+ entry = merged.setdefault(
+ row.llm_model or 'Unknown',
+ {'conv_count': 0, 'token_count': 0, 'total_cost': 0.0},
+ )
+ entry['conv_count'] += int(row.conv_count or 0)
+ entry['token_count'] += int(row.token_count or 0)
+ entry['total_cost'] += float(row.total_cost or 0.0)
+
+ model_usage = []
+ for model_name, entry in sorted(
+ merged.items(), key=lambda kv: kv[1]['total_cost'], reverse=True
+ ):
+ model_usage.append(
+ ModelUsageData(
+ model_name=model_name,
+ conversation_count=int(entry['conv_count']),
+ total_tokens=int(entry['token_count']),
+ total_cost=float(entry['total_cost']),
+ )
+ )
+ return model_usage
+
+ async def _get_usage_totals(
+ self, base_filter: list, cutoff: datetime
+ ) -> tuple[float, int, int, int]:
+ """Windowed cost, tokens, and conversation count."""
+ ledger_query = (
+ select(
+ func.coalesce(func.sum(StoredConversationCostEvent.cost_delta), 0),
+ func.coalesce(
+ func.sum(
+ func.coalesce(StoredConversationCostEvent.prompt_tokens, 0)
+ ),
+ 0,
+ ),
+ func.coalesce(
+ func.sum(
+ func.coalesce(StoredConversationCostEvent.completion_tokens, 0)
+ ),
+ 0,
+ ),
+ func.count(func.distinct(StoredConversationCostEvent.conversation_id)),
+ )
+ .select_from(StoredConversationCostEvent)
+ .join(
+ StoredConversationMetadata,
+ StoredConversationMetadata.conversation_id
+ == StoredConversationCostEvent.conversation_id,
+ )
.join(
StoredConversationMetadataSaas,
StoredConversationMetadata.conversation_id
== StoredConversationMetadataSaas.conversation_id,
)
.where(*base_filter)
- .where(StoredConversationMetadata.created_at >= cutoff)
+ .where(StoredConversationCostEvent.occurred_at >= cutoff)
+ )
+ result = await self.db_session.execute(ledger_query)
+ ledger_cost, ledger_prompt, ledger_completion, ledger_conversations = (
+ result.one()
)
- result = await self.db_session.execute(agent_runs_query)
- agent_runs = result.scalar() or 0
- # 3. Total tokens and cost in time window
- totals_query = (
+ legacy_query = (
select(
func.coalesce(func.sum(StoredConversationMetadata.accumulated_cost), 0),
func.coalesce(func.sum(StoredConversationMetadata.prompt_tokens), 0),
func.coalesce(
func.sum(StoredConversationMetadata.completion_tokens), 0
),
+ func.count(StoredConversationMetadata.conversation_id),
)
.select_from(StoredConversationMetadata)
.join(
@@ -643,17 +752,71 @@ class OrgConversationService:
)
.where(*base_filter)
.where(StoredConversationMetadata.created_at >= cutoff)
+ .where(
+ ~select(StoredConversationCostEvent.id)
+ .where(
+ StoredConversationCostEvent.conversation_id
+ == StoredConversationMetadata.conversation_id
+ )
+ .exists()
+ )
+ )
+ result = await self.db_session.execute(legacy_query)
+ legacy_cost, legacy_prompt, legacy_completion, legacy_conversations = (
+ result.one()
+ )
+
+ return (
+ float(ledger_cost or 0) + float(legacy_cost or 0),
+ int(ledger_prompt or 0) + int(legacy_prompt or 0),
+ int(ledger_completion or 0) + int(legacy_completion or 0),
+ int(ledger_conversations or 0) + int(legacy_conversations or 0),
)
- result = await self.db_session.execute(totals_query)
- total_cost, total_prompt_tokens, total_completion_tokens = result.one()
- # 4. Daily usage breakdown (single query instead of N queries)
- from sqlalchemy import Date, cast
+ async def _get_daily_token_usage(
+ self, base_filter: list, cutoff: datetime
+ ) -> dict[str, int]:
+ """Tokens per ISO day on spend time; legacy fallback by created_at."""
+ tokens: dict[str, int] = {}
- daily_query = (
+ ledger_day = func.date(StoredConversationCostEvent.occurred_at)
+ ledger_query = (
select(
- cast(StoredConversationMetadata.created_at, Date).label('day'),
- func.count(StoredConversationMetadata.conversation_id),
+ ledger_day.label('day'),
+ func.coalesce(
+ func.sum(
+ func.coalesce(StoredConversationCostEvent.prompt_tokens, 0)
+ + func.coalesce(
+ StoredConversationCostEvent.completion_tokens, 0
+ )
+ ),
+ 0,
+ ),
+ )
+ .select_from(StoredConversationCostEvent)
+ .join(
+ StoredConversationMetadata,
+ StoredConversationMetadata.conversation_id
+ == StoredConversationCostEvent.conversation_id,
+ )
+ .join(
+ StoredConversationMetadataSaas,
+ StoredConversationMetadata.conversation_id
+ == StoredConversationMetadataSaas.conversation_id,
+ )
+ .where(*base_filter)
+ .where(StoredConversationCostEvent.occurred_at >= cutoff)
+ .group_by(ledger_day)
+ )
+ result = await self.db_session.execute(ledger_query)
+ for row in result.all():
+ key = str(row[0])[:10]
+ tokens[key] = tokens.get(key, 0) + int(row[1] or 0)
+
+ legacy_day = func.date(StoredConversationMetadata.created_at)
+ legacy_query = (
+ select(
+ legacy_day.label('day'),
func.coalesce(
func.sum(
StoredConversationMetadata.prompt_tokens
@@ -670,47 +833,78 @@ class OrgConversationService:
)
.where(*base_filter)
.where(StoredConversationMetadata.created_at >= cutoff)
- .group_by(cast(StoredConversationMetadata.created_at, Date))
- .order_by(cast(StoredConversationMetadata.created_at, Date).asc())
+ .where(
+ ~select(StoredConversationCostEvent.id)
+ .where(
+ StoredConversationCostEvent.conversation_id
+ == StoredConversationMetadata.conversation_id
+ )
+ .exists()
+ )
+ .group_by(legacy_day)
)
- result = await self.db_session.execute(daily_query)
- daily_rows = result.all()
+ result = await self.db_session.execute(legacy_query)
+ for row in result.all():
+ key = str(row[0])[:10]
+ tokens[key] = tokens.get(key, 0) + int(row[1] or 0)
- # Build a map of date -> (conv_count, token_count)
- daily_map = {row[0]: (row[1], row[2]) for row in daily_rows}
+ return tokens
- # Generate entries for all days in range (including days with no data)
- daily_usage = []
- for i in range(days - 1, -1, -1):
- day_start = (now - timedelta(days=i)).replace(
- hour=0, minute=0, second=0, microsecond=0
- )
- day_date = day_start.date() if hasattr(day_start, 'date') else day_start
- conv_count, token_count = daily_map.get(day_date, (0, 0))
- daily_usage.append(
- DailyUsageData(
- date=day_start.strftime('%Y-%m-%d'),
- tokens=int(token_count or 0),
- conversations=int(conv_count or 0),
- )
- )
+ async def _get_team_usage(
+ self, base_filter: list, cutoff: datetime
+ ) -> dict[str, tuple[int, int, str | None, str | None]]:
+ """Per-user conversation and token usage on spend time."""
+ merged: dict[str, list] = {}
- # 5. Team usage (by user)
- team_query = (
+ ledger_query = (
select(
StoredConversationMetadataSaas.user_id,
User.email,
User.git_user_name,
- func.count(StoredConversationMetadata.conversation_id).label(
- 'conv_count'
+ func.count(func.distinct(StoredConversationCostEvent.conversation_id)),
+ func.coalesce(
+ func.sum(
+ func.coalesce(StoredConversationCostEvent.prompt_tokens, 0)
+ + func.coalesce(
+ StoredConversationCostEvent.completion_tokens, 0
+ )
+ ),
+ 0,
),
+ )
+ .select_from(StoredConversationCostEvent)
+ .join(
+ StoredConversationMetadata,
+ StoredConversationMetadata.conversation_id
+ == StoredConversationCostEvent.conversation_id,
+ )
+ .join(
+ StoredConversationMetadataSaas,
+ StoredConversationMetadata.conversation_id
+ == StoredConversationMetadataSaas.conversation_id,
+ )
+ .outerjoin(User, StoredConversationMetadataSaas.user_id == User.id)
+ .where(*base_filter)
+ .where(StoredConversationCostEvent.occurred_at >= cutoff)
+ .group_by(
+ StoredConversationMetadataSaas.user_id,
+ User.email,
+ User.git_user_name,
+ )
+ )
+ legacy_query = (
+ select(
+ StoredConversationMetadataSaas.user_id,
+ User.email,
+ User.git_user_name,
+ func.count(StoredConversationMetadata.conversation_id),
func.coalesce(
func.sum(
StoredConversationMetadata.prompt_tokens
+ StoredConversationMetadata.completion_tokens
),
0,
- ).label('token_count'),
+ ),
)
.select_from(StoredConversationMetadata)
.join(
@@ -721,51 +915,69 @@ class OrgConversationService:
.outerjoin(User, StoredConversationMetadataSaas.user_id == User.id)
.where(*base_filter)
.where(StoredConversationMetadata.created_at >= cutoff)
+ .where(
+ ~select(StoredConversationCostEvent.id)
+ .where(
+ StoredConversationCostEvent.conversation_id
+ == StoredConversationMetadata.conversation_id
+ )
+ .exists()
+ )
.group_by(
StoredConversationMetadataSaas.user_id,
User.email,
User.git_user_name,
)
- .order_by(func.count(StoredConversationMetadata.conversation_id).desc())
)
- result = await self.db_session.execute(team_query)
- team_rows = result.all()
-
- # Calculate percentages
- total_team_convs = sum(row.conv_count or 0 for row in team_rows)
- team_usage = []
- for row in team_rows:
- pct = (
- (row.conv_count / total_team_convs * 100) if total_team_convs > 0 else 0
+ for query in (ledger_query, legacy_query):
+ result = await self.db_session.execute(query)
+ for row in result.all():
+ entry = merged.setdefault(str(row[0]), [0, 0, row[1], row[2]])
+ entry[0] += int(row[3] or 0)
+ entry[1] += int(row[4] or 0)
+
+ return {key: (val[0], val[1], val[2], val[3]) for key, val in merged.items()}
+
+ async def _get_agent_usage(
+ self, base_filter: list, cutoff: datetime
+ ) -> dict[str, tuple[int, float]]:
+ """Conversation count and cost per agent label on spend time."""
+ usage: dict[str, tuple[int, float]] = {}
+
+ model_label = func.coalesce(
+ StoredConversationCostEvent.llm_model,
+ StoredConversationMetadata.llm_model,
+ )
+ ledger_agent_label = _agent_label_expression(model_label)
+ ledger_query = (
+ select(
+ ledger_agent_label.label('agent_name'),
+ func.count(func.distinct(StoredConversationCostEvent.conversation_id)),
+ func.coalesce(func.sum(StoredConversationCostEvent.cost_delta), 0),
)
- team_usage.append(
- TeamUsageData(
- user_id=str(row.user_id),
- user_email=row.email,
- user_name=row.git_user_name,
- conversation_count=int(row.conv_count or 0),
- total_tokens=int(row.token_count or 0),
- percentage=round(pct, 1),
- )
+ .select_from(StoredConversationCostEvent)
+ .join(
+ StoredConversationMetadata,
+ StoredConversationMetadata.conversation_id
+ == StoredConversationCostEvent.conversation_id,
)
-
- # 6. Model usage (by model)
- model_query = (
+ .join(
+ StoredConversationMetadataSaas,
+ StoredConversationMetadata.conversation_id
+ == StoredConversationMetadataSaas.conversation_id,
+ )
+ .where(*base_filter)
+ .where(StoredConversationCostEvent.occurred_at >= cutoff)
+ .group_by(ledger_agent_label)
+ )
+ legacy_agent_label = _agent_label_expression(
+ StoredConversationMetadata.llm_model
+ )
+ legacy_query = (
select(
- StoredConversationMetadata.llm_model,
- func.count(StoredConversationMetadata.conversation_id).label(
- 'conv_count'
- ),
- func.coalesce(
- func.sum(
- StoredConversationMetadata.prompt_tokens
- + StoredConversationMetadata.completion_tokens
- ),
- 0,
- ).label('token_count'),
- func.coalesce(
- func.sum(StoredConversationMetadata.accumulated_cost), 0
- ).label('total_cost'),
+ legacy_agent_label.label('agent_name'),
+ func.count(StoredConversationMetadata.conversation_id),
+ func.coalesce(func.sum(StoredConversationMetadata.accumulated_cost), 0),
)
.select_from(StoredConversationMetadata)
.join(
@@ -775,36 +987,80 @@ class OrgConversationService:
)
.where(*base_filter)
.where(StoredConversationMetadata.created_at >= cutoff)
- .group_by(StoredConversationMetadata.llm_model)
- .order_by(
- func.coalesce(
- func.sum(StoredConversationMetadata.accumulated_cost), 0
- ).desc()
+ .where(
+ ~select(StoredConversationCostEvent.id)
+ .where(
+ StoredConversationCostEvent.conversation_id
+ == StoredConversationMetadata.conversation_id
+ )
+ .exists()
)
+ .group_by(legacy_agent_label)
)
- result = await self.db_session.execute(model_query)
- model_rows = result.all()
- model_usage = []
- for row in model_rows:
- model_usage.append(
- ModelUsageData(
- model_name=row.llm_model or 'Unknown',
- conversation_count=int(row.conv_count or 0),
- total_tokens=int(row.token_count or 0),
- total_cost=float(row.total_cost or 0.0),
+ for query in (ledger_query, legacy_query):
+ result = await self.db_session.execute(query)
+ for row in result.all():
+ conversation_count, total_cost = usage.get(row[0], (0, 0.0))
+ usage[row[0]] = (
+ conversation_count + int(row[1] or 0),
+ total_cost + float(row[2] or 0.0),
)
+
+ return usage
+
+ async def get_usage_stats(
+ self,
+ org_id: UUID,
+ days: int = 7,
+ ) -> OrgUsageStats:
+ """Get detailed usage statistics for organization dashboard.
+
+ Args:
+ org_id: The organization ID
+ days: Number of days to look back (default 7)
+
+ Returns:
+ OrgUsageStats with detailed usage data
+ """
+ now = datetime.now(UTC)
+ cutoff = now - timedelta(days=days)
+
+ # Base filter for org conversations
+ base_filter = [
+ StoredConversationMetadata.conversation_version == 'V1',
+ StoredConversationMetadataSaas.org_id == org_id,
+ ]
+
+ # 1. Total agent runs (conversations started) in time window
+ agent_runs_query = (
+ select(func.count(StoredConversationMetadata.conversation_id))
+ .select_from(StoredConversationMetadata)
+ .join(
+ StoredConversationMetadataSaas,
+ StoredConversationMetadata.conversation_id
+ == StoredConversationMetadataSaas.conversation_id,
)
+ .where(*base_filter)
+ .where(StoredConversationMetadata.created_at >= cutoff)
+ )
+ result = await self.db_session.execute(agent_runs_query)
+ agent_runs = result.scalar() or 0
- agent_query = (
+ # 2. Spend-time totals, with a no-ledger fallback.
+ (
+ total_cost,
+ total_prompt_tokens,
+ total_completion_tokens,
+ usage_conversation_count,
+ ) = await self._get_usage_totals(base_filter, cutoff)
+
+ # 3. Daily usage breakdown: conversation counts stay on created_at
+ # ("runs started that day"); tokens follow spend time via the ledger.
+ created_day = func.date(StoredConversationMetadata.created_at)
+ daily_query = (
select(
- StoredConversationMetadata.agent_kind,
- StoredConversationMetadata.llm_model,
- func.count(StoredConversationMetadata.conversation_id).label(
- 'conv_count'
- ),
- func.coalesce(
- func.sum(StoredConversationMetadata.accumulated_cost), 0
- ).label('total_cost'),
+ created_day.label('day'),
+ func.count(StoredConversationMetadata.conversation_id),
)
.select_from(StoredConversationMetadata)
.join(
@@ -814,40 +1070,64 @@ class OrgConversationService:
)
.where(*base_filter)
.where(StoredConversationMetadata.created_at >= cutoff)
- .group_by(
- StoredConversationMetadata.agent_kind,
- StoredConversationMetadata.llm_model,
+ .group_by(created_day)
+ )
+ result = await self.db_session.execute(daily_query)
+ count_map = {str(row[0])[:10]: int(row[1] or 0) for row in result.all()}
+ daily_tokens = await self._get_daily_token_usage(base_filter, cutoff)
+
+ # Generate entries for all days in range (including days with no data)
+ daily_usage = []
+ for i in range(days - 1, -1, -1):
+ day_start = (now - timedelta(days=i)).replace(
+ hour=0, minute=0, second=0, microsecond=0
)
- .order_by(
- func.coalesce(
- func.sum(StoredConversationMetadata.accumulated_cost), 0
- ).desc()
+ day_key = day_start.strftime('%Y-%m-%d')
+ daily_usage.append(
+ DailyUsageData(
+ date=day_key,
+ tokens=daily_tokens.get(day_key, 0),
+ conversations=count_map.get(day_key, 0),
+ )
)
- )
- result = await self.db_session.execute(agent_query)
- agent_rows = result.all()
- agent_counts: dict[str, int] = {}
- agent_costs: dict[str, float] = {}
- for row in agent_rows:
- label = _format_agent_label(row.agent_kind, row.llm_model)
- agent_counts[label] = agent_counts.get(label, 0) + int(row.conv_count or 0)
- agent_costs[label] = agent_costs.get(label, 0.0) + float(
- row.total_cost or 0.0
+
+ # 4. Team usage follows spend time for both counts and tokens.
+ team_stats = await self._get_team_usage(base_filter, cutoff)
+ active_users = len(team_stats)
+ total_team_convs = sum(entry[0] for entry in team_stats.values())
+ team_usage = []
+ for user_id, (conv_count, tokens, email, git_user_name) in team_stats.items():
+ pct = (conv_count / total_team_convs * 100) if total_team_convs > 0 else 0
+ team_usage.append(
+ TeamUsageData(
+ user_id=user_id,
+ user_email=email,
+ user_name=git_user_name,
+ conversation_count=conv_count,
+ total_tokens=tokens,
+ percentage=round(pct, 1),
+ )
)
+ team_usage.sort(key=lambda item: item.conversation_count, reverse=True)
+
+ model_usage = await self._get_model_usage(base_filter, cutoff)
+ # Agent counts and costs use the same spend-time cohort.
+ agent_stats = await self._get_agent_usage(base_filter, cutoff)
agent_usage = [
AgentUsageData(
agent_name=label,
- conversation_count=agent_counts[label],
- total_cost=agent_costs[label],
+ conversation_count=conversation_count,
+ total_cost=total_cost,
)
- for label in agent_counts
+ for label, (conversation_count, total_cost) in agent_stats.items()
]
agent_usage.sort(key=lambda item: item.total_cost, reverse=True)
return OrgUsageStats(
active_users=int(active_users),
agent_runs=int(agent_runs),
+ usage_conversation_count=usage_conversation_count,
total_tokens=int(total_prompt_tokens or 0)
+ int(total_completion_tokens or 0),
estimated_spend=float(total_cost or 0),
diff --git a/enterprise/server/utils/saas_app_conversation_info_injector.py b/enterprise/server/utils/saas_app_conversation_info_injector.py
index 533e1bbdcd..cad191a396 100644
--- a/enterprise/server/utils/saas_app_conversation_info_injector.py
+++ b/enterprise/server/utils/saas_app_conversation_info_injector.py
@@ -24,7 +24,7 @@ from openhands.app_server.app_conversation.sql_app_conversation_info_service imp
)
from openhands.app_server.errors import AuthError
from openhands.app_server.services.injector import InjectorState
-from openhands.app_server.user.specifiy_user_context import ADMIN
+from openhands.app_server.user.specifiy_user_context import ADMIN, SandboxUserContext
class SaasSQLAppConversationInfoService(SQLAppConversationInfoService):
@@ -49,11 +49,12 @@ class SaasSQLAppConversationInfoService(SQLAppConversationInfoService):
return result.scalars().first()
async def _apply_user_and_org_filter(self, query):
- """Apply user_id and org_id filters to ensure conversation isolation.
+ """Apply tenant filters to ensure conversation isolation.
Filters conversations by:
- user_id: Only show conversations belonging to the current user
- - org_id: Only show conversations belonging to the request's
+ - sandbox_id: Authenticated sandbox webhooks stay within that sandbox
+ - org_id: Other requests use the request's
*effective* organization (honors ``X-Org-Id`` and API-key org
binding; falls back to ``user.current_org_id``).
@@ -80,6 +81,11 @@ class SaasSQLAppConversationInfoService(SQLAppConversationInfoService):
user_id_uuid = UUID(user_id_str)
query = query.where(StoredConversationMetadataSaas.user_id == user_id_uuid)
+ if isinstance(self.user_context, SandboxUserContext):
+ return query.where(
+ StoredConversationMetadata.sandbox_id == self.user_context.sandbox_id
+ )
+
# Filter by the *effective* organization id (X-Org-Id override or
# API-key binding take precedence over user.current_org_id).
effective_org_id = await self._get_effective_org_id()
@@ -354,6 +360,37 @@ class SaasSQLAppConversationInfoService(SQLAppConversationInfoService):
self, info: AppConversationInfo
) -> AppConversationInfo:
"""Save conversation info and create/update SAAS metadata with user_id and org_id."""
+ if isinstance(self.user_context, SandboxUserContext):
+ user_id_str = await self.user_context.get_user_id()
+ if not user_id_str:
+ raise AuthError('Sandbox owner required')
+ if (
+ info.sandbox_id != self.user_context.sandbox_id
+ or info.created_by_user_id != user_id_str
+ ):
+ raise AuthError()
+ existing_query = (
+ select(
+ StoredConversationMetadata.sandbox_id,
+ StoredConversationMetadataSaas.user_id,
+ )
+ .outerjoin(
+ StoredConversationMetadataSaas,
+ StoredConversationMetadata.conversation_id
+ == StoredConversationMetadataSaas.conversation_id,
+ )
+ .where(
+ StoredConversationMetadata.conversation_id == str(info.id),
+ StoredConversationMetadata.conversation_version == 'V1',
+ )
+ )
+ existing = (await self.db_session.execute(existing_query)).first()
+ if existing and (
+ existing.sandbox_id != self.user_context.sandbox_id
+ or existing.user_id != UUID(user_id_str)
+ ):
+ raise AuthError()
+
# Save the base conversation metadata
await super().save_app_conversation_info(info)
diff --git a/enterprise/tests/unit/storage/test_saas_sql_app_conversation_info_service.py b/enterprise/tests/unit/storage/test_saas_sql_app_conversation_info_service.py
index 12344f67e5..d075f6d625 100644
--- a/enterprise/tests/unit/storage/test_saas_sql_app_conversation_info_service.py
+++ b/enterprise/tests/unit/storage/test_saas_sql_app_conversation_info_service.py
@@ -24,8 +24,18 @@ from openhands.app_server.app_conversation.app_conversation_models import (
AppConversationInfo,
ConversationTrigger,
)
+from openhands.app_server.app_conversation.sql_app_conversation_info_service import (
+ StoredConversationCostEvent,
+ StoredConversationMetadata,
+)
+from openhands.app_server.errors import AuthError
from openhands.app_server.integrations.service_types import ProviderType
-from openhands.app_server.user.specifiy_user_context import SpecifyUserContext
+from openhands.app_server.user.specifiy_user_context import (
+ SandboxUserContext,
+ SpecifyUserContext,
+)
+from openhands.sdk import ConversationStats
+from openhands.sdk.llm import Metrics, TokenUsage
# Test UUIDs
USER1_ID = UUID('a1111111-1111-1111-1111-111111111111')
@@ -444,6 +454,107 @@ class TestSaasSQLAppConversationInfoService:
assert len(page_back_in_org1.items) == 1
assert page_back_in_org1.items[0].title == 'Conversation in Org 1'
+ @pytest.mark.asyncio
+ async def test_sandbox_context_updates_conversation_after_org_switch(
+ self,
+ async_session_with_users: AsyncSession,
+ ):
+ owner_service = SaasSQLAppConversationInfoService(
+ db_session=async_session_with_users,
+ user_context=SpecifyUserContext(user_id=str(USER1_ID)),
+ )
+ conversation = AppConversationInfo(
+ id=uuid4(),
+ created_by_user_id=str(USER1_ID),
+ sandbox_id='authenticated-sandbox',
+ title='Org 1 conversation',
+ )
+ other_conversation = AppConversationInfo(
+ id=uuid4(),
+ created_by_user_id=str(USER1_ID),
+ sandbox_id='other-sandbox',
+ title='Other sandbox conversation',
+ )
+ await owner_service.save_app_conversation_info(conversation)
+ await owner_service.save_app_conversation_info(other_conversation)
+
+ user = await async_session_with_users.get(User, USER1_ID)
+ assert user is not None
+ user.current_org_id = ORG2_ID
+ await async_session_with_users.commit()
+ async_session_with_users.expire_all()
+
+ webhook_service = SaasSQLAppConversationInfoService(
+ db_session=async_session_with_users,
+ user_context=SandboxUserContext(
+ user_id=str(USER1_ID),
+ sandbox_id=conversation.sandbox_id,
+ ),
+ )
+ assert (
+ await webhook_service.get_app_conversation_info(conversation.id)
+ ) is not None
+ assert (
+ await webhook_service.get_app_conversation_info(other_conversation.id)
+ ) is None
+
+ stats = ConversationStats(
+ usage_to_metrics={
+ 'agent': Metrics(
+ accumulated_cost=0.75,
+ accumulated_token_usage=TokenUsage(
+ prompt_tokens=100,
+ completion_tokens=20,
+ ),
+ )
+ }
+ )
+ await webhook_service.update_conversation_statistics(conversation.id, stats)
+ await webhook_service.update_execution_status(conversation.id, 'finished')
+
+ stored = await async_session_with_users.get(
+ StoredConversationMetadata, str(conversation.id)
+ )
+ assert stored is not None
+ assert stored.accumulated_cost == 0.75
+ assert stored.prompt_tokens == 100
+ assert stored.completion_tokens == 20
+ assert stored.execution_status == 'finished'
+
+ cost_events = (
+ await async_session_with_users.scalars(
+ select(StoredConversationCostEvent).where(
+ StoredConversationCostEvent.conversation_id == str(conversation.id)
+ )
+ )
+ ).all()
+ assert [event.cost_delta for event in cost_events] == [0.75]
+
+ wrong_owner_service = SaasSQLAppConversationInfoService(
+ db_session=async_session_with_users,
+ user_context=SandboxUserContext(
+ user_id=str(USER2_ID),
+ sandbox_id=conversation.sandbox_id,
+ ),
+ )
+ assert (
+ await wrong_owner_service.get_app_conversation_info(conversation.id)
+ ) is None
+
+ wrong_sandbox_service = SaasSQLAppConversationInfoService(
+ db_session=async_session_with_users,
+ user_context=SandboxUserContext(
+ user_id=str(USER1_ID),
+ sandbox_id=other_conversation.sandbox_id,
+ ),
+ )
+ with pytest.raises(AuthError):
+ await wrong_sandbox_service.save_app_conversation_info(
+ conversation.model_copy(update={'title': 'Overwritten'})
+ )
+ await async_session_with_users.refresh(stored)
+ assert stored.title == conversation.title
+
@pytest.mark.asyncio
async def test_count_respects_org_isolation(
self,
diff --git a/enterprise/tests/unit/test_org_conversation_usage_models.py b/enterprise/tests/unit/test_org_conversation_usage_models.py
new file mode 100644
index 0000000000..01fe8113c5
--- /dev/null
+++ b/enterprise/tests/unit/test_org_conversation_usage_models.py
@@ -0,0 +1,325 @@
+"""Query-level tests for the model-usage aggregation in org usage stats."""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime, timedelta
+from uuid import uuid4
+
+import pytest
+from server.services.org_conversation_service import OrgConversationService
+from storage.stored_conversation_cost_event import StoredConversationCostEvent
+from storage.stored_conversation_metadata import StoredConversationMetadata
+from storage.stored_conversation_metadata_saas import StoredConversationMetadataSaas
+
+ORG_ID = uuid4()
+USER_ID = uuid4()
+
+
+def _conversation(
+ session,
+ conversation_id,
+ llm_model,
+ cost,
+ prompt,
+ completion,
+ *,
+ created_at=None,
+ agent_kind=None,
+):
+ session.add(
+ StoredConversationMetadata(
+ conversation_id=conversation_id,
+ conversation_version='V1',
+ llm_model=llm_model,
+ accumulated_cost=cost,
+ prompt_tokens=prompt,
+ completion_tokens=completion,
+ created_at=created_at or datetime.now(UTC) - timedelta(days=1),
+ agent_kind=agent_kind,
+ )
+ )
+ session.add(
+ StoredConversationMetadataSaas(
+ conversation_id=conversation_id,
+ user_id=USER_ID,
+ org_id=ORG_ID,
+ )
+ )
+
+
+async def test_model_usage_ledger_legacy_and_no_event_rows(async_session_maker):
+ occurred = datetime.now(UTC) - timedelta(hours=2)
+ async with async_session_maker() as session:
+ # A: attributed ledger rows across two models; the conversation's own
+ # llm_model label must NOT override per-event attribution.
+ _conversation(session, 'conv-a', 'litellm_proxy/current-label', 0.25, 300, 30)
+ session.add(
+ StoredConversationCostEvent(
+ conversation_id='conv-a',
+ cost_delta=0.08,
+ occurred_at=occurred,
+ usage_id='agent',
+ llm_model='litellm_proxy/gpt-5.5',
+ prompt_tokens=100,
+ completion_tokens=10,
+ )
+ )
+ session.add(
+ StoredConversationCostEvent(
+ conversation_id='conv-a',
+ cost_delta=0.17,
+ occurred_at=occurred,
+ usage_id='profile:opus:x1',
+ llm_model='litellm_proxy/claude-opus-4-8',
+ prompt_tokens=200,
+ completion_tokens=20,
+ )
+ )
+ # B: pre-migration NULL rows fall back to the conversation label;
+ # NULL token fields contribute zero tokens.
+ _conversation(session, 'conv-b', 'legacy-model', 0.30, 999, 99)
+ session.add(
+ StoredConversationCostEvent(
+ conversation_id='conv-b',
+ cost_delta=0.30,
+ occurred_at=occurred,
+ )
+ )
+ # C: no ledger rows at all β€” kept via the legacy aggregation.
+ _conversation(session, 'conv-c', 'old-model', 0.55, 50, 5)
+ await session.commit()
+
+ async with async_session_maker() as session:
+ service = OrgConversationService(db_session=session)
+ base_filter = [
+ StoredConversationMetadata.conversation_version == 'V1',
+ StoredConversationMetadataSaas.org_id == ORG_ID,
+ ]
+ cutoff = datetime.now(UTC) - timedelta(days=30)
+ model_usage = await service._get_model_usage(base_filter, cutoff)
+ agent_usage = await service._get_agent_usage(base_filter, cutoff)
+
+ by_model = {m.model_name: m for m in model_usage}
+ assert by_model['litellm_proxy/gpt-5.5'].total_cost == pytest.approx(0.08)
+ assert by_model['litellm_proxy/gpt-5.5'].total_tokens == 110
+ assert by_model['litellm_proxy/claude-opus-4-8'].total_cost == pytest.approx(0.17)
+ assert by_model['litellm_proxy/claude-opus-4-8'].total_tokens == 220
+ assert by_model['legacy-model'].total_cost == pytest.approx(0.30)
+ assert by_model['legacy-model'].total_tokens == 0
+ assert by_model['old-model'].total_cost == pytest.approx(0.55)
+ assert by_model['old-model'].total_tokens == 55
+ # The relabel-prone conversation label never appears as its own row.
+ assert 'litellm_proxy/current-label' not in by_model
+ # Ordered by spend, descending.
+ costs = [m.total_cost for m in model_usage]
+ assert costs == sorted(costs, reverse=True)
+ assert agent_usage['OpenHands'][0] == 3
+ assert agent_usage['OpenHands'][1] == pytest.approx(1.10)
+
+
+async def test_agent_usage_groups_acp_models_and_deduplicates_conversations(
+ async_session_maker,
+):
+ occurred = datetime.now(UTC) - timedelta(hours=2)
+ async with async_session_maker() as session:
+ _conversation(
+ session,
+ 'acp-openai',
+ 'gpt-current',
+ 0.30,
+ 30,
+ 3,
+ agent_kind='acp',
+ )
+ session.add_all(
+ [
+ StoredConversationCostEvent(
+ conversation_id='acp-openai',
+ cost_delta=0.10,
+ occurred_at=occurred,
+ llm_model='openai/gpt-5',
+ ),
+ StoredConversationCostEvent(
+ conversation_id='acp-openai',
+ cost_delta=0.20,
+ occurred_at=occurred,
+ llm_model='gpt-4.1',
+ ),
+ ]
+ )
+ _conversation(
+ session,
+ 'acp-claude',
+ 'claude-current',
+ 0.40,
+ 40,
+ 4,
+ agent_kind='acp',
+ )
+ session.add(
+ StoredConversationCostEvent(
+ conversation_id='acp-claude',
+ cost_delta=0.40,
+ occurred_at=occurred,
+ llm_model='claude-sonnet',
+ )
+ )
+ _conversation(
+ session,
+ 'acp-codex-legacy',
+ 'codex-mini',
+ 0.50,
+ 50,
+ 5,
+ agent_kind='acp',
+ )
+ await session.commit()
+
+ async with async_session_maker() as session:
+ service = OrgConversationService(db_session=session)
+ agent_usage = await service._get_agent_usage(
+ [
+ StoredConversationMetadata.conversation_version == 'V1',
+ StoredConversationMetadataSaas.org_id == ORG_ID,
+ ],
+ datetime.now(UTC) - timedelta(days=30),
+ )
+
+ assert set(agent_usage) == {'OpenAI', 'Claude', 'Codex'}
+ assert agent_usage['OpenAI'][0] == 1
+ assert agent_usage['OpenAI'][1] == pytest.approx(0.30)
+ assert agent_usage['Claude'][0] == 1
+ assert agent_usage['Claude'][1] == pytest.approx(0.40)
+ assert agent_usage['Codex'][0] == 1
+ assert agent_usage['Codex'][1] == pytest.approx(0.50)
+
+
+async def _seed_spend_time_boundary_scenario(async_session_maker):
+ now = datetime.now(UTC)
+ recent = now - timedelta(hours=2)
+ old = now - timedelta(days=45)
+ async with async_session_maker() as session:
+ _conversation(
+ session,
+ 'old-active-a',
+ 'litellm_proxy/current-label',
+ 0.40,
+ 100,
+ 10,
+ created_at=old,
+ )
+ session.add(
+ StoredConversationCostEvent(
+ conversation_id='old-active-a',
+ cost_delta=0.40,
+ occurred_at=recent,
+ usage_id='agent',
+ llm_model='litellm_proxy/gpt-5.5',
+ prompt_tokens=100,
+ completion_tokens=10,
+ )
+ )
+ _conversation(
+ session,
+ 'old-active-b',
+ 'litellm_proxy/current-label',
+ 0.10,
+ 20,
+ 2,
+ created_at=old,
+ )
+ session.add(
+ StoredConversationCostEvent(
+ conversation_id='old-active-b',
+ cost_delta=0.10,
+ occurred_at=recent,
+ usage_id='agent',
+ llm_model='litellm_proxy/gpt-5.5',
+ prompt_tokens=20,
+ completion_tokens=2,
+ )
+ )
+ _conversation(
+ session,
+ 'recent-stale',
+ 'stale-model',
+ 0.60,
+ 600,
+ 60,
+ created_at=now - timedelta(days=1),
+ )
+ session.add(
+ StoredConversationCostEvent(
+ conversation_id='recent-stale',
+ cost_delta=0.60,
+ occurred_at=old,
+ usage_id='agent',
+ llm_model='stale-model',
+ prompt_tokens=600,
+ completion_tokens=60,
+ )
+ )
+ _conversation(
+ session,
+ 'recent-no-ledger',
+ 'legacy-model',
+ 0.25,
+ 20,
+ 2,
+ created_at=now - timedelta(days=1),
+ )
+ _conversation(
+ session,
+ 'old-no-ledger',
+ 'old-model',
+ 0.90,
+ 900,
+ 90,
+ created_at=old,
+ )
+ await session.commit()
+ return now
+
+
+async def test_usage_stats_follow_spend_time_across_window_boundaries(
+ async_session_maker,
+):
+ """Spend metrics include recent usage, independent of conversation age."""
+ seeded_at = await _seed_spend_time_boundary_scenario(async_session_maker)
+
+ async with async_session_maker() as session:
+ service = OrgConversationService(db_session=session)
+ stats = await service.get_usage_stats(ORG_ID, days=30)
+
+ assert stats.agent_runs == 2
+ assert stats.usage_conversation_count == 3
+ assert stats.active_users == 1
+ assert stats.estimated_spend == pytest.approx(0.75)
+ assert stats.total_tokens == 154
+
+ assert len(stats.team_usage) == 1
+ assert stats.team_usage[0].conversation_count == 3
+ assert stats.team_usage[0].total_tokens == 154
+
+ assert len(stats.agent_usage) == 1
+ assert stats.agent_usage[0].agent_name == 'OpenHands'
+ assert stats.agent_usage[0].conversation_count == 3
+ assert stats.agent_usage[0].total_cost == pytest.approx(0.75)
+
+ assert sum(row.conversations for row in stats.daily_usage) == 2
+ assert sum(row.tokens for row in stats.daily_usage) == 154
+ daily_tokens = {row.date: row.tokens for row in stats.daily_usage}
+ expected_daily: dict[str, int] = {}
+ recent_day = (seeded_at - timedelta(hours=2)).strftime('%Y-%m-%d')
+ legacy_day = (seeded_at - timedelta(days=1)).strftime('%Y-%m-%d')
+ expected_daily[recent_day] = expected_daily.get(recent_day, 0) + 132
+ expected_daily[legacy_day] = expected_daily.get(legacy_day, 0) + 22
+ for day, tokens in expected_daily.items():
+ assert daily_tokens[day] == tokens
+
+ assert sum(row.total_cost for row in stats.model_usage) == pytest.approx(0.75)
+ assert sum(row.total_tokens for row in stats.model_usage) == 154
+ assert sum(row.conversation_count for row in stats.model_usage) == 3
diff --git a/frontend/__tests__/components/features/admin-dashboard/usage-dashboard-utils.test.ts b/frontend/__tests__/components/features/admin-dashboard/usage-dashboard-utils.test.ts
new file mode 100644
index 0000000000..24537bd712
--- /dev/null
+++ b/frontend/__tests__/components/features/admin-dashboard/usage-dashboard-utils.test.ts
@@ -0,0 +1,18 @@
+import { describe, expect, it } from "vitest";
+
+import { formatShortDate } from "#/components/features/admin-dashboard/usage-dashboard-utils";
+
+describe("formatShortDate", () => {
+ // Date-only API strings parse as UTC midnight; labels must not shift
+ // with the viewer's timezone (previously "2026-07-01" showed "Jun 30"
+ // for viewers west of UTC).
+ it("renders the UTC calendar date regardless of local timezone", () => {
+ expect(formatShortDate("2026-07-01")).toBe("Jul 1");
+ expect(formatShortDate("2026-06-23")).toBe("Jun 23");
+ expect(formatShortDate("2026-12-31")).toBe("Dec 31");
+ });
+
+ it("renders UTC-midnight timestamps on their UTC day", () => {
+ expect(formatShortDate("2026-07-01T00:00:00Z")).toBe("Jul 1");
+ });
+});
diff --git a/frontend/__tests__/hooks/query/use-org-usage-stats.test.tsx b/frontend/__tests__/hooks/query/use-org-usage-stats.test.tsx
index bbac7ea4ae..14574669a4 100644
--- a/frontend/__tests__/hooks/query/use-org-usage-stats.test.tsx
+++ b/frontend/__tests__/hooks/query/use-org-usage-stats.test.tsx
@@ -32,6 +32,7 @@ const createWrapper = () => {
const mockUsageStats = {
active_users: 0,
agent_runs: 0,
+ usage_conversation_count: 0,
total_tokens: 0,
estimated_spend: 0,
daily_usage: [],
diff --git a/frontend/src/api/organization-service/organization-service.api.ts b/frontend/src/api/organization-service/organization-service.api.ts
index 006c9cb064..d82e558655 100644
--- a/frontend/src/api/organization-service/organization-service.api.ts
+++ b/frontend/src/api/organization-service/organization-service.api.ts
@@ -583,6 +583,7 @@ interface AgentUsageData {
interface OrgUsageStats {
active_users: number;
agent_runs: number;
+ usage_conversation_count: number;
total_tokens: number;
estimated_spend: number;
daily_usage: DailyUsageData[];
diff --git a/frontend/src/components/features/admin-dashboard/usage-dashboard-tabs.tsx b/frontend/src/components/features/admin-dashboard/usage-dashboard-tabs.tsx
index 5064a787f3..cd1acbcd51 100644
--- a/frontend/src/components/features/admin-dashboard/usage-dashboard-tabs.tsx
+++ b/frontend/src/components/features/admin-dashboard/usage-dashboard-tabs.tsx
@@ -28,7 +28,7 @@ export type AgentSpendRow = {
};
export function OverviewTab({
- totalConversations,
+ usageConversations,
activeConversations,
avgCostPerConversation,
totalSpend,
@@ -37,7 +37,7 @@ export function OverviewTab({
agentSpendRows,
agentSpendTotal,
}: {
- totalConversations: number;
+ usageConversations: number;
activeConversations: number;
avgCostPerConversation: number;
totalSpend: string;
@@ -50,8 +50,8 @@ export function OverviewTab({
<div className="space-y-6">
<div className="grid grid-cols-4 gap-4">
<KPICard
- label="Total Conversations"
- value={totalConversations.toLocaleString()}
+ label="Conversations With Usage"
+ value={usageConversations.toLocaleString()}
/>
<KPICard
label="Active Conversations"
@@ -72,7 +72,7 @@ export function OverviewTab({
<div className="flex items-start justify-between mb-6">
<div>
<h2 className="text-lg font-medium text-white">
- Conversations per day
+ Conversations started per day
</h2>
<p className="text-sm text-zinc-500">
{timeWindowLabel} Β· all users
diff --git a/frontend/src/components/features/admin-dashboard/usage-dashboard-utils.ts b/frontend/src/components/features/admin-dashboard/usage-dashboard-utils.ts
index e05e619137..4281e80c66 100644
--- a/frontend/src/components/features/admin-dashboard/usage-dashboard-utils.ts
+++ b/frontend/src/components/features/admin-dashboard/usage-dashboard-utils.ts
@@ -33,6 +33,9 @@ export const formatShortDate = (dateStr: string) => {
return date.toLocaleDateString("en-US", {
month: "short",
day: "numeric",
+ // Date-only strings parse as UTC midnight; local rendering shifts
+ // every label a day back for viewers west of UTC.
+ timeZone: "UTC",
});
};
diff --git a/frontend/src/components/features/admin-dashboard/usage-dashboard.tsx b/frontend/src/components/features/admin-dashboard/usage-dashboard.tsx
index 84f0750557..20c8cecd02 100644
--- a/frontend/src/components/features/admin-dashboard/usage-dashboard.tsx
+++ b/frontend/src/components/features/admin-dashboard/usage-dashboard.tsx
@@ -103,11 +103,11 @@ export function UsageDashboard() {
(org) => org.id === organizationId,
);
- const totalConversations = usageStats?.agent_runs ?? 0;
+ const usageConversations = usageStats?.usage_conversation_count ?? 0;
const activeConversations = stats?.active_conversations ?? 0;
const avgCostPerConversation =
- totalConversations > 0
- ? (usageStats?.estimated_spend ?? 0) / totalConversations
+ usageConversations > 0
+ ? (usageStats?.estimated_spend ?? 0) / usageConversations
: 0;
const totalSpend = formatCost(usageStats?.estimated_spend ?? 0);
@@ -267,7 +267,7 @@ export function UsageDashboard() {
{/* Overview Tab */}
{activeTab === "overview" && (
<OverviewTab
- totalConversations={totalConversations}
+ usageConversations={usageConversations}
activeConversations={activeConversations}
avgCostPerConversation={avgCostPerConversation}
totalSpend={totalSpend}
diff --git a/frontend/src/components/features/budgets/budgets.tsx b/frontend/src/components/features/budgets/budgets.tsx
index 87bc811400..8d69d8dd5f 100644
--- a/frontend/src/components/features/budgets/budgets.tsx
+++ b/frontend/src/components/features/budgets/budgets.tsx
@@ -155,6 +155,9 @@ export function Budgets() {
const cycleLabel = budgetData?.cycle_start_at
? new Date(budgetData.cycle_start_at).toLocaleDateString("en-US", {
month: "long",
+ // Cycle boundaries are UTC midnights; local rendering shifts the
+ // month for viewers west of UTC (July cycle labeled "June").
+ timeZone: "UTC",
})
: "this cycle";
const defaultUserLimit = budgetData?.default_user_monthly_limit ?? null;
diff --git a/openhands/app_server/app_conversation/app_conversation_info_service.py b/openhands/app_server/app_conversation/app_conversation_info_service.py
index cf7cd05b88..c65b44bff9 100644
--- a/openhands/app_server/app_conversation/app_conversation_info_service.py
+++ b/openhands/app_server/app_conversation/app_conversation_info_service.py
@@ -9,6 +9,7 @@ from openhands.app_server.app_conversation.app_conversation_models import (
AppConversationSortOrder,
)
from openhands.app_server.services.injector import Injector
+from openhands.sdk import ConversationStats
from openhands.sdk.event import ConversationStateUpdateEvent
from openhands.sdk.utils.models import DiscriminatedUnionMixin
@@ -116,6 +117,15 @@ class AppConversationInfoService(ABC):
conversation_id: The ID of the conversation to update
"""
+ @abstractmethod
+ async def update_conversation_statistics(
+ self,
+ conversation_id: UUID,
+ stats: ConversationStats,
+ event_timestamp: datetime | None = None,
+ ) -> None:
+ """Update persisted statistics from a ConversationStats snapshot."""
+
@abstractmethod
async def update_execution_status(
self,
diff --git a/openhands/app_server/app_conversation/sql_app_conversation_info_service.py b/openhands/app_server/app_conversation/sql_app_conversation_info_service.py
index b535562bd2..39b2b5c80c 100644
--- a/openhands/app_server/app_conversation/sql_app_conversation_info_service.py
+++ b/openhands/app_server/app_conversation/sql_app_conversation_info_service.py
@@ -19,6 +19,7 @@ from __future__ import annotations
import logging
import uuid
+from collections.abc import Mapping
from dataclasses import dataclass
from datetime import UTC, datetime
from typing import AsyncGenerator, cast
@@ -31,6 +32,7 @@ from sqlalchemy import (
Float,
ForeignKey,
Identity,
+ Integer,
Select,
String,
func,
@@ -85,6 +87,45 @@ def _normalize_event_timestamp(value: datetime | None) -> datetime:
return value
+def _combine_usage_metrics(
+ usage_to_metrics: Mapping[str, MetricsSnapshot],
+) -> tuple[float, TokenUsage | None]:
+ """Sum cost and token usage across every usage bucket in the registry."""
+ total_cost = 0.0
+ combined: TokenUsage | None = None
+ for snapshot in usage_to_metrics.values():
+ total_cost += snapshot.accumulated_cost or 0.0
+ usage = snapshot.accumulated_token_usage
+ if usage is None:
+ continue
+ if combined is None:
+ combined = usage.model_copy()
+ else:
+ combined = TokenUsage(
+ model=combined.model,
+ prompt_tokens=combined.prompt_tokens + usage.prompt_tokens,
+ completion_tokens=combined.completion_tokens + usage.completion_tokens,
+ cache_read_tokens=combined.cache_read_tokens + usage.cache_read_tokens,
+ cache_write_tokens=combined.cache_write_tokens
+ + usage.cache_write_tokens,
+ reasoning_tokens=combined.reasoning_tokens + usage.reasoning_tokens,
+ context_window=max(combined.context_window, usage.context_window),
+ per_turn_token=combined.per_turn_token,
+ )
+ # context_window/per_turn_token are per-model snapshots, not additive;
+ # prefer the primary agent's when present.
+ agent_snapshot = usage_to_metrics.get('agent')
+ agent_usage = agent_snapshot.accumulated_token_usage if agent_snapshot else None
+ if combined is not None and agent_usage is not None:
+ combined = combined.model_copy(
+ update={
+ 'context_window': agent_usage.context_window,
+ 'per_turn_token': agent_usage.per_turn_token,
+ }
+ )
+ return total_cost, combined
+
+
logger = logging.getLogger(__name__)
@@ -162,6 +203,11 @@ class StoredConversationCostEvent(Base):
occurred_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, default=utc_now, index=True
)
+ # Attribution is nullable for rows written before these columns existed.
+ usage_id: Mapped[str | None] = mapped_column(String, nullable=True)
+ llm_model: Mapped[str | None] = mapped_column(String, nullable=True)
+ prompt_tokens: Mapped[int | None] = mapped_column(Integer, nullable=True)
+ completion_tokens: Mapped[int | None] = mapped_column(Integer, nullable=True)
@dataclass
@@ -469,21 +515,21 @@ class SQLAppConversationInfoService(AppConversationInfoService):
stats: ConversationStats object containing usage_to_metrics data from stats event
event_timestamp: Timestamp of the stats event (UTC if naive)
"""
- # Extract agent metrics from usage_to_metrics
usage_to_metrics = stats.usage_to_metrics
- agent_metrics = usage_to_metrics.get('agent')
-
- if not agent_metrics:
+ if not usage_to_metrics:
logger.debug(
- 'No agent metrics found in stats for conversation %s', conversation_id
+ 'No usage metrics found in stats for conversation %s', conversation_id
)
return
# Query existing record using secure select (filters for V1 and user if available)
+ # Row-lock so concurrent snapshots (stats events, run-end pull)
+ # serialize per conversation instead of racing the guard/ledger.
+ # No-op on SQLite.
query = await self._secure_select()
query = query.where(
StoredConversationMetadata.conversation_id == str(conversation_id)
- )
+ ).with_for_update()
result = await self.db_session.execute(query)
stored = result.scalar_one_or_none()
@@ -496,73 +542,141 @@ class SQLAppConversationInfoService(AppConversationInfoService):
event_timestamp = _normalize_event_timestamp(event_timestamp)
- # Extract accumulated_cost and max_budget_per_task from Metrics object
- accumulated_cost = agent_metrics.accumulated_cost
- max_budget_per_task = agent_metrics.max_budget_per_task
-
- if accumulated_cost is not None:
- previous_cost = stored.accumulated_cost or 0.0
- delta_cost = float(accumulated_cost) - float(previous_cost)
- if delta_cost > 0:
- self.db_session.add(
- StoredConversationCostEvent(
- conversation_id=stored.conversation_id,
- cost_delta=delta_cost,
- occurred_at=event_timestamp,
- )
- )
- elif delta_cost < 0:
- logger.debug(
- 'Accumulated cost decreased for conversation %s (prev=%s, new=%s)',
- conversation_id,
- previous_cost,
- accumulated_cost,
- )
+ # Combine ALL usage buckets: switched-in LLMs ("profile:*") and ACP
+ # agents ("acp-managed") accrue spend outside the "agent" bucket, so
+ # an agent-only read freezes persisted cost at the pre-switch value.
+ accumulated_cost, accumulated_token_usage = _combine_usage_metrics(
+ usage_to_metrics
+ )
+ agent_metrics = usage_to_metrics.get('agent')
- # Extract accumulated_token_usage from Metrics object
- accumulated_token_usage = agent_metrics.accumulated_token_usage
- if accumulated_token_usage:
- prompt_tokens = accumulated_token_usage.prompt_tokens
- completion_tokens = accumulated_token_usage.completion_tokens
- cache_read_tokens = accumulated_token_usage.cache_read_tokens
- cache_write_tokens = accumulated_token_usage.cache_write_tokens
- reasoning_tokens = accumulated_token_usage.reasoning_tokens
- context_window = accumulated_token_usage.context_window
- per_turn_token = accumulated_token_usage.per_turn_token
- else:
- prompt_tokens = None
- completion_tokens = None
- cache_read_tokens = None
- cache_write_tokens = None
- reasoning_tokens = None
- context_window = None
- per_turn_token = None
-
- # Update fields only if values are provided (not None)
- if accumulated_cost is not None:
- stored.accumulated_cost = accumulated_cost
- if max_budget_per_task is not None:
- stored.max_budget_per_task = max_budget_per_task
- if prompt_tokens is not None:
- stored.prompt_tokens = prompt_tokens
- if completion_tokens is not None:
- stored.completion_tokens = completion_tokens
- if cache_read_tokens is not None:
- stored.cache_read_tokens = cache_read_tokens
- if cache_write_tokens is not None:
- stored.cache_write_tokens = cache_write_tokens
- if reasoning_tokens is not None:
- stored.reasoning_tokens = reasoning_tokens
- if context_window is not None:
- stored.context_window = context_window
- if per_turn_token is not None:
- stored.per_turn_token = per_turn_token
+ previous_cost = stored.accumulated_cost or 0.0
+ delta_cost = accumulated_cost - float(previous_cost)
+ if delta_cost < 0:
+ # Stale/out-of-order snapshot; never regress the running total.
+ logger.debug(
+ 'Accumulated cost decreased for conversation %s (prev=%s, new=%s)',
+ conversation_id,
+ previous_cost,
+ accumulated_cost,
+ )
+ return
+ # Ledger deltas are computed against the ledger's own per-bucket sums
+ # (not the column), so it self-heals if the column ever advances
+ # through a path that skips the ledger.
+ await self._record_bucket_cost_deltas(stored, usage_to_metrics, event_timestamp)
+
+ stored.accumulated_cost = accumulated_cost
+ if agent_metrics is not None and agent_metrics.max_budget_per_task is not None:
+ stored.max_budget_per_task = agent_metrics.max_budget_per_task
+ # Preserve each cumulative counter independently when snapshots arrive
+ # partially out of order.
+ if accumulated_token_usage is not None:
+ stored.prompt_tokens = max(
+ accumulated_token_usage.prompt_tokens, stored.prompt_tokens or 0
+ )
+ stored.completion_tokens = max(
+ accumulated_token_usage.completion_tokens,
+ stored.completion_tokens or 0,
+ )
+ stored.cache_read_tokens = max(
+ accumulated_token_usage.cache_read_tokens,
+ stored.cache_read_tokens or 0,
+ )
+ stored.cache_write_tokens = max(
+ accumulated_token_usage.cache_write_tokens,
+ stored.cache_write_tokens or 0,
+ )
+ stored.reasoning_tokens = max(
+ accumulated_token_usage.reasoning_tokens,
+ stored.reasoning_tokens or 0,
+ )
+ # Gauges, not counters: only follow snapshots that are at least
+ # as fresh as the stored totals.
+ if (
+ accumulated_token_usage.prompt_tokens
+ + accumulated_token_usage.completion_tokens
+ >= (stored.prompt_tokens or 0) + (stored.completion_tokens or 0)
+ ):
+ stored.context_window = accumulated_token_usage.context_window
+ stored.per_turn_token = accumulated_token_usage.per_turn_token
# Update last_updated_at timestamp
stored.last_updated_at = utc_now()
await self.db_session.commit()
+ async def _record_bucket_cost_deltas(
+ self,
+ stored: StoredConversationMetadata,
+ usage_to_metrics: Mapping[str, MetricsSnapshot],
+ event_timestamp: datetime,
+ ) -> None:
+ """Write per-bucket deltas without replaying legacy unattributed cost."""
+ result = await self.db_session.execute(
+ select(
+ StoredConversationCostEvent.usage_id,
+ func.sum(StoredConversationCostEvent.cost_delta),
+ func.sum(StoredConversationCostEvent.prompt_tokens),
+ func.sum(StoredConversationCostEvent.completion_tokens),
+ )
+ .where(
+ StoredConversationCostEvent.conversation_id == stored.conversation_id
+ )
+ .group_by(StoredConversationCostEvent.usage_id)
+ )
+ prior = {row[0]: row for row in result.all()}
+ unattributed = prior.pop(None, None)
+ cost_drain = float(unattributed[1] or 0.0) if unattributed is not None else 0.0
+
+ pending: list[dict] = []
+ for usage_id, snapshot in usage_to_metrics.items():
+ prior_row = prior.get(usage_id)
+ prior_cost = float(prior_row[1] or 0.0) if prior_row is not None else 0.0
+ prior_prompt = int(prior_row[2] or 0) if prior_row is not None else 0
+ prior_completion = int(prior_row[3] or 0) if prior_row is not None else 0
+ usage = snapshot.accumulated_token_usage
+ pending.append(
+ {
+ 'usage_id': usage_id,
+ 'snapshot': snapshot,
+ 'cost': max(
+ float(snapshot.accumulated_cost or 0.0) - prior_cost, 0.0
+ ),
+ 'prompt': max(usage.prompt_tokens - prior_prompt, 0)
+ if usage
+ else 0,
+ 'completion': max(usage.completion_tokens - prior_completion, 0)
+ if usage
+ else 0,
+ 'has_usage': usage is not None,
+ }
+ )
+
+ for item in sorted(pending, key=lambda i: i['cost'], reverse=True):
+ covered = min(item['cost'], cost_drain)
+ item['cost'] -= covered
+ cost_drain -= covered
+
+ for item in pending:
+ if item['cost'] <= 0 and item['prompt'] <= 0 and item['completion'] <= 0:
+ continue
+ snapshot = item['snapshot']
+ model: str | None = snapshot.model_name
+ if not model or model == 'default':
+ model = stored.llm_model if item['usage_id'] == 'agent' else None
+ self.db_session.add(
+ StoredConversationCostEvent(
+ conversation_id=stored.conversation_id,
+ cost_delta=item['cost'],
+ occurred_at=event_timestamp,
+ usage_id=item['usage_id'],
+ llm_model=model,
+ prompt_tokens=item['prompt'] if item['has_usage'] else None,
+ completion_tokens=item['completion'] if item['has_usage'] else None,
+ )
+ )
+
async def process_stats_event(
self,
event: ConversationStateUpdateEvent,
diff --git a/openhands/app_server/event_callback/webhook_router.py b/openhands/app_server/event_callback/webhook_router.py
index 0364da81d3..78cc9317d0 100644
--- a/openhands/app_server/event_callback/webhook_router.py
+++ b/openhands/app_server/event_callback/webhook_router.py
@@ -7,6 +7,7 @@ import pkgutil
from typing import Any
from uuid import UUID
+import httpx
from fastapi import (
APIRouter,
BackgroundTasks,
@@ -36,6 +37,8 @@ from openhands.app_server.config import (
depends_app_conversation_info_service,
depends_event_service,
depends_jwt_service,
+ get_app_conversation_info_service,
+ get_app_conversation_service,
get_event_callback_service,
get_global_config,
get_sandbox_service,
@@ -62,12 +65,16 @@ from openhands.app_server.user.auth_user_context import AuthUserContext
from openhands.app_server.user.specifiy_user_context import (
ADMIN,
USER_CONTEXT_ATTR,
+ SandboxUserContext,
SpecifyUserContext,
)
from openhands.app_server.user_auth.default_user_auth import DefaultUserAuth
from openhands.app_server.user_auth.user_auth import (
get_for_user as get_user_auth_for_user,
)
+from openhands.app_server.utils.docker_utils import (
+ replace_localhost_hostname_for_docker,
+)
from openhands.sdk import ConversationExecutionStatus, Event
from openhands.sdk.event import ConversationStateUpdateEvent, ObservationEvent
from openhands.sdk.settings import ACPAgentSettings
@@ -274,13 +281,16 @@ async def valid_sandbox(
status.HTTP_401_UNAUTHORIZED, detail='Invalid session API key'
)
- # In SAAS Mode there is always a user, so we set the owner of the sandbox
- # as the current user (Validated by the session_api_key they provided)
+ # Scope webhook services to the authenticated sandbox, independent of the
+ # owner's currently selected organization.
if sandbox_record.created_by_user_id:
setattr(
request.state,
USER_CONTEXT_ATTR,
- SpecifyUserContext(sandbox_record.created_by_user_id),
+ SandboxUserContext(
+ user_id=sandbox_record.created_by_user_id,
+ sandbox_id=sandbox_record.id,
+ ),
)
elif app_mode == AppMode.SAAS:
_logger.error(
@@ -296,11 +306,11 @@ async def valid_sandbox(
async def valid_conversation(
conversation_id: UUID,
sandbox_record: SandboxRecord = Depends(valid_sandbox),
- app_conversation_info_service: AppConversationInfoService = app_conversation_info_service_dependency,
) -> AppConversationInfo:
- app_conversation_info = (
- await app_conversation_info_service.get_app_conversation_info(conversation_id)
- )
+ state = InjectorState()
+ setattr(state, USER_CONTEXT_ATTR, ADMIN)
+ async with get_app_conversation_info_service(state) as service:
+ app_conversation_info = await service.get_app_conversation_info(conversation_id)
if not app_conversation_info:
# Conversation does not yet exist - create a stub
return AppConversationInfo(
@@ -309,8 +319,10 @@ async def valid_conversation(
created_by_user_id=sandbox_record.created_by_user_id,
)
- # Sanity check - Make sure that the conversation and sandbox were created by the same user
- if app_conversation_info.created_by_user_id != sandbox_record.created_by_user_id:
+ if (
+ app_conversation_info.created_by_user_id != sandbox_record.created_by_user_id
+ or app_conversation_info.sandbox_id != sandbox_record.id
+ ):
raise AuthError()
return app_conversation_info
@@ -358,7 +370,8 @@ async def on_conversation_update(
accepted on this single endpoint.
"""
existing = await valid_conversation(
- conversation_info.id, sandbox_record, app_conversation_info_service
+ conversation_info.id,
+ sandbox_record,
)
# If the conversation is being deleted, no action is required...
@@ -465,6 +478,58 @@ async def on_conversation_update(
return Success()
+_LIVE_STATS_PULL_STATUSES = {
+ ConversationExecutionStatus.FINISHED,
+ ConversationExecutionStatus.IDLE,
+ ConversationExecutionStatus.ERROR,
+ ConversationExecutionStatus.STUCK,
+}
+
+
+async def _sync_live_conversation_stats(
+ conversation_id: UUID,
+ app_conversation_info: AppConversationInfo,
+ app_conversation_info_service: AppConversationInfoService,
+) -> None:
+ """Pull the registry's combined stats from the agent-server at run end.
+
+ Switched-in LLMs and ACP agents never attach a stats callback, so their
+ spend emits no stats events; without this pull the persisted totals
+ under-report whatever ran outside the startup-wired LLMs.
+ """
+ state = InjectorState()
+ setattr(
+ state,
+ USER_CONTEXT_ATTR,
+ SandboxUserContext(
+ user_id=app_conversation_info.created_by_user_id,
+ sandbox_id=app_conversation_info.sandbox_id,
+ ),
+ )
+ async with get_app_conversation_service(state) as app_conversation_service:
+ conversation = await app_conversation_service.get_app_conversation(
+ conversation_id
+ )
+ if conversation is None or not conversation.conversation_url:
+ return
+ # Local-docker sandboxes advertise localhost URLs; normalize like the
+ # title processor so the pull reaches the host, not this container.
+ conversation_url = replace_localhost_hostname_for_docker(
+ conversation.conversation_url
+ )
+ headers = {}
+ if conversation.session_api_key:
+ headers['X-Session-API-Key'] = conversation.session_api_key
+ async with httpx.AsyncClient(timeout=10.0) as client:
+ response = await client.get(conversation_url, headers=headers)
+ response.raise_for_status()
+ info = ConversationInfo.model_validate(response.json())
+ if info.stats and info.stats.usage_to_metrics:
+ await app_conversation_info_service.update_conversation_statistics(
+ conversation_id, info.stats
+ )
+
+
@router.post('/events/{conversation_id}')
async def on_event(
background_tasks: BackgroundTasks,
@@ -513,6 +578,7 @@ async def on_event(
# Analytics: conversation terminal state detection
# Also persist execution status to database for dashboard queries
+ run_ended = False
for event in events:
if not isinstance(event, ConversationStateUpdateEvent):
continue
@@ -524,6 +590,8 @@ async def on_event(
await app_conversation_info_service.update_execution_status(
conversation_id, exec_status.value
)
+ if exec_status in _LIVE_STATS_PULL_STATUSES:
+ run_ended = True
if exec_status.is_terminal():
await _track_conversation_terminal(
conversation_id, app_conversation_info, events, exec_status
@@ -533,6 +601,16 @@ async def on_event(
'analytics:conversation_terminal:failed', stack_info=True
)
+ if run_ended:
+ try:
+ await _sync_live_conversation_stats(
+ conversation_id,
+ app_conversation_info,
+ app_conversation_info_service,
+ )
+ except Exception:
+ _logger.warning('live_stats_pull_failed', exc_info=True)
+
background_tasks.add_task(
_run_callbacks_in_bg_and_close,
conversation_id,
diff --git a/openhands/app_server/user/specifiy_user_context.py b/openhands/app_server/user/specifiy_user_context.py
index e4f7de3b79..e6697e7193 100644
--- a/openhands/app_server/user/specifiy_user_context.py
+++ b/openhands/app_server/user/specifiy_user_context.py
@@ -56,6 +56,13 @@ class SpecifyUserContext(UserContext):
return None
+@dataclass(frozen=True)
+class SandboxUserContext(SpecifyUserContext):
+ """User context scoped to an authenticated sandbox."""
+
+ sandbox_id: str
+
+
USER_CONTEXT_ATTR = 'user_context'
ADMIN = SpecifyUserContext(user_id=None)
diff --git a/tests/unit/app_server/test_webhook_router_auth.py b/tests/unit/app_server/test_webhook_router_auth.py
index 9a8cb41bf8..706b34e4d8 100644
--- a/tests/unit/app_server/test_webhook_router_auth.py
+++ b/tests/unit/app_server/test_webhook_router_auth.py
@@ -22,7 +22,7 @@ from openhands.app_server.event_callback.webhook_router import (
from openhands.app_server.sandbox.sandbox_models import SandboxRecord
from openhands.app_server.user.specifiy_user_context import (
USER_CONTEXT_ATTR,
- SpecifyUserContext,
+ SandboxUserContext,
)
@@ -64,6 +64,17 @@ def create_sandbox_service_context_manager(sandbox_service):
return _context_manager
+async def call_valid_conversation(conversation_id, sandbox_record, service):
+ with patch(
+ 'openhands.app_server.event_callback.webhook_router.get_app_conversation_info_service',
+ create_sandbox_service_context_manager(service),
+ ):
+ return await valid_conversation(
+ conversation_id=conversation_id,
+ sandbox_record=sandbox_record,
+ )
+
+
class TestValidSandbox:
"""Test suite for valid_sandbox function."""
@@ -104,8 +115,9 @@ class TestValidSandbox:
# Verify user_context is set correctly on request.state
assert USER_CONTEXT_ATTR in mock_request.state._attributes
user_context = mock_request.state._attributes[USER_CONTEXT_ATTR]
- assert isinstance(user_context, SpecifyUserContext)
+ assert isinstance(user_context, SandboxUserContext)
assert user_context.user_id == user_id
+ assert user_context.sandbox_id == expected_sandbox.id
@pytest.mark.asyncio
async def test_valid_sandbox_sets_user_context_to_sandbox_owner(self):
@@ -138,8 +150,9 @@ class TestValidSandbox:
# Assert - user_context should be set to the sandbox owner
assert USER_CONTEXT_ATTR in mock_request.state._attributes
user_context = mock_request.state._attributes[USER_CONTEXT_ATTR]
- assert isinstance(user_context, SpecifyUserContext)
+ assert isinstance(user_context, SandboxUserContext)
assert user_context.user_id == sandbox_owner_id
+ assert user_context.sandbox_id == expected_sandbox.id
@pytest.mark.asyncio
async def test_valid_sandbox_no_user_context_when_no_user_id(self):
@@ -289,15 +302,16 @@ class TestValidConversation:
expected_info = MagicMock()
expected_info.created_by_user_id = 'user-123'
+ expected_info.sandbox_id = sandbox_record.id
mock_service = AsyncMock()
mock_service.get_app_conversation_info = AsyncMock(return_value=expected_info)
# Act
- result = await valid_conversation(
- conversation_id=conversation_id,
- sandbox_record=sandbox_record,
- app_conversation_info_service=mock_service,
+ result = await call_valid_conversation(
+ conversation_id,
+ sandbox_record,
+ mock_service,
)
# Assert
@@ -317,10 +331,10 @@ class TestValidConversation:
mock_service.get_app_conversation_info = AsyncMock(return_value=None)
# Act
- result = await valid_conversation(
- conversation_id=conversation_id,
- sandbox_record=sandbox_record,
- app_conversation_info_service=mock_service,
+ result = await call_valid_conversation(
+ conversation_id,
+ sandbox_record,
+ mock_service,
)
# Assert
@@ -341,6 +355,7 @@ class TestValidConversation:
# Conversation created by different user
different_user_info = MagicMock()
different_user_info.created_by_user_id = 'different-user-id'
+ different_user_info.sandbox_id = sandbox_record.id
mock_service = AsyncMock()
mock_service.get_app_conversation_info = AsyncMock(
@@ -351,10 +366,10 @@ class TestValidConversation:
from openhands.app_server.errors import AuthError
with pytest.raises(AuthError):
- await valid_conversation(
- conversation_id=conversation_id,
- sandbox_record=sandbox_record,
- app_conversation_info_service=mock_service,
+ await call_valid_conversation(
+ conversation_id,
+ sandbox_record,
+ mock_service,
)
@pytest.mark.asyncio
@@ -371,20 +386,47 @@ class TestValidConversation:
# Conversation created by same user
same_user_info = MagicMock()
same_user_info.created_by_user_id = user_id
+ same_user_info.sandbox_id = sandbox_record.id
mock_service = AsyncMock()
mock_service.get_app_conversation_info = AsyncMock(return_value=same_user_info)
# Act
- result = await valid_conversation(
- conversation_id=conversation_id,
- sandbox_record=sandbox_record,
- app_conversation_info_service=mock_service,
+ result = await call_valid_conversation(
+ conversation_id,
+ sandbox_record,
+ mock_service,
)
# Assert
assert result == same_user_info
+ @pytest.mark.asyncio
+ async def test_valid_conversation_different_sandbox_raises_auth_error(self):
+ """A sandbox key cannot authorize another sandbox's conversation."""
+ conversation_id = uuid4()
+ sandbox_record = SandboxRecord(
+ id='sandbox-123',
+ created_by_user_id='user-123',
+ )
+ other_sandbox_info = MagicMock()
+ other_sandbox_info.created_by_user_id = sandbox_record.created_by_user_id
+ other_sandbox_info.sandbox_id = 'sandbox-456'
+
+ mock_service = AsyncMock()
+ mock_service.get_app_conversation_info = AsyncMock(
+ return_value=other_sandbox_info
+ )
+
+ from openhands.app_server.errors import AuthError
+
+ with pytest.raises(AuthError):
+ await call_valid_conversation(
+ conversation_id,
+ sandbox_record,
+ mock_service,
+ )
+
class TestWebhookAuthenticationIntegration:
"""Integration tests for webhook authentication flow."""
@@ -406,6 +448,7 @@ class TestWebhookAuthenticationIntegration:
conversation_info = MagicMock()
conversation_info.created_by_user_id = 'user-123'
+ conversation_info.sandbox_id = sandbox_record.id
mock_conversation_service = AsyncMock()
mock_conversation_service.get_app_conversation_info = AsyncMock(
@@ -425,10 +468,10 @@ class TestWebhookAuthenticationIntegration:
)
# Then call valid_conversation
- conversation_result = await valid_conversation(
- conversation_id=uuid4(),
- sandbox_record=sandbox_result,
- app_conversation_info_service=mock_conversation_service,
+ conversation_result = await call_valid_conversation(
+ uuid4(),
+ sandbox_result,
+ mock_conversation_service,
)
# Assert
@@ -478,6 +521,7 @@ class TestWebhookAuthenticationIntegration:
# Conversation created by different user
different_user_info = MagicMock()
different_user_info.created_by_user_id = 'different-user'
+ different_user_info.sandbox_id = sandbox_record.id
mock_conversation_service = AsyncMock()
mock_conversation_service.get_app_conversation_info = AsyncMock(
@@ -500,10 +544,10 @@ class TestWebhookAuthenticationIntegration:
from openhands.app_server.errors import AuthError
with pytest.raises(AuthError):
- await valid_conversation(
- conversation_id=uuid4(),
- sandbox_record=sandbox_result,
- app_conversation_info_service=mock_conversation_service,
+ await call_valid_conversation(
+ uuid4(),
+ sandbox_result,
+ mock_conversation_service,
)
diff --git a/tests/unit/app_server/test_webhook_router_stats.py b/tests/unit/app_server/test_webhook_router_stats.py
index 56d314ce6f..7d6aff88b9 100644
--- a/tests/unit/app_server/test_webhook_router_stats.py
+++ b/tests/unit/app_server/test_webhook_router_stats.py
@@ -23,7 +23,11 @@ from openhands.app_server.app_conversation.sql_app_conversation_info_service imp
StoredConversationCostEvent,
StoredConversationMetadata,
)
-from openhands.app_server.user.specifiy_user_context import SpecifyUserContext
+from openhands.app_server.user.specifiy_user_context import (
+ USER_CONTEXT_ATTR,
+ SandboxUserContext,
+ SpecifyUserContext,
+)
from openhands.app_server.utils.sql_utils import Base
from openhands.sdk import ConversationStats
from openhands.sdk.event import ConversationStateUpdateEvent
@@ -236,8 +240,14 @@ class TestUpdateConversationStatistics:
"""Test that cost deltas are recorded for stats updates."""
conversation_id, stored = v1_conversation_metadata
- stored.accumulated_cost = 0.01
- await async_session.commit()
+ await service.update_conversation_statistics(
+ conversation_id,
+ ConversationStats(
+ usage_to_metrics={
+ 'agent': Metrics(model_name='test-model', accumulated_cost=0.01)
+ }
+ ),
+ )
event_timestamp = datetime(2025, 1, 15, tzinfo=timezone.utc)
agent_metrics = Metrics(
@@ -251,11 +261,11 @@ class TestUpdateConversationStatistics:
)
result = await async_session.execute(
- select(StoredConversationCostEvent).where(
- StoredConversationCostEvent.conversation_id == str(conversation_id)
- )
+ select(StoredConversationCostEvent)
+ .where(StoredConversationCostEvent.conversation_id == str(conversation_id))
+ .order_by(StoredConversationCostEvent.id)
)
- cost_event = result.scalar_one()
+ cost_event = result.scalars().all()[-1]
assert cost_event.cost_delta == pytest.approx(0.04)
occurred_at = cost_event.occurred_at
if occurred_at.tzinfo is None:
@@ -263,12 +273,11 @@ class TestUpdateConversationStatistics:
assert occurred_at == event_timestamp
@pytest.mark.asyncio
- async def test_update_statistics_no_agent_metrics(
- self, service, v1_conversation_metadata
+ async def test_update_statistics_non_agent_bucket_counted(
+ self, service, async_session, v1_conversation_metadata
):
- """Test that update is skipped when no agent metrics are present."""
+ """Spend outside the "agent" bucket (ACP, switched LLMs) is persisted."""
conversation_id, stored = v1_conversation_metadata
- original_cost = stored.accumulated_cost
condenser_metrics = Metrics(
model_name='test-model',
@@ -278,8 +287,339 @@ class TestUpdateConversationStatistics:
await service.update_conversation_statistics(conversation_id, stats)
- # Verify no update occurred
- assert stored.accumulated_cost == original_cost
+ await async_session.refresh(stored)
+ assert stored.accumulated_cost == pytest.approx(0.1)
+
+ @pytest.mark.asyncio
+ async def test_update_statistics_combines_switch_buckets(
+ self, service, async_session, v1_conversation_metadata
+ ):
+ """A mid-conversation LLM switch accrues under "profile:*"; totals combine."""
+ conversation_id, stored = v1_conversation_metadata
+
+ stored.accumulated_cost = 0.0824
+ await async_session.commit()
+
+ agent_metrics = Metrics(
+ model_name='gpt-5.5',
+ accumulated_cost=0.0824,
+ accumulated_token_usage=TokenUsage(
+ model='gpt-5.5', prompt_tokens=100, completion_tokens=10
+ ),
+ )
+ profile_metrics = Metrics(
+ model_name='claude-opus-4-8',
+ accumulated_cost=0.1741,
+ accumulated_token_usage=TokenUsage(
+ model='claude-opus-4-8', prompt_tokens=200, completion_tokens=20
+ ),
+ )
+ stats = ConversationStats(
+ usage_to_metrics={
+ 'agent': agent_metrics,
+ 'profile:opus-repro:abc123': profile_metrics,
+ }
+ )
+
+ await service.update_conversation_statistics(conversation_id, stats)
+
+ await async_session.refresh(stored)
+ assert stored.accumulated_cost == pytest.approx(0.2565)
+ assert stored.prompt_tokens == 300
+ assert stored.completion_tokens == 30
+
+ result = await async_session.execute(
+ select(StoredConversationCostEvent)
+ .where(StoredConversationCostEvent.conversation_id == str(conversation_id))
+ .order_by(StoredConversationCostEvent.id)
+ )
+ events = result.scalars().all()
+ # The ledger self-heals the pre-seeded agent spend, then records opus.
+ assert [(e.usage_id, e.llm_model) for e in events] == [
+ ('agent', 'gpt-5.5'),
+ ('profile:opus-repro:abc123', 'claude-opus-4-8'),
+ ]
+ assert events[1].cost_delta == pytest.approx(0.1741)
+ assert events[1].prompt_tokens == 200
+ assert events[1].completion_tokens == 20
+
+ @pytest.mark.asyncio
+ async def test_update_statistics_bucket_attribution_across_turns(
+ self, service, async_session, v1_conversation_metadata
+ ):
+ """Each bucket's spend lands in its own ledger rows, per model."""
+ conversation_id, stored = v1_conversation_metadata
+
+ agent_turn1 = Metrics(model_name='gpt-5.5', accumulated_cost=0.08)
+ await service.update_conversation_statistics(
+ conversation_id, ConversationStats(usage_to_metrics={'agent': agent_turn1})
+ )
+
+ agent_turn2 = Metrics(model_name='gpt-5.5', accumulated_cost=0.08)
+ profile_turn2 = Metrics(model_name='claude-opus-4-8', accumulated_cost=0.17)
+ await service.update_conversation_statistics(
+ conversation_id,
+ ConversationStats(
+ usage_to_metrics={
+ 'agent': agent_turn2,
+ 'profile:opus:xyz': profile_turn2,
+ }
+ ),
+ )
+
+ await async_session.refresh(stored)
+ assert stored.accumulated_cost == pytest.approx(0.25)
+
+ result = await async_session.execute(
+ select(StoredConversationCostEvent)
+ .where(StoredConversationCostEvent.conversation_id == str(conversation_id))
+ .order_by(StoredConversationCostEvent.id)
+ )
+ events = result.scalars().all()
+ assert [(e.usage_id, e.llm_model) for e in events] == [
+ ('agent', 'gpt-5.5'),
+ ('profile:opus:xyz', 'claude-opus-4-8'),
+ ]
+ assert events[0].cost_delta == pytest.approx(0.08)
+ assert events[1].cost_delta == pytest.approx(0.17)
+
+ @pytest.mark.asyncio
+ async def test_update_statistics_transition_from_combined_null_events(
+ self, service, async_session, v1_conversation_metadata
+ ):
+ """Pre-attribution NULL rows (incl. interim combined deltas) drain, not double-count."""
+ conversation_id, stored = v1_conversation_metadata
+
+ # Pre-migration state: cost history in a NULL row (no token data),
+ # token history only in the columns.
+ stored.accumulated_cost = 0.30
+ stored.prompt_tokens = 100
+ stored.completion_tokens = 10
+ async_session.add(
+ StoredConversationCostEvent(
+ conversation_id=str(conversation_id),
+ cost_delta=0.30,
+ occurred_at=datetime(2025, 1, 10, tzinfo=timezone.utc),
+ )
+ )
+ await async_session.commit()
+
+ stats = ConversationStats(
+ usage_to_metrics={
+ 'agent': Metrics(
+ model_name='gpt-5.5',
+ accumulated_cost=0.12,
+ accumulated_token_usage=TokenUsage(
+ model='gpt-5.5', prompt_tokens=150, completion_tokens=15
+ ),
+ ),
+ 'profile:opus:x1': Metrics(
+ model_name='claude-opus-4-8', accumulated_cost=0.25
+ ),
+ }
+ )
+ await service.update_conversation_statistics(conversation_id, stats)
+
+ await async_session.refresh(stored)
+ assert stored.accumulated_cost == pytest.approx(0.37)
+ assert stored.prompt_tokens == 150
+
+ result = await async_session.execute(
+ select(StoredConversationCostEvent).where(
+ StoredConversationCostEvent.conversation_id == str(conversation_id)
+ )
+ )
+ events = result.scalars().all()
+ # Ledger cost tracks the combined column: 0.30 legacy + 0.07 new.
+ assert sum(e.cost_delta for e in events) == pytest.approx(0.37)
+ new_events = [e for e in events if e.usage_id is not None]
+ assert sum(e.cost_delta for e in new_events) == pytest.approx(0.07)
+ # Token history was never ledgered, so the ledger absorbs the full
+ # cumulative totals rather than discarding the unledgered baseline.
+ assert sum(e.prompt_tokens or 0 for e in events) == 150
+ assert sum(e.completion_tokens or 0 for e in events) == 15
+
+ @pytest.mark.asyncio
+ async def test_update_statistics_token_only_growth_recorded(
+ self, service, async_session, v1_conversation_metadata
+ ):
+ """Zero-cost models still get token deltas in the ledger."""
+ conversation_id, stored = v1_conversation_metadata
+
+ def stats(prompt, completion):
+ return ConversationStats(
+ usage_to_metrics={
+ 'agent': Metrics(
+ model_name='custom-llm',
+ accumulated_cost=0.0,
+ accumulated_token_usage=TokenUsage(
+ model='custom-llm',
+ prompt_tokens=prompt,
+ completion_tokens=completion,
+ ),
+ )
+ }
+ )
+
+ await service.update_conversation_statistics(conversation_id, stats(100, 10))
+ await service.update_conversation_statistics(conversation_id, stats(300, 30))
+
+ result = await async_session.execute(
+ select(StoredConversationCostEvent)
+ .where(StoredConversationCostEvent.conversation_id == str(conversation_id))
+ .order_by(StoredConversationCostEvent.id)
+ )
+ events = result.scalars().all()
+ assert [(e.prompt_tokens, e.completion_tokens) for e in events] == [
+ (100, 10),
+ (200, 20),
+ ]
+ assert all(e.cost_delta == 0 for e in events)
+ assert all(e.llm_model == 'custom-llm' for e in events)
+
+ @pytest.mark.asyncio
+ async def test_update_statistics_zero_cost_stale_tokens_ignored(
+ self, service, async_session, v1_conversation_metadata
+ ):
+ """Equal-cost out-of-order snapshots must not regress token totals."""
+ conversation_id, stored = v1_conversation_metadata
+
+ def stats(prompt, completion):
+ return ConversationStats(
+ usage_to_metrics={
+ 'agent': Metrics(
+ model_name='custom-llm',
+ accumulated_cost=0.0,
+ accumulated_token_usage=TokenUsage(
+ model='custom-llm',
+ prompt_tokens=prompt,
+ completion_tokens=completion,
+ ),
+ )
+ }
+ )
+
+ await service.update_conversation_statistics(conversation_id, stats(200, 20))
+ # Stale snapshot with the same (zero) cost but older token counts.
+ await service.update_conversation_statistics(conversation_id, stats(100, 10))
+
+ await async_session.refresh(stored)
+ assert stored.prompt_tokens == 200
+ assert stored.completion_tokens == 20
+
+ @pytest.mark.asyncio
+ async def test_update_statistics_acp_managed_bucket_persisted(
+ self, service, async_session, v1_conversation_metadata
+ ):
+ """ACP agents accrue under 'acp-managed'; cost must persist and ledger."""
+ conversation_id, stored = v1_conversation_metadata
+
+ stats = ConversationStats(
+ usage_to_metrics={
+ 'acp-managed': Metrics(
+ model_name='claude-opus-4-8',
+ accumulated_cost=0.42,
+ accumulated_token_usage=TokenUsage(
+ model='claude-opus-4-8',
+ prompt_tokens=500,
+ completion_tokens=50,
+ ),
+ )
+ }
+ )
+ await service.update_conversation_statistics(conversation_id, stats)
+
+ await async_session.refresh(stored)
+ assert stored.accumulated_cost == pytest.approx(0.42)
+ assert stored.prompt_tokens == 500
+
+ result = await async_session.execute(
+ select(StoredConversationCostEvent).where(
+ StoredConversationCostEvent.conversation_id == str(conversation_id)
+ )
+ )
+ event = result.scalar_one()
+ assert event.usage_id == 'acp-managed'
+ assert event.llm_model == 'claude-opus-4-8'
+ assert event.cost_delta == pytest.approx(0.42)
+
+ @pytest.mark.asyncio
+ async def test_update_statistics_mixed_counter_snapshot_stays_monotonic(
+ self, service, async_session, v1_conversation_metadata
+ ):
+ """Each cumulative counter is monotonic even when others rise."""
+ conversation_id, stored = v1_conversation_metadata
+
+ def stats(prompt, completion, cache_read):
+ return ConversationStats(
+ usage_to_metrics={
+ 'agent': Metrics(
+ model_name='custom-llm',
+ accumulated_cost=0.0,
+ accumulated_token_usage=TokenUsage(
+ model='custom-llm',
+ prompt_tokens=prompt,
+ completion_tokens=completion,
+ cache_read_tokens=cache_read,
+ ),
+ )
+ }
+ )
+
+ await service.update_conversation_statistics(
+ conversation_id, stats(200, 20, 50)
+ )
+ # Same combined sum, but prompt and cache regress while completion rises.
+ await service.update_conversation_statistics(
+ conversation_id, stats(100, 120, 10)
+ )
+
+ await async_session.refresh(stored)
+ assert stored.prompt_tokens == 200
+ assert stored.completion_tokens == 120
+ assert stored.cache_read_tokens == 50
+
+ result = await async_session.execute(
+ select(StoredConversationCostEvent).where(
+ StoredConversationCostEvent.conversation_id == str(conversation_id)
+ )
+ )
+ events = result.scalars().all()
+ assert sum(event.prompt_tokens or 0 for event in events) == 200
+ assert sum(event.completion_tokens or 0 for event in events) == 120
+
+ @pytest.mark.asyncio
+ async def test_update_statistics_stale_snapshot_ignored(
+ self, service, async_session, v1_conversation_metadata
+ ):
+ """A snapshot below the stored total never regresses it."""
+ conversation_id, stored = v1_conversation_metadata
+
+ stored.accumulated_cost = 0.5
+ stored.prompt_tokens = 1000
+ await async_session.commit()
+
+ agent_metrics = Metrics(
+ model_name='test-model',
+ accumulated_cost=0.3,
+ accumulated_token_usage=TokenUsage(
+ model='test-model', prompt_tokens=50, completion_tokens=5
+ ),
+ )
+ stats = ConversationStats(usage_to_metrics={'agent': agent_metrics})
+
+ await service.update_conversation_statistics(conversation_id, stats)
+
+ await async_session.refresh(stored)
+ assert stored.accumulated_cost == pytest.approx(0.5)
+ assert stored.prompt_tokens == 1000
+
+ result = await async_session.execute(
+ select(StoredConversationCostEvent).where(
+ StoredConversationCostEvent.conversation_id == str(conversation_id)
+ )
+ )
+ assert result.scalars().all() == []
@pytest.mark.asyncio
async def test_update_statistics_conversation_not_found(self, service):
@@ -609,3 +949,131 @@ class TestOnEventStatsProcessing:
# Verify stats update was NOT called
mock_app_conversation_info_service.update_conversation_statistics.assert_not_called()
+
+
+# ---------------------------------------------------------------------------
+# Tests for the run-end live-stats pull
+# ---------------------------------------------------------------------------
+
+
+class TestRunEndLiveStatsPull:
+ """Run-end statuses trigger a pull of combined stats from the agent-server."""
+
+ @pytest.mark.asyncio
+ async def test_on_event_pulls_live_stats_at_run_end(self):
+ from types import SimpleNamespace
+
+ from openhands.app_server.event_callback import webhook_router
+ from openhands.app_server.event_callback.webhook_router import on_event
+
+ conversation_id = uuid4()
+ events = [
+ ConversationStateUpdateEvent(key='execution_status', value='finished')
+ ]
+ mock_info = AppConversationInfo(
+ id=conversation_id, sandbox_id='sb1', created_by_user_id='user_123'
+ )
+ mock_info_service = AsyncMock()
+ mock_event_service = AsyncMock()
+
+ conversation = SimpleNamespace(
+ conversation_url='http://localhost:12345/api/conversations/abc',
+ session_api_key='key123',
+ )
+ app_conv_service = AsyncMock()
+ app_conv_service.get_app_conversation.return_value = conversation
+ service_ctx = MagicMock()
+ service_ctx.__aenter__ = AsyncMock(return_value=app_conv_service)
+ service_ctx.__aexit__ = AsyncMock(return_value=False)
+
+ pulled_stats = ConversationStats.model_validate(
+ {
+ 'usage_to_metrics': {
+ 'agent': {'accumulated_cost': 0.1},
+ 'profile:opus:x1': {'accumulated_cost': 0.2},
+ }
+ }
+ )
+ response = MagicMock()
+ response.raise_for_status = MagicMock()
+ response.json.return_value = {'stats': 'raw'}
+ http_client = AsyncMock()
+ http_client.get.return_value = response
+ client_ctx = MagicMock()
+ client_ctx.__aenter__ = AsyncMock(return_value=http_client)
+ client_ctx.__aexit__ = AsyncMock(return_value=False)
+
+ with (
+ patch.object(
+ webhook_router, 'get_app_conversation_service', return_value=service_ctx
+ ) as mock_get_app_conversation_service,
+ patch.object(
+ webhook_router,
+ 'replace_localhost_hostname_for_docker',
+ side_effect=lambda url: url.replace('localhost', 'docker-host'),
+ ) as mock_normalize,
+ patch.object(webhook_router.httpx, 'AsyncClient', return_value=client_ctx),
+ patch.object(webhook_router, 'ConversationInfo') as mock_ci,
+ patch(
+ 'openhands.app_server.event_callback.webhook_router._run_callbacks_in_bg_and_close'
+ ),
+ ):
+ mock_ci.model_validate.return_value = SimpleNamespace(stats=pulled_stats)
+ await on_event(
+ background_tasks=BackgroundTasks(),
+ events=events,
+ conversation_id=conversation_id,
+ app_conversation_info=mock_info,
+ app_conversation_info_service=mock_info_service,
+ event_service=mock_event_service,
+ )
+
+ # URL was normalized for docker-local sandboxes and used for the GET
+ mock_normalize.assert_called_once_with(
+ 'http://localhost:12345/api/conversations/abc'
+ )
+ service_state = mock_get_app_conversation_service.call_args.args[0]
+ service_user_context = getattr(service_state, USER_CONTEXT_ATTR)
+ assert service_user_context == SandboxUserContext(
+ user_id=mock_info.created_by_user_id,
+ sandbox_id=mock_info.sandbox_id,
+ )
+ http_client.get.assert_awaited_once()
+ assert http_client.get.await_args.args[0] == (
+ 'http://docker-host:12345/api/conversations/abc'
+ )
+ assert http_client.get.await_args.kwargs['headers'] == {
+ 'X-Session-API-Key': 'key123'
+ }
+ # Pulled combined stats were persisted
+ mock_info_service.update_conversation_statistics.assert_awaited_once_with(
+ conversation_id, pulled_stats
+ )
+
+ @pytest.mark.asyncio
+ async def test_on_event_no_pull_while_running(self):
+ from openhands.app_server.event_callback import webhook_router
+ from openhands.app_server.event_callback.webhook_router import on_event
+
+ conversation_id = uuid4()
+ events = [ConversationStateUpdateEvent(key='execution_status', value='running')]
+ mock_info = AppConversationInfo(
+ id=conversation_id, sandbox_id='sb1', created_by_user_id='user_123'
+ )
+ mock_info_service = AsyncMock()
+
+ with (
+ patch.object(webhook_router, 'get_app_conversation_service') as mock_svc,
+ patch(
+ 'openhands.app_server.event_callback.webhook_router._run_callbacks_in_bg_and_close'
+ ),
+ ):
+ await on_event(
+ background_tasks=BackgroundTasks(),
+ events=events,
+ conversation_id=conversation_id,
+ app_conversation_info=mock_info,
+ app_conversation_info_service=mock_info_service,
+ event_service=AsyncMock(),
+ )
+ mock_svc.assert_not_called()