π OpenHands: AI-Driven Development
fix(frontend): Support V1 conversations in MetricsModal (#12678)
Tim O'Farrell3d4cb89441a7
diff --git a/frontend/src/api/conversation-service/v1-conversation-service.api.ts b/frontend/src/api/conversation-service/v1-conversation-service.api.tsindex e9e4315397..8f40424275 100644--- a/frontend/src/api/conversation-service/v1-conversation-service.api.ts+++ b/frontend/src/api/conversation-service/v1-conversation-service.api.ts@@ -12,6 +12,7 @@ import type {V1AppConversationStartTaskPage,V1AppConversation,GetSkillsResponse,+ V1RuntimeConversationInfo,} from "./v1-conversation-service.types";class V1ConversationService {@@ -360,6 +361,32 @@ class V1ConversationService {);return data;}++ /**+ * Get conversation info directly from the runtime for a V1 conversation+ * Uses the custom runtime URL from the conversation+ *+ * @param conversationId The conversation ID+ * @param conversationUrl The conversation URL (e.g., "http://localhost:54928/api/conversations/...")+ * @param sessionApiKey Session API key for authentication (required for V1)+ * @returns Conversation info from the runtime+ */+ static async getRuntimeConversation(+ conversationId: string,+ conversationUrl: string | null | undefined,+ sessionApiKey?: string | null,+ ): Promise<V1RuntimeConversationInfo> {+ const url = this.buildRuntimeUrl(+ conversationUrl,+ `/api/conversations/${conversationId}`,+ );+ const headers = buildSessionHeaders(sessionApiKey);++ const { data } = await axios.get<V1RuntimeConversationInfo>(url, {+ headers,+ });+ return data;+ }}export default V1ConversationService;diff --git a/frontend/src/api/conversation-service/v1-conversation-service.types.ts b/frontend/src/api/conversation-service/v1-conversation-service.types.tsindex 35684efcc0..bf37b8ef2d 100644--- a/frontend/src/api/conversation-service/v1-conversation-service.types.ts+++ b/frontend/src/api/conversation-service/v1-conversation-service.types.ts@@ -2,6 +2,22 @@ import { ConversationTrigger } from "../open-hands.types";import { Provider } from "#/types/settings";import { V1SandboxStatus } from "../sandbox-service/sandbox-service.types";+// V1 Metrics Types+export interface V1TokenUsage {+ prompt_tokens: number;+ completion_tokens: number;+ cache_read_tokens: number;+ cache_write_tokens: number;+ context_window: number;+ per_turn_token: number;+}++export interface V1MetricsSnapshot {+ accumulated_cost: number | null;+ max_budget_per_task: number | null;+ accumulated_token_usage: V1TokenUsage | null;+}+// V1 API Types for requests// These types match the SDK's TextContent and ImageContent formatsexport interface V1TextContent {@@ -91,7 +107,7 @@ export interface V1AppConversation {trigger: ConversationTrigger | null;pr_number: number[];llm_model: string | null;- metrics: unknown | null;+ metrics: V1MetricsSnapshot | null;created_at: string;updated_at: string;sandbox_status: V1SandboxStatus;@@ -111,3 +127,40 @@ export interface Skill {export interface GetSkillsResponse {skills: Skill[];}++// Runtime conversation types (from agent server)+export interface V1RuntimeConversationStats {+ usage_to_metrics: Record<string, V1RuntimeMetrics>;+}++export interface V1RuntimeMetrics {+ model_name: string;+ accumulated_cost: number;+ max_budget_per_task: number | null;+ accumulated_token_usage: V1TokenUsage | null;+ costs: V1Cost[];+ response_latencies: V1ResponseLatency[];+ token_usages: V1TokenUsage[];+}++export interface V1Cost {+ model: string;+ cost: number;+ timestamp: number;+}++export interface V1ResponseLatency {+ model: string;+ latency: number;+ response_id: string;+}++export interface V1RuntimeConversationInfo {+ id: string;+ title: string | null;+ metrics: V1MetricsSnapshot | null;+ created_at: string;+ updated_at: string;+ status: V1ConversationExecutionStatus;+ stats: V1RuntimeConversationStats;+}diff --git a/frontend/src/components/features/conversation/metrics-modal/metrics-modal.tsx b/frontend/src/components/features/conversation/metrics-modal/metrics-modal.tsxindex df794be537..42523a7c5c 100644--- a/frontend/src/components/features/conversation/metrics-modal/metrics-modal.tsx+++ b/frontend/src/components/features/conversation/metrics-modal/metrics-modal.tsx@@ -1,3 +1,4 @@+import { useMemo } from "react";import { useTranslation } from "react-i18next";import { BaseModalTitle } from "#/components/shared/modals/confirmation-modals/base-modal";import { ModalBackdrop } from "#/components/shared/modals/modal-backdrop";@@ -8,6 +9,8 @@ import { UsageSection } from "./usage-section";import { ContextWindowSection } from "./context-window-section";import { EmptyState } from "./empty-state";import useMetricsStore from "#/stores/metrics-store";+import { useActiveConversation } from "#/hooks/query/use-active-conversation";+import { useSandboxMetrics } from "#/hooks/query/use-sandbox-metrics";interface MetricsModalProps {isOpen: boolean;@@ -16,7 +19,52 @@ interface MetricsModalProps {export function MetricsModal({ isOpen, onOpenChange }: MetricsModalProps) {const { t } = useTranslation();- const metrics = useMetricsStore();+ const storeMetrics = useMetricsStore();+ const { data: conversation } = useActiveConversation();++ const isV1 = conversation?.conversation_version === "V1";+ const conversationId = conversation?.conversation_id;+ const conversationUrl = conversation?.url;+ const sessionApiKey = conversation?.session_api_key;++ // For V1 conversations, fetch metrics directly from the sandbox+ // Only fetch when the modal is open to avoid unnecessary requests+ const { data: sandboxMetrics } = useSandboxMetrics(+ conversationId,+ conversationUrl,+ sessionApiKey,+ isV1 && isOpen, // Only enable when modal is open+ );++ // Compute the metrics based on conversation version+ const metrics = useMemo(() => {+ if (isV1 && sandboxMetrics) {+ return {+ cost: sandboxMetrics.accumulated_cost,+ max_budget_per_task: sandboxMetrics.max_budget_per_task,+ usage: sandboxMetrics.accumulated_token_usage+ ? {+ prompt_tokens:+ sandboxMetrics.accumulated_token_usage.prompt_tokens ?? 0,+ completion_tokens:+ sandboxMetrics.accumulated_token_usage.completion_tokens ?? 0,+ cache_read_tokens:+ sandboxMetrics.accumulated_token_usage.cache_read_tokens ?? 0,+ cache_write_tokens:+ sandboxMetrics.accumulated_token_usage.cache_write_tokens ?? 0,+ context_window:+ sandboxMetrics.accumulated_token_usage.context_window ?? 0,+ per_turn_token:+ sandboxMetrics.accumulated_token_usage.per_turn_token ?? 0,+ }+ : null,+ };+ }++ // For non-V1 conversations, use the store metrics+ return storeMetrics;+ }, [isV1, sandboxMetrics, storeMetrics]);+if (!isOpen) return null;return (diff --git a/frontend/src/hooks/query/use-sandbox-metrics.ts b/frontend/src/hooks/query/use-sandbox-metrics.tsnew file mode 100644index 0000000000..56053f0c33--- /dev/null+++ b/frontend/src/hooks/query/use-sandbox-metrics.ts@@ -0,0 +1,53 @@+import { useQuery } from "@tanstack/react-query";+import V1ConversationService from "#/api/conversation-service/v1-conversation-service.api";+import { getCombinedMetrics } from "#/utils/conversation-metrics";+import type { V1MetricsSnapshot } from "#/api/conversation-service/v1-conversation-service.types";++/**+ * Hook to fetch metrics directly from the sandbox for V1 conversations+ * @param conversationId The conversation ID+ * @param conversationUrl The conversation URL from the active conversation+ * @param sessionApiKey The session API key from the active conversation+ * @param enabled Whether the query should be enabled (typically when modal is open and conversation is V1)+ */+export const useSandboxMetrics = (+ conversationId: string | null | undefined,+ conversationUrl: string | null | undefined,+ sessionApiKey: string | null | undefined,+ enabled: boolean = true,+): {+ data: V1MetricsSnapshot | undefined;+ isLoading: boolean;+ error: unknown;+} => {+ const query = useQuery({+ queryKey: [+ "sandbox-metrics",+ conversationId,+ conversationUrl,+ sessionApiKey,+ ],+ queryFn: async () => {+ if (!conversationId) throw new Error("Conversation ID is required");+ const conversationInfo =+ await V1ConversationService.getRuntimeConversation(+ conversationId,+ conversationUrl,+ sessionApiKey,+ );+ return getCombinedMetrics(conversationInfo);+ },+ enabled:+ enabled && !!conversationId && !!conversationUrl && !!sessionApiKey,+ staleTime: 1000 * 30, // 30 seconds+ gcTime: 1000 * 60 * 5, // 5 minutes+ refetchInterval: 1000 * 30, // Refetch every 30 seconds+ retry: false, // Don't retry on failure since this is a new endpoint+ });++ return {+ data: query.data,+ isLoading: query.isLoading,+ error: query.error,+ };+};diff --git a/frontend/src/utils/conversation-metrics.ts b/frontend/src/utils/conversation-metrics.tsnew file mode 100644index 0000000000..69ef293174--- /dev/null+++ b/frontend/src/utils/conversation-metrics.ts@@ -0,0 +1,71 @@+import type {+ V1MetricsSnapshot,+ V1RuntimeConversationInfo,+ V1TokenUsage,+} from "#/api/conversation-service/v1-conversation-service.types";++/**+ * TypeScript equivalent of the get_combined_metrics method from the Python SDK+ * Combines metrics from all LLM usage IDs in the conversation stats+ */+export function getCombinedMetrics(+ conversationInfo: V1RuntimeConversationInfo,+): V1MetricsSnapshot {+ const { stats } = conversationInfo;++ if (!stats?.usage_to_metrics) {+ return {+ accumulated_cost: 0,+ max_budget_per_task: null,+ accumulated_token_usage: null,+ };+ }++ let totalCost = 0;+ let maxBudgetPerTask: number | null = null;+ let combinedTokenUsage: V1TokenUsage | null = null;++ // Iterate through all metrics and combine them+ for (const metrics of Object.values(stats.usage_to_metrics)) {+ // Add up costs+ totalCost += metrics.accumulated_cost;++ // Keep the max budget per task if any is set+ if (maxBudgetPerTask === null && metrics.max_budget_per_task !== null) {+ maxBudgetPerTask = metrics.max_budget_per_task;+ }++ // Combine token usage+ if (metrics.accumulated_token_usage) {+ if (combinedTokenUsage === null) {+ combinedTokenUsage = { ...metrics.accumulated_token_usage };+ } else {+ combinedTokenUsage = {+ prompt_tokens:+ combinedTokenUsage.prompt_tokens ++ metrics.accumulated_token_usage.prompt_tokens,+ completion_tokens:+ combinedTokenUsage.completion_tokens ++ metrics.accumulated_token_usage.completion_tokens,+ cache_read_tokens:+ combinedTokenUsage.cache_read_tokens ++ metrics.accumulated_token_usage.cache_read_tokens,+ cache_write_tokens:+ combinedTokenUsage.cache_write_tokens ++ metrics.accumulated_token_usage.cache_write_tokens,+ context_window: Math.max(+ combinedTokenUsage.context_window,+ metrics.accumulated_token_usage.context_window,+ ),+ per_turn_token: metrics.accumulated_token_usage.per_turn_token, // Use the latest per_turn_token+ };+ }+ }+ }++ return {+ accumulated_cost: totalCost,+ max_budget_per_task: maxBudgetPerTask,+ accumulated_token_usage: combinedTokenUsage,+ };+}diff --git a/openhands/app_server/sandbox/remote_sandbox_service.py b/openhands/app_server/sandbox/remote_sandbox_service.pyindex 7249d2e38e..12284bf115 100644--- a/openhands/app_server/sandbox/remote_sandbox_service.py+++ b/openhands/app_server/sandbox/remote_sandbox_service.py@@ -711,8 +711,18 @@ async def refresh_conversation(updated_conversation_info = ConversationInfo.model_validate(response.json())- # TODO: As of writing, ConversationInfo from AgentServer does not have a title to update...app_conversation_info.updated_at = updated_conversation_info.updated_at++ # TODO: This is a temp fix - the agent server is storing metrics in a new format+ # We should probably update the data structures and to store / display the more+ # explicit metrics+ try:+ app_conversation_info.metrics = (+ updated_conversation_info.stats.get_combined_metrics()+ )+ except Exception:+ _logger.exception('error_updating_conversation_metrics', stack_info=True)+# TODO: Update other appropriate attributes...await app_conversation_info_service.save_app_conversation_info(