Skip to content

🙌 OpenHands: AI-Driven Development

feat(frontend): add user feedback Likert scale for agent performance rating (only on OH Cloud) (#8992)

Xingyao Wang2c4ecd02f7af
diff --git a/frontend/src/api/open-hands.ts b/frontend/src/api/open-hands.ts
index d8aafe0cf7..62ce7af33c 100644
--- a/frontend/src/api/open-hands.ts
+++ b/frontend/src/api/open-hands.ts
@@ -111,6 +111,59 @@ class OpenHands {
return data;
}
+ /**
+ * Submit conversation feedback with rating
+ * @param conversationId The conversation ID
+ * @param rating The rating (1-5)
+ * @param eventId Optional event ID this feedback corresponds to
+ * @param reason Optional reason for the rating
+ * @returns Response from the feedback endpoint
+ */
+ static async submitConversationFeedback(
+ conversationId: string,
+ rating: number,
+ eventId?: number,
+ reason?: string,
+ ): Promise<{ status: string; message: string }> {
+ const url = `/feedback/conversation`;
+ const payload = {
+ conversation_id: conversationId,
+ event_id: eventId,
+ rating,
+ reason,
+ metadata: { source: "likert-scale" },
+ };
+ const { data } = await openHands.post<{ status: string; message: string }>(
+ url,
+ payload,
+ );
+ return data;
+ }
+
+ /**
+ * Check if feedback exists for a specific conversation and event
+ * @param conversationId The conversation ID
+ * @param eventId The event ID to check
+ * @returns Feedback data including existence, rating, and reason
+ */
+ static async checkFeedbackExists(
+ conversationId: string,
+ eventId: number,
+ ): Promise<{ exists: boolean; rating?: number; reason?: string }> {
+ try {
+ const url = `/feedback/conversation/${conversationId}/${eventId}`;
+ const { data } = await openHands.get<{
+ exists: boolean;
+ rating?: number;
+ reason?: string;
+ }>(url);
+ return data;
+ } catch (error) {
+ // Error checking if feedback exists
+ return { exists: false };
+ }
+ }
+
/**
* Authenticate with GitHub token
* @returns Response with authentication status and user info if successful
diff --git a/frontend/src/components/features/chat/chat-interface.tsx b/frontend/src/components/features/chat/chat-interface.tsx
index 9cc2c214bc..28b6e63983 100644
--- a/frontend/src/components/features/chat/chat-interface.tsx
+++ b/frontend/src/components/features/chat/chat-interface.tsx
@@ -18,6 +18,7 @@ import { useWsClient } from "#/context/ws-client-provider";
import { Messages } from "./messages";
import { ChatSuggestions } from "./chat-suggestions";
import { ActionSuggestions } from "./action-suggestions";
+import { ScrollProvider } from "#/context/scroll-context";
import { ScrollToBottomButton } from "#/components/shared/buttons/scroll-to-bottom-button";
import { LoadingSpinner } from "#/components/shared/loading-spinner";
@@ -28,6 +29,7 @@ import { useOptimisticUserMessage } from "#/hooks/use-optimistic-user-message";
import { useWSErrorMessage } from "#/hooks/use-ws-error-message";
import { ErrorMessageBanner } from "./error-message-banner";
import { shouldRenderEvent } from "./event-content-helpers/should-render-event";
+import { useConfig } from "#/hooks/query/use-config";
function getEntryPoint(
hasRepository: boolean | null,
@@ -45,8 +47,15 @@ export function ChatInterface() {
useOptimisticUserMessage();
const { t } = useTranslation();
const scrollRef = React.useRef<HTMLDivElement>(null);
- const { scrollDomToBottom, onChatBodyScroll, hitBottom } =
- useScrollToBottom(scrollRef);
+ const {
+ scrollDomToBottom,
+ onChatBodyScroll,
+ hitBottom,
+ autoScroll,
+ setAutoScroll,
+ setHitBottom,
+ } = useScrollToBottom(scrollRef);
+ const { data: config } = useConfig();
const { curAgentState } = useSelector((state: RootState) => state.agent);
@@ -126,80 +135,97 @@ export function ChatInterface() {
curAgentState === AgentState.AWAITING_USER_INPUT ||
curAgentState === AgentState.FINISHED;
+ // Create a ScrollProvider with the scroll hook values
+ const scrollProviderValue = {
+ scrollRef,
+ autoScroll,
+ setAutoScroll,
+ scrollDomToBottom,
+ hitBottom,
+ setHitBottom,
+ onChatBodyScroll,
+ };
+
return (
- <div className="h-full flex flex-col justify-between">
- {events.length === 0 && !optimisticUserMessage && (
- <ChatSuggestions onSuggestionsClick={setMessageToSend} />
- )}
-
- <div
- ref={scrollRef}
- onScroll={(e) => onChatBodyScroll(e.currentTarget)}
- className="scrollbar scrollbar-thin scrollbar-thumb-gray-400 scrollbar-thumb-rounded-full scrollbar-track-gray-800 hover:scrollbar-thumb-gray-300 flex flex-col grow overflow-y-auto overflow-x-hidden px-4 pt-4 gap-2 fast-smooth-scroll"
- >
- {isLoadingMessages && (
- <div className="flex justify-center">
- <LoadingSpinner size="small" />
- </div>
+ <ScrollProvider value={scrollProviderValue}>
+ <div className="h-full flex flex-col justify-between">
+ {events.length === 0 && !optimisticUserMessage && (
+ <ChatSuggestions onSuggestionsClick={setMessageToSend} />
)}
- {!isLoadingMessages && (
- <Messages
- messages={events}
- isAwaitingUserConfirmation={
- curAgentState === AgentState.AWAITING_USER_CONFIRMATION
- }
- />
- )}
+ <div
+ ref={scrollRef}
+ onScroll={(e) => onChatBodyScroll(e.currentTarget)}
+ className="scrollbar scrollbar-thin scrollbar-thumb-gray-400 scrollbar-thumb-rounded-full scrollbar-track-gray-800 hover:scrollbar-thumb-gray-300 flex flex-col grow overflow-y-auto overflow-x-hidden px-4 pt-4 gap-2 fast-smooth-scroll"
+ >
+ {isLoadingMessages && (
+ <div className="flex justify-center">
+ <LoadingSpinner size="small" />
+ </div>
+ )}
- {isWaitingForUserInput &&
- events.length > 0 &&
- !optimisticUserMessage && (
- <ActionSuggestions
- onSuggestionsClick={(value) => handleSendMessage(value, [])}
+ {!isLoadingMessages && (
+ <Messages
+ messages={events}
+ isAwaitingUserConfirmation={
+ curAgentState === AgentState.AWAITING_USER_CONFIRMATION
+ }
/>
)}
- </div>
- <div className="flex flex-col gap-[6px] px-4 pb-4">
- <div className="flex justify-between relative">
- <TrajectoryActions
- onPositiveFeedback={() =>
- onClickShareFeedbackActionButton("positive")
- }
- onNegativeFeedback={() =>
- onClickShareFeedbackActionButton("negative")
- }
- onExportTrajectory={() => onClickExportTrajectoryButton()}
- />
+ {isWaitingForUserInput &&
+ events.length > 0 &&
+ !optimisticUserMessage && (
+ <ActionSuggestions
+ onSuggestionsClick={(value) => handleSendMessage(value, [])}
+ />
+ )}
+ </div>
- <div className="absolute left-1/2 transform -translate-x-1/2 bottom-0">
- {curAgentState === AgentState.RUNNING && <TypingIndicator />}
+ <div className="flex flex-col gap-[6px] px-4 pb-4">
+ <div className="flex justify-between relative">
+ {config?.APP_MODE !== "saas" && (
+ <TrajectoryActions
+ onPositiveFeedback={() =>
+ onClickShareFeedbackActionButton("positive")
+ }
+ onNegativeFeedback={() =>
+ onClickShareFeedbackActionButton("negative")
+ }
+ onExportTrajectory={() => onClickExportTrajectoryButton()}
+ />
+ )}
+
+ <div className="absolute left-1/2 transform -translate-x-1/2 bottom-0">
+ {curAgentState === AgentState.RUNNING && <TypingIndicator />}
+ </div>
+
+ {!hitBottom && <ScrollToBottomButton onClick={scrollDomToBottom} />}
</div>
- {!hitBottom && <ScrollToBottomButton onClick={scrollDomToBottom} />}
+ {errorMessage && <ErrorMessageBanner message={errorMessage} />}
+
+ <InteractiveChatBox
+ onSubmit={handleSendMessage}
+ onStop={handleStop}
+ isDisabled={
+ curAgentState === AgentState.LOADING ||
+ curAgentState === AgentState.AWAITING_USER_CONFIRMATION
+ }
+ mode={curAgentState === AgentState.RUNNING ? "stop" : "submit"}
+ value={messageToSend ?? undefined}
+ onChange={setMessageToSend}
+ />
</div>
- {errorMessage && <ErrorMessageBanner message={errorMessage} />}
-
- <InteractiveChatBox
- onSubmit={handleSendMessage}
- onStop={handleStop}
- isDisabled={
- curAgentState === AgentState.LOADING ||
- curAgentState === AgentState.AWAITING_USER_CONFIRMATION
- }
- mode={curAgentState === AgentState.RUNNING ? "stop" : "submit"}
- value={messageToSend ?? undefined}
- onChange={setMessageToSend}
- />
+ {config?.APP_MODE !== "saas" && (
+ <FeedbackModal
+ isOpen={feedbackModalIsOpen}
+ onClose={() => setFeedbackModalIsOpen(false)}
+ polarity={feedbackPolarity}
+ />
+ )}
</div>
-
- <FeedbackModal
- isOpen={feedbackModalIsOpen}
- onClose={() => setFeedbackModalIsOpen(false)}
- polarity={feedbackPolarity}
- />
- </div>
+ </ScrollProvider>
);
}
diff --git a/frontend/src/components/features/chat/event-message.tsx b/frontend/src/components/features/chat/event-message.tsx
index 2739872ef2..531f266b3d 100644
--- a/frontend/src/components/features/chat/event-message.tsx
+++ b/frontend/src/components/features/chat/event-message.tsx
@@ -1,3 +1,4 @@
+import React from "react";
import { ConfirmationButtons } from "#/components/shared/buttons/confirmation-buttons";
import { OpenHandsAction } from "#/types/core/actions";
import {
@@ -18,6 +19,10 @@ import { MCPObservationContent } from "./mcp-observation-content";
import { getObservationResult } from "./event-content-helpers/get-observation-result";
import { getEventContent } from "./event-content-helpers/get-event-content";
import { GenericEventMessage } from "./generic-event-message";
+import { LikertScale } from "../feedback/likert-scale";
+
+import { useConfig } from "#/hooks/query/use-config";
+import { useFeedbackExists } from "#/hooks/query/use-feedback-exists";
const hasThoughtProperty = (
obj: Record<string, unknown>,
@@ -39,6 +44,14 @@ export function EventMessage({
const shouldShowConfirmationButtons =
isLastMessage && event.source === "agent" && isAwaitingUserConfirmation;
+ const { data: config } = useConfig();
+
+ // Use our query hook to check if feedback exists and get rating/reason
+ const {
+ data: feedbackData = { exists: false },
+ isLoading: isCheckingFeedback,
+ } = useFeedbackExists(isFinishAction(event) ? event.id : undefined);
+
if (isErrorObservation(event)) {
return (
<ErrorMessage
@@ -55,9 +68,25 @@ export function EventMessage({
return null;
}
+ const showLikertScale =
+ config?.APP_MODE === "saas" &&
+ isFinishAction(event) &&
+ isLastMessage &&
+ !isCheckingFeedback;
+
if (isFinishAction(event)) {
return (
- <ChatMessage type="agent" message={getEventContent(event).details} />
+ <>
+ <ChatMessage type="agent" message={getEventContent(event).details} />
+ {showLikertScale && (
+ <LikertScale
+ eventId={event.id}
+ initiallySubmitted={feedbackData.exists}
+ initialRating={feedbackData.rating}
+ initialReason={feedbackData.reason}
+ />
+ )}
+ </>
);
}
diff --git a/frontend/src/components/features/feedback/likert-scale.tsx b/frontend/src/components/features/feedback/likert-scale.tsx
new file mode 100644
index 0000000000..e47e825e5c
--- /dev/null
+++ b/frontend/src/components/features/feedback/likert-scale.tsx
@@ -0,0 +1,248 @@
+import React, { useState, useEffect, useContext } from "react";
+import { cn } from "#/utils/utils";
+import i18n from "#/i18n";
+import { useSubmitConversationFeedback } from "#/hooks/mutation/use-submit-conversation-feedback";
+import { ScrollContext } from "#/context/scroll-context";
+
+// Global timeout duration in milliseconds
+const AUTO_SUBMIT_TIMEOUT = 10000;
+
+interface LikertScaleProps {
+ eventId?: number;
+ initiallySubmitted?: boolean;
+ initialRating?: number;
+ initialReason?: string;
+}
+
+const FEEDBACK_REASONS = [
+ i18n.t("FEEDBACK$REASON_MISUNDERSTOOD_INSTRUCTION"),
+ i18n.t("FEEDBACK$REASON_FORGOT_CONTEXT"),
+ i18n.t("FEEDBACK$REASON_UNNECESSARY_CHANGES"),
+ i18n.t("FEEDBACK$REASON_OTHER"),
+];
+
+export function LikertScale({
+ eventId,
+ initiallySubmitted = false,
+ initialRating,
+ initialReason,
+}: LikertScaleProps) {
+ const [selectedRating, setSelectedRating] = useState<number | null>(
+ initialRating || null,
+ );
+ const [selectedReason, setSelectedReason] = useState<string | null>(
+ initialReason || null,
+ );
+ const [showReasons, setShowReasons] = useState(false);
+ const [reasonTimeout, setReasonTimeout] = useState<NodeJS.Timeout | null>(
+ null,
+ );
+ const [isSubmitted, setIsSubmitted] = useState(initiallySubmitted);
+ const [countdown, setCountdown] = useState<number>(0);
+
+ // Get scroll context
+ const scrollContext = useContext(ScrollContext);
+
+ // If scrollContext is undefined, we're not inside a ScrollProvider
+ const scrollToBottom = scrollContext?.scrollDomToBottom;
+ const autoScroll = scrollContext?.autoScroll;
+
+ // Use our mutation hook
+ const { mutate: submitConversationFeedback } =
+ useSubmitConversationFeedback();
+
+ // Update isSubmitted if initiallySubmitted changes
+ useEffect(() => {
+ setIsSubmitted(initiallySubmitted);
+ }, [initiallySubmitted]);
+
+ // Update selectedRating if initialRating changes
+ useEffect(() => {
+ if (initialRating) {
+ setSelectedRating(initialRating);
+ }
+ }, [initialRating]);
+
+ // Update selectedReason if initialReason changes
+ useEffect(() => {
+ if (initialReason) {
+ setSelectedReason(initialReason);
+ }
+ }, [initialReason]);
+
+ // Submit feedback and disable the component
+ const submitFeedback = (rating: number, reason?: string) => {
+ submitConversationFeedback(
+ {
+ rating,
+ eventId,
+ reason,
+ },
+ {
+ onSuccess: () => {
+ setSelectedReason(reason || null);
+ setShowReasons(false);
+ setIsSubmitted(true);
+ },
+ },
+ );
+ };
+
+ // Handle star rating selection
+ const handleRatingClick = (rating: number) => {
+ if (isSubmitted) return; // Prevent changes after submission
+
+ setSelectedRating(rating);
+
+ // Only show reasons if rating is 3 or less (1, 2, or 3 stars)
+ // For ratings > 3 (4 or 5 stars), submit immediately without showing reasons
+ if (rating <= 3) {
+ setShowReasons(true);
+ setCountdown(Math.ceil(AUTO_SUBMIT_TIMEOUT / 1000));
+
+ // Set a timeout to auto-submit if no reason is selected
+ const timeout = setTimeout(() => {
+ submitFeedback(rating);
+ }, AUTO_SUBMIT_TIMEOUT);
+
+ setReasonTimeout(timeout);
+
+ // Only scroll to bottom if the user is already at the bottom (autoScroll is true)
+ if (scrollToBottom && autoScroll) {
+ // Small delay to ensure the reasons are fully rendered
+ setTimeout(() => {
+ scrollToBottom();
+ }, 100);
+ }
+ } else {
+ // For ratings > 3 (4 or 5 stars), submit immediately without showing reasons
+ setShowReasons(false);
+ submitFeedback(rating);
+ }
+ };
+
+ // Handle reason selection
+ const handleReasonClick = (reason: string) => {
+ if (selectedRating && reasonTimeout && !isSubmitted) {
+ clearTimeout(reasonTimeout);
+ setCountdown(0);
+ submitFeedback(selectedRating, reason);
+ }
+ };
+
+ // Countdown effect
+ useEffect(() => {
+ if (countdown > 0 && showReasons && !isSubmitted) {
+ const timer = setTimeout(() => {
+ setCountdown(countdown - 1);
+ }, 1000);
+ return () => clearTimeout(timer);
+ }
+ return () => {};
+ }, [countdown, showReasons, isSubmitted]);
+
+ // Clean up timeout on unmount
+ useEffect(
+ () => () => {
+ if (reasonTimeout) {
+ clearTimeout(reasonTimeout);
+ }
+ },
+ [reasonTimeout],
+ );
+
+ // Scroll to bottom when component mounts, but only if user is already at the bottom
+ useEffect(() => {
+ if (scrollToBottom && autoScroll && !isSubmitted) {
+ // Small delay to ensure the component is fully rendered
+ setTimeout(() => {
+ scrollToBottom();
+ }, 100);
+ }
+ }, [scrollToBottom, autoScroll, isSubmitted]);
+
+ // Scroll to bottom when reasons are shown, but only if user is already at the bottom
+ useEffect(() => {
+ if (scrollToBottom && autoScroll && showReasons) {
+ // Small delay to ensure the reasons are fully rendered
+ setTimeout(() => {
+ scrollToBottom();
+ }, 100);
+ }
+ }, [scrollToBottom, autoScroll, showReasons]);
+
+ // Helper function to get button class based on state
+ const getButtonClass = (rating: number) => {
+ if (isSubmitted) {
+ return selectedRating && selectedRating >= rating
+ ? "text-yellow-400 cursor-not-allowed"
+ : "text-gray-300 opacity-50 cursor-not-allowed";
+ }
+
+ return selectedRating && selectedRating >= rating
+ ? "text-yellow-400"
+ : "text-gray-300 hover:text-yellow-200";
+ };
+
+ return (
+ <div className="mt-3 flex flex-col gap-1">
+ <div className="text-sm text-gray-500 mb-1">
+ {isSubmitted
+ ? i18n.t("FEEDBACK$THANK_YOU_FOR_FEEDBACK")
+ : i18n.t("FEEDBACK$RATE_AGENT_PERFORMANCE")}
+ </div>
+ <div className="flex flex-col gap-1">
+ <span className="flex gap-2 items-center flex-wrap">
+ {[1, 2, 3, 4, 5].map((rating) => (
+ <button
+ type="button"
+ key={rating}
+ onClick={() => handleRatingClick(rating)}
+ disabled={isSubmitted}
+ className={cn("text-xl transition-all", getButtonClass(rating))}
+ aria-label={`Rate ${rating} stars`}
+ >
+ ★
+ </button>
+ ))}
+ {/* Show selected reason inline with stars when submitted (only for ratings <= 3) */}
+ {isSubmitted &&
+ selectedReason &&
+ selectedRating &&
+ selectedRating <= 3 && (
+ <span className="text-sm text-gray-500 italic">
+ {selectedReason}
+ </span>
+ )}
+ </span>
+ </div>
+
+ {showReasons && !isSubmitted && (
+ <div className="mt-1 flex flex-col gap-1">
+ <div className="text-xs text-gray-500 mb-1">
+ {i18n.t("FEEDBACK$SELECT_REASON")}
+ </div>
+ {countdown > 0 && (
+ <div className="text-xs text-gray-400 mb-1 italic">
+ {i18n.t("FEEDBACK$SELECT_REASON_COUNTDOWN", {
+ countdown,
+ })}
+ </div>
+ )}
+ <div className="flex flex-col gap-0.5">
+ {FEEDBACK_REASONS.map((reason) => (
+ <button
+ type="button"
+ key={reason}
+ onClick={() => handleReasonClick(reason)}
+ className="text-sm text-left py-1 px-2 rounded hover:bg-gray-700 transition-colors"
+ >
+ {reason}
+ </button>
+ ))}
+ </div>
+ </div>
+ )}
+ </div>
+ );
+}
diff --git a/frontend/src/context/scroll-context.tsx b/frontend/src/context/scroll-context.tsx
new file mode 100644
index 0000000000..2394bc07f2
--- /dev/null
+++ b/frontend/src/context/scroll-context.tsx
@@ -0,0 +1,42 @@
+import React, { createContext, useContext, ReactNode, RefObject } from "react";
+import { useScrollToBottom } from "#/hooks/use-scroll-to-bottom";
+
+interface ScrollContextType {
+ scrollRef: RefObject<HTMLDivElement | null>;
+ autoScroll: boolean;
+ setAutoScroll: (value: boolean) => void;
+ scrollDomToBottom: () => void;
+ hitBottom: boolean;
+ setHitBottom: (value: boolean) => void;
+ onChatBodyScroll: (e: HTMLElement) => void;
+}
+
+export const ScrollContext = createContext<ScrollContextType | undefined>(
+ undefined,
+);
+
+interface ScrollProviderProps {
+ children: ReactNode;
+ value?: ScrollContextType;
+}
+
+export function ScrollProvider({ children, value }: ScrollProviderProps) {
+ const scrollHook = useScrollToBottom(React.useRef<HTMLDivElement>(null));
+
+ // Use provided value or default to the hook
+ const contextValue = value || scrollHook;
+
+ return (
+ <ScrollContext.Provider value={contextValue}>
+ {children}
+ </ScrollContext.Provider>
+ );
+}
+
+export function useScrollContext() {
+ const context = useContext(ScrollContext);
+ if (context === undefined) {
+ throw new Error("useScrollContext must be used within a ScrollProvider");
+ }
+ return context;
+}
diff --git a/frontend/src/hooks/mutation/use-submit-conversation-feedback.ts b/frontend/src/hooks/mutation/use-submit-conversation-feedback.ts
new file mode 100644
index 0000000000..ff5ae004d1
--- /dev/null
+++ b/frontend/src/hooks/mutation/use-submit-conversation-feedback.ts
@@ -0,0 +1,39 @@
+import { useMutation, useQueryClient } from "@tanstack/react-query";
+import { useTranslation } from "react-i18next";
+import OpenHands from "#/api/open-hands";
+import { useConversationId } from "#/hooks/use-conversation-id";
+
+type SubmitConversationFeedbackArgs = {
+ rating: number;
+ eventId?: number;
+ reason?: string;
+};
+
+export const useSubmitConversationFeedback = () => {
+ const { conversationId } = useConversationId();
+ const queryClient = useQueryClient();
+ const { t } = useTranslation();
+
+ return useMutation({
+ mutationFn: ({ rating, eventId, reason }: SubmitConversationFeedbackArgs) =>
+ OpenHands.submitConversationFeedback(
+ conversationId,
+ rating,
+ eventId,
+ reason,
+ ),
+ onSuccess: (_, { eventId }) => {
+ // Invalidate the feedback existence query to trigger a refetch
+ if (eventId) {
+ queryClient.invalidateQueries({
+ queryKey: ["feedback", "exists", conversationId, eventId],
+ });
+ }
+ },
+ onError: (error) => {
+ // Log error but don't show toast - user will just see the UI stay in unsubmitted state
+ // eslint-disable-next-line no-console
+ console.error(t("FEEDBACK$FAILED_TO_SUBMIT"), error);
+ },
+ });
+};
diff --git a/frontend/src/hooks/query/use-feedback-exists.ts b/frontend/src/hooks/query/use-feedback-exists.ts
new file mode 100644
index 0000000000..804417e433
--- /dev/null
+++ b/frontend/src/hooks/query/use-feedback-exists.ts
@@ -0,0 +1,24 @@
+import { useQuery } from "@tanstack/react-query";
+import OpenHands from "#/api/open-hands";
+import { useConversationId } from "#/hooks/use-conversation-id";
+
+export interface FeedbackData {
+ exists: boolean;
+ rating?: number;
+ reason?: string;
+}
+
+export const useFeedbackExists = (eventId?: number) => {
+ const { conversationId } = useConversationId();
+
+ return useQuery<FeedbackData>({
+ queryKey: ["feedback", "exists", conversationId, eventId],
+ queryFn: () => {
+ if (!eventId) return { exists: false };
+ return OpenHands.checkFeedbackExists(conversationId, eventId);
+ },
+ enabled: !!eventId,
+ staleTime: 1000 * 60 * 5, // 5 minutes
+ gcTime: 1000 * 60 * 15, // 15 minutes
+ });
+};
diff --git a/frontend/src/i18n/declaration.ts b/frontend/src/i18n/declaration.ts
index 8694683f1d..dbc7856990 100644
--- a/frontend/src/i18n/declaration.ts
+++ b/frontend/src/i18n/declaration.ts
@@ -583,4 +583,13 @@ export enum I18nKey {
SETTINGS$EMAIL_VERIFICATION_RESTRICTION_MESSAGE = "SETTINGS$EMAIL_VERIFICATION_RESTRICTION_MESSAGE",
SETTINGS$RESEND_VERIFICATION = "SETTINGS$RESEND_VERIFICATION",
SETTINGS$FAILED_TO_RESEND_VERIFICATION = "SETTINGS$FAILED_TO_RESEND_VERIFICATION",
+ FEEDBACK$RATE_AGENT_PERFORMANCE = "FEEDBACK$RATE_AGENT_PERFORMANCE",
+ FEEDBACK$SELECT_REASON = "FEEDBACK$SELECT_REASON",
+ FEEDBACK$SELECT_REASON_COUNTDOWN = "FEEDBACK$SELECT_REASON_COUNTDOWN",
+ FEEDBACK$REASON_MISUNDERSTOOD_INSTRUCTION = "FEEDBACK$REASON_MISUNDERSTOOD_INSTRUCTION",
+ FEEDBACK$REASON_FORGOT_CONTEXT = "FEEDBACK$REASON_FORGOT_CONTEXT",
+ FEEDBACK$REASON_UNNECESSARY_CHANGES = "FEEDBACK$REASON_UNNECESSARY_CHANGES",
+ FEEDBACK$REASON_OTHER = "FEEDBACK$REASON_OTHER",
+ FEEDBACK$THANK_YOU_FOR_FEEDBACK = "FEEDBACK$THANK_YOU_FOR_FEEDBACK",
+ FEEDBACK$FAILED_TO_SUBMIT = "FEEDBACK$FAILED_TO_SUBMIT",
}
diff --git a/frontend/src/i18n/translation.json b/frontend/src/i18n/translation.json
index a240e83237..3e08c6ecf0 100644
--- a/frontend/src/i18n/translation.json
+++ b/frontend/src/i18n/translation.json
@@ -9326,5 +9326,149 @@
"tr": "Doğrulama e-postası yeniden gönderilemedi",
"de": "Bestätigungs-E-Mail konnte nicht erneut gesendet werden",
"uk": "Не вдалося повторно надіслати лист підтвердження"
+ },
+ "FEEDBACK$RATE_AGENT_PERFORMANCE": {
+ "en": "Rate the agent's performance:",
+ "ja": "エージェントのパフォーマンスを評価してください:",
+ "zh-CN": "评价代理的表现:",
+ "zh-TW": "評價代理的表現:",
+ "ko-KR": "에이전트의 성능을 평가하세요:",
+ "no": "Vurder agentens ytelse:",
+ "it": "Valuta le prestazioni dell'agente:",
+ "pt": "Avalie o desempenho do agente:",
+ "es": "Evalúe el rendimiento del agente:",
+ "ar": "قيم أداء الوكيل:",
+ "fr": "Évaluez la performance de l'agent :",
+ "tr": "Ajanın performansını değerlendirin:",
+ "de": "Bewerten Sie die Leistung des Agenten:",
+ "uk": "Оцініть продуктивність агента:"
+ },
+ "FEEDBACK$SELECT_REASON": {
+ "en": "Select a reason (optional):",
+ "ja": "理由を選択してください(任意):",
+ "zh-CN": "选择原因(可选):",
+ "zh-TW": "選擇原因(可選):",
+ "ko-KR": "이유 선택 (선택 사항):",
+ "no": "Velg en grunn (valgfritt):",
+ "it": "Seleziona un motivo (opzionale):",
+ "pt": "Selecione um motivo (opcional):",
+ "es": "Seleccione un motivo (opcional):",
+ "ar": "حدد سببًا (اختياري):",
+ "fr": "Sélectionnez une raison (facultatif) :",
+ "tr": "Bir neden seçin (isteğe bağlı):",
+ "de": "Wählen Sie einen Grund (optional):",
+ "uk": "Виберіть причину (необов'язково):"
+ },
+ "FEEDBACK$SELECT_REASON_COUNTDOWN": {
+ "en": "Auto-submitting in {{countdown}} seconds...",
+ "ja": "{{countdown}}秒後に自動送信されます...",
+ "zh-CN": "{{countdown}}秒后自动提交...",
+ "zh-TW": "{{countdown}}秒後自動提交...",
+ "ko-KR": "{{countdown}}초 후 자동 제출...",
+ "no": "Sender automatisk om {{countdown}} sekunder...",
+ "it": "Invio automatico tra {{countdown}} secondi...",
+ "pt": "Enviando automaticamente em {{countdown}} segundos...",
+ "es": "Enviando automáticamente en {{countdown}} segundos...",
+ "ar": "الإرسال التلقائي خلال {{countdown}} ثانية...",
+ "fr": "Envoi automatique dans {{countdown}} secondes...",
+ "tr": "{{countdown}} saniye içinde otomatik gönderilecek...",
+ "de": "Automatische Übermittlung in {{countdown}} Sekunden...",
+ "uk": "Автоматична відправка через {{countdown}} секунд..."
+ },
+ "FEEDBACK$REASON_MISUNDERSTOOD_INSTRUCTION": {
+ "en": "The agent misunderstood my instruction",
+ "ja": "エージェントは私の指示を誤解しました",
+ "zh-CN": "代理误解了我的指示",
+ "zh-TW": "代理誤解了我的指示",
+ "ko-KR": "에이전트가 내 지시를 잘못 이해했습니다",
+ "no": "Agenten misforsto instruksjonene mine",
+ "it": "L'agente ha frainteso le mie istruzioni",
+ "pt": "O agente não entendeu minhas instruções",
+ "es": "El agente malinterpretó mis instrucciones",
+ "ar": "أساء الوكيل فهم تعليماتي",
+ "fr": "L'agent a mal compris mes instructions",
+ "tr": "Ajan talimatlarımı yanlış anladı",
+ "de": "Der Agent hat meine Anweisungen missverstanden",
+ "uk": "Агент неправильно зрозумів мої інструкції"
+ },
+ "FEEDBACK$REASON_FORGOT_CONTEXT": {
+ "en": "The agent forgot about the earlier context",
+ "ja": "エージェントは以前のコンテキストを忘れました",
+ "zh-CN": "代理忘记了之前的上下文",
+ "zh-TW": "代理忘記了之前的上下文",
+ "ko-KR": "에이전트가 이전 컨텍스트를 잊었습니다",
+ "no": "Agenten glemte den tidligere konteksten",
+ "it": "L'agente ha dimenticato il contesto precedente",
+ "pt": "O agente esqueceu o contexto anterior",
+ "es": "El agente olvidó el contexto anterior",
+ "ar": "نسي الوكيل السياق السابق",
+ "fr": "L'agent a oublié le contexte précédent",
+ "tr": "Ajan önceki bağlamı unuttu",
+ "de": "Der Agent hat den früheren Kontext vergessen",
+ "uk": "Агент забув про попередній контекст"
+ },
+ "FEEDBACK$REASON_UNNECESSARY_CHANGES": {
+ "en": "The agent made unnecessary changes",
+ "ja": "エージェントは不要な変更を行いました",
+ "zh-CN": "代理进行了不必要的更改",
+ "zh-TW": "代理進行了不必要的更改",
+ "ko-KR": "에이전트가 불필요한 변경을 했습니다",
+ "no": "Agenten gjorde unødvendige endringer",
+ "it": "L'agente ha apportato modifiche non necessarie",
+ "pt": "O agente fez alterações desnecessárias",
+ "es": "El agente hizo cambios innecesarios",
+ "ar": "قام الوكيل بتغييرات غير ضرورية",
+ "fr": "L'agent a apporté des modifications inutiles",
+ "tr": "Ajan gereksiz değişiklikler yaptı",
+ "de": "Der Agent hat unnötige Änderungen vorgenommen",
+ "uk": "Агент зробив непотрібні зміни"
+ },
+ "FEEDBACK$REASON_OTHER": {
+ "en": "Other",
+ "ja": "その他",
+ "zh-CN": "其他",
+ "zh-TW": "其他",
+ "ko-KR": "기타",
+ "no": "Annet",
+ "it": "Altro",
+ "pt": "Outro",
+ "es": "Otro",
+ "ar": "أخرى",
+ "fr": "Autre",
+ "tr": "Diğer",
+ "de": "Andere",
+ "uk": "Інше"
+ },
+ "FEEDBACK$THANK_YOU_FOR_FEEDBACK": {
+ "en": "Thank you for your feedback! This will help us improve OpenHands going forward.",
+ "ja": "フィードバックをありがとうございます!これにより、今後OpenHandsを改善していくことができます。",
+ "zh-CN": "感谢您的反馈!这将帮助我们改进OpenHands。",
+ "zh-TW": "感謝您的反饋!這將幫助我們改進OpenHands。",
+ "ko-KR": "피드백 감사합니다! 이를 통해 OpenHands를 개선해 나가겠습니다.",
+ "no": "Takk for tilbakemeldingen! Dette vil hjelpe oss med å forbedre OpenHands fremover.",
+ "it": "Grazie per il tuo feedback! Questo ci aiuterà a migliorare OpenHands in futuro.",
+ "pt": "Obrigado pelo seu feedback! Isso nos ajudará a melhorar o OpenHands no futuro.",
+ "es": "¡Gracias por su comentario! Esto nos ayudará a mejorar OpenHands en el futuro.",
+ "ar": "شكرا على ملاحظاتك! سيساعدنا هذا في تحسين OpenHands في المستقبل.",
+ "fr": "Merci pour votre retour ! Cela nous aidera à améliorer OpenHands à l'avenir.",
+ "tr": "Geri bildiriminiz için teşekkürler! Bu, OpenHands'i ileride geliştirmemize yardımcı olacak.",
+ "de": "Vielen Dank für Ihr Feedback! Das hilft uns, OpenHands in Zukunft zu verbessern.",
+ "uk": "Дякуємо за ваш відгук! Це допоможе нам покращити OpenHands у майбутньому."
+ },
+ "FEEDBACK$FAILED_TO_SUBMIT": {
+ "en": "Failed to submit feedback",
+ "ja": "フィードバックの送信に失敗しました",
+ "zh-CN": "提交反馈失败",
+ "zh-TW": "提交反饋失敗",
+ "ko-KR": "피드백 제출 실패",
+ "no": "Kunne ikke sende tilbakemelding",
+ "it": "Impossibile inviare feedback",
+ "pt": "Falha ao enviar feedback",
+ "es": "Error al enviar comentarios",
+ "ar": "فشل في تقديم التعليقات",
+ "fr": "Échec de l'envoi des commentaires",
+ "tr": "Geri bildirim gönderilemedi",
+ "de": "Feedback konnte nicht gesendet werden",
+ "uk": "Не вдалося надіслати відгук"
}
}
diff --git a/openhands/agenthub/browsing_agent/browsing_agent.py b/openhands/agenthub/browsing_agent/browsing_agent.py
index 7721d000da..9fb4441e19 100644
--- a/openhands/agenthub/browsing_agent/browsing_agent.py
+++ b/openhands/agenthub/browsing_agent/browsing_agent.py
@@ -125,8 +125,7 @@ class BrowsingAgent(Agent):
self.reset()
def reset(self) -> None:
- """Resets the Browsing Agent's internal state.
- """
+ """Resets the Browsing Agent's internal state."""
super().reset()
# Reset agent-specific counters but not LLM metrics
self.error_accumulator = 0
diff --git a/openhands/agenthub/codeact_agent/codeact_agent.py b/openhands/agenthub/codeact_agent/codeact_agent.py
index 8366b4f354..eaa9ca4c7a 100644
--- a/openhands/agenthub/codeact_agent/codeact_agent.py
+++ b/openhands/agenthub/codeact_agent/codeact_agent.py
@@ -136,8 +136,7 @@ class CodeActAgent(Agent):
return tools
def reset(self) -> None:
- """Resets the CodeAct Agent's internal state.
- """
+ """Resets the CodeAct Agent's internal state."""
super().reset()
# Only clear pending actions, not LLM metrics
self.pending_actions.clear()
diff --git a/openhands/agenthub/visualbrowsing_agent/visualbrowsing_agent.py b/openhands/agenthub/visualbrowsing_agent/visualbrowsing_agent.py
index 3cd5a6fa3d..7ae484f018 100644
--- a/openhands/agenthub/visualbrowsing_agent/visualbrowsing_agent.py
+++ b/openhands/agenthub/visualbrowsing_agent/visualbrowsing_agent.py
@@ -176,8 +176,7 @@ Note:
self.reset()
def reset(self) -> None:
- """Resets the VisualBrowsingAgent's internal state.
- """
+ """Resets the VisualBrowsingAgent's internal state."""
super().reset()
# Reset agent-specific counters but not LLM metrics
self.error_accumulator = 0
diff --git a/openhands/controller/state/control_flags.py b/openhands/controller/state/control_flags.py
index 902aef7750..c41cbb927a 100644
--- a/openhands/controller/state/control_flags.py
+++ b/openhands/controller/state/control_flags.py
@@ -8,7 +8,6 @@ T = TypeVar(
) # Type for the value (int for iterations, float for budget)
-
@dataclass
class ControlFlag(Generic[T]):
"""Base class for control flags that manage limits and state transitions."""
@@ -31,7 +30,6 @@ class ControlFlag(Generic[T]):
"""Expand the limit when needed."""
raise NotImplementedError
-
def step(self):
"""Determine the next state based on the current state and mode.
@@ -56,7 +54,6 @@ class IterationControlFlag(ControlFlag[int]):
self.max_value += self.limit_increase_amount
self._hit_limit = False
-
def step(self):
if self.reached_limit():
raise RuntimeError(
@@ -68,9 +65,6 @@ class IterationControlFlag(ControlFlag[int]):
self.current_value += 1
-
-
-
@dataclass
class BudgetControlFlag(ControlFlag[float]):
"""Control flag for managing budget limits."""
@@ -99,6 +93,3 @@ class BudgetControlFlag(ControlFlag[float]):
f'Agent reached maximum budget for conversation.'
f'Current budget: {current_str}, max budget: {max_str}'
)
-
-
-
diff --git a/openhands/controller/state/state_tracker.py b/openhands/controller/state/state_tracker.py
index 13dd838ad0..c16d688480 100644
--- a/openhands/controller/state/state_tracker.py
+++ b/openhands/controller/state/state_tracker.py
@@ -1,5 +1,8 @@
from openhands.controller.agent import Agent
-from openhands.controller.state.control_flags import BudgetControlFlag, IterationControlFlag
+from openhands.controller.state.control_flags import (
+ BudgetControlFlag,
+ IterationControlFlag,
+)
from openhands.controller.state.state import State
from openhands.core.logger import openhands_logger as logger
from openhands.events.action.agent import AgentDelegateAction, ChangeAgentStateAction
@@ -71,9 +74,19 @@ class StateTracker:
self.state = State(
session_id=id.removesuffix('-delegate'),
inputs={},
- iteration_flag=IterationControlFlag(limit_increase_amount=max_iterations, current_value=0, max_value= max_iterations),
- budget_flag=None if not max_budget_per_task else BudgetControlFlag(limit_increase_amount=max_budget_per_task, current_value=0, max_value=max_budget_per_task),
- confirmation_mode=confirmation_mode
+ iteration_flag=IterationControlFlag(
+ limit_increase_amount=max_iterations,
+ current_value=0,
+ max_value=max_iterations,
+ ),
+ budget_flag=None
+ if not max_budget_per_task
+ else BudgetControlFlag(
+ limit_increase_amount=max_budget_per_task,
+ current_value=0,
+ max_value=max_budget_per_task,
+ ),
+ confirmation_mode=confirmation_mode,
)
self.state.start_id = 0
@@ -89,7 +102,6 @@ class StateTracker:
f'AgentController {id} initializing history from event {self.state.start_id}',
)
-
# Share the state metrics with the agent's LLM metrics
# This ensures that all accumulated metrics are always in sync between controller and llm
agent.llm.metrics = self.state.metrics
@@ -221,9 +233,7 @@ class StateTracker:
for event in self.state.history
]
- def maybe_increase_control_flags_limits(
- self, headless_mode: bool
- ):
+ def maybe_increase_control_flags_limits(self, headless_mode: bool):
# Iteration and budget extensions are independent of each other
# An error will be thrown if any one of the control flags have reached or exceeded its limit
self.state.iteration_flag.increase_limit(headless_mode)
@@ -247,7 +257,6 @@ class StateTracker:
if self.sid and self.file_store:
self.state.save_to_session(self.sid, self.file_store, self.user_id)
-
def run_control_flags(self):
"""
Performs one step of the control flags
@@ -256,25 +265,24 @@ class StateTracker:
if self.state.budget_flag:
self.state.budget_flag.step()
-
def sync_budget_flag_with_metrics(self):
"""
- Ensures that budget flag is up to date with accumulated costs from llm completions
- Budget flag will monitor for when budget is exceeded
+ Ensures that budget flag is up to date with accumulated costs from llm completions
+ Budget flag will monitor for when budget is exceeded
"""
if self.state.budget_flag:
self.state.budget_flag.current_value = self.state.metrics.accumulated_cost
def merge_metrics(self, metrics: Metrics):
"""
- Merges metrics with the state metrics
+ Merges metrics with the state metrics
- NOTE: this should be refactored in the future. We should have services (draft llm, title autocomplete, condenser, etc)
- use their own LLMs, but the metrics object should be shared. This way we have one source of truth for accumulated costs from
- all services
+ NOTE: this should be refactored in the future. We should have services (draft llm, title autocomplete, condenser, etc)
+ use their own LLMs, but the metrics object should be shared. This way we have one source of truth for accumulated costs from
+ all services
- This would prevent having fragmented stores for metrics, and we don't have the burden of deciding where and how to store them
- if we decide introduce more specialized services that require llm completions
+ This would prevent having fragmented stores for metrics, and we don't have the burden of deciding where and how to store them
+ if we decide introduce more specialized services that require llm completions
"""
self.state.metrics.merge(metrics)