Skip to content

🙌 OpenHands: AI-Driven Development

feat(frontend): Add critic result types, component, and event rendering (#14133)

Xingyao Wang0c2e3dc0eb62
diff --git a/frontend/__tests__/components/features/settings/sdk-settings/sdk-section-page.test.tsx b/frontend/__tests__/components/features/settings/sdk-settings/sdk-section-page.test.tsx
index 6acc8a6920..fd2c5ad7f0 100644
--- a/frontend/__tests__/components/features/settings/sdk-settings/sdk-section-page.test.tsx
+++ b/frontend/__tests__/components/features/settings/sdk-settings/sdk-section-page.test.tsx
@@ -154,7 +154,7 @@ describe("SdkSectionPage", () => {
);
renderSdkSectionPage({
- sectionKeys: ["llm"],
+ settingsSources: [{ settingsSource: "agent_settings", sectionKeys: ["llm"] }],
getInitialView: () => "advanced",
});
@@ -222,7 +222,7 @@ describe("SdkSectionPage", () => {
return (
<SdkSectionPage
- sectionKeys={["llm"]}
+ settingsSources={[{ settingsSource: "agent_settings", sectionKeys: ["llm"] }]}
header={() => (
<input
data-testid="external-state-input"
@@ -321,7 +321,7 @@ describe("SdkSectionPage", () => {
return true;
});
- renderSdkSectionPage({ sectionKeys: ["llm"] });
+ renderSdkSectionPage({ settingsSources: [{ settingsSource: "agent_settings", sectionKeys: ["llm"] }] });
await screen.findByTestId("sdk-section-advanced-toggle");
await userEvent.click(screen.getByTestId("sdk-section-advanced-toggle"));
@@ -413,7 +413,7 @@ describe("SdkSectionPage", () => {
return true;
});
- renderSdkSectionPage({ sectionKeys: ["llm"] });
+ renderSdkSectionPage({ settingsSources: [{ settingsSource: "agent_settings", sectionKeys: ["llm"] }] });
await screen.findByTestId("sdk-section-all-toggle");
await userEvent.click(screen.getByTestId("sdk-section-all-toggle"));
@@ -439,7 +439,7 @@ describe("SdkSectionPage", () => {
vi.spyOn(SettingsService, "getSettings").mockResolvedValue(buildSavableSettings());
renderSdkSectionPage({
- sectionKeys: ["llm"],
+ settingsSources: [{ settingsSource: "agent_settings", sectionKeys: ["llm"] }],
forceShowAdvancedView: true,
});
@@ -498,7 +498,7 @@ describe("SdkSectionPage", () => {
}),
);
- renderSdkSectionPage({ sectionKeys: ["condenser"] });
+ renderSdkSectionPage({ settingsSources: [{ settingsSource: "agent_settings", sectionKeys: ["condenser"] }] });
await screen.findByTestId("sdk-section-basic-toggle");
expect(
@@ -550,14 +550,16 @@ describe("SdkSectionPage", () => {
buildSettings({
agent_settings_schema: schema,
agent_settings: {
- "verification.critic_enabled": true,
- "verification.critic_server_url": "https://critic.example.com",
+ verification: {
+ critic_enabled: true,
+ critic_server_url: "https://critic.example.com",
+ },
},
}),
);
renderSdkSectionPage({
- sectionKeys: ["verification"],
+ settingsSources: [{ settingsSource: "agent_settings", sectionKeys: ["verification"] }],
getInitialView: () => "all",
});
@@ -566,6 +568,79 @@ describe("SdkSectionPage", () => {
).toHaveAttribute("type", "url");
});
+ it("shows iterative refinement in the basic view once critic is enabled", async () => {
+ const schema: NonNullable<Settings["agent_settings_schema"]> = {
+ model_name: "AgentSettings",
+ sections: [
+ {
+ key: "verification",
+ label: "Verification",
+ fields: [
+ {
+ key: "verification.critic_enabled",
+ label: "Enable critic",
+ section: "verification",
+ section_label: "Verification",
+ value_type: "boolean",
+ default: true,
+ choices: [],
+ depends_on: [],
+ prominence: "critical",
+ secret: false,
+ required: true,
+ },
+ {
+ key: "verification.enable_iterative_refinement",
+ label: "Enable iterative refinement",
+ section: "verification",
+ section_label: "Verification",
+ value_type: "boolean",
+ default: false,
+ choices: [],
+ depends_on: ["verification.critic_enabled"],
+ prominence: "critical",
+ secret: false,
+ required: false,
+ },
+ ],
+ },
+ ],
+ };
+
+ vi.spyOn(SettingsService, "getSettings").mockResolvedValue(
+ buildSettings({
+ agent_settings_schema: schema,
+ agent_settings: {
+ verification: {
+ critic_enabled: false,
+ enable_iterative_refinement: false,
+ },
+ },
+ }),
+ );
+
+ renderSdkSectionPage({ settingsSources: [{ settingsSource: "agent_settings", sectionKeys: ["verification"] }] });
+
+ expect(
+ await screen.findByTestId("sdk-settings-verification.critic_enabled"),
+ ).toBeInTheDocument();
+ expect(
+ screen.queryByTestId(
+ "sdk-settings-verification.enable_iterative_refinement",
+ ),
+ ).not.toBeInTheDocument();
+
+ await userEvent.click(
+ screen.getByTestId("sdk-settings-verification.critic_enabled"),
+ );
+
+ expect(
+ await screen.findByTestId(
+ "sdk-settings-verification.enable_iterative_refinement",
+ ),
+ ).toBeInTheDocument();
+ });
+
it("shows a success toast after saving settings", async () => {
vi.spyOn(SettingsService, "getSettings").mockResolvedValue(
buildSavableSettings(),
@@ -576,7 +651,7 @@ describe("SdkSectionPage", () => {
"displaySuccessToast",
);
- renderSdkSectionPage({ sectionKeys: ["llm"] });
+ renderSdkSectionPage({ settingsSources: [{ settingsSource: "agent_settings", sectionKeys: ["llm"] }] });
const endpointInput = await screen.findByTestId(
"sdk-settings-llm.endpoint",
@@ -599,7 +674,7 @@ describe("SdkSectionPage", () => {
);
const displayErrorToastSpy = vi.spyOn(ToastHandlers, "displayErrorToast");
- renderSdkSectionPage({ sectionKeys: ["llm"] });
+ renderSdkSectionPage({ settingsSources: [{ settingsSource: "agent_settings", sectionKeys: ["llm"] }] });
const endpointInput = await screen.findByTestId(
"sdk-settings-llm.endpoint",
@@ -619,7 +694,7 @@ describe("SdkSectionPage", () => {
);
renderSdkSectionPage({
- sectionKeys: ["llm"],
+ settingsSources: [{ settingsSource: "agent_settings", sectionKeys: ["llm"] }],
trailingActions: (
<button type="button" data-testid="trailing-action">
Extra
@@ -642,7 +717,7 @@ describe("SdkSectionPage", () => {
.mockResolvedValue(true);
renderSdkSectionPage({
- sectionKeys: ["llm"],
+ settingsSources: [{ settingsSource: "agent_settings", sectionKeys: ["llm"] }],
extraDirty: true,
buildPayload: (payload) => ({
...payload,
diff --git a/frontend/__tests__/components/v1/chat/event-message-components/critic-result-display.test.tsx b/frontend/__tests__/components/v1/chat/event-message-components/critic-result-display.test.tsx
new file mode 100644
index 0000000000..f1e6e5c278
--- /dev/null
+++ b/frontend/__tests__/components/v1/chat/event-message-components/critic-result-display.test.tsx
@@ -0,0 +1,262 @@
+import { screen } from "@testing-library/react";
+import userEvent from "@testing-library/user-event";
+import { beforeEach, describe, it, expect, vi } from "vitest";
+import { renderWithProviders } from "test-utils";
+import { CriticResultDisplay } from "#/components/v1/chat/event-message-components/critic-result-display";
+import type { CriticResult } from "#/types/v1/core/base/critic";
+
+const mockUseSettings = vi.hoisted(() =>
+ vi.fn(() => ({
+ data: {
+ agent_settings: {
+ verification: {
+ enable_iterative_refinement: true,
+ },
+ },
+ },
+ })),
+);
+
+vi.mock("#/hooks/query/use-settings", () => ({
+ useSettings: () => mockUseSettings(),
+}));
+
+const makeCriticResult = (
+ overrides: Partial<CriticResult> = {},
+): CriticResult => ({
+ score: 0.85,
+ message: null,
+ metadata: null,
+ ...overrides,
+});
+
+beforeEach(() => {
+ mockUseSettings.mockReturnValue({
+ data: {
+ agent_settings: {
+ verification: {
+ enable_iterative_refinement: true,
+ },
+ },
+ },
+ });
+});
+
+describe("CriticResultDisplay", () => {
+ it("renders score as percentage", () => {
+ renderWithProviders(
+ <CriticResultDisplay criticResult={makeCriticResult({ score: 0.72 })} />,
+ );
+
+ expect(screen.getByText("(72.0%)")).toBeInTheDocument();
+ });
+
+ it("renders 5 stars for a perfect score", () => {
+ renderWithProviders(
+ <CriticResultDisplay criticResult={makeCriticResult({ score: 1.0 })} />,
+ );
+
+ expect(screen.getByText("★★★★★")).toBeInTheDocument();
+ });
+
+ it("renders 0 stars for a zero score", () => {
+ renderWithProviders(
+ <CriticResultDisplay criticResult={makeCriticResult({ score: 0 })} />,
+ );
+
+ expect(screen.getByText("☆☆☆☆☆")).toBeInTheDocument();
+ });
+
+ it("renders green color for high score", () => {
+ renderWithProviders(
+ <CriticResultDisplay criticResult={makeCriticResult({ score: 0.8 })} />,
+ );
+
+ const stars = screen.getByText("★★★★☆");
+ expect(stars.className).toContain("text-green-400");
+ });
+
+ it("renders yellow color for medium score", () => {
+ renderWithProviders(
+ <CriticResultDisplay criticResult={makeCriticResult({ score: 0.5 })} />,
+ );
+
+ const stars = screen.getByText("★★★☆☆");
+ expect(stars.className).toContain("text-yellow-400");
+ });
+
+ it("renders red color for low score", () => {
+ renderWithProviders(
+ <CriticResultDisplay criticResult={makeCriticResult({ score: 0.2 })} />,
+ );
+
+ const stars = screen.getByText("★☆☆☆☆");
+ expect(stars.className).toContain("text-red-400");
+ });
+
+ it("renders label text", () => {
+ renderWithProviders(
+ <CriticResultDisplay criticResult={makeCriticResult()} />,
+ );
+
+ expect(
+ screen.getByText("CRITIC$SUCCESS_LIKELIHOOD_LABEL"),
+ ).toBeInTheDocument();
+ });
+
+ it("does not render expand button without features", () => {
+ renderWithProviders(
+ <CriticResultDisplay criticResult={makeCriticResult()} />,
+ );
+
+ expect(
+ screen.queryByLabelText("Expand details"),
+ ).not.toBeInTheDocument();
+ });
+
+ it("prompts users to enable iterative refinement when it is disabled", () => {
+ mockUseSettings.mockReturnValue({
+ data: {
+ agent_settings: {
+ verification: {
+ enable_iterative_refinement: false,
+ },
+ },
+ },
+ });
+
+ renderWithProviders(
+ <CriticResultDisplay criticResult={makeCriticResult()} />,
+ );
+
+ expect(
+ screen.getByTestId("critic-iterative-refinement-hint"),
+ ).toHaveTextContent("CRITIC$ITERATIVE_REFINEMENT_HINT");
+ });
+
+ it("does not prompt users when iterative refinement is enabled", () => {
+ renderWithProviders(
+ <CriticResultDisplay criticResult={makeCriticResult()} />,
+ );
+
+ expect(
+ screen.queryByTestId("critic-iterative-refinement-hint"),
+ ).not.toBeInTheDocument();
+ });
+
+ it("renders expand button when features are present", () => {
+ const result = makeCriticResult({
+ metadata: {
+ categorized_features: {
+ agent_behavioral_issues: [
+ {
+ name: "insufficient_testing",
+ display_name: "Insufficient Testing",
+ probability: 0.75,
+ },
+ ],
+ },
+ },
+ });
+
+ renderWithProviders(<CriticResultDisplay criticResult={result} />);
+
+ expect(screen.getByLabelText("Expand details")).toBeInTheDocument();
+ });
+
+ it("expands features on click", async () => {
+ const user = userEvent.setup();
+ const result = makeCriticResult({
+ metadata: {
+ categorized_features: {
+ agent_behavioral_issues: [
+ {
+ name: "insufficient_testing",
+ display_name: "Insufficient Testing",
+ probability: 0.75,
+ },
+ ],
+ },
+ },
+ });
+
+ renderWithProviders(<CriticResultDisplay criticResult={result} />);
+
+ expect(
+ screen.queryByText("Insufficient Testing"),
+ ).not.toBeInTheDocument();
+
+ await user.click(screen.getByLabelText("Expand details"));
+
+ expect(screen.getByText("Insufficient Testing")).toBeInTheDocument();
+ expect(screen.getByText("(75%)")).toBeInTheDocument();
+ expect(screen.getByText("CRITIC$POTENTIAL_ISSUES")).toBeInTheDocument();
+ });
+
+ it("collapses features on second click", async () => {
+ const user = userEvent.setup();
+ const result = makeCriticResult({
+ metadata: {
+ categorized_features: {
+ agent_behavioral_issues: [
+ {
+ name: "loop_behavior",
+ display_name: "Loop Behavior",
+ probability: 0.6,
+ },
+ ],
+ },
+ },
+ });
+
+ renderWithProviders(<CriticResultDisplay criticResult={result} />);
+
+ await user.click(screen.getByLabelText("Expand details"));
+ expect(screen.getByText("Loop Behavior")).toBeInTheDocument();
+
+ await user.click(screen.getByLabelText("Collapse details"));
+ expect(screen.queryByText("Loop Behavior")).not.toBeInTheDocument();
+ });
+
+ it("renders multiple categories of features", async () => {
+ const user = userEvent.setup();
+ const result = makeCriticResult({
+ metadata: {
+ categorized_features: {
+ agent_behavioral_issues: [
+ {
+ name: "incomplete_changes",
+ display_name: "Incomplete Changes",
+ probability: 0.8,
+ },
+ ],
+ infrastructure_issues: [
+ {
+ name: "build_failure",
+ display_name: "Build Failure",
+ probability: 0.4,
+ },
+ ],
+ user_followup_patterns: [
+ {
+ name: "will_ask_refinement",
+ display_name: "Will Ask Refinement",
+ probability: 0.55,
+ },
+ ],
+ },
+ },
+ });
+
+ renderWithProviders(<CriticResultDisplay criticResult={result} />);
+
+ await user.click(screen.getByLabelText("Expand details"));
+
+ expect(screen.getByText("CRITIC$POTENTIAL_ISSUES")).toBeInTheDocument();
+ expect(screen.getByText("Incomplete Changes")).toBeInTheDocument();
+ expect(screen.getByText("CRITIC$INFRASTRUCTURE")).toBeInTheDocument();
+ expect(screen.getByText("Build Failure")).toBeInTheDocument();
+ expect(screen.getByText("CRITIC$LIKELY_FOLLOWUP")).toBeInTheDocument();
+ expect(screen.getByText("Will Ask Refinement")).toBeInTheDocument();
+ });
+});
diff --git a/frontend/__tests__/routes/verification-settings.test.tsx b/frontend/__tests__/routes/verification-settings.test.tsx
index 4067356007..992c99acbb 100644
--- a/frontend/__tests__/routes/verification-settings.test.tsx
+++ b/frontend/__tests__/routes/verification-settings.test.tsx
@@ -9,14 +9,31 @@ import VerificationSettingsScreen, {
} from "#/routes/verification-settings";
import { Settings } from "#/types/settings";
+const mockUseConfig = vi.fn();
+
+vi.mock("#/hooks/query/use-config", () => ({
+ useConfig: () => mockUseConfig(),
+}));
+
+vi.mock("#/hooks/query/use-is-authed", () => ({
+ useIsAuthed: () => ({ data: true }),
+}));
+
function buildSettings(overrides: Partial<Settings> = {}): Settings {
return {
...MOCK_DEFAULT_USER_SETTINGS,
...overrides,
+ agent_settings: {
+ ...MOCK_DEFAULT_USER_SETTINGS.agent_settings,
+ ...overrides.agent_settings,
+ },
conversation_settings: {
...MOCK_DEFAULT_USER_SETTINGS.conversation_settings,
...overrides.conversation_settings,
},
+ agent_settings_schema:
+ overrides.agent_settings_schema ??
+ MOCK_DEFAULT_USER_SETTINGS.agent_settings_schema,
conversation_settings_schema:
overrides.conversation_settings_schema ??
MOCK_DEFAULT_USER_SETTINGS.conversation_settings_schema,
@@ -39,25 +56,51 @@ function renderVerificationSettingsScreen() {
beforeEach(() => {
vi.restoreAllMocks();
+ mockUseConfig.mockReturnValue({
+ data: { app_mode: "oss" },
+ isLoading: false,
+ });
});
describe("VerificationSettingsScreen", () => {
- it("keeps confirmation mode visible in the basic view", async () => {
- vi.spyOn(SettingsService, "getSettings").mockResolvedValue(buildSettings());
+ it("renders critic controls alongside existing confirmation settings", async () => {
+ vi.spyOn(SettingsService, "getSettings").mockResolvedValue(
+ buildSettings({
+ agent_settings: {
+ ...MOCK_DEFAULT_USER_SETTINGS.agent_settings,
+ verification: {
+ critic_enabled: true,
+ enable_iterative_refinement: false,
+ },
+ },
+ }),
+ );
renderVerificationSettingsScreen();
await screen.findByTestId("verification-settings-screen");
+ expect(
+ screen.getByTestId("sdk-settings-verification.critic_enabled"),
+ ).toBeInTheDocument();
+ expect(
+ screen.getByTestId(
+ "sdk-settings-verification.enable_iterative_refinement",
+ ),
+ ).toBeInTheDocument();
expect(
screen.getByTestId("sdk-settings-confirmation_mode"),
).toBeInTheDocument();
+ expect(
+ screen.queryByTestId("sdk-settings-verification.critic_mode"),
+ ).not.toBeInTheDocument();
+ // Security analyzer is hidden when confirmation mode is off
expect(
screen.queryByTestId("sdk-settings-security_analyzer"),
).not.toBeInTheDocument();
});
- it("shows the security analyzer only in advanced view", async () => {
+ it("keeps security analyzer in the existing advanced view", async () => {
vi.spyOn(SettingsService, "getSettings").mockResolvedValue(
buildSettings({
conversation_settings: {
@@ -65,6 +108,13 @@ describe("VerificationSettingsScreen", () => {
confirmation_mode: true,
security_analyzer: "llm",
},
+ agent_settings: {
+ ...MOCK_DEFAULT_USER_SETTINGS.agent_settings,
+ verification: {
+ critic_enabled: true,
+ enable_iterative_refinement: false,
+ },
+ },
}),
);
@@ -81,8 +131,10 @@ describe("VerificationSettingsScreen", () => {
expect(
await screen.findByTestId("sdk-settings-security_analyzer"),
).toBeInTheDocument();
+ expect(
+ await screen.findByTestId("sdk-settings-verification.critic_mode"),
+ ).toBeInTheDocument();
});
-
});
describe("clientLoader permission checks", () => {
diff --git a/frontend/__tests__/utils/sdk-settings-schema.test.ts b/frontend/__tests__/utils/sdk-settings-schema.test.ts
index 7e49677c48..09f931a036 100644
--- a/frontend/__tests__/utils/sdk-settings-schema.test.ts
+++ b/frontend/__tests__/utils/sdk-settings-schema.test.ts
@@ -76,16 +76,16 @@ const BASE_SETTINGS: Settings = {
],
},
{
- key: "critic",
- label: "Critic",
+ key: "verification",
+ label: "Verification",
fields: [
{
- key: "critic.enabled",
+ key: "verification.critic_enabled",
label: "Enable critic",
- section: "critic",
- section_label: "Critic",
+ section: "verification",
+ section_label: "Verification",
value_type: "boolean",
- default: false,
+ default: true,
choices: [],
depends_on: [],
prominence: "critical",
@@ -93,17 +93,17 @@ const BASE_SETTINGS: Settings = {
required: true,
},
{
- key: "critic.mode",
+ key: "verification.critic_mode",
label: "Mode",
- section: "critic",
- section_label: "Critic",
+ section: "verification",
+ section_label: "Verification",
value_type: "string",
default: "finish_and_message",
choices: [
{ label: "finish_and_message", value: "finish_and_message" },
{ label: "all_actions", value: "all_actions" },
],
- depends_on: ["critic.enabled"],
+ depends_on: ["verification.critic_enabled"],
prominence: "minor",
secret: false,
required: true,
@@ -133,15 +133,13 @@ const BASE_SETTINGS: Settings = {
},
agent_settings: {
agent: "CodeActAgent",
- critic: {
- mode: "finish_and_message",
- enabled: false,
- },
llm: {
api_key: null,
model: "openai/gpt-4o",
},
verification: {
+ critic_enabled: false,
+ critic_mode: "finish_and_message",
confirmation_mode: false,
},
condenser: {
@@ -154,13 +152,13 @@ const BASE_SETTINGS: Settings = {
describe("sdk settings schema helpers", () => {
it("builds initial form values from the current settings", () => {
expect(buildInitialSettingsFormValues(BASE_SETTINGS)).toEqual({
- "critic.mode": "finish_and_message",
- "critic.enabled": false,
"llm.api_key": "",
"llm.base_url": "",
"llm.litellm_extra_body": "{}",
"llm.model": "openai/gpt-4o",
mcp_config: "",
+ "verification.critic_enabled": false,
+ "verification.critic_mode": "finish_and_message",
});
});
@@ -172,10 +170,10 @@ describe("sdk settings schema helpers", () => {
...BASE_SETTINGS,
agent_settings: {
...BASE_SETTINGS.agent_settings,
- critic: {
+ verification: {
...(BASE_SETTINGS.agent_settings as Record<string, unknown>)
- .critic as Record<string, unknown>,
- mode: "all_actions",
+ .verification as Record<string, unknown>,
+ critic_mode: "all_actions",
},
},
};
@@ -213,11 +211,13 @@ describe("sdk settings schema helpers", () => {
const allSections = getVisibleSettingsSections(
BASE_SETTINGS.agent_settings_schema!,
- { ...values, "critic.enabled": true },
+ { ...values, "verification.critic_enabled": true },
"all",
);
- const criticSection = allSections.find((s) => s.key === "critic");
- expect(criticSection?.fields).toHaveLength(2);
+ const verificationSection = allSections.find(
+ (s) => s.key === "verification",
+ );
+ expect(verificationSection?.fields).toHaveLength(2);
});
it("passes through all fields when excludeKeys is empty", () => {
@@ -238,7 +238,7 @@ describe("sdk settings schema helpers", () => {
BASE_SETTINGS.agent_settings_schema!,
{
...buildInitialSettingsFormValues(BASE_SETTINGS),
- "critic.enabled": true,
+ "verification.critic_enabled": true,
"llm.api_key": "new-key",
"llm.litellm_extra_body": JSON.stringify(
{ metadata: { tier: "enterprise" } },
@@ -247,7 +247,7 @@ describe("sdk settings schema helpers", () => {
),
},
{
- "critic.enabled": true,
+ "verification.critic_enabled": true,
"llm.api_key": true,
"llm.litellm_extra_body": true,
"llm.model": false,
@@ -255,11 +255,11 @@ describe("sdk settings schema helpers", () => {
);
expect(payload).toEqual({
- critic: { enabled: true },
llm: {
api_key: "new-key",
litellm_extra_body: { metadata: { tier: "enterprise" } },
},
+ verification: { critic_enabled: true },
});
});
@@ -286,8 +286,8 @@ describe("sdk settings schema helpers", () => {
}),
"llm.model": "anthropic/claude-sonnet-4-20250514",
"llm.timeout": "90",
- "critic.enabled": true,
- "critic.mode": "all_actions",
+ "verification.critic_enabled": true,
+ "verification.critic_mode": "all_actions",
"llm.litellm_extra_body": JSON.stringify(
{ metadata: { tier: "enterprise" } },
null,
@@ -298,8 +298,8 @@ describe("sdk settings schema helpers", () => {
const dirty = {
"llm.model": true,
"llm.timeout": true,
- "critic.enabled": true,
- "critic.mode": true,
+ "verification.critic_enabled": true,
+ "verification.critic_mode": true,
"llm.litellm_extra_body": true,
};
@@ -309,8 +309,8 @@ describe("sdk settings schema helpers", () => {
timeout: 30,
litellm_extra_body: {},
},
- critic: { enabled: true, mode: "finish_and_message" },
mcp_config: null,
+ verification: { critic_enabled: true, critic_mode: "finish_and_message" },
});
expect(
@@ -321,8 +321,8 @@ describe("sdk settings schema helpers", () => {
timeout: 90,
litellm_extra_body: {},
},
- critic: { enabled: true, mode: "finish_and_message" },
mcp_config: null,
+ verification: { critic_enabled: true, critic_mode: "finish_and_message" },
});
expect(buildSdkSettingsPayloadForView(schema, values, dirty, "all")).toEqual({
@@ -331,7 +331,7 @@ describe("sdk settings schema helpers", () => {
timeout: 90,
litellm_extra_body: { metadata: { tier: "enterprise" } },
},
- critic: { enabled: true, mode: "all_actions" },
+ verification: { critic_enabled: true, critic_mode: "all_actions" },
});
});
});
diff --git a/frontend/src/components/features/settings/sdk-settings/sdk-section-page.tsx b/frontend/src/components/features/settings/sdk-settings/sdk-section-page.tsx
index 03cce8630f..cbcc707c46 100644
--- a/frontend/src/components/features/settings/sdk-settings/sdk-section-page.tsx
+++ b/frontend/src/components/features/settings/sdk-settings/sdk-section-page.tsx
@@ -49,6 +49,12 @@ const getLessDetailedView = (
): SettingsView =>
VIEW_ORDER[nextView] < VIEW_ORDER[currentView] ? nextView : currentView;
+const getMoreDetailedView = (
+ currentView: SettingsView,
+ nextView: SettingsView,
+): SettingsView =>
+ VIEW_ORDER[nextView] > VIEW_ORDER[currentView] ? nextView : currentView;
+
const normalizeView = (
view: SettingsView,
{
@@ -78,6 +84,20 @@ const normalizeView = (
return "basic";
};
+const PAYLOAD_DIFF_KEY: Record<SettingsValueSource, string> = {
+ agent_settings: "agent_settings_diff",
+ conversation_settings: "conversation_settings_diff",
+};
+
+export interface SettingsSourceConfig {
+ /** Which schema/values bucket on `settings` this source pulls from. */
+ settingsSource: SettingsValueSource;
+ /** Section keys (e.g. ["llm"]) within that schema to render. */
+ sectionKeys: string[];
+ /** Field keys to skip (rendered elsewhere by the caller). */
+ excludeKeys?: Set<string>;
+}
+
export interface SdkSectionHeaderProps {
values: SettingsFormValues;
isDisabled: boolean;
@@ -85,20 +105,29 @@ export interface SdkSectionHeaderProps {
onChange: (key: string, value: string | boolean) => void;
}
+interface ResolvedSource extends SettingsSourceConfig {
+ filteredSchema: SettingsSchema | null;
+}
+
/**
- * A generic SDK-schema–driven settings page that renders fields
- * from one or more schema sections.
+ * A generic SDK-schema–driven settings page that renders fields from one or
+ * more schema sections.
*
- * @param sectionKeys - which schema section(s) this page owns (e.g. ["condenser"])
- * @param excludeKeys - field keys to skip (rendered elsewhere by the caller)
- * @param header - optional render prop receiving shared state to render above fields
- * @param testId - data-testid for the page wrapper
+ * The `settingsSources` array specifies which schema(s)/section(s) the page
+ * owns. The page tracks values/dirty state per source, renders sections from
+ * each source in order (filtered by the schema's `prominence` field for the
+ * selected view), and emits a combined save payload like
+ * `{ conversation_settings_diff: {...}, agent_settings_diff: {...} }` —
+ * including only the keys for sources that actually have dirty changes.
+ *
+ * @param settingsSources one or more schemas to render fields from
+ * @param header render prop above the fields (receives unified state)
+ * @param buildPayload customize the save payload before submission
+ * @param testId data-testid on the page wrapper
*/
export function SdkSectionPage({
- sectionKeys,
- excludeKeys = EMPTY_EXCLUDE_KEYS,
+ settingsSources,
scope = "personal",
- settingsSource = "agent_settings",
header,
extraDirty = false,
buildPayload,
@@ -109,15 +138,18 @@ export function SdkSectionPage({
trailingActions,
testId = "sdk-section-settings-screen",
}: {
- sectionKeys: string[];
- excludeKeys?: Set<string>;
+ settingsSources: SettingsSourceConfig[];
scope?: SettingsScope;
- settingsSource?: SettingsValueSource;
header?: (props: SdkSectionHeaderProps) => React.ReactNode;
extraDirty?: boolean;
+ /**
+ * Customize the save payload. Receives the wrapped default payload (e.g.
+ * `{ agent_settings_diff: { llm: { model: "gpt-4" } } }`) plus the unified
+ * form context. Return the payload to actually send.
+ */
buildPayload?: (
- payload: ReturnType<typeof buildSdkSettingsPayloadForView>,
+ defaultPayload: Record<string, unknown>,
context: {
values: SettingsFormValues;
dirty: SettingsDirtyState;
@@ -125,6 +157,10 @@ export function SdkSectionPage({
},
) => Record<string, unknown>;
onSaveSuccess?: () => void;
+ /**
+ * Override the initial view per source. Called once per source on
+ * hydration; the most-detailed result wins across sources.
+ */
getInitialView?: (
settings: Settings,
filteredSchema: SettingsSchema,
@@ -146,14 +182,6 @@ export function SdkSectionPage({
const conversationSchemaQuery = useConversationSettingsSchema(
settings?.conversation_settings_schema,
);
- const schema =
- settingsSource === "conversation_settings"
- ? conversationSchemaQuery.data
- : agentSchemaQuery.data;
- const isSchemaLoading =
- settingsSource === "conversation_settings"
- ? conversationSchemaQuery.isLoading
- : agentSchemaQuery.isLoading;
const { data: config } = useConfig();
const { data: me } = useMe();
const { hasPermission } = usePermission(me?.role ?? "member");
@@ -162,72 +190,130 @@ export function SdkSectionPage({
const isReadOnly =
scope === "org" && !isOssMode ? !hasPermission("edit_llm_settings") : false;
- const [view, setView] = React.useState<SettingsView>("basic");
- const [values, setValues] = React.useState<SettingsFormValues>({});
- const [dirty, setDirty] = React.useState<SettingsDirtyState>({});
- const hasHydratedViewRef = React.useRef(false);
+ // Route all downstream memos through a JSON signature so callers passing
+ // a fresh `settingsSources` array reference on every render don't
+ // invalidate component state (e.g. the selected view).
+ const sourcesSignature = React.useMemo(
+ () =>
+ JSON.stringify(
+ settingsSources.map((s) => ({
+ source: s.settingsSource,
+ sectionKeys: s.sectionKeys,
+ excludeKeys: s.excludeKeys ? Array.from(s.excludeKeys).sort() : null,
+ })),
+ ),
+ [settingsSources],
+ );
- const sectionKeysSignature = React.useMemo(
- () => JSON.stringify(sectionKeys),
- [sectionKeys],
+ // Stable list of source configs; reference only changes when the
+ // signature changes (i.e. semantic content has actually changed).
+ const resolvedSourceConfigs = React.useMemo<SettingsSourceConfig[]>(() => {
+ const parsed = JSON.parse(sourcesSignature) as Array<{
+ source: SettingsValueSource;
+ sectionKeys: string[];
+ excludeKeys: string[] | null;
+ }>;
+ return parsed.map((p) => ({
+ settingsSource: p.source,
+ sectionKeys: p.sectionKeys,
+ excludeKeys: p.excludeKeys ? new Set(p.excludeKeys) : undefined,
+ }));
+ }, [sourcesSignature]);
+
+ const getSchemaForSource = React.useCallback(
+ (source: SettingsValueSource) =>
+ source === "conversation_settings"
+ ? conversationSchemaQuery.data
+ : agentSchemaQuery.data,
+ [agentSchemaQuery.data, conversationSchemaQuery.data],
);
- const stableSectionKeys = React.useMemo(
- () => JSON.parse(sectionKeysSignature) as string[],
- [sectionKeysSignature],
+
+ // Are we waiting on any schema this page actually uses?
+ const isSchemaLoading = resolvedSourceConfigs.some((src) =>
+ src.settingsSource === "conversation_settings"
+ ? conversationSchemaQuery.isLoading
+ : agentSchemaQuery.isLoading,
);
- // Build a filtered schema containing only the requested sections
- const filteredSchema = React.useMemo(() => {
- if (!schema) return null;
- const sectionSet = new Set(stableSectionKeys);
- return {
- ...schema,
- sections: schema.sections.filter((s) => sectionSet.has(s.key)),
- };
- }, [schema, stableSectionKeys]);
+ // Build a per-source filtered schema (sections matching its sectionKeys).
+ const resolvedSources = React.useMemo<ResolvedSource[]>(
+ () =>
+ resolvedSourceConfigs.map((src) => {
+ const schema = getSchemaForSource(src.settingsSource);
+ if (!schema) {
+ return { ...src, filteredSchema: null };
+ }
+ const sectionSet = new Set(src.sectionKeys);
+ const filteredSchema: SettingsSchema = {
+ ...schema,
+ sections: schema.sections.filter((s) => sectionSet.has(s.key)),
+ };
+ return { ...src, filteredSchema };
+ }),
+ [resolvedSourceConfigs, getSchemaForSource],
+ );
const showAdvanced =
- forceShowAdvancedView || hasAdvancedSettings(filteredSchema);
- const showAll = allowAllView && hasMinorSettings(filteredSchema);
-
- const initialValues = React.useMemo(() => {
- if (!settings || !filteredSchema) return null;
- return buildInitialSettingsFormValues(
- settings,
- filteredSchema,
- settingsSource,
- );
- }, [settings, filteredSchema, settingsSource]);
+ forceShowAdvancedView ||
+ resolvedSources.some((src) => hasAdvancedSettings(src.filteredSchema));
+ const showAll =
+ allowAllView &&
+ resolvedSources.some((src) => hasMinorSettings(src.filteredSchema));
+
+ const [view, setView] = React.useState<SettingsView>("basic");
+ const [valuesBySource, setValuesBySource] = React.useState<
+ Partial<Record<SettingsValueSource, SettingsFormValues>>
+ >({});
+ const [dirtyBySource, setDirtyBySource] = React.useState<
+ Partial<Record<SettingsValueSource, SettingsDirtyState>>
+ >({});
+ const hasHydratedViewRef = React.useRef(false);
+
+ const initialValuesBySource = React.useMemo<Partial<
+ Record<SettingsValueSource, SettingsFormValues>
+ > | null>(() => {
+ if (!settings) return null;
+ const result: Partial<Record<SettingsValueSource, SettingsFormValues>> = {};
+ for (const src of resolvedSources) {
+ if (!src.filteredSchema) return null;
+ result[src.settingsSource] = {
+ ...(result[src.settingsSource] ?? {}),
+ ...buildInitialSettingsFormValues(
+ settings,
+ src.filteredSchema,
+ src.settingsSource,
+ ),
+ };
+ }
+ return result;
+ }, [settings, resolvedSources]);
const initialView = React.useMemo(() => {
- if (!settings || !filteredSchema) return null;
-
- const resolvedInitialView = getInitialView
- ? getInitialView(settings, filteredSchema)
- : inferInitialView(settings, filteredSchema, settingsSource);
-
- return normalizeView(resolvedInitialView, { showAdvanced, showAll });
- }, [
- settings,
- filteredSchema,
- getInitialView,
- settingsSource,
- showAdvanced,
- showAll,
- ]);
+ if (!settings) return null;
+ let result: SettingsView | null = null;
+ for (const src of resolvedSources) {
+ if (!src.filteredSchema) return null;
+ const perSource = getInitialView
+ ? getInitialView(settings, src.filteredSchema)
+ : inferInitialView(settings, src.filteredSchema, src.settingsSource);
+ result = result ? getMoreDetailedView(result, perSource) : perSource;
+ }
+ if (!result) return null;
+ return normalizeView(result, { showAdvanced, showAll });
+ }, [settings, resolvedSources, getInitialView, showAdvanced, showAll]);
React.useEffect(() => {
hasHydratedViewRef.current = false;
setView("basic");
- setValues({});
- setDirty({});
- }, [scope, settingsSource, sectionKeysSignature]);
+ setValuesBySource({});
+ setDirtyBySource({});
+ }, [scope, sourcesSignature]);
React.useEffect(() => {
- if (!initialValues || !initialView) return;
+ if (!initialValuesBySource || !initialView) return;
- setValues(initialValues);
- setDirty({});
+ setValuesBySource(initialValuesBySource);
+ setDirtyBySource({});
setView((currentView) => {
if (!hasHydratedViewRef.current) {
hasHydratedViewRef.current = true;
@@ -236,24 +322,64 @@ export function SdkSectionPage({
return getLessDetailedView(currentView, initialView);
});
- }, [initialValues, initialView]);
-
- const visibleSections = React.useMemo(() => {
- if (!filteredSchema) return [];
- return getVisibleSettingsSections(
- filteredSchema,
- values,
- view,
- excludeKeys,
- );
- }, [filteredSchema, values, view, excludeKeys]);
+ }, [initialValuesBySource, initialView]);
+
+ // Map from field key → source it belongs to. Used by the header callback
+ // to route generic onChange calls to the right source's bucket.
+ const fieldKeyToSource = React.useMemo(() => {
+ const map = new Map<string, SettingsValueSource>();
+ for (const src of resolvedSources) {
+ if (src.filteredSchema) {
+ for (const section of src.filteredSchema.sections) {
+ for (const field of section.fields) {
+ if (!map.has(field.key)) {
+ map.set(field.key, src.settingsSource);
+ }
+ }
+ }
+ }
+ }
+ return map;
+ }, [resolvedSources]);
+
+ // Unified views over per-source state, used for header callbacks and for
+ // dependency-resolution in `isSettingsFieldVisible`.
+ const flatValues = React.useMemo<SettingsFormValues>(() => {
+ const merged: SettingsFormValues = {};
+ for (const src of resolvedSources) {
+ Object.assign(merged, valuesBySource[src.settingsSource] ?? {});
+ }
+ return merged;
+ }, [resolvedSources, valuesBySource]);
+
+ const flatDirty = React.useMemo<SettingsDirtyState>(() => {
+ const merged: SettingsDirtyState = {};
+ for (const src of resolvedSources) {
+ Object.assign(merged, dirtyBySource[src.settingsSource] ?? {});
+ }
+ return merged;
+ }, [resolvedSources, dirtyBySource]);
const handleFieldChange = React.useCallback(
(fieldKey: string, nextValue: string | boolean) => {
- setValues((prev) => ({ ...prev, [fieldKey]: nextValue }));
- setDirty((prev) => ({ ...prev, [fieldKey]: true }));
+ const sourceKey = fieldKeyToSource.get(fieldKey);
+ if (!sourceKey) return;
+ setValuesBySource((prev) => ({
+ ...prev,
+ [sourceKey]: {
+ ...(prev[sourceKey] ?? {}),
+ [fieldKey]: nextValue,
+ },
+ }));
+ setDirtyBySource((prev) => ({
+ ...prev,
+ [sourceKey]: {
+ ...(prev[sourceKey] ?? {}),
+ [fieldKey]: true,
+ },
+ }));
},
- [],
+ [fieldKeyToSource],
);
const handleError = React.useCallback(
@@ -265,24 +391,39 @@ export function SdkSectionPage({
);
const handleSave = () => {
- if (!filteredSchema || isReadOnly) return;
+ if (isReadOnly) return;
+ if (resolvedSources.some((src) => !src.filteredSchema)) return;
let payload: Record<string, unknown>;
try {
- const basePayload = buildSdkSettingsPayloadForView(
- filteredSchema,
- values,
- dirty,
- view,
- );
- let defaultPayload: Record<string, unknown>;
- if (settingsSource === "conversation_settings") {
- defaultPayload = { conversation_settings_diff: basePayload };
- } else {
- defaultPayload = { agent_settings_diff: basePayload };
+ const defaultPayload: Record<string, unknown> = {};
+ for (const src of resolvedSources) {
+ const schema = src.filteredSchema!;
+ const sourceValues = valuesBySource[src.settingsSource] ?? {};
+ const sourceDirty = dirtyBySource[src.settingsSource] ?? {};
+ const diff = buildSdkSettingsPayloadForView(
+ schema,
+ sourceValues,
+ sourceDirty,
+ view,
+ );
+ if (Object.keys(diff).length > 0) {
+ const diffKey = PAYLOAD_DIFF_KEY[src.settingsSource];
+ defaultPayload[diffKey] = {
+ ...((defaultPayload[diffKey] as
+ | Record<string, unknown>
+ | undefined) ?? {}),
+ ...diff,
+ };
+ }
}
+
payload = buildPayload
- ? buildPayload(basePayload, { values, dirty, view })
+ ? buildPayload(defaultPayload, {
+ values: flatValues,
+ dirty: flatDirty,
+ view,
+ })
: defaultPayload;
} catch (error) {
displayErrorToast(
@@ -297,7 +438,7 @@ export function SdkSectionPage({
onError: handleError,
onSuccess: () => {
displaySuccessToast(t(I18nKey.SETTINGS$SAVED_WARNING));
- setDirty({});
+ setDirtyBySource({});
onSaveSuccess?.();
},
});
@@ -307,7 +448,11 @@ export function SdkSectionPage({
return <LlmSettingsInputsSkeleton />;
}
- if (!filteredSchema || filteredSchema.sections.length === 0) {
+ const hasAnyVisibleSection = resolvedSources.some(
+ (src) => src.filteredSchema && src.filteredSchema.sections.length > 0,
+ );
+
+ if (!hasAnyVisibleSection) {
return (
<Typography.Paragraph className="text-tertiary-alt">
{t(I18nKey.SETTINGS$SDK_SCHEMA_UNAVAILABLE)}
@@ -315,7 +460,11 @@ export function SdkSectionPage({
);
}
- if (Object.keys(values).length === 0) return <LlmSettingsInputsSkeleton />;
+ if (Object.keys(flatValues).length === 0) {
+ return <LlmSettingsInputsSkeleton />;
+ }
+
+ const isDirty = Object.keys(flatDirty).length > 0;
return (
<div data-testid={testId} className="h-full relative">
@@ -330,29 +479,45 @@ export function SdkSectionPage({
<div className="flex flex-col gap-8 pb-20">
{header?.({
- values,
+ values: flatValues,
isDisabled: isReadOnly,
view,
onChange: handleFieldChange,
})}
- {visibleSections.map((section) => (
- <section key={section.key} className="flex flex-col gap-4">
- <div className="grid gap-4 xl:grid-cols-2">
- {section.fields.map((field) => (
- <SchemaField
- key={field.key}
- field={field}
- value={values[field.key]}
- isDisabled={isReadOnly}
- onChange={(nextValue) =>
- handleFieldChange(field.key, nextValue)
- }
- />
- ))}
- </div>
- </section>
- ))}
+ {resolvedSources.map((src) => {
+ if (!src.filteredSchema) return null;
+ const sourceValues = valuesBySource[src.settingsSource] ?? {};
+ const visibleSections = getVisibleSettingsSections(
+ src.filteredSchema,
+ // Use flatValues for dependency resolution so cross-source
+ // `depends_on` works; in practice fields only depend on keys
+ // within the same source.
+ { ...flatValues, ...sourceValues },
+ view,
+ src.excludeKeys ?? EMPTY_EXCLUDE_KEYS,
+ );
+ return visibleSections.map((section) => (
+ <section
+ key={`${src.settingsSource}:${section.key}`}
+ className="flex flex-col gap-4"
+ >
+ <div className="grid gap-4 xl:grid-cols-2">
+ {section.fields.map((field) => (
+ <SchemaField
+ key={field.key}
+ field={field}
+ value={sourceValues[field.key]}
+ isDisabled={isReadOnly}
+ onChange={(nextValue) =>
+ handleFieldChange(field.key, nextValue)
+ }
+ />
+ ))}
+ </div>
+ </section>
+ ));
+ })}
</div>
{!isReadOnly ? (
@@ -361,9 +526,7 @@ export function SdkSectionPage({
testId="save-button"
type="button"
variant="primary"
- isDisabled={
- isPending || (Object.keys(dirty).length === 0 && !extraDirty)
- }
+ isDisabled={isPending || (!isDirty && !extraDirty)}
onClick={handleSave}
>
{isPending
diff --git a/frontend/src/components/v1/chat/event-message-components/critic-result-display.tsx b/frontend/src/components/v1/chat/event-message-components/critic-result-display.tsx
new file mode 100644
index 0000000000..4f71cf7558
--- /dev/null
+++ b/frontend/src/components/v1/chat/event-message-components/critic-result-display.tsx
@@ -0,0 +1,224 @@
+import React from "react";
+import { useTranslation } from "react-i18next";
+import ArrowDown from "#/icons/angle-down-solid.svg?react";
+import ArrowUp from "#/icons/angle-up-solid.svg?react";
+import { useSettings } from "#/hooks/query/use-settings";
+import { I18nKey } from "#/i18n/declaration";
+import type {
+ CriticResult,
+ CriticFeature,
+ CriticCategorizedFeatures,
+} from "#/types/v1/core/base/critic";
+
+/**
+ * Convert a score (0-1) to a 5-star rating string.
+ */
+function getStarRating(score: number): { filled: number; empty: number } {
+ const filled = Math.round(score * 5);
+ return { filled, empty: 5 - filled };
+}
+
+/**
+ * Get the color class for the star rating based on score.
+ */
+function getScoreColorClass(score: number): string {
+ if (score >= 0.6) return "text-green-400";
+ if (score >= 0.4) return "text-yellow-400";
+ return "text-red-400";
+}
+
+/**
+ * Get the color class for an issue probability.
+ */
+function getIssueColorClass(probability: number): string {
+ if (probability >= 0.7) return "text-red-400 font-semibold";
+ if (probability >= 0.5) return "text-yellow-400";
+ return "text-neutral-400";
+}
+
+function getIterativeRefinementEnabled(
+ agentSettings: Record<string, unknown> | null | undefined,
+): boolean | null {
+ const verification = agentSettings?.verification;
+ if (
+ verification == null ||
+ typeof verification !== "object" ||
+ Array.isArray(verification)
+ ) {
+ return null;
+ }
+
+ const value = (verification as Record<string, unknown>)
+ .enable_iterative_refinement;
+ return typeof value === "boolean" ? value : null;
+}
+
+/**
+ * Renders a single issue feature with its probability.
+ */
+function FeatureItem({ feature }: { feature: CriticFeature }) {
+ const percentage = Math.round(feature.probability * 100);
+ const colorClass = getIssueColorClass(feature.probability);
+
+ return (
+ <span className="inline-flex items-center gap-1">
+ <span className="text-neutral-200">{feature.display_name}</span>
+ <span className={colorClass}>({percentage}%)</span>
+ </span>
+ );
+}
+
+/**
+ * Renders a category of features (e.g., "Potential Issues", "Infrastructure").
+ */
+function FeatureCategory({
+ label,
+ features,
+}: {
+ label: string;
+ features: CriticFeature[];
+}) {
+ if (!features || features.length === 0) return null;
+
+ return (
+ <div className="flex flex-wrap items-center gap-x-1 text-xs">
+ <span className="font-semibold text-neutral-300">{label}</span>
+ {features.map((feature, i) => (
+ <React.Fragment key={feature.name}>
+ {i > 0 && <span className="text-neutral-500">·</span>}
+ <FeatureItem feature={feature} />
+ </React.Fragment>
+ ))}
+ </div>
+ );
+}
+
+/**
+ * Renders the categorized features breakdown.
+ */
+function FeaturesBreakdown({
+ categorized,
+}: {
+ categorized: CriticCategorizedFeatures;
+}) {
+ const { t } = useTranslation();
+
+ const hasAgentIssues =
+ categorized.agent_behavioral_issues &&
+ categorized.agent_behavioral_issues.length > 0;
+ const hasUserPatterns =
+ categorized.user_followup_patterns &&
+ categorized.user_followup_patterns.length > 0;
+ const hasInfra =
+ categorized.infrastructure_issues &&
+ categorized.infrastructure_issues.length > 0;
+ const hasOther = categorized.other && categorized.other.length > 0;
+
+ if (!hasAgentIssues && !hasUserPatterns && !hasInfra && !hasOther) {
+ return null;
+ }
+
+ return (
+ <div className="flex flex-col gap-1 mt-1.5">
+ {hasAgentIssues && (
+ <FeatureCategory
+ label={t(I18nKey.CRITIC$POTENTIAL_ISSUES)}
+ features={categorized.agent_behavioral_issues!}
+ />
+ )}
+ {hasInfra && (
+ <FeatureCategory
+ label={t(I18nKey.CRITIC$INFRASTRUCTURE)}
+ features={categorized.infrastructure_issues!}
+ />
+ )}
+ {hasUserPatterns && (
+ <FeatureCategory
+ label={t(I18nKey.CRITIC$LIKELY_FOLLOWUP)}
+ features={categorized.user_followup_patterns!}
+ />
+ )}
+ {hasOther && (
+ <FeatureCategory
+ label={t(I18nKey.CRITIC$OTHER)}
+ features={categorized.other!}
+ />
+ )}
+ </div>
+ );
+}
+
+interface CriticResultDisplayProps {
+ criticResult: CriticResult;
+}
+
+/**
+ * Displays a critic evaluation result with star rating, score percentage,
+ * and expandable categorized feature breakdown.
+ */
+export function CriticResultDisplay({
+ criticResult,
+}: CriticResultDisplayProps) {
+ const { t } = useTranslation();
+ const { data: settings } = useSettings();
+ const [expanded, setExpanded] = React.useState(false);
+
+ const { filled, empty } = getStarRating(criticResult.score);
+ const colorClass = getScoreColorClass(criticResult.score);
+ const percentage = (criticResult.score * 100).toFixed(1);
+ const iterativeRefinementEnabled = getIterativeRefinementEnabled(
+ settings?.agent_settings as Record<string, unknown> | null | undefined,
+ );
+ const showIterativeRefinementHint = iterativeRefinementEnabled === false;
+
+ const categorized = criticResult.metadata?.categorized_features;
+ const hasDetails =
+ categorized != null &&
+ ((categorized.agent_behavioral_issues ?? []).length > 0 ||
+ (categorized.user_followup_patterns ?? []).length > 0 ||
+ (categorized.infrastructure_issues ?? []).length > 0 ||
+ (categorized.other ?? []).length > 0);
+
+ return (
+ <div className="border-l-2 border-neutral-600 pl-2 my-2 py-1.5 text-sm">
+ <div className="flex items-center gap-1">
+ <span className="font-semibold text-neutral-300 text-xs">
+ {t(I18nKey.CRITIC$SUCCESS_LIKELIHOOD_LABEL)}
+ </span>
+ <span className={`${colorClass} text-xs tracking-wide`}>
+ {"★".repeat(filled)}
+ {"☆".repeat(empty)}
+ </span>
+ <span className="text-neutral-500 text-xs">({percentage}%)</span>
+
+ {hasDetails && (
+ <button
+ type="button"
+ onClick={() => setExpanded((prev) => !prev)}
+ className="cursor-pointer ml-1"
+ aria-label={expanded ? "Collapse details" : "Expand details"}
+ >
+ {expanded ? (
+ <ArrowUp className="h-3 w-3 inline fill-neutral-400" />
+ ) : (
+ <ArrowDown className="h-3 w-3 inline fill-neutral-400" />
+ )}
+ </button>
+ )}
+ </div>
+
+ {expanded && hasDetails && (
+ <FeaturesBreakdown categorized={categorized!} />
+ )}
+
+ {showIterativeRefinementHint && (
+ <p
+ className="mt-1.5 text-xs leading-5 text-neutral-500"
+ data-testid="critic-iterative-refinement-hint"
+ >
+ {t(I18nKey.CRITIC$ITERATIVE_REFINEMENT_HINT)}
+ </p>
+ )}
+ </div>
+ );
+}
diff --git a/frontend/src/components/v1/chat/event-message-components/finish-event-message.tsx b/frontend/src/components/v1/chat/event-message-components/finish-event-message.tsx
index 8950876054..c7d45e9864 100644
--- a/frontend/src/components/v1/chat/event-message-components/finish-event-message.tsx
+++ b/frontend/src/components/v1/chat/event-message-components/finish-event-message.tsx
@@ -2,6 +2,7 @@ import { ActionEvent } from "#/types/v1/core";
import { FinishAction } from "#/types/v1/core/base/action";
import { ChatMessage } from "../../../features/chat/chat-message";
import { getEventContent } from "../event-content-helpers/get-event-content";
+import { CriticResultDisplay } from "./critic-result-display";
interface FinishEventMessageProps {
event: ActionEvent<FinishAction>;
@@ -19,10 +20,15 @@ export function FinishEventMessage({
: String(eventContent.details);
return (
- <ChatMessage
- type="agent"
- message={message}
- isFromPlanningAgent={isFromPlanningAgent}
- />
+ <>
+ <ChatMessage
+ type="agent"
+ message={message}
+ isFromPlanningAgent={isFromPlanningAgent}
+ />
+ {event.critic_result != null && (
+ <CriticResultDisplay criticResult={event.critic_result} />
+ )}
+ </>
);
}
diff --git a/frontend/src/components/v1/chat/event-message-components/index.ts b/frontend/src/components/v1/chat/event-message-components/index.ts
index 52df9ab096..abc1cba992 100644
--- a/frontend/src/components/v1/chat/event-message-components/index.ts
+++ b/frontend/src/components/v1/chat/event-message-components/index.ts
@@ -5,3 +5,4 @@ export { FinishEventMessage } from "./finish-event-message";
export { GenericEventMessageWrapper } from "./generic-event-message-wrapper";
export { ThoughtEventMessage } from "./thought-event-message";
export { HookExecutionEventMessage } from "./hook-execution-event-message";
+export { CriticResultDisplay } from "./critic-result-display";
diff --git a/frontend/src/components/v1/chat/event-message-components/user-assistant-event-message.tsx b/frontend/src/components/v1/chat/event-message-components/user-assistant-event-message.tsx
index cf60af0143..e285075435 100644
--- a/frontend/src/components/v1/chat/event-message-components/user-assistant-event-message.tsx
+++ b/frontend/src/components/v1/chat/event-message-components/user-assistant-event-message.tsx
@@ -4,6 +4,7 @@ import { ChatMessage } from "../../../features/chat/chat-message";
import { ImageCarousel } from "../../../features/images/image-carousel";
import { V1ConfirmationButtons } from "#/components/shared/buttons/v1-confirmation-buttons";
import { parseMessageFromEvent } from "../event-content-helpers/parse-message-from-event";
+import { CriticResultDisplay } from "./critic-result-display";
interface UserAssistantEventMessageProps {
event: MessageEvent;
@@ -28,15 +29,20 @@ export function UserAssistantEventMessage({
}
return (
- <ChatMessage
- type={event.source}
- message={message}
- isFromPlanningAgent={isFromPlanningAgent}
- >
- {imageUrls.length > 0 && (
- <ImageCarousel size="small" images={imageUrls} />
+ <>
+ <ChatMessage
+ type={event.source}
+ message={message}
+ isFromPlanningAgent={isFromPlanningAgent}
+ >
+ {imageUrls.length > 0 && (
+ <ImageCarousel size="small" images={imageUrls} />
+ )}
+ {isLastMessage && <V1ConfirmationButtons />}
+ </ChatMessage>
+ {event.source === "agent" && event.critic_result != null && (
+ <CriticResultDisplay criticResult={event.critic_result} />
)}
- {isLastMessage && <V1ConfirmationButtons />}
- </ChatMessage>
+ </>
);
}
diff --git a/frontend/src/i18n/declaration.ts b/frontend/src/i18n/declaration.ts
index 3e01fabfff..44e744d06a 100644
--- a/frontend/src/i18n/declaration.ts
+++ b/frontend/src/i18n/declaration.ts
@@ -1448,4 +1448,10 @@ export enum I18nKey {
SETTINGS$PROFILE_EDIT = "SETTINGS$PROFILE_EDIT",
SETTINGS$PROFILE_SET_ACTIVE = "SETTINGS$PROFILE_SET_ACTIVE",
SETTINGS$PROFILE_MENU = "SETTINGS$PROFILE_MENU",
+ CRITIC$SUCCESS_LIKELIHOOD_LABEL = "CRITIC$SUCCESS_LIKELIHOOD_LABEL",
+ CRITIC$ITERATIVE_REFINEMENT_HINT = "CRITIC$ITERATIVE_REFINEMENT_HINT",
+ CRITIC$POTENTIAL_ISSUES = "CRITIC$POTENTIAL_ISSUES",
+ CRITIC$INFRASTRUCTURE = "CRITIC$INFRASTRUCTURE",
+ CRITIC$LIKELY_FOLLOWUP = "CRITIC$LIKELY_FOLLOWUP",
+ CRITIC$OTHER = "CRITIC$OTHER",
}
diff --git a/frontend/src/i18n/translation.json b/frontend/src/i18n/translation.json
index 92f95ca9fd..096fb196d4 100644
--- a/frontend/src/i18n/translation.json
+++ b/frontend/src/i18n/translation.json
@@ -24614,5 +24614,107 @@
"ca": "Accions del perfil",
"tr": "Profil işlemleri",
"uk": "Дії профілю"
+ },
+ "CRITIC$SUCCESS_LIKELIHOOD_LABEL": {
+ "en": "Critic: agent success likelihood",
+ "ja": "クリティック: エージェント成功確率",
+ "zh-CN": "评估: 代理成功概率",
+ "zh-TW": "評估: 代理成功機率",
+ "ko-KR": "평가: 에이전트 성공 가능성",
+ "no": "Kritiker: sannsynlighet for agentsuksess",
+ "it": "Critico: probabilità di successo dell'agente",
+ "pt": "Crítico: probabilidade de sucesso do agente",
+ "es": "Crítico: probabilidad de éxito del agente",
+ "ar": "الناقد: احتمالية نجاح الوكيل",
+ "fr": "Critique : probabilité de succès de l'agent",
+ "tr": "Eleştirmen: ajan başarı olasılığı",
+ "de": "Kritiker: Erfolgswahrscheinlichkeit des Agenten",
+ "uk": "Критик: ймовірність успіху агента",
+ "ca": "Crític: probabilitat d'èxit de l'agent"
+ },
+ "CRITIC$ITERATIVE_REFINEMENT_HINT": {
+ "en": "Want the agent to automatically fix potential issues? Enable iterative refinement in settings.",
+ "ja": "エージェントに潜在的な問題を自動修正させますか?設定で反復改善を有効にしてください。",
+ "zh-CN": "想让代理自动修复可能出现的问题?请在设置里开启迭代优化。",
+ "zh-TW": "想讓代理自動修復可能出現的問題?請在設定中開啟迭代最佳化。",
+ "ko-KR": "에이전트가 잠재적인 문제를 자동으로 수정하게 할까요? 설정에서 반복 개선을 켜세요.",
+ "no": "Vil du at agenten automatisk skal fikse mulige problemer? Slå på iterativ forbedring i innstillingene.",
+ "it": "Vuoi che l'agente corregga automaticamente i potenziali problemi? Attiva il perfezionamento iterativo nelle impostazioni.",
+ "pt": "Quer que o agente corrija automaticamente possíveis problemas? Ative o refinamento iterativo nas configurações.",
+ "es": "¿Quieres que el agente corrija automáticamente posibles problemas? Activa el refinamiento iterativo en la configuración.",
+ "ar": "هل تريد أن يصلح الوكيل المشكلات المحتملة تلقائيًا؟ فعّل التحسين التكراري من الإعدادات.",
+ "fr": "Vous voulez que l'agent corrige automatiquement les problèmes possibles ? Activez l'affinage itératif dans les paramètres.",
+ "tr": "Aracının olası sorunları otomatik olarak düzeltmesini ister misiniz? Ayarlardan yinelemeli iyileştirmeyi etkinleştirin.",
+ "de": "Soll der Agent mögliche Probleme automatisch beheben? Aktiviere iterative Verfeinerung in den Einstellungen.",
+ "uk": "Хочете, щоб агент автоматично виправляв можливі проблеми? Увімкніть ітеративне вдосконалення в налаштуваннях.",
+ "ca": "Vols que l'agent corregeixi automàticament possibles problemes? Activa el refinament iteratiu a la configuració."
+ },
+ "CRITIC$POTENTIAL_ISSUES": {
+ "en": "Potential Issues:",
+ "ja": "潜在的な問題:",
+ "zh-CN": "潜在问题:",
+ "zh-TW": "潛在問題:",
+ "ko-KR": "잠재적 문제:",
+ "no": "Potensielle problemer:",
+ "it": "Problemi potenziali:",
+ "pt": "Problemas potenciais:",
+ "es": "Problemas potenciales:",
+ "ar": "مشاكل محتملة:",
+ "fr": "Problèmes potentiels :",
+ "tr": "Olası sorunlar:",
+ "de": "Mögliche Probleme:",
+ "uk": "Потенційні проблеми:",
+ "ca": "Problemes potencials:"
+ },
+ "CRITIC$INFRASTRUCTURE": {
+ "en": "Infrastructure:",
+ "ja": "インフラストラクチャ:",
+ "zh-CN": "基础设施:",
+ "zh-TW": "基礎設施:",
+ "ko-KR": "인프라:",
+ "no": "Infrastruktur:",
+ "it": "Infrastruttura:",
+ "pt": "Infraestrutura:",
+ "es": "Infraestructura:",
+ "ar": "البنية التحتية:",
+ "fr": "Infrastructure :",
+ "tr": "Altyapı:",
+ "de": "Infrastruktur:",
+ "uk": "Інфраструктура:",
+ "ca": "Infraestructura:"
+ },
+ "CRITIC$LIKELY_FOLLOWUP": {
+ "en": "Likely Follow-up:",
+ "ja": "予想されるフォローアップ:",
+ "zh-CN": "可能的后续操作:",
+ "zh-TW": "可能的後續操作:",
+ "ko-KR": "예상 후속 조치:",
+ "no": "Sannsynlig oppfølging:",
+ "it": "Probabile seguito:",
+ "pt": "Provável acompanhamento:",
+ "es": "Probable seguimiento:",
+ "ar": "متابعة محتملة:",
+ "fr": "Suivi probable :",
+ "tr": "Olası takip:",
+ "de": "Wahrscheinliche Nachverfolgung:",
+ "uk": "Ймовірне продовження:",
+ "ca": "Probable seguiment:"
+ },
+ "CRITIC$OTHER": {
+ "en": "Other:",
+ "ja": "その他:",
+ "zh-CN": "其他:",
+ "zh-TW": "其他:",
+ "ko-KR": "기타:",
+ "no": "Annet:",
+ "it": "Altro:",
+ "pt": "Outros:",
+ "es": "Otros:",
+ "ar": "أخرى:",
+ "fr": "Autre :",
+ "tr": "Diğer:",
+ "de": "Sonstiges:",
+ "uk": "Інше:",
+ "ca": "Altres:"
}
}
diff --git a/frontend/src/mocks/settings-handlers.ts b/frontend/src/mocks/settings-handlers.ts
index c56e11e06c..184f422633 100644
--- a/frontend/src/mocks/settings-handlers.ts
+++ b/frontend/src/mocks/settings-handlers.ts
@@ -147,42 +147,122 @@ const MOCK_AGENT_SETTINGS_SCHEMA: NonNullable<
],
},
{
- key: "critic",
- label: "Critic",
+ key: "verification",
+ label: "Verification",
fields: [
{
+ key: "verification.critic_enabled",
+ label: "Enable Critic",
description:
"Enable an additional critic pass to review the agent's work.",
-
- key: "critic.enabled",
- label: "Enable critic",
- section: "critic",
- section_label: "Critic",
+ section: "verification",
+ section_label: "Verification",
value_type: "boolean",
default: false,
choices: [],
depends_on: [],
prominence: "critical",
secret: false,
- required: true,
+ required: false,
},
{
+ key: "verification.critic_mode",
+ label: "Critic Mode",
description: "Choose when the critic should review and intervene.",
-
- key: "critic.mode",
- label: "Mode",
- section: "critic",
- section_label: "Critic",
+ section: "verification",
+ section_label: "Verification",
value_type: "string",
default: "finish_and_message",
choices: [
- { label: "finish_and_message", value: "finish_and_message" },
+ {
+ label: "finish_and_message",
+ value: "finish_and_message",
+ },
{ label: "all_actions", value: "all_actions" },
],
- depends_on: ["critic.enabled"],
+ depends_on: ["verification.critic_enabled"],
+ prominence: "major",
+ secret: false,
+ required: false,
+ },
+ {
+ key: "verification.enable_iterative_refinement",
+ label: "Enable Iterative Refinement",
+ description:
+ "Let the critic send the agent back to refine its work when issues are found.",
+ section: "verification",
+ section_label: "Verification",
+ value_type: "boolean",
+ default: false,
+ choices: [],
+ depends_on: ["verification.critic_enabled"],
+ prominence: "critical",
+ secret: false,
+ required: false,
+ },
+ {
+ key: "verification.critic_threshold",
+ label: "Critic Threshold",
+ description:
+ "Critic success threshold used for iterative refinement.",
+ section: "verification",
+ section_label: "Verification",
+ value_type: "number",
+ default: 0.6,
+ choices: [],
+ depends_on: [
+ "verification.critic_enabled",
+ "verification.enable_iterative_refinement",
+ ],
prominence: "minor",
secret: false,
- required: true,
+ required: false,
+ },
+ {
+ key: "verification.max_refinement_iterations",
+ label: "Max Refinement Iterations",
+ description:
+ "Maximum number of refinement attempts after critic feedback.",
+ section: "verification",
+ section_label: "Verification",
+ value_type: "integer",
+ default: 3,
+ choices: [],
+ depends_on: [
+ "verification.critic_enabled",
+ "verification.enable_iterative_refinement",
+ ],
+ prominence: "minor",
+ secret: false,
+ required: false,
+ },
+ {
+ key: "verification.critic_server_url",
+ label: "Critic Server URL",
+ description: "Override the critic service URL.",
+ section: "verification",
+ section_label: "Verification",
+ value_type: "string",
+ default: null,
+ choices: [],
+ depends_on: ["verification.critic_enabled"],
+ prominence: "minor",
+ secret: false,
+ required: false,
+ },
+ {
+ key: "verification.critic_model_name",
+ label: "Critic Model Name",
+ description: "Override the critic model name.",
+ section: "verification",
+ section_label: "Verification",
+ value_type: "string",
+ default: null,
+ choices: [],
+ depends_on: ["verification.critic_enabled"],
+ prominence: "minor",
+ secret: false,
+ required: false,
},
],
},
@@ -266,9 +346,9 @@ export const MOCK_DEFAULT_USER_SETTINGS: Settings = {
agent_settings_schema: MOCK_AGENT_SETTINGS_SCHEMA,
agent_settings: {
...DEFAULT_AGENT_SETTINGS,
- critic: {
- mode: "finish_and_message",
- enabled: false,
+ verification: {
+ critic_enabled: false,
+ enable_iterative_refinement: false,
},
llm: {
...(llmDefaults ?? {}),
diff --git a/frontend/src/routes/condenser-settings.tsx b/frontend/src/routes/condenser-settings.tsx
index 8222b18f5c..7a6735929d 100644
--- a/frontend/src/routes/condenser-settings.tsx
+++ b/frontend/src/routes/condenser-settings.tsx
@@ -5,7 +5,9 @@ import { requireOrgDefaultsRedirect } from "#/utils/org/saas-redirect-to-org-def
function CondenserSettingsScreen() {
return (
<SdkSectionPage
- sectionKeys={["condenser"]}
+ settingsSources={[
+ { settingsSource: "agent_settings", sectionKeys: ["condenser"] },
+ ]}
testId="condenser-settings-screen"
/>
);
diff --git a/frontend/src/routes/llm-settings.tsx b/frontend/src/routes/llm-settings.tsx
index 7dac5ba1fa..bda0971841 100644
--- a/frontend/src/routes/llm-settings.tsx
+++ b/frontend/src/routes/llm-settings.tsx
@@ -375,14 +375,18 @@ export function LlmSettingsScreen({
const buildPayload = React.useCallback(
(
- basePayload: Record<string, unknown>,
+ defaultPayload: Record<string, unknown>,
context: {
values: Record<string, string | boolean>;
view: SettingsView;
},
) => {
- // basePayload is a nested dict (e.g. {llm: {model: "gpt-4"}})
- const agentSettings = structuredClone(basePayload);
+ // defaultPayload is the wrapped diff (e.g.
+ // `{ agent_settings_diff: { llm: { model: "gpt-4" } } }`); we only
+ // mutate the inner `llm` object below.
+ const agentSettings = structuredClone(
+ (defaultPayload.agent_settings_diff as Record<string, unknown>) ?? {},
+ );
const modelValue =
typeof context.values["llm.model"] === "string"
@@ -411,9 +415,10 @@ export function LlmSettingsScreen({
// Remember the model currently shown in the form — this is what the
// user is saving regardless of whether `llm.model` was toggled dirty
- // this turn. ``basePayload`` only includes dirty fields, so falling
- // back to ``context.values`` makes the profile auto-creation fire on
- // same-value re-saves (e.g. save → delete profile → save again).
+ // this turn. ``defaultPayload`` only includes dirty fields, so
+ // falling back to ``context.values`` makes the profile auto-creation
+ // fire on same-value re-saves (e.g. save → delete profile → save
+ // again).
lastSavedModelRef.current = modelValue || null;
return { agent_settings_diff: agentSettings };
@@ -515,8 +520,13 @@ export function LlmSettingsScreen({
{backToProfiles}
<SdkSectionPage
scope={scope}
- sectionKeys={["llm"]}
- excludeKeys={LLM_EXCLUDED_KEYS}
+ settingsSources={[
+ {
+ settingsSource: "agent_settings",
+ sectionKeys: ["llm"],
+ excludeKeys: LLM_EXCLUDED_KEYS,
+ },
+ ]}
header={buildHeader}
buildPayload={buildPayload}
extraDirty={profileName.trim() !== initialProfileName.trim()}
diff --git a/frontend/src/routes/org-default-condenser-settings.tsx b/frontend/src/routes/org-default-condenser-settings.tsx
index cba3222e0b..b112f65d93 100644
--- a/frontend/src/routes/org-default-condenser-settings.tsx
+++ b/frontend/src/routes/org-default-condenser-settings.tsx
@@ -8,7 +8,9 @@ function OrgDefaultCondenserSettingsScreen() {
return (
<SdkSectionPage
scope="org"
- sectionKeys={["condenser"]}
+ settingsSources={[
+ { settingsSource: "agent_settings", sectionKeys: ["condenser"] },
+ ]}
header={renderOrgDefaultsBanner}
testId="org-default-condenser-settings-screen"
/>
diff --git a/frontend/src/routes/verification-settings.tsx b/frontend/src/routes/verification-settings.tsx
index 9994ce6976..fd69878de6 100644
--- a/frontend/src/routes/verification-settings.tsx
+++ b/frontend/src/routes/verification-settings.tsx
@@ -1,16 +1,22 @@
-import React from "react";
import { SdkSectionPage } from "#/components/features/settings/sdk-settings/sdk-section-page";
import { SettingsScope } from "#/types/settings";
import { createPermissionGuard } from "#/utils/org/permission-guard";
import { requireOrgDefaultsRedirect } from "#/utils/org/saas-redirect-to-org-defaults-guard";
-function VerificationSettingsHeader({
- renderTopContent,
-}: {
- renderTopContent?: () => React.ReactNode;
-}) {
- return <div className="flex flex-col gap-6">{renderTopContent?.()}</div>;
-}
+// Defensive de-dup: agent_settings.verification still carries
+// `confirmation_mode` and `security_analyzer` for back-compat, but the SDK
+// deprecated them in 1.17.0 (see VerificationSettings field validators in
+// openhands-sdk) and moved the canonical copies to ConversationSettings.
+// Render only the conversation-source versions so these fields don't show
+// up twice on the page.
+//
+// TODO: drop this set once openhands-sdk 1.22.0 lands and the deprecated
+// VerificationSettings.{confirmation_mode,security_analyzer} fields are
+// fully removed.
+const CONVERSATION_OWNED_AGENT_VERIFICATION_FIELD_KEYS = new Set([
+ "verification.confirmation_mode",
+ "verification.security_analyzer",
+]);
export function VerificationSettingsScreen({
scope = "personal",
@@ -21,17 +27,21 @@ export function VerificationSettingsScreen({
renderTopContent?: () => React.ReactNode;
testId?: string;
}) {
- const buildHeader = React.useCallback(
- () => <VerificationSettingsHeader renderTopContent={renderTopContent} />,
- [renderTopContent],
- );
-
return (
<SdkSectionPage
scope={scope}
- settingsSource="conversation_settings"
- sectionKeys={["verification"]}
- header={buildHeader}
+ settingsSources={[
+ {
+ settingsSource: "conversation_settings",
+ sectionKeys: ["verification"],
+ },
+ {
+ settingsSource: "agent_settings",
+ sectionKeys: ["verification"],
+ excludeKeys: CONVERSATION_OWNED_AGENT_VERIFICATION_FIELD_KEYS,
+ },
+ ]}
+ header={renderTopContent ? () => renderTopContent() : undefined}
testId={testId}
/>
);
diff --git a/frontend/src/services/settings.ts b/frontend/src/services/settings.ts
index 36e6313662..1d159e0760 100644
--- a/frontend/src/services/settings.ts
+++ b/frontend/src/services/settings.ts
@@ -47,6 +47,10 @@ export const DEFAULT_SETTINGS: Settings = {
enabled: true,
max_size: 240,
},
+ verification: {
+ critic_enabled: false,
+ enable_iterative_refinement: false,
+ },
mcp_config: {
sse_servers: [],
stdio_servers: [],
diff --git a/frontend/src/types/v1/core/base/critic.ts b/frontend/src/types/v1/core/base/critic.ts
new file mode 100644
index 0000000000..da60cea06f
--- /dev/null
+++ b/frontend/src/types/v1/core/base/critic.ts
@@ -0,0 +1,50 @@
+/**
+ * A single feature detected by the critic (e.g., "Insufficient Testing").
+ */
+export interface CriticFeature {
+ /** Internal feature name (e.g., "insufficient_testing") */
+ name: string;
+ /** Human-readable display name (e.g., "Insufficient Testing") */
+ display_name: string;
+ /** Probability of this feature being present (0-1) */
+ probability: number;
+}
+
+/**
+ * Categorized features from the critic evaluation.
+ */
+export interface CriticCategorizedFeatures {
+ /** Agent behavioral issues (e.g., insufficient testing, loop behavior) */
+ agent_behavioral_issues?: CriticFeature[];
+ /** Likely user follow-up patterns */
+ user_followup_patterns?: CriticFeature[];
+ /** Infrastructure-related issues */
+ infrastructure_issues?: CriticFeature[];
+ /** Other uncategorized metrics */
+ other?: CriticFeature[];
+}
+
+/**
+ * Metadata from a critic evaluation, including categorized features
+ * and event IDs for reproducibility.
+ */
+export interface CriticMetadata {
+ categorized_features?: CriticCategorizedFeatures;
+ event_ids?: string[];
+ [key: string]: unknown;
+}
+
+/**
+ * Result of a critic evaluation on an agent's actions.
+ *
+ * The critic predicts the probability that the agent has successfully
+ * completed the task.
+ */
+export interface CriticResult {
+ /** Predicted probability of success (0-1) */
+ score: number;
+ /** Optional message explaining the score */
+ message: string | null;
+ /** Optional metadata with categorized features and event IDs */
+ metadata: CriticMetadata | null;
+}
diff --git a/frontend/src/types/v1/core/base/index.ts b/frontend/src/types/v1/core/base/index.ts
index 07292f3ed7..6053554e30 100644
--- a/frontend/src/types/v1/core/base/index.ts
+++ b/frontend/src/types/v1/core/base/index.ts
@@ -2,5 +2,6 @@
export * from "./action";
export * from "./base";
export * from "./common";
+export * from "./critic";
export * from "./event";
export * from "./observation";
diff --git a/frontend/src/types/v1/core/events/action-event.ts b/frontend/src/types/v1/core/events/action-event.ts
index fd2408b5d6..03d9153e4e 100644
--- a/frontend/src/types/v1/core/events/action-event.ts
+++ b/frontend/src/types/v1/core/events/action-event.ts
@@ -1,5 +1,6 @@
import { Action } from "../base/action";
import { EventID, ToolCallID, SecurityRisk, TextContent } from "../base/common";
+import { CriticResult } from "../base/critic";
import {
BaseEvent,
ChatCompletionMessageToolCall,
@@ -59,6 +60,11 @@ export interface ActionEvent<T extends Action = Action> extends BaseEvent {
*/
security_risk: SecurityRisk;
+ /**
+ * Optional critic evaluation of this action and preceding history.
+ */
+ critic_result?: CriticResult | null;
+
/**
* Optional LLM-generated summary used to label the tool call in the UI.
*/
diff --git a/frontend/src/types/v1/core/events/message-event.ts b/frontend/src/types/v1/core/events/message-event.ts
index be89631645..954284439d 100644
--- a/frontend/src/types/v1/core/events/message-event.ts
+++ b/frontend/src/types/v1/core/events/message-event.ts
@@ -1,4 +1,5 @@
import { TextContent } from "../base/common";
+import { CriticResult } from "../base/critic";
import { BaseEvent, Message } from "../base/event";
export interface MessageEvent extends BaseEvent {
@@ -16,4 +17,9 @@ export interface MessageEvent extends BaseEvent {
* List of content added by agent context
*/
extended_content: TextContent[];
+
+ /**
+ * Optional critic evaluation of the agent's work at this point.
+ */
+ critic_result?: CriticResult | null;
}
diff --git a/tests/unit/app_server/test_settings_api.py b/tests/unit/app_server/test_settings_api.py
index 4c43804c1f..69db6f2904 100644
--- a/tests/unit/app_server/test_settings_api.py
+++ b/tests/unit/app_server/test_settings_api.py
@@ -121,6 +121,7 @@ def test_get_agent_settings_schema_includes_critic_verification_fields(test_clie
section = next(s for s in schema['sections'] if s['key'] == 'verification')
field_keys = [f['key'] for f in section['fields']]
assert 'verification.critic_enabled' in field_keys
+ assert 'verification.enable_iterative_refinement' in field_keys
assert 'confirmation_mode' not in field_keys
assert 'security_analyzer' not in field_keys