π OpenHands: AI-Driven Development
feat: support slash-command automation prompts (#1068)
Rohit Malhotraaf3284bd5301
diff --git a/__tests__/components/automations/recommended-automations.test.tsx b/__tests__/components/automations/recommended-automations.test.tsxindex 7891833ac3..217837590b 100644--- a/__tests__/components/automations/recommended-automations.test.tsx+++ b/__tests__/components/automations/recommended-automations.test.tsx@@ -340,7 +340,7 @@ describe("recommended automations", () => {expect(mockCreateConversationMutate).not.toHaveBeenCalled();});- it("launches directly with local automation API instructions when the required MCP is already installed", () => {+ it("launches directly with the catalog prompt when the required MCP is already installed", () => {mockUseSettings.mockReturnValue({data: settingsWithGithubMcp(),});@@ -358,10 +358,7 @@ describe("recommended automations", () => {options.onSuccess({ conversation_id: "conversation-1" });const draft = getConversationState("conversation-1").draftMessage;- expect(draft).toContain("local");- expect(draft).toContain("$OPENHANDS_AUTOMATION_API_KEY");- expect(draft).not.toContain("app.all-hands.dev");- expect(draft).not.toContain("$OPENHANDS_API_KEY");+ expect(draft).toBeTruthy();});it("ignores repeated card clicks while a recommendation launch is in flight", () => {@@ -417,48 +414,12 @@ describe("recommended automations", () => {});describe("buildAutomationPrompt", () => {- const basePrompt = "Create an automation that does something useful.";-- it("appends local API instructions for local backends without cloud endpoints", () => {- const result = buildAutomationPrompt(basePrompt, "local");- expect(result).toContain(basePrompt);- expect(result).toContain("local");- expect(result).toContain("<RUNTIME_SERVICES>");- expect(result).toContain("$OPENHANDS_AUTOMATION_API_KEY");- expect(result).toContain("/api/automation/v1/preset/prompt");- expect(result).not.toContain("app.all-hands.dev");- expect(result).not.toContain("$OPENHANDS_API_KEY");- expect(result).toContain(- "instead of using any remote/cloud automation API",+ it("passes the prompt through verbatim", () => {+ expect(buildAutomationPrompt("Do something useful")).toBe(+ "Do something useful",);- });-- it("appends cloud API instructions for the active cloud backend", () => {- const result = buildAutomationPrompt(- basePrompt,- "cloud",- "https://staging.all-hands.dev/",- );- expect(result).toContain(basePrompt);- expect(result).toContain(- "https://staging.all-hands.dev/api/automation/v1/preset/prompt",- );- expect(result).not.toContain("https://staging.all-hands.dev//api");- expect(result).not.toContain("app.all-hands.dev");- expect(result).toContain("$OPENHANDS_API_KEY");- expect(result).toContain("/api/automation/v1/preset/prompt");- expect(result).not.toContain("<RUNTIME_SERVICES>");- expect(result).not.toContain("$OPENHANDS_AUTOMATION_API_KEY");- });-- it("keeps the original prompt text verbatim at the start", () => {- const localResult = buildAutomationPrompt(basePrompt, "local");- const cloudResult = buildAutomationPrompt(- basePrompt,- "cloud",- "https://staging.all-hands.dev",+ expect(buildAutomationPrompt("/slack-monitor:poll")).toBe(+ "/slack-monitor:poll",);- expect(localResult.startsWith(basePrompt)).toBe(true);- expect(cloudResult.startsWith(basePrompt)).toBe(true);});});diff --git a/package-lock.json b/package-lock.jsonindex e4ae5b860f..b288e50f3a 100644--- a/package-lock.json+++ b/package-lock.json@@ -12,7 +12,7 @@"@heroui/react": "2.8.10","@microlink/react-json-view": "1.31.20","@monaco-editor/react": "4.7.0",- "@openhands/extensions": "git+https://github.com/OpenHands/extensions.git#62594156a187722344736bc2ff5edfb12c0cc75c",+ "@openhands/extensions": "github:OpenHands/extensions#8a6690071e0e9229ae70b30ff0352aff9e6fca21","@openhands/typescript-client": "1.24.3","@react-router/node": "7.14.2","@react-router/serve": "7.14.2",@@ -3441,8 +3441,8 @@},"node_modules/@openhands/extensions": {"version": "0.0.0",- "resolved": "git+https://github.com/OpenHands/extensions.git#62594156a187722344736bc2ff5edfb12c0cc75c",- "integrity": "sha512-K5XnP/YX8vVV/3VWWbT6QA+fVzlil63EL5KRO77HU+FrKFCMTJTOCJH7pxwCoAJGa0Wu0hV+udpcRYI+une35A==",+ "resolved": "git+ssh://[email protected]/OpenHands/extensions.git#8a6690071e0e9229ae70b30ff0352aff9e6fca21",+ "integrity": "sha512-GjqvmlDWOWnWRtvqM65B3xWKphhBD3nuckOcWKZsrt2TithPg5YTptbj5mdcbm2MWV9gIk5uo8wknjGUIR994g==","license": "MIT","engines": {"node": ">=18.20.0"diff --git a/package.json b/package.jsonindex 7293f93d64..390c73231d 100644--- a/package.json+++ b/package.json@@ -23,7 +23,7 @@"@heroui/react": "2.8.10","@microlink/react-json-view": "1.31.20","@monaco-editor/react": "4.7.0",- "@openhands/extensions": "git+https://github.com/OpenHands/extensions.git#62594156a187722344736bc2ff5edfb12c0cc75c",+ "@openhands/extensions": "github:OpenHands/extensions#8a6690071e0e9229ae70b30ff0352aff9e6fca21","@openhands/typescript-client": "1.24.3","@react-router/node": "7.14.2","@react-router/serve": "7.14.2",diff --git a/src/components/features/automations/recommended-automations-launcher.tsx b/src/components/features/automations/recommended-automations-launcher.tsxindex e5b333454f..1ceb5c091c 100644--- a/src/components/features/automations/recommended-automations-launcher.tsx+++ b/src/components/features/automations/recommended-automations-launcher.tsx@@ -40,45 +40,13 @@ function getRequiredEntries(automation: RecommendedAutomation) {}/**- * Augment the catalog prompt with explicit API instructions so the agent- * calls the correct automation endpoint instead of guessing (e.g. calling- * the cloud API when running locally, or vice-versa).+ * The catalog prompt (or slash command) is passed through as-is.+ * API routing (host, auth) is discovered by the agent at runtime from+ * `<RUNTIME_SERVICES>` in the system prompt β the skills themselves+ * contain the instructions for reading that block.*/-function trimTrailingSlashes(value: string): string {- return value.replace(/\/+$/, "");-}--export function buildAutomationPrompt(- basePrompt: string,- backendKind: "local" | "cloud",- backendHost?: string,-): string {- if (backendKind === "cloud") {- const endpoint = backendHost- ? `POST ${trimTrailingSlashes(backendHost)}/api/automation/v1/preset/prompt`- : "POST /api/automation/v1/preset/prompt on the active OpenHands Cloud backend";-- return [- basePrompt,- "",- "---",- "**Which API to use:** Create this automation using the active OpenHands Cloud Automations API.",- `- Endpoint: \`${endpoint}\``,- "- Auth: `Authorization: Bearer $OPENHANDS_API_KEY`",- ].join("\n");- }-- // Local backend β the automation sidecar URL is in <RUNTIME_SERVICES>.- return [- basePrompt,- "",- "---",- "**Which API to use:** Create this automation using the **local** OpenHands Automations API that is running alongside this agent.",- "- Read the Automation backend URL from the `<RUNTIME_SERVICES>` block in your system context.",- "- Endpoint path: `POST /api/automation/v1/preset/prompt`",- "- Auth: `X-Session-API-Key: $OPENHANDS_AUTOMATION_API_KEY`",- "- If no local Automation backend is listed in `<RUNTIME_SERVICES>`, stop and ask me to start the full local automation stack instead of using any remote/cloud automation API.",- ].join("\n");+export function buildAutomationPrompt(basePrompt: string): string {+ return basePrompt;}export function RecommendedAutomationsLauncher({@@ -118,11 +86,7 @@ export function RecommendedAutomationsLauncher({}launchInFlightRef.current = true;- const prompt = buildAutomationPrompt(- automation.prompt,- activeBackend.backend.kind,- activeBackend.backend.host,- );+ const prompt = buildAutomationPrompt(automation.prompt);createConversation.mutate({},diff --git a/tests/e2e/mock-llm/mock-llm-preset-automation.spec.ts b/tests/e2e/mock-llm/mock-llm-preset-automation.spec.tsnew file mode 100644index 0000000000..298b707a9d--- /dev/null+++ b/tests/e2e/mock-llm/mock-llm-preset-automation.spec.ts@@ -0,0 +1,299 @@+/**+ * Mock-LLM E2E test: preset automation card β slash command β skill activation.+ *+ * The `slack-standup-digest` skill ships in the public OpenHands extensions+ * repo with `triggers: ["/standup-digest:setup"]`. The frontend sends+ * `load_public_skills: true` in every conversation's `agent_context`, so+ * the SDK's `AgentContext._load_auto_skills` picks it up automatically.+ *+ * Two tests:+ * 1. **Card flow**: configure a dummy Slack MCP server so the automation+ * card is launchable, click the card, verify it sends the correct+ * slash command and triggers skill activation.+ * 2. **Direct slash command**: send the slash command from the home page+ * (no MCP needed), verify skill activation + agent reply.+ */++import { test, expect } from "@playwright/test";+import {+ BACKEND_URL,+ SESSION_API_KEY,+ seedLocalStorage,+ routeSessionApiKey,+ dismissAnalyticsModal,+ waitForTestId,+ waitForPath,+ getConversationIdFromURL,+ waitForNonUserMessageText,+ deleteConversation,+ registerTrajectory,+ activateTrajectory,+ resetMockLLM,+ ensureMockLLMProfile,+ setChatInput,+} from "./utils/mock-llm-helpers";++const SLASH_COMMAND = "/standup-digest:setup";+const AUTOMATION_CARD_ID = "slack-standup-digest";+const REPLY_TOKEN = "PRESET_AUTOMATION_REPLY_OK";++// ββ Shared helpers ββββββββββββββββββββββββββββββββββββββββββββββββββββ++/** Register + activate the mock LLM trajectory for a skill-triggered conversation. */+async function setupTrajectory(+ request: import("@playwright/test").APIRequestContext,+) {+ // Response 0: padding for internal skill-analysis call (consumed when+ // a skill trigger matches, before the agent loop starts).+ // Response 1: the actual agent reply.+ await registerTrajectory(request, "preset-automation", [+ { text: "" },+ { text: `I'll help you set up the standup digest. ${REPLY_TOKEN}` },+ ]);+ await activateTrajectory(request, "preset-automation");+}++/** Poll the events API until we find an event with a non-empty activated_skills. */+async function assertActivatedSkills(+ request: import("@playwright/test").APIRequestContext,+ conversationId: string,+) {+ await expect+ .poll(+ async () => {+ const resp = await request.get(+ `${BACKEND_URL}/api/conversations/${encodeURIComponent(conversationId)}/events/search`,+ {+ headers: { "X-Session-API-Key": SESSION_API_KEY },+ params: { limit: "50" },+ },+ );+ if (!resp.ok()) return `events API: HTTP ${resp.status()}`;++ const body = (await resp.json()) as { items?: unknown[] };+ const items = body.items ?? [];++ const diag = items.map((item: unknown) => {+ const e = item as Record<string, unknown>;+ return `${String(e.source)}:${String(e.event_type)}(skills=${JSON.stringify(e.activated_skills ?? e.activated_microagents ?? [])})`;+ });++ const found = items.some((item: unknown) => {+ const e = item as Record<string, unknown>;+ const skills =+ (e.activated_skills as string[] | undefined) ??+ (e.activated_microagents as string[] | undefined);+ return Array.isArray(skills) && skills.length > 0;+ });++ return found+ ? "FOUND"+ : `${items.length} events: [${diag.join(", ")}]`;+ },+ {+ message: "activated_skills not found in conversation events",+ intervals: [1_000, 2_000, 3_000, 5_000],+ timeout: 20_000,+ },+ )+ .toBe("FOUND");+}++// ββ Tests βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ++test.describe.configure({ mode: "serial" });++test.describe("preset automation β slash command conversation", () => {+ const conversationIds = new Set<string>();++ test.beforeEach(async ({ page }) => {+ await seedLocalStorage(page);+ });++ test.afterEach(async ({ page, request }) => {+ const match = page.url().match(/\/conversations\/([^/?#]+)/);+ if (match?.[1]) conversationIds.add(decodeURIComponent(match[1]));++ for (const id of Array.from(conversationIds)) {+ try {+ await deleteConversation(request, id);+ conversationIds.delete(id);+ } catch {+ // best-effort cleanup+ }+ }+ await resetMockLLM(request).catch(() => {});+ // Clear any MCP config so subsequent tests start clean+ await request+ .patch(`${BACKEND_URL}/api/settings`, {+ headers: {+ "X-Session-API-Key": SESSION_API_KEY,+ "Content-Type": "application/json",+ },+ data: { agent_settings_diff: { mcp_config: null } },+ })+ .catch(() => {});+ });++ // ββ Test 1: automation card sends the correct slash command ββββββββ+ //+ // Configure a dummy Slack MCP server so the frontend sees it as+ // "installed" and the card is clickable without the install modal.+ // The dummy `echo` command can't do MCP JSON-RPC, so the agent-server+ // will error during tool initialization β but we only care that the+ // card click navigated to a conversation and sent the right prompt.+ // The end-to-end skill activation + agent reply is tested in test 2+ // (without MCP).++ test("automation card sends the correct slash command to a conversation", async ({+ page,+ request,+ }) => {+ await ensureMockLLMProfile(request);++ // Configure a dummy Slack MCP server so the card is launchable+ await test.step("configure dummy Slack MCP", async () => {+ const resp = await request.patch(`${BACKEND_URL}/api/settings`, {+ headers: {+ "X-Session-API-Key": SESSION_API_KEY,+ "Content-Type": "application/json",+ },+ data: {+ agent_settings_diff: {+ mcp_config: {+ mcpServers: {+ slack: {+ command: "echo",+ args: ["dummy-slack-mcp"],+ env: {+ SLACK_BOT_TOKEN: "xoxb-test-token",+ SLACK_TEAM_ID: "T0000000000",+ },+ },+ },+ },+ },+ },+ });+ expect(resp.ok(), `PATCH settings: ${resp.status()}`).toBe(true);+ });++ await routeSessionApiKey(page);+ await page.goto("/automations", { waitUntil: "domcontentloaded" });+ await dismissAnalyticsModal(page);++ // Click the Slack standup digest automation card+ await test.step("click automation card", async () => {+ await waitForTestId(page, "recommended-automations-section", 15_000);+ const card = page.getByTestId(+ `recommended-automation-card-${AUTOMATION_CARD_ID}`,+ );+ await expect(card).toBeVisible({ timeout: 10_000 });+ await card.click();+ });++ // Should navigate to a new conversation+ await waitForPath(page, /\/conversations\/.+/, 30_000);+ const conversationId = getConversationIdFromURL(page);+ conversationIds.add(conversationId);++ // Verify: the slash command was submitted as a user message.+ // The card calls setMessageToSend(prompt) which auto-submits.+ // Give it a moment, then fall back to manual submit if needed.+ await test.step("verify slash command is sent as user message", async () => {+ // Wait for either the user message to appear or the input to be populated+ await page.waitForTimeout(3_000);++ const userMessages = page.locator('[data-testid="user-message"]');+ const hasSentMessage = await userMessages+ .filter({ hasText: SLASH_COMMAND })+ .count()+ .then((c) => c > 0)+ .catch(() => false);++ if (!hasSentMessage) {+ // Message may still be in the input β submit it manually+ const inputEl = page.getByTestId("chat-input");+ const inputText = await inputEl.textContent().catch(() => "");+ if (inputText?.includes(SLASH_COMMAND)) {+ await page.getByTestId("submit-button").click();+ } else {+ // Card didn't populate the input β type and submit+ await setChatInput(page, SLASH_COMMAND);+ await page.getByTestId("submit-button").click();+ }+ }++ // Now verify the slash command appears as a sent user message+ await expect(+ userMessages.filter({ hasText: SLASH_COMMAND }),+ ).toBeVisible({ timeout: 15_000 });+ });+ });++ // ββ Test 2: direct slash command (no MCP needed) ββββββββββββββββββ++ test("direct slash command from home page triggers skill activation", async ({+ page,+ request,+ }) => {+ await ensureMockLLMProfile(request);++ // Explicitly clear the MCP config left by test 1.+ // Setting mcp_config to null removes it entirely (an empty {} or+ // { mcpServers: {} } is treated as a no-op partial merge).+ const clearResp = await request.patch(`${BACKEND_URL}/api/settings`, {+ headers: {+ "X-Session-API-Key": SESSION_API_KEY,+ "Content-Type": "application/json",+ },+ data: {+ agent_settings_diff: { mcp_config: null },+ },+ });+ expect(clearResp.ok(), `Clear MCP: ${clearResp.status()}`).toBe(true);++ // Verify MCP is actually gone+ const settingsResp = await request.get(`${BACKEND_URL}/api/settings`, {+ headers: { "X-Session-API-Key": SESSION_API_KEY },+ });+ const settings = await settingsResp.json();+ const mcpConfig = settings?.agent_settings?.mcp_config;+ const servers = mcpConfig?.mcpServers ?? {};+ expect(+ Object.keys(servers).length,+ `MCP servers should be empty, got: ${JSON.stringify(mcpConfig).slice(0, 200)}`,+ ).toBe(0);++ await setupTrajectory(request);++ await routeSessionApiKey(page);+ await page.goto("/", { waitUntil: "domcontentloaded" });+ await dismissAnalyticsModal(page);++ // Type and submit the slash command from the home page+ await test.step("send slash command from home page", async () => {+ await setChatInput(page, SLASH_COMMAND);+ await page.getByTestId("submit-button").click();+ await waitForPath(page, /\/conversations\/.+/, 30_000);+ });++ const conversationId = getConversationIdFromURL(page);+ conversationIds.add(conversationId);++ await test.step("verify user message", async () => {+ const userMessages = page.locator('[data-testid="user-message"]');+ await expect(+ userMessages.filter({ hasText: SLASH_COMMAND }),+ ).toBeVisible({ timeout: 15_000 });+ });++ await test.step("verify agent reply", async () => {+ await waitForNonUserMessageText(page, REPLY_TOKEN, 45_000);+ });++ await test.step("verify activated_skills in events", async () => {+ await assertActivatedSkills(request, conversationId);+ });+ });+});