Skip to content

πŸ™Œ OpenHands: AI-Driven Development

feat: mock the setup contract from the published fixtures (#16221)

Hiep Le5d5a0648db45
diff --git a/__tests__/api/automation-handlers.test.ts b/__tests__/api/automation-handlers.test.ts
index e16c6cd77d..d82477344c 100644
--- a/__tests__/api/automation-handlers.test.ts
+++ b/__tests__/api/automation-handlers.test.ts
@@ -1,12 +1,45 @@
import { http, HttpResponse } from "msw";
import { setupServer } from "msw/node";
import { describe, it, expect, beforeAll, afterAll, afterEach } from "vitest";
+import capabilitiesFixture from "@openhands/extensions/testing/automations/capabilities.json";
+import prReviewerFixture from "@openhands/extensions/testing/automations/github-pr-reviewer.json";
+import repoMonitorFixture from "@openhands/extensions/testing/automations/github-repo-monitor.json";
import {
AUTOMATION_HANDLERS,
resetAutomationMockData,
} from "#/mocks/automation-handlers";
import { MOCK_AUTOMATIONS_RESPONSE } from "#/mocks/automations.mock";
+interface PreflightExchange {
+ request: { method: string; path: string; body: unknown };
+ response: { status: number; body: unknown };
+}
+
+interface FixtureBundle {
+ automationId: string;
+ scenarios: { id: string; preflight?: PreflightExchange }[];
+}
+
+/**
+ * Every preflight exchange the published contract fixtures record. The mock
+ * backend must reproduce each one, so the setup flow exercised against it is
+ * exercised against the reference contract.
+ */
+const PREFLIGHT_EXCHANGES = (
+ [prReviewerFixture, repoMonitorFixture] as FixtureBundle[]
+).flatMap((bundle) =>
+ bundle.scenarios.flatMap((scenario) =>
+ scenario.preflight
+ ? [
+ {
+ name: `${bundle.automationId}/${scenario.id}`,
+ exchange: scenario.preflight,
+ },
+ ]
+ : [],
+ ),
+);
+
const server = setupServer(...AUTOMATION_HANDLERS);
describe("Automation MSW Handlers", () => {
@@ -39,6 +72,95 @@ describe("Automation MSW Handlers", () => {
});
});
+ describe("GET /api/automation/v1/capabilities", () => {
+ it("answers with the contract fixtures' supported deployment", async () => {
+ // Act
+ const res = await fetch("/api/automation/v1/capabilities");
+ const data = await res.json();
+
+ // Assert
+ expect({ status: res.status, body: data }).toEqual({
+ status: 200,
+ body: capabilitiesFixture.responses.supported.body,
+ });
+ });
+ });
+
+ describe("POST /api/automation/v1/validate", () => {
+ it.each(PREFLIGHT_EXCHANGES)(
+ "reproduces the $name exchange from the contract fixtures",
+ async ({ exchange }) => {
+ // Act
+ const res = await fetch(`/api/automation${exchange.request.path}`, {
+ method: exchange.request.method,
+ headers: { "Content-Type": "application/json" },
+ body: JSON.stringify(exchange.request.body),
+ });
+ const data = await res.json();
+
+ // Assert
+ expect({ status: res.status, body: data }).toEqual({
+ status: exchange.response.status,
+ body: exchange.response.body,
+ });
+ },
+ );
+
+ it("rejects an every-minute schedule below the deployment minimum", async () => {
+ // Arrange
+ const draft = {
+ trigger: { type: "cron", schedule: "* * * * *", timezone: "UTC" },
+ };
+
+ // Act
+ const res = await fetch("/api/automation/v1/validate", {
+ method: "POST",
+ headers: { "Content-Type": "application/json" },
+ body: JSON.stringify({
+ automationId: "github-pr-reviewer",
+ endpoint: "/v1/preset/prompt",
+ draft,
+ }),
+ });
+ const data = await res.json();
+
+ // Assert
+ expect(data).toEqual({
+ valid: false,
+ errors: [
+ {
+ field: "trigger.schedule",
+ code: "interval_too_short",
+ message: "Minimum interval for this deployment is 5 minutes.",
+ },
+ ],
+ });
+ });
+
+ it("passes a schedule shape its cron reader does not model", async () => {
+ // Arrange β€” "0 0 31 2 *" can never fire, but only the real service can
+ // say so; the fixtures record that rejection at the create stage.
+ const draft = {
+ trigger: { type: "cron", schedule: "0 0 31 2 *", timezone: "UTC" },
+ };
+
+ // Act
+ const res = await fetch("/api/automation/v1/validate", {
+ method: "POST",
+ headers: { "Content-Type": "application/json" },
+ body: JSON.stringify({
+ automationId: "github-pr-reviewer",
+ endpoint: "/v1/preset/prompt",
+ draft,
+ }),
+ });
+ const data = await res.json();
+
+ // Assert
+ expect(data).toEqual({ valid: true, errors: [] });
+ });
+ });
+
describe("GET /api/automation/v1/:id", () => {
it("returns a single automation by id", async () => {
const id = MOCK_AUTOMATIONS_RESPONSE.automations[0].id;
diff --git a/__tests__/manifests/automation-setup.test.ts b/__tests__/manifests/automation-setup.test.ts
index 74501f9780..aba6986520 100644
--- a/__tests__/manifests/automation-setup.test.ts
+++ b/__tests__/manifests/automation-setup.test.ts
@@ -1,16 +1,27 @@
import { describe, expect, it, vi } from "vitest";
+import incidentFixture from "@openhands/extensions/testing/automations/incident-retrospective-drafter.json";
+import prReviewerFixture from "@openhands/extensions/testing/automations/github-pr-reviewer.json";
+import repoMonitorFixture from "@openhands/extensions/testing/automations/github-repo-monitor.json";
import {
+ AUTOMATION_CREATE_ENDPOINT,
buildAssistedMessage,
buildCreatePayload,
buildPreflightBody,
deriveErrorMap,
} from "#/manifests/automation-setup";
-import { validateSetupEntry } from "#/manifests/manifest-validation";
-import type { SetupEntry } from "#/manifests/types";
+import {
+ mapServiceErrors,
+ normalizeServiceErrors,
+} from "#/manifests/manifest-error-map";
+import { validateFormValues } from "#/manifests/manifest-local-validation";
+import { SETUP_REGISTRY } from "#/manifests/manifest-sources";
+import type { SetupEntry, SetupFormValues } from "#/manifests/types";
-// The command a skill publishes in its own frontmatter, which the host looks up
-// rather than storing. Pinned here so the assertion does not move when the
-// packaged catalog does.
+// The command a skill publishes in its own frontmatter, which the host looks
+// up rather than storing. Pinned so the assertion does not move when the
+// packaged skills catalog does; the automation catalog itself is imported for
+// real, because pinning its derivation to the contract fixtures published
+// beside it is this file's point.
vi.mock("@openhands/extensions/skills", () => ({
SKILLS_CATALOG: [
{
@@ -23,276 +34,176 @@ vi.mock("@openhands/extensions/skills", () => ({
}));
/**
- * The setup blocks published in `OpenHands/extensions`, and the request bodies
- * its contract fixtures say they must produce. Those bodies were verified
- * against the live service, and the create model forbids extra keys, so any
- * divergence here is a 422 in production rather than a cosmetic difference.
+ * The reference fixtures `OpenHands/extensions` publishes with its catalog.
+ * Their request bodies were verified against the live service, and the create
+ * model forbids extra keys, so any divergence between the host's derivation
+ * and a fixture is a 422 in production rather than a cosmetic difference.
*/
-const PR_REVIEWER: SetupEntry = {
- id: "github-pr-reviewer",
- name: "GitHub Code Review Agent",
- description: "Watch for a configurable label on GitHub pull requests.",
- requires: {
- integrations: {
- github: { message: "Used to read pull requests and post comments." },
- },
- features: ["repoClone", "presetPrompt"],
- },
- setup: {
- version: "1.0",
- mode: "direct",
- form: {
- triggers: {
- cron: {
- schedule: {
- type: "cron",
- label: "Check frequency",
- help: "How often to look for newly labelled pull requests.",
- default: "*/15 * * * *",
- required: true,
- },
- timezone: {
- type: "timezone",
- label: "Timezone",
- help: "Timezone the schedule is interpreted in.",
- default: "UTC",
- required: true,
+interface FixtureExchange {
+ request: { method: string; path: string; body: Record<string, unknown> };
+ response: { status: number; body: unknown };
+}
+
+interface FixtureScenario {
+ id: string;
+ formValues?: SetupFormValues;
+ localValidation?: { valid: boolean };
+ preflight?: FixtureExchange;
+ create?: FixtureExchange;
+ conversation?: { request: { action: string; message: string } };
+ expectedFieldErrors?: Record<string, string>;
+ /** False when the recorded request is deliberately not what setup sends. */
+ matchesSetupPayload?: boolean;
+}
+
+interface FixtureBundle {
+ automationId: string;
+ scenarios: FixtureScenario[];
+}
+
+const BUNDLES = [
+ prReviewerFixture,
+ repoMonitorFixture,
+ incidentFixture,
+] as FixtureBundle[];
+
+function requireEntry(automationId: string): SetupEntry {
+ const entry = SETUP_REGISTRY.findById(automationId);
+ if (!entry) throw new Error(`The registry did not admit ${automationId}`);
+ return entry;
+}
+
+function requireScenario(
+ bundle: FixtureBundle,
+ scenarioId: string,
+): FixtureScenario {
+ const scenario = bundle.scenarios.find(({ id }) => id === scenarioId);
+ if (!scenario) {
+ throw new Error(`${bundle.automationId} has no scenario ${scenarioId}`);
+ }
+ return scenario;
+}
+
+const CREATE_CASES = BUNDLES.flatMap((bundle) =>
+ bundle.scenarios.flatMap((scenario) =>
+ scenario.create && scenario.matchesSetupPayload !== false
+ ? [
+ {
+ name: `${bundle.automationId}/${scenario.id}`,
+ automationId: bundle.automationId,
+ formValues: scenario.formValues ?? {},
+ body: scenario.create.request.body,
},
- },
- },
- args: {
- repository: {
- type: "repo-picker",
- label: "Repository",
- help: "The repository whose pull requests will be reviewed.",
- provider: "github",
- required: true,
- },
- triggerLabel: {
- type: "text",
- label: "Trigger label",
- help: "Only pull requests carrying this label are reviewed.",
- default: "openhands-review",
- required: true,
- constraints: { minLength: 1, maxLength: 50 },
- },
- reviewTone: {
- type: "select",
- label: "Review tone",
- help: "How detailed the review comments should be.",
- default: "concise",
- required: true,
- options: [
- { value: "concise", label: "Concise" },
- { value: "thorough", label: "Thorough" },
- ],
- },
- },
- },
- prompt:
- "Review pull requests labeled '{{form.triggerLabel}}' in {{form.repository}}. Review tone: {{form.reviewTone}}.",
- },
-};
-
-const REPO_MONITOR: SetupEntry = {
- id: "github-repo-monitor",
- name: "GitHub repository monitor",
- description: "Watch a repository for @OpenHands mentions.",
- requires: {
- integrations: {
- github: { message: "Used to read comments and post replies." },
- },
- features: ["repoClone", "presetPrompt", "webhookDelivery"],
- },
- setup: {
- version: "1.0",
- mode: "direct",
- form: {
- triggers: {
- event: {
- on: {
- type: "select",
- label: "Respond to",
- help: "Which GitHub comment event starts a conversation.",
- default: "issue_comment.created",
- required: true,
- options: [
- {
- value: "issue_comment.created",
- label: "Comments on issues and pull requests",
- },
- {
- value: "pull_request_review_comment.created",
- label: "Pull request review comments only",
- },
- ],
+ ]
+ : [],
+ ),
+);
+
+const PREFLIGHT_CASES = BUNDLES.flatMap((bundle) =>
+ bundle.scenarios.flatMap((scenario) =>
+ scenario.preflight
+ ? [
+ {
+ name: `${bundle.automationId}/${scenario.id}`,
+ automationId: bundle.automationId,
+ formValues: scenario.formValues ?? {},
+ body: scenario.preflight.request.body,
},
- triggerPhrase: {
- type: "text",
- label: "Trigger phrase",
- help: "Only comments containing this phrase start a conversation.",
- default: "@openhands",
- required: true,
- constraints: {
- minLength: 2,
- maxLength: 50,
- format: "safeExpressionLiteral",
- },
+ ]
+ : [],
+ ),
+);
+
+const CONVERSATION_CASES = BUNDLES.flatMap((bundle) =>
+ bundle.scenarios.flatMap((scenario) =>
+ scenario.conversation
+ ? [
+ {
+ name: `${bundle.automationId}/${scenario.id}`,
+ automationId: bundle.automationId,
+ formValues: scenario.formValues ?? {},
+ message: scenario.conversation.request.message,
},
- },
- },
- args: {
- repository: {
- type: "repo-picker",
- label: "Repository",
- help: "The repository or repositories to watch for mentions.",
- provider: "github",
- required: true,
- },
- ref: {
- type: "text",
- label: "Base branch",
- help: "Branch checked out when the agent responds.",
- default: "main",
- required: true,
- constraints: { minLength: 1, maxLength: 255 },
- },
- },
- },
- prompt:
- "A comment in {{form.repository}} mentions '{{form.triggerPhrase}}'. Read the surrounding issue or pull request context from the event payload, then post a helpful reply as a comment on the same thread.",
- filter:
- "icontains(comment.body, '{{form.triggerPhrase}}') && glob(repository.full_name, '{{form.repository}}')",
- },
-};
-
-const RETRO_DRAFTER: SetupEntry = {
- id: "incident-retrospective-drafter",
- name: "Incident retrospective drafter",
- description: "Collect incident chatter and draft a timeline.",
- requires: {
- integrations: {
- slack: { message: "Reads incident discussion and timestamps." },
- linear: { message: "Reads follow-up tickets, owners, and status." },
- notion: {
- message: "Publishes the drafted retrospective.",
- required: false,
- },
- },
- features: ["mcpTools", "conversationDispatch"],
- },
- skill: "incident-retrospective",
- setup: {
- version: "1.0",
- mode: "assisted",
- form: {
- note: "These answers start the conversation.",
- args: {
- incidentChannel: {
- type: "text",
- label: "Incident channel",
- help: "The Slack channel where incidents are discussed.",
- required: false,
- constraints: { maxLength: 100 },
- },
- linearTeam: {
- type: "text",
- label: "Linear team",
- help: "The team whose follow-up tickets belong in the retrospective.",
- required: false,
- constraints: { maxLength: 100 },
- },
- notionDestination: {
- type: "text",
- label: "Notion destination",
- help: "Page or database where drafts should be published.",
- required: false,
- constraints: { maxLength: 200 },
- },
- triggerPreference: {
- type: "select",
- label: "How should it run?",
- help: "A starting preference only.",
- default: "undecided",
- required: false,
- options: [
- { value: "undecided", label: "Let the agent recommend one" },
- { value: "scheduled", label: "On a schedule" },
- ],
- },
- notes: {
- type: "textarea",
- label: "Anything else the agent should know?",
- help: "Incident naming conventions, who should be notified.",
- required: false,
- constraints: { maxLength: 2000 },
- },
+ ]
+ : [],
+ ),
+);
+
+const SERVICE_ERROR_CASES = BUNDLES.flatMap((bundle) =>
+ bundle.scenarios.flatMap((scenario) => {
+ const exchange = scenario.preflight ?? scenario.create;
+ if (!scenario.expectedFieldErrors || !exchange) return [];
+ return [
+ {
+ name: `${bundle.automationId}/${scenario.id}`,
+ automationId: bundle.automationId,
+ formValues: scenario.formValues ?? {},
+ responseBody: exchange.response.body,
+ expectedFieldErrors: scenario.expectedFieldErrors,
},
+ ];
+ }),
+);
+
+describe("the published catalog", () => {
+ it.each(BUNDLES.map((bundle) => [bundle.automationId]))(
+ "admits %s",
+ (automationId) => {
+ // Act
+ const entry = SETUP_REGISTRY.findById(automationId);
+
+ // Assert
+ expect(entry).not.toBeNull();
},
- message:
- "The user supplied these starting points. Confirm each one, ask about anything left blank, then create the automation.\n\n- Incident channel: {{form.incidentChannel}}\n- Linear team: {{form.linearTeam}}\n- Notion destination: {{form.notionDestination}}\n- Preferred trigger: {{form.triggerPreference}}\n- Notes: {{form.notes}}\n\nStill undecided and required before creation: the incident identifier format and the approved Notion retrospective template.",
- },
-};
-
-const PR_REVIEWER_VALUES = {
- repository: "OpenHands/agent-server-gui",
- triggerLabel: "openhands-review",
- reviewTone: "thorough",
- schedule: "*/15 * * * *",
- timezone: "UTC",
-};
-
-const PR_REVIEWER_DRAFT = {
- name: "GitHub Code Review Agent - OpenHands/agent-server-gui",
- prompt:
- "Review pull requests labeled 'openhands-review' in OpenHands/agent-server-gui. Review tone: thorough.",
- repos: [{ url: "OpenHands/agent-server-gui", provider: "github" }],
- trigger: { type: "cron", schedule: "*/15 * * * *", timezone: "UTC" },
-};
+ );
+});
-describe("buildCreatePayload", () => {
- it("derives the scheduled request body the contract fixture pins", () => {
+describe("the contract fixtures", () => {
+ it("address the endpoints the host calls", () => {
// Act
- const payload = buildCreatePayload(PR_REVIEWER, PR_REVIEWER_VALUES);
+ const createPaths = new Set(
+ BUNDLES.flatMap((bundle) =>
+ bundle.scenarios.flatMap((scenario) =>
+ scenario.create ? [scenario.create.request.path] : [],
+ ),
+ ),
+ );
+ const preflightPaths = new Set(
+ BUNDLES.flatMap((bundle) =>
+ bundle.scenarios.flatMap((scenario) =>
+ scenario.preflight ? [scenario.preflight.request.path] : [],
+ ),
+ ),
+ );
// Assert
- expect(payload).toEqual(PR_REVIEWER_DRAFT);
+ expect({ create: [...createPaths], preflight: [...preflightPaths] }).toEqual(
+ { create: [AUTOMATION_CREATE_ENDPOINT], preflight: ["/v1/validate"] },
+ );
});
+});
- it("derives an event trigger's source and filter, which no manifest states", () => {
- // Act
- const payload = buildCreatePayload(REPO_MONITOR, {
- repository: "OpenHands/agent-server-gui",
- triggerPhrase: "@openhands",
- on: "issue_comment.created",
- ref: "main",
- });
+describe("buildCreatePayload", () => {
+ it.each(CREATE_CASES)(
+ "derives the $name create body its fixture pins",
+ ({ automationId, formValues, body }) => {
+ // Arrange
+ const entry = requireEntry(automationId);
- // Assert
- expect(payload).toEqual({
- name: "GitHub repository monitor - OpenHands/agent-server-gui",
- prompt:
- "A comment in OpenHands/agent-server-gui mentions '@openhands'. Read the surrounding issue or pull request context from the event payload, then post a helpful reply as a comment on the same thread.",
- repos: [
- {
- url: "OpenHands/agent-server-gui",
- ref: "main",
- provider: "github",
- },
- ],
- trigger: {
- type: "event",
- source: "github",
- on: "issue_comment.created",
- filter:
- "icontains(comment.body, '@openhands') && glob(repository.full_name, 'OpenHands/agent-server-gui')",
- },
- });
- });
+ // Act
+ const payload = buildCreatePayload(entry, formValues);
+
+ // Assert
+ expect(payload).toEqual(body);
+ },
+ );
it("sends no request body for an entry that hands setup to a conversation", () => {
+ // Arrange
+ const entry = requireEntry("incident-retrospective-drafter");
+
// Act
- const payload = buildCreatePayload(RETRO_DRAFTER, {});
+ const payload = buildCreatePayload(entry, {});
// Assert
expect(payload).toBeNull();
@@ -300,23 +211,93 @@ describe("buildCreatePayload", () => {
});
describe("buildPreflightBody", () => {
- it("wraps the draft in the envelope the validate endpoint expects", () => {
+ it.each(PREFLIGHT_CASES)(
+ "derives the $name preflight envelope its fixture pins",
+ ({ automationId, formValues, body }) => {
+ // Arrange
+ const entry = requireEntry(automationId);
+
+ // Act
+ const envelope = buildPreflightBody(entry, formValues);
+
+ // Assert
+ expect(envelope).toEqual(body);
+ },
+ );
+});
+
+describe("buildAssistedMessage", () => {
+ it.each(CONVERSATION_CASES)(
+ "opens $name with the skill command and the fixture's seed message",
+ ({ automationId, formValues, message }) => {
+ // Arrange
+ const entry = requireEntry(automationId);
+
+ // Act
+ const seed = buildAssistedMessage(entry, formValues);
+
+ // Assert
+ expect(seed).toBe(`/incident-retro:setup\n\n${message}`);
+ },
+ );
+});
+
+describe("service rejections mapped back to fields", () => {
+ it.each(SERVICE_ERROR_CASES)(
+ "maps the $name rejection to the fields the fixture names",
+ ({ automationId, formValues, responseBody, expectedFieldErrors }) => {
+ // Arrange
+ const entry = requireEntry(automationId);
+ const payload = buildCreatePayload(entry, formValues);
+
+ // Act
+ const mapped = mapServiceErrors(
+ normalizeServiceErrors(responseBody, payload),
+ deriveErrorMap(entry),
+ );
+
+ // Assert
+ expect(mapped).toEqual({ fieldErrors: expectedFieldErrors, formErrors: [] });
+ },
+ );
+});
+
+describe("local validation of fixture form values", () => {
+ it("blocks the unsafe trigger phrase before any request is made", () => {
+ // Arrange β€” the fixture names the failing field; the code is the host's
+ // own vocabulary, rendered through its translations.
+ const scenario = requireScenario(
+ BUNDLES[1],
+ "quote-in-trigger-phrase-blocked-locally",
+ );
+ const entry = requireEntry("github-repo-monitor");
+
// Act
- const body = buildPreflightBody(PR_REVIEWER, PR_REVIEWER_VALUES);
+ const errors = validateFormValues(entry.setup, scenario.formValues ?? {});
// Assert
- expect(body).toEqual({
- automationId: "github-pr-reviewer",
- endpoint: "/v1/preset/prompt",
- draft: PR_REVIEWER_DRAFT,
+ expect(errors).toEqual({
+ triggerPhrase: { code: "unsafeExpressionLiteral" },
});
});
+
+ it("passes an entirely blank assisted form, as its fixture records", () => {
+ // Arrange
+ const scenario = requireScenario(BUNDLES[2], "nothing-filled-in");
+ const entry = requireEntry("incident-retrospective-drafter");
+
+ // Act
+ const errors = validateFormValues(entry.setup, scenario.formValues ?? {});
+
+ // Assert
+ expect(errors).toEqual({});
+ });
});
describe("deriveErrorMap", () => {
it("recovers which fields built each payload path", () => {
// Act
- const errorMap = deriveErrorMap(PR_REVIEWER);
+ const errorMap = deriveErrorMap(requireEntry("github-pr-reviewer"));
// Assert
expect(errorMap).toEqual({
@@ -328,37 +309,3 @@ describe("deriveErrorMap", () => {
});
});
});
-
-describe("buildAssistedMessage", () => {
- it("opens with the owning skill's command and the interpolated answers", () => {
- // Act
- const message = buildAssistedMessage(RETRO_DRAFTER, {
- incidentChannel: "#incidents",
- linearTeam: "Reliability",
- notionDestination: "",
- triggerPreference: "undecided",
- notes: "",
- });
-
- // Assert
- expect(message).toBe(
- "/incident-retro:setup\n\nThe user supplied these starting points. Confirm each one, ask about anything left blank, then create the automation.\n\n- Incident channel: #incidents\n- Linear team: Reliability\n- Notion destination: \n- Preferred trigger: undecided\n- Notes: \n\nStill undecided and required before creation: the incident identifier format and the approved Notion retrospective template.",
- );
- });
-});
-
-describe("the published manifests", () => {
- // The host enforces invariants its own derivation depends on, which the
- // published schema does not. Every shipped entry has to satisfy them too.
- it.each([
- ["github-pr-reviewer", PR_REVIEWER],
- ["github-repo-monitor", REPO_MONITOR],
- ["incident-retrospective-drafter", RETRO_DRAFTER],
- ])("admits %s", (_id, entry) => {
- // Act
- const result = validateSetupEntry(entry);
-
- // Assert
- expect(result).toEqual({ valid: true, errors: [] });
- });
-});
diff --git a/__tests__/manifests/manifest-capabilities.test.ts b/__tests__/manifests/manifest-capabilities.test.ts
index 261f6aba37..3a0406b525 100644
--- a/__tests__/manifests/manifest-capabilities.test.ts
+++ b/__tests__/manifests/manifest-capabilities.test.ts
@@ -1,5 +1,10 @@
import { describe, expect, it } from "vitest";
+import capabilitiesFixture from "@openhands/extensions/testing/automations/capabilities.json";
+import incidentFixture from "@openhands/extensions/testing/automations/incident-retrospective-drafter.json";
+import prReviewerFixture from "@openhands/extensions/testing/automations/github-pr-reviewer.json";
+import repoMonitorFixture from "@openhands/extensions/testing/automations/github-repo-monitor.json";
import { assessCapabilityRequirements } from "#/manifests/manifest-capabilities";
+import { SETUP_REGISTRY } from "#/manifests/manifest-sources";
import type { DeploymentCapabilities } from "#/manifests/types";
import { createSetup, createSetupEntry } from "./manifest-test-data";
@@ -77,3 +82,44 @@ describe("assessCapabilityRequirements", () => {
expect(assessment).toEqual({ supported: false, unmet: [] });
});
});
+
+describe("the published contract fixtures", () => {
+ // Each fixture bundle lists in `blockedBy` exactly the published deployment
+ // shapes its entry cannot run under. The host's assessment must agree with
+ // that record for every entry Γ— shape pair.
+ const shapes = Object.entries(capabilitiesFixture.responses).map(
+ ([shapeName, response]) => ({
+ shapeName,
+ capabilities: response.body as DeploymentCapabilities,
+ }),
+ );
+
+ const bundles = [prReviewerFixture, repoMonitorFixture, incidentFixture] as {
+ automationId: string;
+ blockedBy: string[];
+ }[];
+
+ it.each(
+ bundles.flatMap(({ automationId, blockedBy }) =>
+ shapes.map(({ shapeName, capabilities }) => ({
+ automationId,
+ shapeName,
+ capabilities,
+ blocked: blockedBy.includes(shapeName),
+ })),
+ ),
+ )(
+ "$automationId under the $shapeName deployment: blocked=$blocked",
+ ({ automationId, capabilities, blocked }) => {
+ // Arrange
+ const entry = SETUP_REGISTRY.findById(automationId);
+ if (!entry) throw new Error(`The registry did not admit ${automationId}`);
+
+ // Act
+ const assessment = assessCapabilityRequirements(entry, capabilities);
+
+ // Assert
+ expect(assessment.supported).toBe(!blocked);
+ },
+ );
+});
diff --git a/src/mocks/automation-handlers.ts b/src/mocks/automation-handlers.ts
index e2d4aac70f..c71894b13b 100644
--- a/src/mocks/automation-handlers.ts
+++ b/src/mocks/automation-handlers.ts
@@ -1,4 +1,10 @@
import { http, HttpResponse, delay } from "msw";
+import capabilitiesFixture from "@openhands/extensions/testing/automations/capabilities.json";
+import type {
+ DeploymentCapabilities,
+ DraftValidationError,
+ ValidateDraftResponse,
+} from "#/manifests/types";
import type {
Automation,
AutomationsResponse,
@@ -9,6 +15,79 @@ import { AutomationRunStatus } from "#/types/automation";
import { MOCK_AUTOMATIONS_RESPONSE } from "./automations.mock";
import { MOCK_AUTOMATION_RUNS } from "./automation-runs.mock";
+// The "supported" deployment from the published contract fixtures. Discovery
+// and preflight answer with it, so the setup flow runs against the same
+// reference data the extensions contract is verified against.
+const CAPABILITIES: DeploymentCapabilities =
+ capabilitiesFixture.responses.supported.body;
+
+interface DraftTrigger {
+ type?: string;
+ schedule?: string;
+ on?: string;
+ source?: string;
+}
+
+// The schedules the mock can read: "* * * * *" and "*/N * * * *". Anything
+// else is assumed to satisfy the deployment minimum β€” this stands in for the
+// service's cron parser rather than reimplementing it.
+const STEP_SCHEDULE_PATTERN = /^(?:\*|\*\/(\d+)) \* \* \* \*$/;
+
+function cronIntervalSeconds(schedule: string): number | null {
+ const match = STEP_SCHEDULE_PATTERN.exec(schedule.trim());
+ if (!match) return null;
+ return (match[1] ? Number(match[1]) : 1) * 60;
+}
+
+// Event deliveries registered for the mock organization. Distinct from
+// CAPABILITIES.eventTypes on purpose: a deployment can support an event type
+// that no webhook is registered to deliver, and preflight is what catches
+// that gap β€” the fixtures' event-type-not-delivered scenario depends on one
+// supported type staying unregistered here.
+const REGISTERED_EVENT_TYPES = CAPABILITIES.eventTypes.filter(
+ (type) => type !== "pull_request_review_comment.created",
+);
+
+// The deployment checks the contract fixtures record: a cron schedule below
+// triggers.cron.minIntervalSeconds, and an event type no webhook delivers.
+// Errors address the draft by dotted path, exactly as the fixtures do.
+function validateDraftTrigger(
+ trigger: DraftTrigger | undefined,
+): DraftValidationError[] {
+ if (!trigger) return [];
+
+ const cron = CAPABILITIES.triggers.cron;
+ if (trigger.type === "cron" && trigger.schedule && cron) {
+ const interval = cronIntervalSeconds(trigger.schedule);
+ if (interval !== null && interval < cron.minIntervalSeconds) {
+ return [
+ {
+ field: "trigger.schedule",
+ code: "interval_too_short",
+ message: `Minimum interval for this deployment is ${cron.minIntervalSeconds / 60} minutes.`,
+ },
+ ];
+ }
+ }
+
+ if (
+ trigger.type === "event" &&
+ trigger.on &&
+ !REGISTERED_EVENT_TYPES.includes(trigger.on)
+ ) {
+ const source = trigger.source === "github" ? "GitHub" : trigger.source;
+ return [
+ {
+ field: "trigger.on",
+ code: "event_type_not_delivered",
+ message: `No ${source} webhook delivering ${trigger.on} is registered for this organization.`,
+ },
+ ];
+ }
+
+ return [];
+}
+
// Mutable copy for CRUD operations within the mock session
const automations = new Map<string, Automation>(
MOCK_AUTOMATIONS_RESPONSE.automations.map((a) => [a.id, { ...a }]),
@@ -47,6 +126,31 @@ export const AUTOMATION_HANDLERS = [
return HttpResponse.json(response);
}),
+ // GET /api/automation/v1/capabilities β€” What this deployment supports
+ http.get("*/api/automation/v1/capabilities", async () => {
+ await delay(200);
+ return HttpResponse.json(CAPABILITIES);
+ }),
+
+ // POST /api/automation/v1/validate β€” Preflight a draft without creating it
+ http.post("*/api/automation/v1/validate", async ({ request }) => {
+ await delay(200);
+
+ const body = (await request.clone().json()) as {
+ automationId?: string;
+ endpoint?: string;
+ draft?: { trigger?: DraftTrigger };
+ };
+
+ const errors = validateDraftTrigger(body.draft?.trigger);
+ const response: ValidateDraftResponse = {
+ valid: errors.length === 0,
+ errors,
+ };
+
+ return HttpResponse.json(response);
+ }),
+
// POST /api/automation/v1/preset/:kind β€” Create a prompt/plugin automation
http.post("*/api/automation/v1/preset/:kind", async ({ params, request }) => {
await delay(200);
diff --git a/tests/e2e/mock-llm/test-mapping.json b/tests/e2e/mock-llm/test-mapping.json
index 14bbc81108..57e758cdbb 100644
--- a/tests/e2e/mock-llm/test-mapping.json
+++ b/tests/e2e/mock-llm/test-mapping.json
@@ -47,10 +47,14 @@
"tests": ["files"]
},
{
- "comment": "Automations lifecycle and preset cards",
+ "comment": "Automations lifecycle, preset cards, and the manifest setup host",
"sources": [
"src/components/features/automations/**",
- "src/routes/automations*.tsx"
+ "src/components/features/manifest/**",
+ "src/manifests/**",
+ "src/mocks/automation-handlers.ts",
+ "src/routes/automations*.tsx",
+ "src/routes/automation-setup-route.tsx"
],
"tests": ["automations"]
},