Skip to content

πŸ™Œ OpenHands: AI-Driven Development

test: capture Docker E2E container logs (#16660)

Graham Neubigd66157e689f5
diff --git a/.github/workflows/mock-llm-docker-e2e.yml b/.github/workflows/mock-llm-docker-e2e.yml
index 4375117256..dbc394d5f4 100644
--- a/.github/workflows/mock-llm-docker-e2e.yml
+++ b/.github/workflows/mock-llm-docker-e2e.yml
@@ -393,12 +393,20 @@ jobs:
pw_exit=0
fi
- # Clean up the Docker container (belt-and-suspenders)
- docker ps -q --filter "name=agent-canvas-mock-llm" | xargs -r docker stop 2>/dev/null || true
-
echo "exit_code=$pw_exit" >> "$GITHUB_OUTPUT"
exit 0
+ - name: Capture Docker container logs
+ if: always()
+ run: |
+ docker ps -a --filter "name=agent-canvas-mock-llm" --format '{{.Names}}\t{{.Status}}' | tee docker-container-status.txt || true
+ : > docker-container-logs.txt
+ for container in $(docker ps -a --filter "name=agent-canvas-mock-llm" --format '{{.Names}}'); do
+ echo "=== $container ===" >> docker-container-logs.txt
+ docker logs "$container" >> docker-container-logs.txt 2>&1 || true
+ done
+ docker ps -aq --filter "name=agent-canvas-mock-llm" | xargs -r docker rm -f 2>/dev/null || true
+
# ── Reporting ──────────────────────────────────────────────────────
- name: Upload test artifacts
id: upload_artifacts
@@ -411,6 +419,8 @@ jobs:
path: |
playwright-report-mock-llm-docker/
test-results-mock-llm-docker/
+ docker-container-status.txt
+ docker-container-logs.txt
- name: Detect newly added spec files
if: always() && github.event.pull_request.number
diff --git a/playwright.mock-llm-docker.config.ts b/playwright.mock-llm-docker.config.ts
index d59387a5f6..d0b9791612 100644
--- a/playwright.mock-llm-docker.config.ts
+++ b/playwright.mock-llm-docker.config.ts
@@ -202,7 +202,7 @@ export default defineConfig({
// Stop any leftover container from a previous failed run
`docker rm -f ${CONTAINER_NAME} 2>/dev/null;`,
"exec docker run",
- "--rm",
+ // Keep the container until the workflow captures its logs.
`--name ${CONTAINER_NAME}`,
"--network host",
// Mount the mock ACP server script so the agent-server inside
diff --git a/tests/e2e/mock-llm/automations/mock-llm-automation.spec.ts b/tests/e2e/mock-llm/automations/mock-llm-automation.spec.ts
index 8a274e8ce9..e032614cef 100644
--- a/tests/e2e/mock-llm/automations/mock-llm-automation.spec.ts
+++ b/tests/e2e/mock-llm/automations/mock-llm-automation.spec.ts
@@ -87,27 +87,46 @@ async function listAutomations(
);
}
-/**
- * Wait for a newly-created automation to become visible through the list API.
- * Creation and listing are separate backend operations, so the list can lag
- * briefly after the create request succeeds.
- */
-async function waitForAutomation(
+/** Poll the main conversation for the automation ID returned by the create command. */
+async function waitForCreatedAutomationId(
request: import("@playwright/test").APIRequestContext,
- name: string,
+ conversationId: string,
timeoutMs = 30_000,
) {
const deadline = Date.now() + timeoutMs;
while (Date.now() < deadline) {
- const data = await listAutomations(request, 1);
- const automations = data.automations ?? data.items ?? [];
- const automation = automations.find(
- (candidate: { name: string }) => candidate.name === name,
+ const response = await request.get(
+ `${BACKEND_URL}/api/conversations/${encodeURIComponent(conversationId)}/events/search`,
+ {
+ headers: { "X-Session-API-Key": SESSION_API_KEY },
+ params: { limit: "100", sort_order: "TIMESTAMP_DESC" },
+ },
);
- if (automation) return automation;
- await new Promise((resolve) => setTimeout(resolve, 1_000));
+ if (response.ok()) {
+ const body = (await response.json()) as { items?: unknown[] };
+ const text = JSON.stringify(body.items ?? []);
+ const match = text.match(/automation_id\\?":\\?"([0-9a-f-]{36})/i);
+ if (match) return match[1];
+ }
+ await new Promise((resolve) => setTimeout(resolve, 500));
}
- throw new Error(`Automation "${name}" was not visible after ${timeoutMs}ms`);
+ throw new Error(
+ `Created automation ID was not reported after ${timeoutMs}ms`,
+ );
+}
+
+async function getAutomation(
+ request: import("@playwright/test").APIRequestContext,
+ automationId: string,
+) {
+ const response = await request.get(
+ `${AUTOMATION_API_BASE}/${encodeURIComponent(automationId)}`,
+ { headers: { "X-Session-API-Key": SESSION_API_KEY } },
+ );
+ expect(response.ok(), `GET automation returned ${response.status()}`).toBe(
+ true,
+ );
+ return response.json();
}
/**
@@ -197,7 +216,8 @@ test.describe.configure({ mode: "serial" });
test.describe("mock-LLM automation lifecycle", () => {
const conversationIds = new Set<string>();
const automationIds = new Set<string>();
- /** conversation_id from the completed automation run (set in step 2, verified in step 3) */
+ /** IDs carried between the serial lifecycle steps. */
+ let createdAutomationId: string | null = null;
let runConversationId: string | null = null;
test.beforeEach(async ({ page }) => {
@@ -416,8 +436,13 @@ test.describe("mock-LLM automation lifecycle", () => {
// ── Verify: automation was created in the real automation backend ──
await test.step("verify automation was created", async () => {
- const created = await waitForAutomation(request, AUTOMATION_NAME);
+ createdAutomationId = await waitForCreatedAutomationId(
+ request,
+ conversationId,
+ );
+ const created = await getAutomation(request, createdAutomationId);
automationIds.add(created.id);
+ expect(created.name).toBe(AUTOMATION_NAME);
expect(created.trigger?.schedule).toBe(CRON_SCHEDULE);
expect(created.enabled).toBe(true);
});
@@ -425,7 +450,8 @@ test.describe("mock-LLM automation lifecycle", () => {
// ── Verify: run completed successfully with a conversation link ──
await test.step("verify run completed with conversation link", async () => {
- const automation = await waitForAutomation(request, AUTOMATION_NAME);
+ expect(createdAutomationId).toBeTruthy();
+ const automation = await getAutomation(request, createdAutomationId!);
automationIds.add(automation.id);
// Wait for the run to reach COMPLETED. The trajectory includes extra
@@ -514,26 +540,20 @@ test.describe("mock-LLM automation lifecycle", () => {
await test.step("automation card visible on list page", async () => {
await waitForTestId(page, "automations-add-automation", 15_000);
+ expect(createdAutomationId).toBeTruthy();
- // Scope to the automation card (data-testid `automation-card-<id>`) so
- // the locator only matches the list entry, not the global sidebar's
- // conversation cards. The automation run creates a conversation named
- // ``<AUTOMATION_NAME> β€” <timestamp>`` that the sidebar conversation list
- // surfaces, so an unscoped ``getByText(AUTOMATION_NAME)`` resolves to
- // multiple elements (the card + every run conversation) and trips
- // Playwright strict mode.
- const automationCard = page
- .locator('[data-testid^="automation-card-"]')
- .filter({ hasText: AUTOMATION_NAME });
-
+ const automationCard = page.locator(
+ `[data-testid="automation-card-${createdAutomationId}"]`,
+ );
await expect(automationCard).toBeVisible({ timeout: 15_000 });
+ await expect(automationCard).toContainText(AUTOMATION_NAME);
});
await test.step("click through to automation detail page", async () => {
// The automation card is a link β€” clicking it navigates to /automations/:id.
- const automationCard = page
- .locator('[data-testid^="automation-card-"]')
- .filter({ hasText: AUTOMATION_NAME });
+ const automationCard = page.locator(
+ `[data-testid="automation-card-${createdAutomationId}"]`,
+ );
await automationCard.click();
await waitForPath(page, /\/automations\/.+/, 10_000);