The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
test(e2e): link runner campaign summaries (#12927)
Dotta9ecd93a54dad
diff --git a/.github/workflows/runner-full-stack-e2e.yml b/.github/workflows/runner-full-stack-e2e.ymlindex 3179f5c8e1..52b069de93 100644--- a/.github/workflows/runner-full-stack-e2e.yml+++ b/.github/workflows/runner-full-stack-e2e.yml@@ -1102,6 +1102,8 @@ jobs:PAPERCLIP_E2E_CAMPAIGN_ID: gha-${{ github.run_id }}-${{ github.run_attempt }}PAPERCLIP_RUNNER_E2E_SOURCE_SHA: ${{ needs.authorize.outputs.target_sha }}PAPERCLIP_RUNNER_E2E_SOURCE_REF: ${{ needs.authorize.outputs.target_ref }}+ PAPERCLIP_RUNNER_E2E_HISTORY_PUBLIC_BASE_URL: ${{ vars.RUNNER_E2E_HISTORY_PUBLIC_BASE_URL }}+ PAPERCLIP_RUNNER_E2E_HISTORY_PREFIX: ${{ vars.RUNNER_E2E_HISTORY_PREFIX || 'runner-e2e' }}run: |set +epnpm test:e2e:runner:reportdiff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.mdindex b9ffb1dd6c..c2e29273fe 100644--- a/tests/runner-e2e/README.md+++ b/tests/runner-e2e/README.md@@ -266,6 +266,11 @@ same metrics per campaign/suite/execution, source SHA/ref, definitionfingerprints, completeness, retries, and cleanup. Trend charts compare onlycomplete campaigns by default; partial/manual selections remain browsable.`summary.md` carries the current totals into the GitHub Actions job summary.+In CI, its **View results** section links to the exact immutable public campaign+report, the workflow and per-cell logs, and the access-controlled report+artifacts. Each cell name links to its exact section in the campaign report.+The public campaign links become available after the history publisher+finishes. The artifact links remain available for 30 days.### Iterate on a published dashboard without rerunning paid testsdiff --git a/tests/runner-e2e/dashboard.ts b/tests/runner-e2e/dashboard.tsindex 6fe3f8d782..06a72bd9cb 100644--- a/tests/runner-e2e/dashboard.ts+++ b/tests/runner-e2e/dashboard.ts@@ -247,6 +247,7 @@ function renderCase(</div>`: "";return `<article+ id="execution-${html(execution.id)}"class="case case-${state}"data-execution-id="${html(execution.id)}"data-report-casediff --git a/tests/runner-e2e/history-destination.ts b/tests/runner-e2e/history-destination.tsnew file mode 100644index 0000000000..7e1f60d655--- /dev/null+++ b/tests/runner-e2e/history-destination.ts@@ -0,0 +1,60 @@+export interface RunnerE2EHistoryPublicDestination {+ prefix: string;+ publicBaseUrl: string;+}++export function validateHistoryPublicDestination(input: {+ prefix: string;+ publicBaseUrl: string;+}): RunnerE2EHistoryPublicDestination {+ const prefix = input.prefix.replace(/^\/+|\/+$/g, "");+ const segments = prefix.split("/");+ if (+ !prefix ||+ segments.some(+ (segment) =>+ !segment ||+ segment === "." ||+ segment === ".." ||+ !/^[A-Za-z0-9._~-]+$/.test(segment),+ )+ ) {+ throw new Error(+ "RUNNER_E2E_HISTORY_PREFIX must be a safe non-empty key prefix",+ );+ }+ const publicUrl = new URL(input.publicBaseUrl);+ if (+ publicUrl.protocol !== "https:" ||+ publicUrl.username ||+ publicUrl.password ||+ publicUrl.search ||+ publicUrl.hash+ ) {+ throw new Error(+ "RUNNER_E2E_HISTORY_PUBLIC_BASE_URL must be a credential-free HTTPS URL",+ );+ }+ return {+ prefix,+ publicBaseUrl: publicUrl.href.replace(/\/+$/, ""),+ };+}++export function validateHistoryDestination(input: {+ bucket: string;+ prefix: string;+ publicBaseUrl: string;+}) {+ if (!/^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$/.test(input.bucket)) {+ throw new Error("RUNNER_E2E_HISTORY_S3_BUCKET is not a valid bucket name");+ }+ return validateHistoryPublicDestination(input);+}++export function runnerE2ECampaignPublicUrl(+ destination: RunnerE2EHistoryPublicDestination,+ campaignId: string,+) {+ return `${destination.publicBaseUrl}/${destination.prefix}/campaigns/${encodeURIComponent(campaignId)}/index.html`;+}diff --git a/tests/runner-e2e/history-publish.ts b/tests/runner-e2e/history-publish.tsindex db865998c2..f5dc913873 100644--- a/tests/runner-e2e/history-publish.ts+++ b/tests/runner-e2e/history-publish.ts@@ -19,6 +19,7 @@ import os from "node:os";import { writePublicCampaignSummaryImage } from "./public-summary-image.js";import { regenerateRunnerDashboard } from "./dashboard-regenerate.js";import { renderRunnerHistoryIndex } from "./history-index.js";+import { validateHistoryDestination } from "./history-destination.js";import { PUBLIC_RUNNER_SCREENSHOT_MARKER } from "./screenshot-policy.js";import {campaignHistoryRecord,@@ -259,39 +260,7 @@ function json(value: unknown) {return `${JSON.stringify(value, null, 2)}\n`;}-export function validateHistoryDestination(input: {- bucket: string;- prefix: string;- publicBaseUrl: string;-}) {- if (!/^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$/.test(input.bucket)) {- throw new Error("RUNNER_E2E_HISTORY_S3_BUCKET is not a valid bucket name");- }- const prefix = input.prefix.replace(/^\/+|\/+$/g, "");- if (- !prefix ||- prefix- .split("/")- .some((segment) => !segment || segment === "." || segment === "..")- ) {- throw new Error(- "RUNNER_E2E_HISTORY_PREFIX must be a safe non-empty key prefix",- );- }- const publicUrl = new URL(input.publicBaseUrl);- if (- publicUrl.protocol !== "https:" ||- publicUrl.username ||- publicUrl.password ||- publicUrl.search ||- publicUrl.hash- ) {- throw new Error(- "RUNNER_E2E_HISTORY_PUBLIC_BASE_URL must be a credential-free HTTPS URL",- );- }- return { prefix, publicBaseUrl: publicUrl.href.replace(/\/$/, "") };-}+export { validateHistoryDestination };async function relativeFiles(root: string,diff --git a/tests/runner-e2e/history.test.ts b/tests/runner-e2e/history.test.tsindex 32c6a2bc53..cec6d79128 100644--- a/tests/runner-e2e/history.test.ts+++ b/tests/runner-e2e/history.test.ts@@ -634,6 +634,13 @@ describe("historical publication security", () => {publicBaseUrl: "https://history.paperclip.ai/",}),).toThrow("safe non-empty key prefix");+ expect(() =>+ validateHistoryDestination({+ bucket: "paperclip-runner-e2e-history",+ prefix: "runner-e2e?other",+ publicBaseUrl: "https://history.paperclip.ai/",+ }),+ ).toThrow("safe non-empty key prefix");expect(() =>validateHistoryDestination({bucket: "paperclip-runner-e2e-history",diff --git a/tests/runner-e2e/report.test.ts b/tests/runner-e2e/report.test.tsindex 4ac6453cf0..f1e7b0e21f 100644--- a/tests/runner-e2e/report.test.ts+++ b/tests/runner-e2e/report.test.ts@@ -150,6 +150,9 @@ describe("runner E2E report aggregation", () => {GITHUB_SERVER_URL: "https://github.com",GITHUB_REPOSITORY: "paperclipai/paperclip",GITHUB_RUN_ID: "123456",+ PAPERCLIP_RUNNER_E2E_HISTORY_PUBLIC_BASE_URL:+ "https://reports.example.test/",+ PAPERCLIP_RUNNER_E2E_HISTORY_PREFIX: "/runner-e2e/",},},);@@ -203,6 +206,9 @@ describe("runner E2E report aggregation", () => {expect(dashboard).toContain("<img");expect(dashboard).toContain('class="brand-lockup"');expect(dashboard).toContain("data-gallery-dialog");+ expect(dashboard).toContain(+ `id="execution-core-compatibility.${executionId}"`,+ );expect(dashboard).toContain("data-gallery-previous");expect(dashboard).toContain("data-gallery-next");expect(dashboard).toContain("View gallery · 1");@@ -266,6 +272,20 @@ describe("runner E2E report aggregation", () => {expect(await readFile(path.join(output, "index.html"), "utf8")).toBe(dashboard,);+ const summary = await readFile(path.join(output, "summary.md"), "utf8");+ expect(summary).toContain("## View results");+ expect(summary).toContain(+ "[Open the exact interactive campaign report](https://reports.example.test/runner-e2e/campaigns/gha-123456-1/index.html)",+ );+ expect(summary).toContain(+ "[Open the workflow run and per-cell job logs](https://github.com/paperclipai/paperclip/actions/runs/123456)",+ );+ expect(summary).toContain(+ "[Download the merged report and per-cell evidence](https://github.com/paperclipai/paperclip/actions/runs/123456#artifacts)",+ );+ expect(summary).toContain(+ `[core-compatibility.${executionId}](https://reports.example.test/runner-e2e/campaigns/gha-123456-1/index.html#execution-core-compatibility.${executionId})`,+ );});it("prefers a valid rerun over a higher attempt number from an older campaign", async () => {diff --git a/tests/runner-e2e/report.ts b/tests/runner-e2e/report.tsindex cc58fe73d9..3b61eb5d14 100644--- a/tests/runner-e2e/report.ts+++ b/tests/runner-e2e/report.ts@@ -15,6 +15,7 @@ import {upgradeRunnerResult,} from "./history.js";import { resolveRunnerE2ESource } from "./source.js";+import { runnerE2ESummaryLinks } from "./summary-links.js";import type { RunnerE2EResult } from "./types.js";const repositoryRoot = path.resolve(import.meta.dirname, "../..");@@ -296,11 +297,33 @@ async function main() {writeFile(path.join(output, "index.html"), dashboard, "utf8"),]);+ const summaryLinks = runnerE2ESummaryLinks({+ campaignId: normalized.campaignId,+ workflowRunUrl: normalized.source.workflowRunUrl,+ historyPublicBaseUrl:+ process.env.PAPERCLIP_RUNNER_E2E_HISTORY_PUBLIC_BASE_URL,+ historyPrefix: process.env.PAPERCLIP_RUNNER_E2E_HISTORY_PREFIX,+ });+ const publicCampaignUrl = summaryLinks.find(+ (link) => link.kind === "campaign",+ )?.url;+const summaryLines = ["# Runner Full-Stack E2E","",`Passed: ${normalized.passed}/${selected.length}`,"",+ ...(summaryLinks.length > 0+ ? [+ "## View results",+ "",+ ...summaryLinks.map(+ (link) =>+ `- [${link.label}](${link.url})${link.note ? ` — ${link.note}` : ""}`,+ ),+ "",+ ]+ : []),`Tokens: ${billing.llm.inputTokens} input / ${billing.llm.outputTokens} output / ${billing.llm.cachedInputTokens} cached`,"",`Provider-reported LLM cost: $${billing.reportedLlmCostUsd.toFixed(6)} (${billing.llm.runsWithReportedCost}/${billing.llm.runCount} runs priced)`,@@ -314,7 +337,10 @@ async function main() {const resolved = resolvedResults[index]!;const cellBilling = resolved.billing!;const runtimeCost = cellBilling.runtime.estimatedListCostUsd;- return `| ${resolved.executionId} | ${resolved.attempt} | ${entry.valid ? "pass" : "fail"} | ${resolved.runtimeMode} | ${Math.round(resolved.durationMs / 1000)}s | ${cellBilling.llm.inputTokens}/${cellBilling.llm.outputTokens} | $${cellBilling.reportedCostUsd.toFixed(6)} (${cellBilling.llm.costStatus}) | ${runtimeCost === undefined ? cellBilling.runtime.costStatus : `$${runtimeCost.toFixed(6)} est.`} | ${detail} |`;+ const cell = publicCampaignUrl+ ? `[${resolved.executionId}](${publicCampaignUrl}#execution-${encodeURIComponent(resolved.executionId)})`+ : resolved.executionId;+ return `| ${cell} | ${resolved.attempt} | ${entry.valid ? "pass" : "fail"} | ${resolved.runtimeMode} | ${Math.round(resolved.durationMs / 1000)}s | ${cellBilling.llm.inputTokens}/${cellBilling.llm.outputTokens} | $${cellBilling.reportedCostUsd.toFixed(6)} (${cellBilling.llm.costStatus}) | ${runtimeCost === undefined ? cellBilling.runtime.costStatus : `$${runtimeCost.toFixed(6)} est.`} | ${detail} |`;}),"",];diff --git a/tests/runner-e2e/summary-links.test.ts b/tests/runner-e2e/summary-links.test.tsnew file mode 100644index 0000000000..9ab4185137--- /dev/null+++ b/tests/runner-e2e/summary-links.test.ts@@ -0,0 +1,61 @@+import { describe, expect, it } from "vitest";+import { runnerE2ESummaryLinks } from "./summary-links.js";++describe("runner E2E summary links", () => {+ it("builds exact public, workflow, and artifact links", () => {+ expect(+ runnerE2ESummaryLinks({+ campaignId: "gha-34026735033-1",+ workflowRunUrl:+ "https://github.com/paperclipai/paperclip/actions/runs/34026735033",+ historyPublicBaseUrl: "https://reports.example.test///",+ historyPrefix: "/runner-e2e/",+ }),+ ).toEqual([+ {+ kind: "campaign",+ label: "Open the exact interactive campaign report",+ url: "https://reports.example.test/runner-e2e/campaigns/gha-34026735033-1/index.html",+ note: "available after the history publisher finishes",+ },+ {+ kind: "workflow",+ label: "Open the workflow run and per-cell job logs",+ url: "https://github.com/paperclipai/paperclip/actions/runs/34026735033",+ },+ {+ kind: "artifacts",+ label: "Download the merged report and per-cell evidence",+ url: "https://github.com/paperclipai/paperclip/actions/runs/34026735033#artifacts",+ note: "GitHub access required; retained for 30 days",+ },+ ]);+ });++ it("omits unsafe or incomplete destinations", () => {+ expect(+ runnerE2ESummaryLinks({+ campaignId: "../other-campaign",+ workflowRunUrl: "https://[email protected]/actions/runs/1",+ historyPublicBaseUrl: "http://reports.example.test",+ historyPrefix: "runner-e2e",+ }),+ ).toEqual([]);+ expect(+ runnerE2ESummaryLinks({+ campaignId: "gha-1-1",+ workflowRunUrl: null,+ historyPublicBaseUrl: null,+ historyPrefix: null,+ }),+ ).toEqual([]);+ expect(+ runnerE2ESummaryLinks({+ campaignId: "gha-1-1",+ workflowRunUrl: null,+ historyPublicBaseUrl: "https://reports.example.test",+ historyPrefix: "runner-e2e#other",+ }),+ ).toEqual([]);+ });+});diff --git a/tests/runner-e2e/summary-links.ts b/tests/runner-e2e/summary-links.tsnew file mode 100644index 0000000000..b69d5b8150--- /dev/null+++ b/tests/runner-e2e/summary-links.ts@@ -0,0 +1,73 @@+import {+ runnerE2ECampaignPublicUrl,+ validateHistoryPublicDestination,+} from "./history-destination.js";++export interface RunnerE2ESummaryLink {+ kind: "campaign" | "workflow" | "artifacts";+ label: string;+ url: string;+ note?: string;+}++function safeHttpsUrl(value: string | null | undefined) {+ const input = value?.trim();+ if (!input) return null;+ try {+ const url = new URL(input);+ if (+ url.protocol !== "https:" ||+ url.username ||+ url.password ||+ url.search ||+ url.hash+ ) {+ return null;+ }+ return url;+ } catch {+ return null;+ }+}++export function runnerE2ESummaryLinks(input: {+ campaignId: string;+ workflowRunUrl: string | null | undefined;+ historyPublicBaseUrl: string | null | undefined;+ historyPrefix: string | null | undefined;+}): RunnerE2ESummaryLink[] {+ const links: RunnerE2ESummaryLink[] = [];+ if (/^[A-Za-z0-9][A-Za-z0-9._-]{0,199}$/.test(input.campaignId)) {+ try {+ const destination = validateHistoryPublicDestination({+ publicBaseUrl: input.historyPublicBaseUrl?.trim() ?? "",+ prefix: input.historyPrefix?.trim() ?? "",+ });+ links.push({+ kind: "campaign",+ label: "Open the exact interactive campaign report",+ url: runnerE2ECampaignPublicUrl(destination, input.campaignId),+ note: "available after the history publisher finishes",+ });+ } catch {+ // Invalid or absent history configuration must not prevent report merging.+ }+ }++ const workflowRun = safeHttpsUrl(input.workflowRunUrl);+ if (workflowRun) {+ links.push({+ kind: "workflow",+ label: "Open the workflow run and per-cell job logs",+ url: workflowRun.href,+ });+ workflowRun.hash = "artifacts";+ links.push({+ kind: "artifacts",+ label: "Download the merged report and per-cell evidence",+ url: workflowRun.href,+ note: "GitHub access required; retained for 30 days",+ });+ }+ return links;+}diff --git a/tests/runner-e2e/workflow-security.test.ts b/tests/runner-e2e/workflow-security.test.tsindex 03a68f6af8..4dce7c374a 100644--- a/tests/runner-e2e/workflow-security.test.ts+++ b/tests/runner-e2e/workflow-security.test.ts@@ -671,6 +671,12 @@ describe("public repository paid workflow security", () => {expect(report).toContain("PAPERCLIP_RUNNER_E2E_REPORT_ROOT: ${{ github.workspace }}/selected-runner-e2e",);+ expect(report).toContain(+ "PAPERCLIP_RUNNER_E2E_HISTORY_PUBLIC_BASE_URL: ${{ vars.RUNNER_E2E_HISTORY_PUBLIC_BASE_URL }}",+ );+ expect(report).toContain(+ "PAPERCLIP_RUNNER_E2E_HISTORY_PREFIX: ${{ vars.RUNNER_E2E_HISTORY_PREFIX || 'runner-e2e' }}",+ );expect(report.indexOf("Select latest workflow attempt per cell"),).toBeLessThan(report.indexOf("Collect blob reports"));