Skip to content

The open-source app everyone uses to manage agents at work

test(e2e): link runner campaign summaries (#12927)

Dotta9ecd93a54dad
diff --git a/.github/workflows/runner-full-stack-e2e.yml b/.github/workflows/runner-full-stack-e2e.yml
index 3179f5c8e1..52b069de93 100644
--- a/.github/workflows/runner-full-stack-e2e.yml
+++ b/.github/workflows/runner-full-stack-e2e.yml
@@ -1102,6 +1102,8 @@ jobs:
PAPERCLIP_E2E_CAMPAIGN_ID: gha-${{ github.run_id }}-${{ github.run_attempt }}
PAPERCLIP_RUNNER_E2E_SOURCE_SHA: ${{ needs.authorize.outputs.target_sha }}
PAPERCLIP_RUNNER_E2E_SOURCE_REF: ${{ needs.authorize.outputs.target_ref }}
+ PAPERCLIP_RUNNER_E2E_HISTORY_PUBLIC_BASE_URL: ${{ vars.RUNNER_E2E_HISTORY_PUBLIC_BASE_URL }}
+ PAPERCLIP_RUNNER_E2E_HISTORY_PREFIX: ${{ vars.RUNNER_E2E_HISTORY_PREFIX || 'runner-e2e' }}
run: |
set +e
pnpm test:e2e:runner:report
diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md
index b9ffb1dd6c..c2e29273fe 100644
--- a/tests/runner-e2e/README.md
+++ b/tests/runner-e2e/README.md
@@ -266,6 +266,11 @@ same metrics per campaign/suite/execution, source SHA/ref, definition
fingerprints, completeness, retries, and cleanup. Trend charts compare only
complete campaigns by default; partial/manual selections remain browsable.
`summary.md` carries the current totals into the GitHub Actions job summary.
+In CI, its **View results** section links to the exact immutable public campaign
+report, the workflow and per-cell logs, and the access-controlled report
+artifacts. Each cell name links to its exact section in the campaign report.
+The public campaign links become available after the history publisher
+finishes. The artifact links remain available for 30 days.
### Iterate on a published dashboard without rerunning paid tests
diff --git a/tests/runner-e2e/dashboard.ts b/tests/runner-e2e/dashboard.ts
index 6fe3f8d782..06a72bd9cb 100644
--- a/tests/runner-e2e/dashboard.ts
+++ b/tests/runner-e2e/dashboard.ts
@@ -247,6 +247,7 @@ function renderCase(
</div>`
: "";
return `<article
+ id="execution-${html(execution.id)}"
class="case case-${state}"
data-execution-id="${html(execution.id)}"
data-report-case
diff --git a/tests/runner-e2e/history-destination.ts b/tests/runner-e2e/history-destination.ts
new file mode 100644
index 0000000000..7e1f60d655
--- /dev/null
+++ b/tests/runner-e2e/history-destination.ts
@@ -0,0 +1,60 @@
+export interface RunnerE2EHistoryPublicDestination {
+ prefix: string;
+ publicBaseUrl: string;
+}
+
+export function validateHistoryPublicDestination(input: {
+ prefix: string;
+ publicBaseUrl: string;
+}): RunnerE2EHistoryPublicDestination {
+ const prefix = input.prefix.replace(/^\/+|\/+$/g, "");
+ const segments = prefix.split("/");
+ if (
+ !prefix ||
+ segments.some(
+ (segment) =>
+ !segment ||
+ segment === "." ||
+ segment === ".." ||
+ !/^[A-Za-z0-9._~-]+$/.test(segment),
+ )
+ ) {
+ throw new Error(
+ "RUNNER_E2E_HISTORY_PREFIX must be a safe non-empty key prefix",
+ );
+ }
+ const publicUrl = new URL(input.publicBaseUrl);
+ if (
+ publicUrl.protocol !== "https:" ||
+ publicUrl.username ||
+ publicUrl.password ||
+ publicUrl.search ||
+ publicUrl.hash
+ ) {
+ throw new Error(
+ "RUNNER_E2E_HISTORY_PUBLIC_BASE_URL must be a credential-free HTTPS URL",
+ );
+ }
+ return {
+ prefix,
+ publicBaseUrl: publicUrl.href.replace(/\/+$/, ""),
+ };
+}
+
+export function validateHistoryDestination(input: {
+ bucket: string;
+ prefix: string;
+ publicBaseUrl: string;
+}) {
+ if (!/^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$/.test(input.bucket)) {
+ throw new Error("RUNNER_E2E_HISTORY_S3_BUCKET is not a valid bucket name");
+ }
+ return validateHistoryPublicDestination(input);
+}
+
+export function runnerE2ECampaignPublicUrl(
+ destination: RunnerE2EHistoryPublicDestination,
+ campaignId: string,
+) {
+ return `${destination.publicBaseUrl}/${destination.prefix}/campaigns/${encodeURIComponent(campaignId)}/index.html`;
+}
diff --git a/tests/runner-e2e/history-publish.ts b/tests/runner-e2e/history-publish.ts
index db865998c2..f5dc913873 100644
--- a/tests/runner-e2e/history-publish.ts
+++ b/tests/runner-e2e/history-publish.ts
@@ -19,6 +19,7 @@ import os from "node:os";
import { writePublicCampaignSummaryImage } from "./public-summary-image.js";
import { regenerateRunnerDashboard } from "./dashboard-regenerate.js";
import { renderRunnerHistoryIndex } from "./history-index.js";
+import { validateHistoryDestination } from "./history-destination.js";
import { PUBLIC_RUNNER_SCREENSHOT_MARKER } from "./screenshot-policy.js";
import {
campaignHistoryRecord,
@@ -259,39 +260,7 @@ function json(value: unknown) {
return `${JSON.stringify(value, null, 2)}\n`;
}
-export function validateHistoryDestination(input: {
- bucket: string;
- prefix: string;
- publicBaseUrl: string;
-}) {
- if (!/^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$/.test(input.bucket)) {
- throw new Error("RUNNER_E2E_HISTORY_S3_BUCKET is not a valid bucket name");
- }
- const prefix = input.prefix.replace(/^\/+|\/+$/g, "");
- if (
- !prefix ||
- prefix
- .split("/")
- .some((segment) => !segment || segment === "." || segment === "..")
- ) {
- throw new Error(
- "RUNNER_E2E_HISTORY_PREFIX must be a safe non-empty key prefix",
- );
- }
- const publicUrl = new URL(input.publicBaseUrl);
- if (
- publicUrl.protocol !== "https:" ||
- publicUrl.username ||
- publicUrl.password ||
- publicUrl.search ||
- publicUrl.hash
- ) {
- throw new Error(
- "RUNNER_E2E_HISTORY_PUBLIC_BASE_URL must be a credential-free HTTPS URL",
- );
- }
- return { prefix, publicBaseUrl: publicUrl.href.replace(/\/$/, "") };
-}
+export { validateHistoryDestination };
async function relativeFiles(
root: string,
diff --git a/tests/runner-e2e/history.test.ts b/tests/runner-e2e/history.test.ts
index 32c6a2bc53..cec6d79128 100644
--- a/tests/runner-e2e/history.test.ts
+++ b/tests/runner-e2e/history.test.ts
@@ -634,6 +634,13 @@ describe("historical publication security", () => {
publicBaseUrl: "https://history.paperclip.ai/",
}),
).toThrow("safe non-empty key prefix");
+ expect(() =>
+ validateHistoryDestination({
+ bucket: "paperclip-runner-e2e-history",
+ prefix: "runner-e2e?other",
+ publicBaseUrl: "https://history.paperclip.ai/",
+ }),
+ ).toThrow("safe non-empty key prefix");
expect(() =>
validateHistoryDestination({
bucket: "paperclip-runner-e2e-history",
diff --git a/tests/runner-e2e/report.test.ts b/tests/runner-e2e/report.test.ts
index 4ac6453cf0..f1e7b0e21f 100644
--- a/tests/runner-e2e/report.test.ts
+++ b/tests/runner-e2e/report.test.ts
@@ -150,6 +150,9 @@ describe("runner E2E report aggregation", () => {
GITHUB_SERVER_URL: "https://github.com",
GITHUB_REPOSITORY: "paperclipai/paperclip",
GITHUB_RUN_ID: "123456",
+ PAPERCLIP_RUNNER_E2E_HISTORY_PUBLIC_BASE_URL:
+ "https://reports.example.test/",
+ PAPERCLIP_RUNNER_E2E_HISTORY_PREFIX: "/runner-e2e/",
},
},
);
@@ -203,6 +206,9 @@ describe("runner E2E report aggregation", () => {
expect(dashboard).toContain("<img");
expect(dashboard).toContain('class="brand-lockup"');
expect(dashboard).toContain("data-gallery-dialog");
+ expect(dashboard).toContain(
+ `id="execution-core-compatibility.${executionId}"`,
+ );
expect(dashboard).toContain("data-gallery-previous");
expect(dashboard).toContain("data-gallery-next");
expect(dashboard).toContain("View gallery · 1");
@@ -266,6 +272,20 @@ describe("runner E2E report aggregation", () => {
expect(await readFile(path.join(output, "index.html"), "utf8")).toBe(
dashboard,
);
+ const summary = await readFile(path.join(output, "summary.md"), "utf8");
+ expect(summary).toContain("## View results");
+ expect(summary).toContain(
+ "[Open the exact interactive campaign report](https://reports.example.test/runner-e2e/campaigns/gha-123456-1/index.html)",
+ );
+ expect(summary).toContain(
+ "[Open the workflow run and per-cell job logs](https://github.com/paperclipai/paperclip/actions/runs/123456)",
+ );
+ expect(summary).toContain(
+ "[Download the merged report and per-cell evidence](https://github.com/paperclipai/paperclip/actions/runs/123456#artifacts)",
+ );
+ expect(summary).toContain(
+ `[core-compatibility.${executionId}](https://reports.example.test/runner-e2e/campaigns/gha-123456-1/index.html#execution-core-compatibility.${executionId})`,
+ );
});
it("prefers a valid rerun over a higher attempt number from an older campaign", async () => {
diff --git a/tests/runner-e2e/report.ts b/tests/runner-e2e/report.ts
index cc58fe73d9..3b61eb5d14 100644
--- a/tests/runner-e2e/report.ts
+++ b/tests/runner-e2e/report.ts
@@ -15,6 +15,7 @@ import {
upgradeRunnerResult,
} from "./history.js";
import { resolveRunnerE2ESource } from "./source.js";
+import { runnerE2ESummaryLinks } from "./summary-links.js";
import type { RunnerE2EResult } from "./types.js";
const repositoryRoot = path.resolve(import.meta.dirname, "../..");
@@ -296,11 +297,33 @@ async function main() {
writeFile(path.join(output, "index.html"), dashboard, "utf8"),
]);
+ const summaryLinks = runnerE2ESummaryLinks({
+ campaignId: normalized.campaignId,
+ workflowRunUrl: normalized.source.workflowRunUrl,
+ historyPublicBaseUrl:
+ process.env.PAPERCLIP_RUNNER_E2E_HISTORY_PUBLIC_BASE_URL,
+ historyPrefix: process.env.PAPERCLIP_RUNNER_E2E_HISTORY_PREFIX,
+ });
+ const publicCampaignUrl = summaryLinks.find(
+ (link) => link.kind === "campaign",
+ )?.url;
+
const summaryLines = [
"# Runner Full-Stack E2E",
"",
`Passed: ${normalized.passed}/${selected.length}`,
"",
+ ...(summaryLinks.length > 0
+ ? [
+ "## View results",
+ "",
+ ...summaryLinks.map(
+ (link) =>
+ `- [${link.label}](${link.url})${link.note ? ` — ${link.note}` : ""}`,
+ ),
+ "",
+ ]
+ : []),
`Tokens: ${billing.llm.inputTokens} input / ${billing.llm.outputTokens} output / ${billing.llm.cachedInputTokens} cached`,
"",
`Provider-reported LLM cost: $${billing.reportedLlmCostUsd.toFixed(6)} (${billing.llm.runsWithReportedCost}/${billing.llm.runCount} runs priced)`,
@@ -314,7 +337,10 @@ async function main() {
const resolved = resolvedResults[index]!;
const cellBilling = resolved.billing!;
const runtimeCost = cellBilling.runtime.estimatedListCostUsd;
- return `| ${resolved.executionId} | ${resolved.attempt} | ${entry.valid ? "pass" : "fail"} | ${resolved.runtimeMode} | ${Math.round(resolved.durationMs / 1000)}s | ${cellBilling.llm.inputTokens}/${cellBilling.llm.outputTokens} | $${cellBilling.reportedCostUsd.toFixed(6)} (${cellBilling.llm.costStatus}) | ${runtimeCost === undefined ? cellBilling.runtime.costStatus : `$${runtimeCost.toFixed(6)} est.`} | ${detail} |`;
+ const cell = publicCampaignUrl
+ ? `[${resolved.executionId}](${publicCampaignUrl}#execution-${encodeURIComponent(resolved.executionId)})`
+ : resolved.executionId;
+ return `| ${cell} | ${resolved.attempt} | ${entry.valid ? "pass" : "fail"} | ${resolved.runtimeMode} | ${Math.round(resolved.durationMs / 1000)}s | ${cellBilling.llm.inputTokens}/${cellBilling.llm.outputTokens} | $${cellBilling.reportedCostUsd.toFixed(6)} (${cellBilling.llm.costStatus}) | ${runtimeCost === undefined ? cellBilling.runtime.costStatus : `$${runtimeCost.toFixed(6)} est.`} | ${detail} |`;
}),
"",
];
diff --git a/tests/runner-e2e/summary-links.test.ts b/tests/runner-e2e/summary-links.test.ts
new file mode 100644
index 0000000000..9ab4185137
--- /dev/null
+++ b/tests/runner-e2e/summary-links.test.ts
@@ -0,0 +1,61 @@
+import { describe, expect, it } from "vitest";
+import { runnerE2ESummaryLinks } from "./summary-links.js";
+
+describe("runner E2E summary links", () => {
+ it("builds exact public, workflow, and artifact links", () => {
+ expect(
+ runnerE2ESummaryLinks({
+ campaignId: "gha-34026735033-1",
+ workflowRunUrl:
+ "https://github.com/paperclipai/paperclip/actions/runs/34026735033",
+ historyPublicBaseUrl: "https://reports.example.test///",
+ historyPrefix: "/runner-e2e/",
+ }),
+ ).toEqual([
+ {
+ kind: "campaign",
+ label: "Open the exact interactive campaign report",
+ url: "https://reports.example.test/runner-e2e/campaigns/gha-34026735033-1/index.html",
+ note: "available after the history publisher finishes",
+ },
+ {
+ kind: "workflow",
+ label: "Open the workflow run and per-cell job logs",
+ url: "https://github.com/paperclipai/paperclip/actions/runs/34026735033",
+ },
+ {
+ kind: "artifacts",
+ label: "Download the merged report and per-cell evidence",
+ url: "https://github.com/paperclipai/paperclip/actions/runs/34026735033#artifacts",
+ note: "GitHub access required; retained for 30 days",
+ },
+ ]);
+ });
+
+ it("omits unsafe or incomplete destinations", () => {
+ expect(
+ runnerE2ESummaryLinks({
+ campaignId: "../other-campaign",
+ workflowRunUrl: "https://[email protected]/actions/runs/1",
+ historyPublicBaseUrl: "http://reports.example.test",
+ historyPrefix: "runner-e2e",
+ }),
+ ).toEqual([]);
+ expect(
+ runnerE2ESummaryLinks({
+ campaignId: "gha-1-1",
+ workflowRunUrl: null,
+ historyPublicBaseUrl: null,
+ historyPrefix: null,
+ }),
+ ).toEqual([]);
+ expect(
+ runnerE2ESummaryLinks({
+ campaignId: "gha-1-1",
+ workflowRunUrl: null,
+ historyPublicBaseUrl: "https://reports.example.test",
+ historyPrefix: "runner-e2e#other",
+ }),
+ ).toEqual([]);
+ });
+});
diff --git a/tests/runner-e2e/summary-links.ts b/tests/runner-e2e/summary-links.ts
new file mode 100644
index 0000000000..b69d5b8150
--- /dev/null
+++ b/tests/runner-e2e/summary-links.ts
@@ -0,0 +1,73 @@
+import {
+ runnerE2ECampaignPublicUrl,
+ validateHistoryPublicDestination,
+} from "./history-destination.js";
+
+export interface RunnerE2ESummaryLink {
+ kind: "campaign" | "workflow" | "artifacts";
+ label: string;
+ url: string;
+ note?: string;
+}
+
+function safeHttpsUrl(value: string | null | undefined) {
+ const input = value?.trim();
+ if (!input) return null;
+ try {
+ const url = new URL(input);
+ if (
+ url.protocol !== "https:" ||
+ url.username ||
+ url.password ||
+ url.search ||
+ url.hash
+ ) {
+ return null;
+ }
+ return url;
+ } catch {
+ return null;
+ }
+}
+
+export function runnerE2ESummaryLinks(input: {
+ campaignId: string;
+ workflowRunUrl: string | null | undefined;
+ historyPublicBaseUrl: string | null | undefined;
+ historyPrefix: string | null | undefined;
+}): RunnerE2ESummaryLink[] {
+ const links: RunnerE2ESummaryLink[] = [];
+ if (/^[A-Za-z0-9][A-Za-z0-9._-]{0,199}$/.test(input.campaignId)) {
+ try {
+ const destination = validateHistoryPublicDestination({
+ publicBaseUrl: input.historyPublicBaseUrl?.trim() ?? "",
+ prefix: input.historyPrefix?.trim() ?? "",
+ });
+ links.push({
+ kind: "campaign",
+ label: "Open the exact interactive campaign report",
+ url: runnerE2ECampaignPublicUrl(destination, input.campaignId),
+ note: "available after the history publisher finishes",
+ });
+ } catch {
+ // Invalid or absent history configuration must not prevent report merging.
+ }
+ }
+
+ const workflowRun = safeHttpsUrl(input.workflowRunUrl);
+ if (workflowRun) {
+ links.push({
+ kind: "workflow",
+ label: "Open the workflow run and per-cell job logs",
+ url: workflowRun.href,
+ });
+ workflowRun.hash = "artifacts";
+ links.push({
+ kind: "artifacts",
+ label: "Download the merged report and per-cell evidence",
+ url: workflowRun.href,
+ note: "GitHub access required; retained for 30 days",
+ });
+ }
+ return links;
+}
diff --git a/tests/runner-e2e/workflow-security.test.ts b/tests/runner-e2e/workflow-security.test.ts
index 03a68f6af8..4dce7c374a 100644
--- a/tests/runner-e2e/workflow-security.test.ts
+++ b/tests/runner-e2e/workflow-security.test.ts
@@ -671,6 +671,12 @@ describe("public repository paid workflow security", () => {
expect(report).toContain(
"PAPERCLIP_RUNNER_E2E_REPORT_ROOT: ${{ github.workspace }}/selected-runner-e2e",
);
+ expect(report).toContain(
+ "PAPERCLIP_RUNNER_E2E_HISTORY_PUBLIC_BASE_URL: ${{ vars.RUNNER_E2E_HISTORY_PUBLIC_BASE_URL }}",
+ );
+ expect(report).toContain(
+ "PAPERCLIP_RUNNER_E2E_HISTORY_PREFIX: ${{ vars.RUNNER_E2E_HISTORY_PREFIX || 'runner-e2e' }}",
+ );
expect(
report.indexOf("Select latest workflow attempt per cell"),
).toBeLessThan(report.indexOf("Collect blob reports"));