Skip to content

Automate browser based workflows with AI

Show the test-run count on the copilot not-confirmed checklist stub (#7659)

Celal Zamanoğlud49aa7635c2d
diff --git a/skyvern-frontend/src/routes/workflows/copilot/copilotPhases.test.ts b/skyvern-frontend/src/routes/workflows/copilot/copilotPhases.test.ts
index 7b61e0fbd..6d6aae453 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/copilotPhases.test.ts
+++ b/skyvern-frontend/src/routes/workflows/copilot/copilotPhases.test.ts
@@ -597,3 +597,32 @@ describe("derivePhases — redraft re-activation (SKY-11970 pin)", () => {
expect(phase(rows, "draft").status).not.toBe("active");
});
});
+
+describe("derivePhases — test-run count on the not-confirmed stub (SKY-11339)", () => {
+ const notConfirmedTurn = (runToolNames: string[]): TurnNarrativeState =>
+ turn({
+ terminal: "response",
+ designEnded: true,
+ draft: { blockCount: 1, blockLabels: ["block_1"], summary: null },
+ blocks: [block({ state: "completed", outcome: "not_demonstrated" })],
+ designActivity: runToolNames.map((toolName, i) =>
+ entry({ id: `r${i}`, kind: "tool_call", toolName }),
+ ),
+ });
+
+ it("surfaces the run count on a multi-run redraft loop (else 6 runs read as 1)", () => {
+ const rows = derivePhases(
+ notConfirmedTurn([
+ "update_and_run_blocks",
+ "update_and_run_blocks",
+ "run_blocks_and_collect_debug",
+ ]),
+ );
+ expect(phase(rows, "test").stub).toBe("3 runs · not confirmed");
+ });
+
+ it("omits the count for a single run, keeping today's look", () => {
+ const rows = derivePhases(notConfirmedTurn(["update_and_run_blocks"]));
+ expect(phase(rows, "test").stub).toBe("· not confirmed");
+ });
+});
diff --git a/skyvern-frontend/src/routes/workflows/copilot/copilotPhases.ts b/skyvern-frontend/src/routes/workflows/copilot/copilotPhases.ts
index 12cc62ef1..7c2a5e04c 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/copilotPhases.ts
+++ b/skyvern-frontend/src/routes/workflows/copilot/copilotPhases.ts
@@ -137,13 +137,20 @@ function lastAuthoringToolName(
return undefined;
}
-function countAuthoringToolCalls(designActivity: ActivityEntry[]): number {
+// Raw designActivity count of tool_calls in `tools` — drives the Draft
+// "N drafts" (AUTHORING_TOOLS) and Test-run "N runs" (RUN_TOOLS) stubs, since
+// each update_workflow / update_and_run_blocks / run_blocks_and_collect_debug
+// call lands in designActivity.
+function countToolCalls(
+ designActivity: ActivityEntry[],
+ tools: Set<string>,
+): number {
let count = 0;
for (const entry of designActivity) {
if (
entry.kind === "tool_call" &&
entry.toolName &&
- AUTHORING_TOOLS.has(entry.toolName)
+ tools.has(entry.toolName)
) {
count += 1;
}
@@ -301,7 +308,7 @@ export function derivePhases(turn: TurnNarrativeState): PhaseRowModel[] {
// count: it spans update_workflow (draft bucket) AND
// update_and_run_blocks (test bucket), so there's no single
// condensed bucket to count against the way the explore stub does.
- const drafts = countAuthoringToolCalls(turn.designActivity);
+ const drafts = countToolCalls(turn.designActivity, AUTHORING_TOOLS);
return drafts >= 2 ? `${base} · ${drafts} drafts` : base;
}
// test
@@ -314,7 +321,15 @@ export function derivePhases(turn: TurnNarrativeState): PhaseRowModel[] {
? `${pluralize(latestBlocks.length, "block")} · stopped`
: "stopped";
}
- if (anyNotDemonstrated) return "· not confirmed";
+ if (anyNotDemonstrated) {
+ // Surface how many test runs the redraft loop actually made — otherwise a
+ // 6-run "not confirmed" turn looks identical to a 1-run one. Omit at 1 to
+ // keep today's look (meaningful-or-nothing).
+ const runs = countToolCalls(turn.designActivity, RUN_TOOLS);
+ return runs >= 2
+ ? `${pluralize(runs, "run")} · not confirmed`
+ : "· not confirmed";
+ }
const { start, end } = spanIso(latestBlocks);
const elapsed = formatElapsed(start, end);
return elapsed