Skip to content

armgit / kaneo

Public Imported from GitHub
0

⚡ Project management that doesn't become the project. Boards, Gantt charts, and GitHub integration. Self-host it or use Kaneo Cloud.

fix: ground Peekareq custom-field screenshots in fixtures

Tin4b684e794410
diff --git a/scripts/ui-review-bot/README.md b/scripts/ui-review-bot/README.md
index 9bea89972..9101dc106 100644
--- a/scripts/ui-review-bot/README.md
+++ b/scripts/ui-review-bot/README.md
@@ -44,7 +44,7 @@ Local reports, before/after PNGs, and pixel differences are saved in `.cache/ui-
## Coverage and isolation
-The current fixture adapter supports account settings, a synthetic workspace/project/task, and stateful time-entry Start/Stop controls. Each scenario gets its own fixture state. The capture-only preset is specific to PR #1719. Other features may require fixtures; unknown API reads produce incomplete captures. Screenshots use a 1440×1000 desktop viewport, light theme, English, UTC, and disabled animations. This tests frontend appearance and interactions, not real backend behavior.
+The current fixture adapter supports account settings, a synthetic workspace/project/task, stateful time-entry Start/Stop controls, and custom-field values. Multiselect changes use fixture-backed scenarios for the task overview, option picker, and multiple selected values instead of asking the model to invent controls. Marketing-site and docs-only changes stop during planning because the preview currently runs `apps/web` only. Each scenario gets its own fixture state. The capture-only preset is specific to PR #1719. Other features may require fixtures; unknown API reads produce incomplete captures. Screenshots use a 1440×1000 desktop viewport, light theme, English, UTC, and disabled animations. This tests frontend appearance and interactions, not real backend behavior.
Before images use the PR's merge base. Different routes or interaction states are labeled in the local report rather than treated as regression scores. Each revision uses its own frozen lockfile with install scripts disabled. Browser traffic is restricted to the local preview and synthetic API responses.
diff --git a/scripts/ui-review-bot/core.mjs b/scripts/ui-review-bot/core.mjs
index 6b4910743..18da18f44 100644
--- a/scripts/ui-review-bot/core.mjs
+++ b/scripts/ui-review-bot/core.mjs
@@ -1,5 +1,6 @@
import { execFile } from "node:child_process";
import path from "node:path";
+import { setTimeout as delay } from "node:timers/promises";
import { fileURLToPath } from "node:url";
import { promisify } from "node:util";
@@ -195,12 +196,12 @@ export async function completion({
signal,
maxTokens = 2500,
}) {
- if (run.calls >= 5)
- throw new Error("The five-call limit for this run was reached.");
- run.calls++;
- const response = await fetch(
- "https://openrouter.ai/api/v1/chat/completions",
- {
+ let response;
+ for (let attempt = 0; attempt < 3; attempt++) {
+ if (run.calls >= 5)
+ throw new Error("The five-call limit for this run was reached.");
+ run.calls++;
+ response = await fetch("https://openrouter.ai/api/v1/chat/completions", {
method: "POST",
signal: AbortSignal.any([signal, AbortSignal.timeout(120_000)]),
headers: {
@@ -217,8 +218,26 @@ export async function completion({
response_format: { type: "json_object" },
provider: { require_parameters: true },
}),
- },
- );
+ });
+ if (
+ ![429, 502, 503, 504].includes(response.status) ||
+ attempt === 2 ||
+ run.calls >= 5
+ )
+ break;
+ const retryAfter = response.headers?.get("retry-after");
+ const seconds =
+ retryAfter === null || retryAfter === undefined
+ ? Number.NaN
+ : Number(retryAfter);
+ const wait =
+ Number.isFinite(seconds) && seconds >= 0
+ ? seconds * 1000
+ : 2000 * 2 ** attempt;
+ if (wait > 30000) break;
+ await response.body?.cancel();
+ await delay(wait, undefined, { signal });
+ }
const data = await response.json();
if (!response.ok || data.error) {
const code = data.error?.code || response.status;
diff --git a/scripts/ui-review-bot/fixtures.mjs b/scripts/ui-review-bot/fixtures.mjs
index e56adf317..cfbe5ecfa 100644
--- a/scripts/ui-review-bot/fixtures.mjs
+++ b/scripts/ui-review-bot/fixtures.mjs
@@ -81,7 +81,32 @@ const columns = [
},
];
-export function createFixtureSession() {
+export function createFixtureSession({
+ customFields = false,
+ multiselect = false,
+} = {}) {
+ const field = {
+ id: "ui-review-audience",
+ projectId: project.id,
+ name: "Audience",
+ type: multiselect ? "multiselect" : "dropdown",
+ required: false,
+ defaultValue: null,
+ options: ["Design", "Engineering", "Product"],
+ position: 0,
+ createdAt: date,
+ updatedAt: date,
+ };
+ const fieldValue = {
+ id: "ui-review-field-value",
+ taskId: task.id,
+ fieldId: field.id,
+ value: multiselect ? JSON.stringify(["Design"]) : "Design",
+ fieldName: field.name,
+ fieldType: field.type,
+ fieldPosition: 0,
+ fieldOptions: field.options,
+ };
const entries = [
{
id: "ui-review-time-entry",
@@ -97,6 +122,18 @@ export function createFixtureSession() {
];
return (url, method, body = {}) => {
const p = new URL(url).pathname;
+ if (customFields) {
+ if (p === `/api/custom-field/project/${project.id}` && method === "GET")
+ return structuredClone([field]);
+ if (p === `/api/custom-field/task/${task.id}` && method === "GET")
+ return structuredClone([fieldValue]);
+ if (p === "/api/custom-field/value" && method === "PUT") {
+ if (body.taskId !== task.id || body.fieldId !== field.id)
+ return undefined;
+ fieldValue.value = body.value;
+ return structuredClone(fieldValue);
+ }
+ }
if (p === `/api/time-entry/task/${task.id}` && method === "GET")
return entries.map((entry) => ({ ...entry }));
if (
@@ -219,8 +256,8 @@ export function fixture(url, method) {
return undefined;
}
-export async function installFixtures(context, origin, diagnostics) {
- const sessionFixture = createFixtureSession();
+export async function installFixtures(context, origin, diagnostics, options) {
+ const sessionFixture = createFixtureSession(options);
await context.addInitScript(() => {
localStorage.setItem("theme", "light");
localStorage.setItem("vite-ui-theme", "light");
diff --git a/scripts/ui-review-bot/runner.mjs b/scripts/ui-review-bot/runner.mjs
index 2996fef76..7ab3b9539 100644
--- a/scripts/ui-review-bot/runner.mjs
+++ b/scripts/ui-review-bot/runner.mjs
@@ -26,6 +26,7 @@ import {
import { installFixtures } from "./fixtures.mjs";
import { readScreenshot } from "./images.mjs";
import { ATTRIBUTION } from "./publish.mjs";
+import { checkSupportedSurface, customFieldPlan } from "./scenarios.mjs";
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
const exists = (p) =>
@@ -229,7 +230,15 @@ function locator(page, a) {
return page.getByText(a.name, { exact: true }).first();
}
-export async function capture(browser, origin, scenario, side, file, signal) {
+export async function capture(
+ browser,
+ origin,
+ scenario,
+ side,
+ file,
+ signal,
+ fixtures,
+) {
const context = await browser.newContext({
viewport: { width: 1440, height: 1000 },
deviceScaleFactor: 1,
@@ -249,7 +258,7 @@ export async function capture(browser, origin, scenario, side, file, signal) {
const close = () => context.close().catch(() => {});
signal.addEventListener("abort", close, { once: true });
try {
- await installFixtures(context, origin, diagnostics);
+ await installFixtures(context, origin, diagnostics, fixtures);
const page = await context.newPage();
page.on("pageerror", (e) =>
diagnostics.errors.push(e.message.slice(0, 700)),
@@ -402,6 +411,7 @@ export async function planRun(run, token, signal, log) {
throw new Error(
"Capture-only preset is prepared for PR #1719. Use AI review for another UI PR.",
);
+ checkSupportedSurface(run.pr.files);
const revisions = await fetchRevisions(run.pr, log, signal);
const snapshots = {
before: { sha: revisions.before },
@@ -416,6 +426,25 @@ export async function planRun(run, token, signal, log) {
"Each revision uses its locked dependencies with install scripts disabled.",
"Coverage is limited to the selected scenarios at a 1440 × 1000 desktop viewport.",
];
+ const taskDetailsPath =
+ "apps/web/src/components/task/task-details-content.tsx";
+ if (
+ run.pr.files.some((file) =>
+ /custom-field|task-details-content/.test(file.path),
+ )
+ ) {
+ const source = await command(
+ "git",
+ ["show", `${revisions.after}:${taskDetailsPath}`],
+ { signal },
+ );
+ if (source.includes('field.type === "multiselect"')) {
+ run.fixtureProfile = "custom-fields";
+ run.plan = customFieldPlan(source.includes("<Accordion"));
+ log("plan", "Using fixture-backed custom-field interaction scenarios");
+ return;
+ }
+ }
const context = await planningContext(run.pr, snapshots, signal);
if (run.mode === "ai") {
log(
@@ -477,6 +506,20 @@ export async function captureRun(run, signal, log, folder, update = () => {}) {
diff: path.join(folder, `${i}-diff.png`),
};
for (const [j, side] of ["before", "after"].entries()) {
+ let fixtures;
+ if (run.fixtureProfile === "custom-fields") {
+ const source = await readFile(
+ path.join(
+ snapshots[side].dir,
+ "apps/web/src/components/task/task-details-content.tsx",
+ ),
+ "utf8",
+ );
+ fixtures = {
+ customFields: true,
+ multiselect: source.includes('field.type === "multiselect"'),
+ };
+ }
log(
"capture",
`Capturing ${side === "before" ? "base" : "PR"}: ${scenario.name}`,
@@ -488,6 +531,7 @@ export async function captureRun(run, signal, log, folder, update = () => {}) {
side,
files[side],
signal,
+ fixtures,
);
item[side].image = `${i}-${side}.png`;
}
@@ -506,6 +550,10 @@ export async function captureRun(run, signal, log, folder, update = () => {}) {
}
export async function reviewRun(run, token, signal, log, folder) {
+ if (run.results.some((item) => item.status !== "captured"))
+ throw new Error(
+ "Capture incomplete; no model review or publishing will run.",
+ );
if (run.mode !== "ai") return;
for (const item of run.results) {
log("review", `AI is reviewing before/after screenshots: ${item.name}`);
diff --git a/scripts/ui-review-bot/scenarios.mjs b/scripts/ui-review-bot/scenarios.mjs
new file mode 100644
index 000000000..69a0365f8
--- /dev/null
+++ b/scripts/ui-review-bot/scenarios.mjs
@@ -0,0 +1,72 @@
+import { validatePlan } from "./core.mjs";
+
+export function checkSupportedSurface(files) {
+ if (
+ files.some(
+ ({ path }) =>
+ path.startsWith("apps/site/") || path.startsWith("apps/docs/"),
+ ) &&
+ !files.some(
+ ({ path }) =>
+ path.startsWith("apps/web/src/") && !path.endsWith("/main.tsx"),
+ )
+ ) {
+ throw new Error(
+ "This PR changes the marketing site or docs. Peekareq currently previews apps/web only; refusing unrelated app screenshots.",
+ );
+ }
+}
+
+export function customFieldPlan(collapsible) {
+ const route =
+ "/dashboard/workspace/ui-review-workspace/project/ui-review-project/task/ui-review-task";
+ const click = (name, by = "text", role = "button") => ({
+ type: "click",
+ by,
+ role,
+ name,
+ only: "after",
+ });
+ const open = [
+ ...(collapsible ? [click("Custom Fields")] : []),
+ click("Design"),
+ ];
+ return validatePlan({
+ summary:
+ "Synthetic Audience field: task overview, available options, and multiple selected values. Base uses a dropdown when multiselect is unavailable.",
+ scenarios: [
+ {
+ name: "Task overview",
+ reason: "Initial task layout with the custom-field section.",
+ beforePath: route,
+ afterPath: route,
+ actions: [],
+ },
+ {
+ name: "Custom-field options",
+ reason: "Expanded Audience multiselect with synthetic options.",
+ beforePath: route,
+ afterPath: route,
+ actions: open,
+ },
+ {
+ name: "Multiple selected values",
+ reason: "Audience with Design and Engineering selected.",
+ beforePath: route,
+ afterPath: route,
+ actions: [
+ ...open,
+ click("Engineering", "role", "option"),
+ {
+ type: "press",
+ by: "role",
+ role: "option",
+ name: "Engineering",
+ value: "Escape",
+ only: "after",
+ },
+ ],
+ },
+ ],
+ });
+}
diff --git a/scripts/ui-review-bot/tests/core.test.mjs b/scripts/ui-review-bot/tests/core.test.mjs
index f9fa0cf51..c2526dbaf 100644
--- a/scripts/ui-review-bot/tests/core.test.mjs
+++ b/scripts/ui-review-bot/tests/core.test.mjs
@@ -208,3 +208,47 @@ test("CLI requires an explicit PR and rejects misspelled flags before spending o
assert.equal(options.post, false);
assert.equal(options.mode, "capture");
});
+
+test("temporary provider throttling retries within the five-request budget", async () => {
+ const original = globalThis.fetch;
+ const run = {
+ calls: 0,
+ usage: { input: 0, output: 0, cost: 0, costKnown: true },
+ };
+ let calls = 0;
+ globalThis.fetch = async () => {
+ calls++;
+ return calls === 1
+ ? new Response('{"error":{"code":429}}', {
+ status: 429,
+ headers: { "retry-after": "0" },
+ })
+ : Response.json({
+ choices: [{ message: { content: '{"caption":"Captured"}' } }],
+ usage: { cost: 0 },
+ });
+ };
+ try {
+ const args = {
+ token: "test",
+ model: "test",
+ messages: [],
+ run,
+ signal: new AbortController().signal,
+ };
+ assert.deepEqual(await completion(args), { caption: "Captured" });
+ assert.equal(run.calls, 2);
+ globalThis.fetch = async () =>
+ new Response('{"error":{"code":429}}', {
+ status: 429,
+ headers: { "retry-after": "0" },
+ });
+ run.calls = 4;
+ await assert.rejects(completion(args), /429/);
+ assert.equal(run.calls, 5);
+ await assert.rejects(completion(args), /five-call limit/);
+ assert.equal(run.calls, 5);
+ } finally {
+ globalThis.fetch = original;
+ }
+});
diff --git a/scripts/ui-review-bot/tests/fixtures.test.mjs b/scripts/ui-review-bot/tests/fixtures.test.mjs
index f70b824e9..590b78477 100644
--- a/scripts/ui-review-bot/tests/fixtures.test.mjs
+++ b/scripts/ui-review-bot/tests/fixtures.test.mjs
@@ -35,3 +35,47 @@ test("timer start and stop persist within one browser scenario and never leak be
});
assert.equal(list(a).at(-1).duration, 120);
});
+
+test("custom-field values persist, preserve the API shape, and reset for each scenario", () => {
+ const a = createFixtureSession({ customFields: true, multiselect: true });
+ const b = createFixtureSession({ customFields: true, multiselect: true });
+ const api = (session, route, method = "GET", body = {}) =>
+ session(`http://app/api/${route}`, method, body);
+ const [field] = api(a, "custom-field/project/ui-review-project");
+ assert.equal(field.type, "multiselect");
+ assert.deepEqual(field.options, ["Design", "Engineering", "Product"]);
+ const value = JSON.stringify(["Design", "Engineering"]);
+ api(a, "custom-field/value", "PUT", {
+ taskId: "ui-review-task",
+ fieldId: field.id,
+ value,
+ });
+ assert.equal(api(a, "custom-field/task/ui-review-task")[0].value, value);
+ assert.equal(
+ api(b, "custom-field/task/ui-review-task")[0].value,
+ '["Design"]',
+ );
+ assert.equal(
+ api(a, "custom-field/value", "PUT", {
+ taskId: "unknown",
+ fieldId: field.id,
+ value,
+ }),
+ undefined,
+ );
+ const base = createFixtureSession({ customFields: true });
+ assert.equal(
+ api(base, "custom-field/project/ui-review-project")[0].type,
+ "dropdown",
+ );
+ assert.equal(
+ api(base, "custom-field/task/ui-review-task")[0].value,
+ "Design",
+ );
+ const result = api(a, "custom-field/project/ui-review-project");
+ result[0].options.length = 0;
+ assert.equal(
+ api(a, "custom-field/project/ui-review-project")[0].options.length,
+ 3,
+ );
+});
diff --git a/scripts/ui-review-bot/tests/scenarios.test.mjs b/scripts/ui-review-bot/tests/scenarios.test.mjs
new file mode 100644
index 000000000..0e0a71834
--- /dev/null
+++ b/scripts/ui-review-bot/tests/scenarios.test.mjs
@@ -0,0 +1,34 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import { checkSupportedSurface, customFieldPlan } from "../scenarios.mjs";
+
+test("site changes cannot generate unrelated task screenshots", () => {
+ assert.throws(
+ () =>
+ checkSupportedSurface([
+ { path: "apps/site/components/landing/hero.tsx" },
+ { path: "apps/web/src/main.tsx" },
+ { path: "apps/web/index.html" },
+ ]),
+ /previews apps\/web only/,
+ );
+ assert.doesNotThrow(() =>
+ checkSupportedSurface([
+ { path: "apps/docs/openapi.json" },
+ { path: "apps/web/src/components/task/task-details-content.tsx" },
+ ]),
+ );
+});
+
+test("custom-field scenarios use fixture values and handle the new collapsed section", () => {
+ const plan = customFieldPlan(true);
+ assert.equal(plan.scenarios.length, 3);
+ assert.deepEqual(plan.scenarios[0].actions, []);
+ assert.equal(plan.scenarios[1].actions[0].name, "Custom Fields");
+ assert.equal(customFieldPlan(false).scenarios[1].actions[0].name, "Design");
+ assert.ok(
+ plan.scenarios[2].actions.some(
+ (action) => action.role === "option" && action.name === "Engineering",
+ ),
+ );
+});