Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(annotation-queues): make score config setup visible and ordered (#18819)

Nikita Kabardin18dcd0b4c70b
diff --git a/packages/shared/scripts/seeder/README.md b/packages/shared/scripts/seeder/README.md
index 774f0f2b3f..75d57e1d3a 100644
--- a/packages/shared/scripts/seeder/README.md
+++ b/packages/shared/scripts/seeder/README.md
@@ -59,6 +59,21 @@ Common flags: `--project` (defaults to the seeded example project),
`--environment`, `--seed`, `--id-prefix`, `--dry-run` (instant, arithmetic
counts, writes nothing), `--json` (machine mode: pure-JSON stdout).
+For queue setup testing, use `annotation-queue --setup-demo --core-items 4`.
+It seeds Answer quality and Safety review queues with different, nonalphabetical
+score config orders and a shared Feedback config, plus a Start here queue with
+four pending items and intentionally empty `scoreConfigIds`. This empty queue is
+a test fixture; the application still requires a config when saving. The demo
+uses separate queue names and adds `-setup` to the ID prefix, preserving the
+default keyboard QA fixtures. It includes observation, session, and completed
+examples but skips the missing-object item.
+
+Add `--score-config-count 60` to seed a larger, varied project catalog. With
+`--setup-demo`, Answer quality selects 35 fields and Safety review selects 34,
+using different subsets and orders; Start here stays unconfigured. The default
+is nine configs, and the supported range is 9–500. Use a fresh project and
+`--id-prefix` for an independent fixture: queue names are reused within a project.
+
Scenarios compose: e.g. a session where one trace has zero observations is
two `long-session` runs sharing a `--session-id` with different
`--id-prefix` values.
diff --git a/packages/shared/scripts/seeder/scenarios/annotation-queue.ts b/packages/shared/scripts/seeder/scenarios/annotation-queue.ts
index c239e256da..86a5081af9 100644
--- a/packages/shared/scripts/seeder/scenarios/annotation-queue.ts
+++ b/packages/shared/scripts/seeder/scenarios/annotation-queue.ts
@@ -179,9 +179,29 @@ const run = async (
params: Record<string, string | number | boolean>,
): Promise<SeedSummary> => {
const startedAt = Date.now();
- const coreItems = Math.max(1, Number(params["core-items"] ?? 12));
+ const setupDemo = params["setup-demo"] === true;
+ const scoreConfigCount = Number(
+ params["score-config-count"] ?? CONFIG_DEFS.length,
+ );
+ if (
+ !Number.isInteger(scoreConfigCount) ||
+ scoreConfigCount < CONFIG_DEFS.length ||
+ scoreConfigCount > 500
+ ) {
+ throw new SeedError(
+ "--score-config-count must be an integer from 9 to 500",
+ );
+ }
+ const coreItems = Math.max(
+ setupDemo ? 4 : 1,
+ Number(params["core-items"] ?? 12),
+ );
const withV4 = params.v4 !== false; // default true — local dev renders v4 events
- const pfx = ctx.idPrefix && ctx.idPrefix.length > 0 ? ctx.idPrefix : "annoqa";
+ const idPrefix =
+ ctx.idPrefix && ctx.idPrefix.length > 0 ? ctx.idPrefix : "annoqa";
+ const pfx = setupDemo ? `${idPrefix}-setup` : idPrefix;
+ const queueCount = setupDemo ? 3 : 2;
+ const edgeItems = setupDemo ? 7 : 8;
// Deterministic time window (6h before today's UTC midnight); jitter() adds
// stateless per-row variation so re-runs overwrite in place.
@@ -194,6 +214,13 @@ const run = async (
// --id-prefix is reused — items must reference its real id, not ours).
let coreQueueId = `${pfx}-q-core`;
let edgeQueueId = `${pfx}-q-edge`;
+ let startQueueId = `${pfx}-q-start`;
+
+ const queueLinks = () => [
+ annotationQueueLink(ctx, coreQueueId),
+ annotationQueueLink(ctx, edgeQueueId),
+ ...(setupDemo ? [annotationQueueLink(ctx, startQueueId)] : []),
+ ];
if (ctx.dryRun) {
return {
@@ -208,27 +235,28 @@ const run = async (
// Mirror the real-run counts: 8 edge items (partial1/2, stale, 2 obs,
// session, missing, done), of which 8 have traces (missing has none),
// 2 carry observations, and 9 pre-existing scores (2+3+1+3) are seeded.
- scoreConfigs: CONFIG_DEFS.length,
- queues: 2,
- items: coreItems + 8,
+ scoreConfigs: scoreConfigCount,
+ queues: queueCount,
+ items: coreItems + edgeItems + (setupDemo ? 4 : 0),
traces: coreItems + 8,
observations: 2,
scores: 9,
events: withV4 ? coreItems + 10 : 0,
},
verified: {},
- links: [
- annotationQueueLink(ctx, coreQueueId),
- annotationQueueLink(ctx, edgeQueueId),
- ],
+ links: queueLinks(),
dryRun: true,
durationMs: Date.now() - startedAt,
};
}
// --- 1. Score configs (Postgres) ----------------------------------------
+ const extraConfigs = createExtraConfigDefinitions(
+ scoreConfigCount - CONFIG_DEFS.length,
+ );
+ const scoreConfigs = [...CONFIG_DEFS, ...extraConfigs];
const configId: Record<string, string> = {};
- for (const def of CONFIG_DEFS) {
+ for (const def of scoreConfigs) {
const id = `${pfx}-cfg-${def.key}`;
configId[def.key] = id;
const base = {
@@ -248,25 +276,53 @@ const run = async (
}
// --- 2. Queues (Postgres) -----------------------------------------------
- const coreConfigIds = CORE_KEYS.map((k) => configId[k]);
- const edgeConfigIds = CONFIG_DEFS.map((d) => configId[d.key]);
+ const selectedExtraCount = Math.ceil(extraConfigs.length * 0.6);
+ const coreConfigIds = [
+ ...(setupDemo
+ ? ["helpfulness", "accuracy", "quality", "feedback"]
+ : CORE_KEYS),
+ ...extraConfigs.slice(0, selectedExtraCount).map((def) => def.key),
+ ].map((key) => configId[key]);
+ const edgeConfigIds = setupDemo
+ ? [
+ "toxicity",
+ "confidence",
+ "feedback",
+ ...extraConfigs
+ .slice(extraConfigs.length - selectedExtraCount)
+ .reverse()
+ .map((def) => def.key),
+ ].map((key) => configId[key])
+ : scoreConfigs.map((def) => configId[def.key]);
+ const coreQueueName = setupDemo
+ ? "Queue setup demo · Answer quality"
+ : "Keyboard QA · core types";
+ const coreQueueDescription = setupDemo
+ ? "Review answer helpfulness, accuracy, and quality, then leave feedback."
+ : "Every score-field type, fresh items — fast keyboard run.";
+ const edgeQueueName = setupDemo
+ ? "Queue setup demo · Safety review"
+ : "Keyboard QA · edge cases";
+ const edgeQueueDescription = setupDemo
+ ? "Review toxicity and confidence, with the same project-wide feedback config."
+ : "Many fields + partial/stale/archived scores, observation/session/deleted/completed items.";
const coreQueue = await prisma.annotationQueue.upsert({
where: {
projectId_name: {
projectId: ctx.projectId,
- name: "Keyboard QA · core types",
+ name: coreQueueName,
},
},
update: {
- description: "Every score-field type, fresh items — fast keyboard run.",
+ description: coreQueueDescription,
scoreConfigIds: coreConfigIds,
},
create: {
id: coreQueueId,
projectId: ctx.projectId,
- name: "Keyboard QA · core types",
- description: "Every score-field type, fresh items — fast keyboard run.",
+ name: coreQueueName,
+ description: coreQueueDescription,
scoreConfigIds: coreConfigIds,
},
});
@@ -275,25 +331,48 @@ const run = async (
where: {
projectId_name: {
projectId: ctx.projectId,
- name: "Keyboard QA · edge cases",
+ name: edgeQueueName,
},
},
update: {
- description:
- "Many fields + partial/stale/archived scores, observation/session/deleted/completed items.",
+ description: edgeQueueDescription,
scoreConfigIds: edgeConfigIds,
},
create: {
id: edgeQueueId,
projectId: ctx.projectId,
- name: "Keyboard QA · edge cases",
- description:
- "Many fields + partial/stale/archived scores, observation/session/deleted/completed items.",
+ name: edgeQueueName,
+ description: edgeQueueDescription,
scoreConfigIds: edgeConfigIds,
},
});
edgeQueueId = edgeQueue.id;
+ if (setupDemo) {
+ // An intentionally unconfigured fixture exercises setup before fields are
+ // selected; the application still requires a config when saving a queue.
+ const startQueue = await prisma.annotationQueue.upsert({
+ where: {
+ projectId_name: {
+ projectId: ctx.projectId,
+ name: "Queue setup demo · Start here",
+ },
+ },
+ update: {
+ description: "Add score configs to start annotating these items.",
+ scoreConfigIds: [],
+ },
+ create: {
+ id: startQueueId,
+ projectId: ctx.projectId,
+ name: "Queue setup demo · Start here",
+ description: "Add score configs to start annotating these items.",
+ scoreConfigIds: [],
+ },
+ });
+ startQueueId = startQueue.id;
+ }
+
// --- 3. ClickHouse rows + queue items -----------------------------------
const traces: TraceRecordInsertType[] = [];
const observations: ObservationRecordInsertType[] = [];
@@ -528,12 +607,14 @@ const run = async (
pushItem(`${pfx}-edge-i-session`, edgeQueueId, sessionId, "SESSION");
}
// (v) Deleted/missing object — no trace created → ObjectNotFound card.
- pushItem(
- `${pfx}-edge-i-missing`,
- edgeQueueId,
- `${pfx}-edge-missing-trace`,
- "TRACE",
- );
+ if (!setupDemo) {
+ pushItem(
+ `${pfx}-edge-i-missing`,
+ edgeQueueId,
+ `${pfx}-edge-missing-trace`,
+ "TRACE",
+ );
+ }
// (vi) An already-completed item, fully scored.
{
const { timestamp } = newTrace(`${pfx}-edge-done`);
@@ -570,6 +651,17 @@ const run = async (
);
}
+ if (setupDemo) {
+ for (let i = 0; i < 4; i++) {
+ pushItem(
+ `${pfx}-start-i${i}`,
+ startQueueId,
+ `${pfx}-core-t${i}`,
+ "TRACE",
+ );
+ }
+ }
+
// --- 4. Persist queue items (Postgres) ----------------------------------
for (const it of items) {
const data = {
@@ -590,7 +682,7 @@ const run = async (
// --- 5. Persist ClickHouse rows -----------------------------------------
ctx.log(
- `writing ${CONFIG_DEFS.length} score configs, 2 queues, ${items.length} items, ${traces.length} traces, ${observations.length} observations, ${scores.length} scores${withV4 ? `, ${events.length} events` : ""}`,
+ `writing ${scoreConfigs.length} score configs, ${queueCount} queues, ${items.length} items, ${traces.length} traces, ${observations.length} observations, ${scores.length} scores${withV4 ? `, ${events.length} events` : ""}`,
);
for (const batch of chunk(traces, 1000)) await createTracesCh(batch);
for (const batch of chunk(observations, 1000))
@@ -617,7 +709,9 @@ const run = async (
const pgItems = await prisma.annotationQueueItem.count({
where: {
projectId: ctx.projectId,
- queueId: { in: [coreQueueId, edgeQueueId] },
+ queueId: {
+ in: [coreQueueId, edgeQueueId, ...(setupDemo ? [startQueueId] : [])],
+ },
},
});
verified.queueItems = pgItems;
@@ -647,8 +741,8 @@ const run = async (
traceIds: traceIds.slice(0, 5),
sessionIds: [sessionId],
counts: {
- scoreConfigs: CONFIG_DEFS.length,
- queues: 2,
+ scoreConfigs: scoreConfigs.length,
+ queues: queueCount,
items: items.length,
traces: traces.length,
observations: observations.length,
@@ -656,10 +750,7 @@ const run = async (
events: events.length,
},
verified,
- links: [
- annotationQueueLink(ctx, coreQueueId),
- annotationQueueLink(ctx, edgeQueueId),
- ],
+ links: queueLinks(),
dryRun: false,
durationMs: Date.now() - startedAt,
};
@@ -677,6 +768,20 @@ export const annotationQueueScenario: ScenarioDefinition = {
default: 12,
description: "number of fresh trace items on the core-types queue",
},
+ {
+ flag: "setup-demo",
+ type: "boolean",
+ default: false,
+ description:
+ "seed two differently configured demo queues and an unconfigured queue with four pending items (at least four core traces)",
+ },
+ {
+ flag: "score-config-count",
+ type: "number",
+ default: CONFIG_DEFS.length,
+ description:
+ "total project score configs (9–500); extra varied fields are attached in different subsets and orders to exercise long queue editors",
+ },
{
flag: "v4",
type: "boolean",
@@ -687,3 +792,63 @@ export const annotationQueueScenario: ScenarioDefinition = {
],
run,
};
+
+function createExtraConfigDefinitions(count: number): ConfigDef[] {
+ const names = [
+ "Response clarity",
+ "Source relevance",
+ "Policy compliance",
+ "Review notes",
+ "Answer completeness",
+ "Citation quality",
+ "Sensitive content",
+ "Improvement suggestions",
+ "Instruction following",
+ "Reasoning quality",
+ "Escalation needed",
+ "Reviewer rationale",
+ ];
+
+ return Array.from({ length: count }, (_, index) => {
+ const key = `additional-${index + 1}`;
+ const name = `${names[index % names.length]} ${index + 1}`;
+ const description =
+ "Synthetic score config for testing long queue field lists.";
+ switch (index % 4) {
+ case 0:
+ return {
+ key,
+ name,
+ description,
+ dataType: "NUMERIC",
+ minValue: 0,
+ maxValue: index % 8 === 0 ? 5 : 100,
+ };
+ case 1:
+ return {
+ key,
+ name,
+ description,
+ dataType: "CATEGORICAL",
+ categories: [
+ { label: "Meets expectations", value: 2 },
+ { label: "Needs improvement", value: 1 },
+ { label: "Does not meet expectations", value: 0 },
+ ],
+ };
+ case 2:
+ return {
+ key,
+ name,
+ description,
+ dataType: "BOOLEAN",
+ categories: [
+ { label: "True", value: 1 },
+ { label: "False", value: 0 },
+ ],
+ };
+ default:
+ return { key, name, description, dataType: "TEXT" };
+ }
+ });
+}
diff --git a/web/src/__tests__/server/annotation-queue-items-trpc.servertest.ts b/web/src/__tests__/server/annotation-queue-items-trpc.servertest.ts
index fd4eb1ccad..7067881d8e 100644
--- a/web/src/__tests__/server/annotation-queue-items-trpc.servertest.ts
+++ b/web/src/__tests__/server/annotation-queue-items-trpc.servertest.ts
@@ -222,6 +222,150 @@ describe("annotation queues trpc", () => {
});
describe("annotationQueues.update", () => {
+ it("updates queue fields independently without overwriting concurrent settings or config order", async () => {
+ const setup = await createOrgProjectAndApiKey();
+ orgIds.push(setup.org.id);
+ const { caller } = createCallerForProjectRole(
+ setup,
+ "ADMIN",
+ "cloud:pro",
+ );
+ const first = await createScoreConfig(setup.project.id);
+ const second = await createScoreConfig(setup.project.id);
+ const queue = await caller.annotationQueues.create({
+ projectId: setup.project.id,
+ name: "configured-queue",
+ description: "Initial description",
+ scoreConfigIds: [first.id, second.id],
+ });
+ await caller.annotationQueues.create({
+ projectId: setup.project.id,
+ name: "another-queue",
+ scoreConfigIds: [first.id],
+ });
+ await prisma.annotationQueue.update({
+ where: { id: queue.id, projectId: setup.project.id },
+ data: { name: "Latest name", description: "Latest description" },
+ });
+
+ const configured = await caller.annotationQueues.update({
+ projectId: setup.project.id,
+ queueId: queue.id,
+ scoreConfigIds: [second.id, first.id],
+ });
+ expect(configured).toMatchObject({
+ name: "Latest name",
+ description: "Latest description",
+ scoreConfigIds: [second.id, first.id],
+ });
+
+ const settings = await caller.annotationQueues.update({
+ projectId: setup.project.id,
+ queueId: queue.id,
+ name: "Changed settings",
+ description: "Changed description",
+ });
+ expect(settings).toMatchObject({
+ name: "Changed settings",
+ description: "Changed description",
+ scoreConfigIds: [second.id, first.id],
+ });
+ });
+
+ it("deduplicates resolved queue configs in first-occurrence order", async () => {
+ const setup = await createOrgProjectAndApiKey();
+ orgIds.push(setup.org.id);
+ const { caller } = createCallerForProjectRole(setup);
+ const first = await createScoreConfig(setup.project.id);
+ const second = await createScoreConfig(setup.project.id);
+ const queue = await prisma.annotationQueue.create({
+ data: {
+ projectId: setup.project.id,
+ name: "legacy-queue-configs",
+ scoreConfigIds: [second.id, uuidv4(), second.id, first.id, first.id],
+ },
+ });
+
+ const [details, list] = await Promise.all([
+ caller.annotationQueues.byId({
+ projectId: setup.project.id,
+ queueId: queue.id,
+ }),
+ caller.annotationQueues.all({ projectId: setup.project.id }),
+ ]);
+ expect(details.scoreConfigs.map((config) => config.id)).toEqual([
+ second.id,
+ first.id,
+ ]);
+ expect(
+ list.queues
+ .find((entry) => entry.id === queue.id)
+ ?.scoreConfigs.map((config) => config.id),
+ ).toEqual([second.id, first.id]);
+ });
+
+ it("persists score config order through queue reads without changing completed items", async () => {
+ const setup = await createOrgProjectAndApiKey();
+ orgIds.push(setup.org.id);
+ const { caller } = createCallerForProjectRole(
+ setup,
+ "ADMIN",
+ "cloud:pro",
+ );
+ const first = await createScoreConfig(setup.project.id);
+ const second = await createScoreConfig(setup.project.id);
+ const queue = await caller.annotationQueues.create({
+ projectId: setup.project.id,
+ name: "ordered-queue",
+ scoreConfigIds: [second.id, first.id],
+ });
+ const completedAt = new Date();
+ const item = await prisma.annotationQueueItem.create({
+ data: {
+ queueId: queue.id,
+ projectId: setup.project.id,
+ objectId: uuidv4(),
+ objectType: AnnotationQueueObjectType.TRACE,
+ status: "COMPLETED",
+ completedAt,
+ },
+ });
+
+ const assertQueueOrder = async (expected: string[]) => {
+ const details = await caller.annotationQueues.byId({
+ projectId: setup.project.id,
+ queueId: queue.id,
+ });
+ expect(details.scoreConfigIds).toEqual(expected);
+ expect(details.scoreConfigs.map((config) => config.id)).toEqual(
+ expected,
+ );
+ const list = await caller.annotationQueues.all({
+ projectId: setup.project.id,
+ });
+ expect(
+ list.queues
+ .find((entry) => entry.id === queue.id)
+ ?.scoreConfigs.map((config) => config.id),
+ ).toEqual(expected);
+ };
+
+ await assertQueueOrder([second.id, first.id]);
+ await caller.annotationQueues.update({
+ projectId: setup.project.id,
+ queueId: queue.id,
+ name: queue.name,
+ scoreConfigIds: [first.id, second.id],
+ });
+ await assertQueueOrder([first.id, second.id]);
+ expect(
+ await prisma.annotationQueueItem.findUnique({
+ where: { id: item.id },
+ select: { status: true, completedAt: true },
+ }),
+ ).toEqual({ status: "COMPLETED", completedAt });
+ });
+
it("rejects a rename that collides with another queue in the project", async () => {
const setup = await createOrgProjectAndApiKey();
orgIds.push(setup.org.id);
@@ -273,6 +417,65 @@ describe("annotation queues trpc", () => {
});
});
+ describe.each(["create", "update"] as const)(
+ "annotationQueues.%s score config validation",
+ (operation) => {
+ it.each(["empty", "duplicate", "other-project"] as const)(
+ "rejects %s score config IDs without changing the queue",
+ async (invalidSelection) => {
+ const setup = await createOrgProjectAndApiKey();
+ const otherSetup = await createOrgProjectAndApiKey();
+ orgIds.push(setup.org.id, otherSetup.org.id);
+ const { caller } = createCallerForProjectRole(
+ setup,
+ "ADMIN",
+ "cloud:pro",
+ );
+ const config = await createScoreConfig(setup.project.id);
+ const otherConfig = await createScoreConfig(otherSetup.project.id);
+ const queue = await caller.annotationQueues.create({
+ projectId: setup.project.id,
+ name: "existing-queue",
+ scoreConfigIds: [config.id],
+ });
+ const invalidIds = {
+ empty: [],
+ duplicate: [config.id, config.id],
+ "other-project": [config.id, otherConfig.id],
+ };
+ const scoreConfigIds = invalidIds[invalidSelection];
+ const input = {
+ projectId: setup.project.id,
+ name: "changed-queue",
+ scoreConfigIds,
+ };
+
+ await expect(
+ operation === "create"
+ ? caller.annotationQueues.create(input)
+ : caller.annotationQueues.update({
+ projectId: setup.project.id,
+ queueId: queue.id,
+ scoreConfigIds,
+ }),
+ ).rejects.toMatchObject({ code: "BAD_REQUEST" });
+ expect(
+ await prisma.annotationQueue.findMany({
+ where: { projectId: setup.project.id },
+ select: { id: true, name: true, scoreConfigIds: true },
+ }),
+ ).toEqual([
+ {
+ id: queue.id,
+ name: queue.name,
+ scoreConfigIds: [config.id],
+ },
+ ]);
+ },
+ );
+ },
+ );
+
describe("annotationQueues.delete", () => {
it("returns not found when the queue does not exist in the project", async () => {
const setup = await createOrgProjectAndApiKey();
diff --git a/web/src/__tests__/server/annotation-queues-api.servertest.ts b/web/src/__tests__/server/annotation-queues-api.servertest.ts
index e8077fb4f2..1eb26dc231 100644
--- a/web/src/__tests__/server/annotation-queues-api.servertest.ts
+++ b/web/src/__tests__/server/annotation-queues-api.servertest.ts
@@ -18,6 +18,7 @@ import {
AnnotationQueueStatus,
} from "@langfuse/shared";
import { createOrgProjectAndApiKey } from "@langfuse/shared/src/server";
+import { createAnnotationQueueForApi } from "@/src/features/annotation-queues/server/publicAnnotationQueueService";
import { v4 as uuidv4 } from "uuid";
const TOTAL_TEST_QUEUES = 10; // Create enough queues to test pagination
@@ -26,6 +27,7 @@ const TOTAL_TEST_QUEUE_ITEMS = 6; // Create enough queue items to test filtering
describe("Annotation Queues API Endpoints", () => {
let auth: string;
let projectId: string;
+ let organizationId: string;
const createQueue = (overrides: Partial<{ name: string }> = {}) =>
prisma.annotationQueue.create({
@@ -92,10 +94,14 @@ describe("Annotation Queues API Endpoints", () => {
beforeEach(async () => {
// Create organization, project, and API key for testing
- const { auth: newAuth, projectId: newProjectId } =
- await createOrgProjectAndApiKey();
+ const {
+ auth: newAuth,
+ projectId: newProjectId,
+ org,
+ } = await createOrgProjectAndApiKey();
auth = newAuth;
projectId = newProjectId;
+ organizationId = org.id;
});
describe("GET /annotation-queues", () => {
@@ -182,6 +188,27 @@ describe("Annotation Queues API Endpoints", () => {
});
describe("POST /annotation-queues", () => {
+ it("rejects duplicated IDs before API queue creation", async () => {
+ const config = await createScoreConfig();
+
+ try {
+ await expect(
+ createAnnotationQueueForApi({
+ projectId,
+ input: {
+ name: "Duplicate configs queue",
+ scoreConfigIds: [config.id, config.id],
+ },
+ }),
+ ).rejects.toThrow("Score config IDs must be unique");
+ expect(
+ await prisma.annotationQueue.count({ where: { projectId } }),
+ ).toBe(0);
+ } finally {
+ await prisma.organization.delete({ where: { id: organizationId } });
+ }
+ });
+
it("should create a new annotation queue", async () => {
const scoreConfig = await createScoreConfig();
diff --git a/web/src/components/design-system/Dialog/Dialog.stories.tsx b/web/src/components/design-system/Dialog/Dialog.stories.tsx
index f4a10330b4..aaeb7a31e2 100644
--- a/web/src/components/design-system/Dialog/Dialog.stories.tsx
+++ b/web/src/components/design-system/Dialog/Dialog.stories.tsx
@@ -3,6 +3,7 @@ import { expect, fn, userEvent, waitFor, within } from "storybook/test";
import preview from "../../../../.storybook/preview";
import { DialogController } from "@/src/components/design-system/DialogController/DialogController";
+import { Button } from "@/src/components/design-system/Button/Button";
import { Dialog } from "./Dialog";
@@ -164,3 +165,62 @@ export const DestructiveOnlyAutofocus = meta.story({
);
},
});
+
+export const ScrollableBodyKeepsControlsAboveFooter = meta.story({
+ name: "(Test) Scrollable body keeps controls above footer",
+ render: () => (
+ <DialogController
+ renderDialog={() => (
+ <Dialog
+ title="Long dialog"
+ actions={[{ label: "Save", onClick: fn() }]}
+ >
+ <Dialog.Body>
+ {Array.from({ length: 20 }, (_, index) => (
+ <p key={index}>Review instruction {index + 1}.</p>
+ ))}
+ <Button text="Additional action" onClick={fn()} />
+ </Dialog.Body>
+ </Dialog>
+ )}
+ >
+ {({ openDialog }) => <Button text="Open dialog" onClick={openDialog} />}
+ </DialogController>
+ ),
+ play: async ({ canvasElement }) => {
+ const document = canvasElement.ownerDocument;
+ const canvas = within(canvasElement);
+ const body = within(document.body);
+
+ await userEvent.click(canvas.getByRole("button", { name: "Open dialog" }));
+ const lowerControl = body.getByRole("button", {
+ name: "Additional action",
+ });
+ const scrollBody = lowerControl.parentElement!;
+ const footer = body.getByRole("button", { name: "Save" }).parentElement!;
+
+ await expect(scrollBody.scrollHeight).toBeGreaterThan(
+ scrollBody.clientHeight,
+ );
+ await expect(scrollBody.getBoundingClientRect().bottom).toBeLessThanOrEqual(
+ footer.getBoundingClientRect().top + 1,
+ );
+
+ lowerControl.scrollIntoView({ block: "center" });
+ await waitFor(() => expect(scrollBody.scrollTop).toBeGreaterThan(0));
+ const controlBounds = lowerControl.getBoundingClientRect();
+ await expect(controlBounds.bottom).toBeLessThanOrEqual(
+ footer.getBoundingClientRect().top + 1,
+ );
+ await expect(
+ lowerControl.contains(
+ document.elementFromPoint(
+ controlBounds.left + controlBounds.width / 2,
+ controlBounds.top + controlBounds.height / 2,
+ ),
+ ),
+ ).toBe(true);
+ await userEvent.click(lowerControl);
+ await expect(lowerControl).toHaveFocus();
+ },
+});
diff --git a/web/src/components/design-system/Dialog/Dialog.tsx b/web/src/components/design-system/Dialog/Dialog.tsx
index beaa9d9006..0497da4ad1 100644
--- a/web/src/components/design-system/Dialog/Dialog.tsx
+++ b/web/src/components/design-system/Dialog/Dialog.tsx
@@ -7,12 +7,13 @@ import * as React from "react";
import { Button } from "@/src/components/design-system/Button/Button";
import { useLayerContainer } from "@/src/context/LayerContext/LayerContext";
+import { PopoverLayerProvider } from "@/src/context/LayerContext/PopoverLayerProvider";
import { cn } from "@/src/utils/tailwind";
import motionStyles from "./Dialog.module.css";
const dialogContentVariants = cva(
- "fixed top-0 left-0 flex h-dvh w-full flex-col overflow-hidden bg-modal shadow-lg sm:top-[50%] sm:left-[50%] sm:h-auto sm:max-h-[85vh] sm:translate-x-[-50%] sm:translate-y-[-50%] sm:rounded-lg",
+ "fixed top-0 left-0 flex h-dvh w-full flex-col bg-modal shadow-lg sm:top-[50%] sm:left-[50%] sm:h-auto sm:max-h-[85vh] sm:translate-x-[-50%] sm:translate-y-[-50%] sm:rounded-lg",
{
variants: {
size: {
@@ -42,6 +43,8 @@ type DialogAction = {
type DialogProps = {
actions?: ReadonlyArray<DialogAction>;
closeOnInteractionOutside?: boolean;
+ /** Disable when a child interaction handles activation keys on the document. */
+ stopPropagationOnEnterSpace?: boolean;
size?: VariantProps<typeof dialogContentVariants>["size"];
title: string;
} & (
@@ -53,11 +56,14 @@ function DialogRoot({
actions,
children,
closeOnInteractionOutside = false,
+ stopPropagationOnEnterSpace = true,
size,
text,
title,
}: DialogProps) {
const container = useLayerContainer("modal");
+ const [popoverContainer, setPopoverContainer] =
+ React.useState<HTMLDivElement | null>(null);
const cancelButtonRef = React.useRef<HTMLButtonElement>(null);
const actionButtonRefs = React.useRef<Array<HTMLButtonElement | null>>([]);
@@ -103,7 +109,10 @@ function DialogRoot({
}}
onClick={(event) => event.stopPropagation()}
onKeyDown={(event) => {
- if (event.key === "Enter" || event.key === " ") {
+ if (
+ stopPropagationOnEnterSpace &&
+ (event.key === "Enter" || event.key === " ")
+ ) {
event.stopPropagation();
}
}}
@@ -114,44 +123,57 @@ function DialogRoot({
if (!closeOnInteractionOutside) event.preventDefault();
}}
>
- <header className="bg-modal sticky top-0 z-30 flex shrink-0 flex-col gap-1 rounded-t-lg p-4">
- <div className="flex w-full items-center justify-between gap-4 text-left">
- <DialogPrimitive.Title className="min-w-0 flex-1 text-lg leading-none font-bold tracking-tight">
- {title}
- </DialogPrimitive.Title>
- <DialogPrimitive.Close
- className="ring-offset-background focus:ring-ring data-[state=open]:bg-accent data-[state=open]:text-muted-foreground hover:bg-accent z-20 -mt-2 -mr-2 inline-flex size-8 shrink-0 items-center justify-center rounded-md opacity-70 transition-opacity hover:opacity-100 focus:ring-2 focus:ring-offset-2 focus:outline-hidden disabled:pointer-events-none"
- tabIndex={-1}
- >
- <X className="icon-base" />
- <span className="sr-only">Close</span>
- </DialogPrimitive.Close>
- </div>
- </header>
- {text !== undefined ? <DialogBody>{text}</DialogBody> : children}
- {actions?.length ? (
- <DialogFooter>
- <DialogPrimitive.Close asChild>
- <Button ref={cancelButtonRef} variant="secondary" text="Cancel" />
- </DialogPrimitive.Close>
- {actions.map(
- ({ icon: Icon, label, variant, ...buttonProps }, index) => (
+ <div className="isolate flex min-h-0 flex-1 flex-col overflow-hidden">
+ <header className="bg-modal flex shrink-0 flex-col gap-1 rounded-t-lg p-4">
+ <div className="flex w-full items-center justify-between gap-4 text-left">
+ <DialogPrimitive.Title className="min-w-0 flex-1 text-lg leading-none font-bold tracking-tight">
+ {title}
+ </DialogPrimitive.Title>
+ <DialogPrimitive.Close
+ className="ring-offset-background focus:ring-ring data-[state=open]:bg-accent data-[state=open]:text-muted-foreground hover:bg-accent z-20 -mt-2 -mr-2 inline-flex size-8 shrink-0 items-center justify-center rounded-md opacity-70 transition-opacity hover:opacity-100 focus:ring-2 focus:ring-offset-2 focus:outline-hidden disabled:pointer-events-none"
+ tabIndex={-1}
+ >
+ <X className="icon-base" />
+ <span className="sr-only">Close</span>
+ </DialogPrimitive.Close>
+ </div>
+ </header>
+ <PopoverLayerProvider container={popoverContainer}>
+ {text !== undefined ? <DialogBody>{text}</DialogBody> : children}
+ </PopoverLayerProvider>
+ {actions?.length ? (
+ <DialogFooter>
+ <DialogPrimitive.Close asChild>
<Button
- key={label}
- icon={Icon}
- ref={(button) => {
- actionButtonRefs.current[index] = button;
- }}
- text={label}
- {...buttonProps}
- variant={
- variant === "destructive" ? "destructive" : "primary"
- }
+ ref={cancelButtonRef}
+ variant="secondary"
+ text="Cancel"
/>
- ),
- )}
- </DialogFooter>
- ) : null}
+ </DialogPrimitive.Close>
+ {actions.map(
+ ({ icon: Icon, label, variant, ...buttonProps }, index) => (
+ <Button
+ key={label}
+ icon={Icon}
+ ref={(button) => {
+ actionButtonRefs.current[index] = button;
+ }}
+ text={label}
+ {...buttonProps}
+ variant={
+ variant === "destructive" ? "destructive" : "primary"
+ }
+ />
+ ),
+ )}
+ </DialogFooter>
+ ) : null}
+ </div>
+ <div
+ className="isolate"
+ data-layer="popover"
+ ref={setPopoverContainer}
+ />
</DialogPrimitive.Content>
</DialogPrimitive.Portal>
);
@@ -159,7 +181,7 @@ function DialogRoot({
function DialogBody({ children }: { children: React.ReactNode }) {
return (
- <div className="flex flex-1 flex-col gap-4 overflow-y-auto p-4 text-sm leading-relaxed">
+ <div className="flex min-h-0 flex-1 flex-col gap-4 overflow-y-auto p-4 text-sm leading-relaxed">
{children}
</div>
);
@@ -167,7 +189,7 @@ function DialogBody({ children }: { children: React.ReactNode }) {
function DialogFooter({ children }: { children: React.ReactNode }) {
return (
- <div className="bg-modal sticky bottom-0 z-10 flex shrink-0 flex-col-reverse rounded-b-lg p-4 sm:flex-row sm:justify-end sm:space-x-2 [&>button]:w-full sm:[&>button]:w-auto">
+ <div className="bg-modal flex shrink-0 flex-col-reverse rounded-b-lg p-4 sm:flex-row sm:justify-end sm:space-x-2 [&>button]:w-full sm:[&>button]:w-auto">
{children}
</div>
);
diff --git a/web/src/components/design-system/DialogController/DialogController.tsx b/web/src/components/design-system/DialogController/DialogController.tsx
index 69240be491..11da79305f 100644
--- a/web/src/components/design-system/DialogController/DialogController.tsx
+++ b/web/src/components/design-system/DialogController/DialogController.tsx
@@ -13,6 +13,8 @@ type DialogControllerProps<State = void> = {
onDismiss?: () => void;
renderDialog: (control: {
state: State;
+ /** Visibility is separate from the payload retained during closing. */
+ isOpen: boolean;
closeDialog: () => void;
}) => React.ReactNode;
};
@@ -76,7 +78,11 @@ function DialogController<State = void>({
// Keep the closed content mounted for Radix's exit animation, but
// start a fresh content instance whenever the dialog opens again.
<React.Fragment key={controllerState.openCount}>
- {renderDialog({ state: controllerState.state, closeDialog })}
+ {renderDialog({
+ state: controllerState.state,
+ isOpen: controllerState.isOpen,
+ closeDialog,
+ })}
</React.Fragment>
) : null}
</DialogPrimitive.Root>
diff --git a/web/src/components/design-system/Form/Form.tsx b/web/src/components/design-system/Form/Form.tsx
index 575499b10d..c5e1383682 100644
--- a/web/src/components/design-system/Form/Form.tsx
+++ b/web/src/components/design-system/Form/Form.tsx
@@ -21,43 +21,59 @@ export type FormAction = {
);
export type FormProps = {
+ id?: FormHTMLAttributes<HTMLFormElement>["id"];
+ density?: "default" | "compact";
actions: FormAction[];
children: ReactNode;
onSubmit: FormHTMLAttributes<HTMLFormElement>["onSubmit"];
};
-function FormRoot({ actions, children, onSubmit }: FormProps) {
+function FormRoot({
+ actions,
+ children,
+ density = "default",
+ id,
+ onSubmit,
+}: FormProps) {
return (
- <form className="space-y-8" onSubmit={onSubmit}>
- <div className="space-y-5">{children}</div>
- <div className="flex gap-2">
- {actions.map((action) => {
- if (action.type === "submit") {
+ <form id={id} className="flex flex-col gap-8" onSubmit={onSubmit}>
+ <div
+ className={
+ density === "compact" ? "flex flex-col gap-4" : "flex flex-col gap-5"
+ }
+ >
+ {children}
+ </div>
+ {actions.length > 0 && (
+ <div className="flex gap-2">
+ {actions.map((action) => {
+ if (action.type === "submit") {
+ return (
+ <Button
+ key={action.id}
+ type="submit"
+ text={action.text}
+ variant={action.variant}
+ disabled={action.disabled}
+ loading={action.loading}
+ />
+ );
+ }
+
return (
<Button
key={action.id}
- type="submit"
+ type="button"
text={action.text}
variant={action.variant}
disabled={action.disabled}
loading={action.loading}
+ onClick={action.onClick}
/>
);
- }
-
- return (
- <Button
- key={action.id}
- type="button"
- text={action.text}
- variant={action.variant}
- disabled={action.disabled}
- loading={action.loading}
- onClick={action.onClick}
- />
- );
- })}
- </div>
+ })}
+ </div>
+ )}
</form>
);
}
diff --git a/web/src/components/design-system/SortableList/SortableList.stories.tsx b/web/src/components/design-system/SortableList/SortableList.stories.tsx
new file mode 100644
index 0000000000..db089eda37
--- /dev/null
+++ b/web/src/components/design-system/SortableList/SortableList.stories.tsx
@@ -0,0 +1,142 @@
+import { useState } from "react";
+import { arrayMove } from "@dnd-kit/sortable";
+import { expect, fn, userEvent, waitFor, within } from "storybook/test";
+import preview from "../../../../.storybook/preview";
+import { Dialog } from "../Dialog/Dialog";
+import { DialogController } from "../DialogController/DialogController";
+import { SortableList } from "./SortableList";
+
+const meta = preview.meta({ component: SortableList });
+
+export const TestMixedHeights = meta.story({
+ name: "(Test) Different Height Rows Keep Their Size While Dragging",
+ args: {
+ items: ["Short", "Tall"],
+ getId: String,
+ getLabel: String,
+ onReorder: fn(),
+ renderItem: (item) => (
+ <div className={item === "Tall" ? "h-40 border p-2" : "h-10 border p-2"}>
+ {String(item)}
+ </div>
+ ),
+ },
+ play: async ({ canvasElement }) => {
+ const canvas = within(canvasElement);
+ const handle = canvas.getByRole("button", { name: "Reorder Short" });
+ const row = handle.closest("li")!;
+ const target = canvas
+ .getByRole("button", { name: "Reorder Tall" })
+ .closest("li")!;
+ const original = row.getBoundingClientRect();
+ const destination = target.getBoundingClientRect();
+ await userEvent.pointer({
+ target: handle,
+ keys: "[MouseLeft>]",
+ coords: { clientX: original.left + 10, clientY: original.top + 16 },
+ });
+ await userEvent.pointer({
+ target: handle,
+ coords: { clientX: original.left + 10, clientY: original.top + 24 },
+ });
+ await waitFor(() => expect(handle).toHaveAttribute("aria-pressed", "true"));
+ await userEvent.pointer({
+ target,
+ coords: {
+ clientX: original.left + 10,
+ clientY: destination.top + destination.height / 2,
+ },
+ });
+ await waitFor(() =>
+ expect(target.getBoundingClientRect().top).toBeLessThan(destination.top),
+ );
+ await expect(
+ Math.abs(row.getBoundingClientRect().height - original.height),
+ ).toBeLessThan(1);
+ await userEvent.pointer({ keys: "[/MouseLeft]" });
+ },
+});
+
+export const TestSortingInDialog = meta.story({
+ name: "(Test) Sorting Inside a Dialog",
+ args: {
+ items: ["First", "Second"],
+ getId: String,
+ getLabel: String,
+ onReorder: fn(),
+ renderItem: String,
+ },
+ render: (args) => {
+ const [items, setItems] = useState(args.items);
+ return (
+ <DialogController<boolean>
+ initialState={() => true}
+ renderDialog={() => (
+ <Dialog title="Reorder choices" stopPropagationOnEnterSpace={false}>
+ <Dialog.Body>
+ <SortableList
+ {...args}
+ items={items}
+ aria-label="Choices"
+ onReorder={(from, to) => {
+ setItems(arrayMove(items, from, to));
+ args.onReorder(from, to);
+ }}
+ />
+ </Dialog.Body>
+ </Dialog>
+ )}
+ >
+ {() => null}
+ </DialogController>
+ );
+ },
+ play: async ({ canvasElement, args }) => {
+ const body = within(canvasElement.ownerDocument.body);
+ body.getByRole("button", { name: "Reorder Second" }).focus();
+ await userEvent.keyboard(" ");
+ await userEvent.keyboard("{ArrowUp}");
+ await userEvent.keyboard(" ");
+ await waitFor(() => expect(args.onReorder).toHaveBeenCalledWith(1, 0));
+ const rows = within(
+ body.getByRole("list", { name: "Choices" }),
+ ).getAllByRole("listitem");
+ await expect(rows[0]).toHaveTextContent("Second");
+ body.getByRole("button", { name: "Reorder Second" }).focus();
+ await userEvent.keyboard(" ");
+ await userEvent.keyboard("{ArrowDown}");
+ await userEvent.keyboard("{Escape}");
+ await expect(args.onReorder).toHaveBeenCalledTimes(1);
+ await expect(
+ body.getByRole("dialog", { name: "Reorder choices" }),
+ ).toHaveAttribute("data-state", "open");
+ await expect(
+ body.getByRole("button", { name: "Reorder Second" }),
+ ).not.toHaveAttribute("aria-pressed", "true");
+ const handle = body.getByRole("button", { name: "Reorder Second" });
+ const bounds = handle.getBoundingClientRect();
+ await userEvent.pointer({
+ target: handle,
+ keys: "[MouseLeft>]",
+ coords: { clientX: bounds.left + 5, clientY: bounds.top + 5 },
+ });
+ await userEvent.pointer({
+ target: handle,
+ coords: { clientX: bounds.left + 5, clientY: bounds.top + 15 },
+ });
+ await waitFor(() => expect(handle).toHaveAttribute("aria-pressed", "true"));
+ await userEvent.keyboard("{Escape}");
+ await expect(
+ body.getByRole("dialog", { name: "Reorder choices" }),
+ ).toHaveAttribute("data-state", "open");
+ await expect(handle).not.toHaveAttribute("aria-pressed", "true");
+ await expect(args.onReorder).toHaveBeenCalledTimes(1);
+ await userEvent.pointer({ keys: "[/MouseLeft]" });
+ await userEvent.keyboard("{Escape}");
+ await waitFor(() =>
+ expect(
+ body.queryByRole("dialog", { name: "Reorder choices" }),
+ ).not.toBeInTheDocument(),
+ );
+ },
+});
diff --git a/web/src/features/evals/v2/components/SortableList/SortableList.tsx b/web/src/components/design-system/SortableList/SortableList.tsx
similarity index 63%
rename from web/src/features/evals/v2/components/SortableList/SortableList.tsx
rename to web/src/components/design-system/SortableList/SortableList.tsx
index 42f3f3c4eb..054968d67c 100644
--- a/web/src/features/evals/v2/components/SortableList/SortableList.tsx
+++ b/web/src/components/design-system/SortableList/SortableList.tsx
@@ -1,4 +1,4 @@
-import { type CSSProperties, type ReactNode } from "react";
+import { type CSSProperties, type ReactNode, useEffect, useState } from "react";
import {
closestCenter,
DndContext,
@@ -26,12 +26,14 @@ function SortableRow({
label,
disabled,
compact,
+ variant,
children,
}: {
id: string;
label: string;
disabled: boolean;
compact: boolean;
+ variant: "plain" | "hover-card";
children: ReactNode;
}) {
const {
@@ -43,8 +45,9 @@ function SortableRow({
transition,
isDragging,
} = useSortable({ id, disabled });
+ const plainHandleHeight = compact ? "h-8" : "h-9";
const style: CSSProperties = {
- transform: CSS.Transform.toString(transform),
+ transform: CSS.Translate.toString(transform),
transition,
};
@@ -53,7 +56,10 @@ function SortableRow({
ref={setNodeRef}
style={style}
className={cn(
- "grid grid-cols-[auto_1fr] items-start gap-1",
+ "group/sortable",
+ variant === "hover-card"
+ ? "hover:border-border focus-within:border-border relative rounded-md border border-transparent p-2"
+ : "grid grid-cols-[auto_minmax(0,1fr)] items-start gap-1",
isDragging && "relative z-10 opacity-70",
)}
>
@@ -63,10 +69,12 @@ function SortableRow({
disabled={disabled}
className={cn(
"text-muted-foreground hover:text-foreground focus-visible:ring-ring flex w-5 cursor-grab touch-none items-center justify-center rounded focus-visible:ring-2 focus-visible:outline-hidden active:cursor-grabbing disabled:cursor-default disabled:opacity-30",
- compact ? "h-8" : "h-9",
+ variant === "hover-card"
+ ? "absolute top-2 right-2 h-6 opacity-100 group-focus-within/sortable:opacity-100 group-hover/sortable:opacity-100 [@media(hover:hover)]:opacity-0"
+ : plainHandleHeight,
)}
aria-label={`Reorder ${label}`}
- title="Drag to reorder"
+ title="Drag to reorder. With a keyboard: Space to pick up or drop, arrow keys to move, Escape to cancel."
{...attributes}
{...listeners}
>
@@ -88,6 +96,9 @@ export function SortableList<T>({
onReorder,
renderItem,
gap = "sm",
+ variant = "plain",
+ disabled = false,
+ "aria-label": ariaLabel,
}: {
items: T[];
getId: (item: T, index: number) => string;
@@ -96,7 +107,22 @@ export function SortableList<T>({
renderItem: (item: T, index: number) => ReactNode;
/** Row spacing: `sm` for compact input rows, `md` for cards. */
gap?: "sm" | "md";
+ variant?: "plain" | "hover-card";
+ disabled?: boolean;
+ "aria-label"?: string;
}) {
+ const [isDragging, setIsDragging] = useState(false);
+ useEffect(() => {
+ if (!isDragging) return;
+ // Consume Escape before modal document listeners; the sensor still
+ // receives the event and cancels the drag without discarding the dialog.
+ const preventDialogDismissal = (event: KeyboardEvent) => {
+ if (event.key === "Escape") event.preventDefault();
+ };
+ window.addEventListener("keydown", preventDialogDismissal, true);
+ return () =>
+ window.removeEventListener("keydown", preventDialogDismissal, true);
+ }, [isDragging]);
const sensors = useSensors(
useSensor(MouseSensor, { activationConstraint: { distance: 4 } }),
useSensor(TouchSensor),
@@ -107,7 +133,8 @@ export function SortableList<T>({
const ids = items.map(getId);
const handleDragEnd = ({ active, over }: DragEndEvent) => {
- if (!over || active.id === over.id) return;
+ setIsDragging(false);
+ if (disabled || !over || active.id === over.id) return;
const fromIndex = ids.indexOf(String(active.id));
const toIndex = ids.indexOf(String(over.id));
if (fromIndex < 0 || toIndex < 0) return;
@@ -119,17 +146,23 @@ export function SortableList<T>({
collisionDetection={closestCenter}
modifiers={[restrictToVerticalAxis]}
sensors={sensors}
+ onDragStart={() => setIsDragging(true)}
+ onDragCancel={() => setIsDragging(false)}
onDragEnd={handleDragEnd}
>
<SortableContext items={ids} strategy={verticalListSortingStrategy}>
- <ol className={cn("flex flex-col", gap === "md" ? "gap-3" : "gap-1.5")}>
+ <ol
+ aria-label={ariaLabel}
+ className={cn("flex flex-col", gap === "md" ? "gap-4" : "gap-1.5")}
+ >
{items.map((item, index) => (
<SortableRow
key={ids[index]}
id={ids[index]!}
label={getLabel(item, index)}
- disabled={items.length < 2}
+ disabled={disabled || items.length < 2}
compact={gap === "sm"}
+ variant={variant}
>
{renderItem(item, index)}
</SortableRow>
diff --git a/web/src/components/design-system/table/columns/createButtonTableColumn.tsx b/web/src/components/design-system/table/columns/createButtonTableColumn.tsx
index 5bc79d6718..15a0e1808c 100644
--- a/web/src/components/design-system/table/columns/createButtonTableColumn.tsx
+++ b/web/src/components/design-system/table/columns/createButtonTableColumn.tsx
@@ -14,6 +14,7 @@ export function createButtonTableColumn<TData extends RowData, TValue>({
}: TableColumnOptions<TData, TValue> & {
getButton: (context: CellContext<TData, TValue | null | undefined>) => {
text: string;
+ disabledReason?: string;
onClick: () => void;
disabled?: boolean;
loading?: boolean;
@@ -24,9 +25,9 @@ export function createButtonTableColumn<TData extends RowData, TValue>({
enableResizing: false,
loadingCell: <Skeleton className="h-8 w-16" />,
renderCell: (_, context) => {
- const buttonProps = getButton(context);
+ const { disabledReason, ...buttonProps } = getButton(context);
return (
- <div>
+ <div title={buttonProps.disabled ? disabledReason : undefined}>
<Button size="sm" {...buttonProps} />
</div>
);
diff --git a/web/src/components/ui/sheet.stories.tsx b/web/src/components/ui/sheet.stories.tsx
new file mode 100644
index 0000000000..c6f9ca1340
--- /dev/null
+++ b/web/src/components/ui/sheet.stories.tsx
@@ -0,0 +1,179 @@
+import { useState } from "react";
+import { expect, fn, userEvent, waitFor, within } from "storybook/test";
+import preview from "../../../.storybook/preview";
+import { Button } from "@/src/components/design-system/Button/Button";
+import { Dialog } from "@/src/components/design-system/Dialog/Dialog";
+import { DialogController } from "@/src/components/design-system/DialogController/DialogController";
+import { Input } from "@/src/components/design-system/Input/Input";
+import { SelectInput } from "@/src/components/design-system/SelectInput/SelectInput";
+import { DropdownMenu } from "@/src/components/design-system/DropdownMenu/DropdownMenu";
+import { Button as TriggerButton } from "./button";
+import { Sheet, SheetContent, SheetHeader, SheetTitle } from "./sheet";
+
+const meta = preview.meta({ component: SheetContent });
+
+export const NestedInteractiveOverlays = meta.story({
+ name: "(Test) Picker and Nested Dialog Remain Accessible",
+ render: () => {
+ const [value, setValue] = useState("");
+ return (
+ <Sheet defaultOpen>
+ <SheetContent className="flex flex-col overflow-hidden">
+ <SheetHeader>
+ <SheetTitle>Setup</SheetTitle>
+ </SheetHeader>
+ <Input aria-label="Draft name" defaultValue="Keep this draft" />
+ <SelectInput
+ aria-label="Choice"
+ value={value}
+ onValueChange={setValue}
+ placeholder="Choose"
+ options={[{ value: "first", label: "First option" }]}
+ />
+ <DialogController
+ renderDialog={() => (
+ <Dialog title="Edit choice" actions={[{ label: "Save" }]}>
+ <Dialog.Body>
+ <Input aria-label="Choice name" defaultValue="First option" />
+ </Dialog.Body>
+ </Dialog>
+ )}
+ >
+ {({ openDialog }) => (
+ <Button text="Edit choice" onClick={openDialog} />
+ )}
+ </DialogController>
+ </SheetContent>
+ </Sheet>
+ );
+ },
+ play: async ({ canvasElement }) => {
+ const body = within(canvasElement.ownerDocument.body);
+ await userEvent.click(body.getByRole("combobox", { name: "Choice" }));
+ const option = await body.findByRole("option", { name: "First option" });
+ await expect(option.closest('[aria-hidden="true"]')).toBeNull();
+ await userEvent.click(option);
+ await userEvent.click(body.getByRole("button", { name: "Edit choice" }));
+ const dialog = await body.findByRole("dialog", { name: "Edit choice" });
+ await expect(dialog.closest('[aria-hidden="true"]')).toBeNull();
+ await waitFor(() => {
+ const rect = dialog.getBoundingClientRect();
+ expect(
+ Math.abs(rect.left + rect.width / 2 - window.innerWidth / 2),
+ ).toBeLessThan(2);
+ });
+ const name = within(dialog).getByRole("textbox", { name: "Choice name" });
+ await userEvent.clear(name);
+ await userEvent.type(name, "Updated choice");
+ await expect(name).toHaveValue("Updated choice");
+ await userEvent.click(
+ within(dialog).getByRole("button", { name: "Cancel" }),
+ );
+ await waitFor(() =>
+ expect(
+ body.queryByRole("dialog", { name: "Edit choice" }),
+ ).not.toBeInTheDocument(),
+ );
+ await expect(body.getByRole("dialog", { name: "Setup" })).toHaveAttribute(
+ "data-state",
+ "open",
+ );
+ await expect(body.getByRole("textbox", { name: "Draft name" })).toHaveValue(
+ "Keep this draft",
+ );
+ await expect(
+ body
+ .getByRole("dialog", { name: "Setup" })
+ .contains(canvasElement.ownerDocument.activeElement),
+ ).toBe(true);
+ },
+});
+
+export const EscapeDismissesNestedMenu = meta.story({
+ name: "(Test) Escape Dismisses Menu Before Sheet",
+ args: { onEscapeKeyDown: fn() },
+ render: (args) => (
+ <Sheet defaultOpen>
+ <SheetContent {...args}>
+ <SheetHeader>
+ <SheetTitle>Setup</SheetTitle>
+ </SheetHeader>
+ <Input aria-label="Draft name" defaultValue="Keep this draft" />
+ <DialogController
+ renderDialog={() => (
+ <Dialog title="Create choice" actions={[{ label: "Save" }]}>
+ <Dialog.Body>
+ <Input aria-label="Choice name" />
+ </Dialog.Body>
+ </Dialog>
+ )}
+ >
+ {({ openDialog }) => (
+ <DropdownMenu
+ ariaLabel="Choices"
+ search={{ placeholder: "Search choices" }}
+ items={[
+ {
+ type: "item",
+ id: "create",
+ title: "Create choice",
+ onClick: openDialog,
+ },
+ ]}
+ >
+ {({ getTriggerProps }) => (
+ <TriggerButton {...getTriggerProps()}>Add choice</TriggerButton>
+ )}
+ </DropdownMenu>
+ )}
+ </DialogController>
+ </SheetContent>
+ </Sheet>
+ ),
+ play: async ({ canvasElement, args }) => {
+ const body = within(canvasElement.ownerDocument.body);
+ await userEvent.click(body.getByRole("button", { name: "Add choice" }));
+ await userEvent.type(
+ body.getByRole("searchbox", { name: "Search choices" }),
+ "create",
+ );
+ await userEvent.keyboard("{Escape}");
+ await expect(
+ body.queryByRole("menu", { name: "Choices" }),
+ ).not.toBeInTheDocument();
+ await expect(body.getByRole("dialog", { name: "Setup" })).toHaveAttribute(
+ "data-state",
+ "open",
+ );
+ await expect(args.onEscapeKeyDown).toHaveBeenCalled();
+ await userEvent.click(body.getByRole("button", { name: "Add choice" }));
+ await userEvent.click(
+ body.getByRole("menuitem", { name: "Create choice" }),
+ );
+ await expect(
+ body.queryByRole("menu", { name: "Choices" }),
+ ).not.toBeInTheDocument();
+ await waitFor(() =>
+ expect(body.getByRole("dialog", { name: "Create choice" })).toBeVisible(),
+ );
+ await userEvent.keyboard("{Escape}");
+ await waitFor(() =>
+ expect(
+ body.queryByRole("dialog", { name: "Create choice" }),
+ ).not.toBeInTheDocument(),
+ );
+ await expect(body.getByRole("dialog", { name: "Setup" })).toHaveAttribute(
+ "data-state",
+ "open",
+ );
+ await expect(body.getByRole("textbox", { name: "Draft name" })).toHaveValue(
+ "Keep this draft",
+ );
+ await userEvent.keyboard("{Escape}");
+ await waitFor(() =>
+ expect(
+ body.queryByRole("dialog", { name: "Setup" }),
+ ).not.toBeInTheDocument(),
+ );
+ },
+});
diff --git a/web/src/components/ui/sheet.tsx b/web/src/components/ui/sheet.tsx
index c88baa5d95..3953ef2ce2 100644
--- a/web/src/components/ui/sheet.tsx
+++ b/web/src/components/ui/sheet.tsx
@@ -7,7 +7,11 @@ import { cva, type VariantProps } from "class-variance-authority";
import { X } from "lucide-react";
import { cn } from "@/src/utils/tailwind";
-import { useLayerContainer } from "@/src/context/LayerContext/LayerContext";
+import {
+ LayerContainerProvider,
+ useLayerContainer,
+} from "@/src/context/LayerContext/LayerContext";
+import { PopoverLayerProvider } from "@/src/context/LayerContext/PopoverLayerProvider";
const Sheet = SheetPrimitive.Root;
@@ -70,22 +74,54 @@ interface SheetContentProps
const SheetContent = React.forwardRef<
React.ComponentRef<typeof SheetPrimitive.Content>,
SheetContentProps
->(({ side = "right", className, children, ...props }, ref) => (
- <SheetPortal>
- <SheetOverlay />
- <SheetPrimitive.Content
- ref={ref}
- className={cn(sheetVariants({ side }), className)}
- {...props}
- >
- {children}
- <SheetPrimitive.Close className="ring-offset-background focus:ring-ring data-[state=open]:bg-secondary absolute top-2 right-2 inline-flex size-8 items-center justify-center rounded-sm opacity-70 transition-opacity hover:opacity-100 focus:ring-2 focus:ring-offset-2 focus:outline-hidden disabled:pointer-events-none">
- <X className="icon-base" />
- <span className="sr-only">Close</span>
- </SheetPrimitive.Close>
- </SheetPrimitive.Content>
- </SheetPortal>
-));
+>(({ side = "right", className, children, onEscapeKeyDown, ...props }, ref) => {
+ const [modalContainer, setModalContainer] =
+ React.useState<HTMLDivElement | null>(null);
+ const [popoverContainer, setPopoverContainer] =
+ React.useState<HTMLDivElement | null>(null);
+ return (
+ <SheetPortal>
+ <SheetOverlay />
+ <SheetPrimitive.Content
+ ref={ref}
+ className={cn(sheetVariants({ side }), className)}
+ {...props}
+ onEscapeKeyDown={(event) => {
+ onEscapeKeyDown?.(event);
+ // Nested menus handle Escape after the Sheet's capture listener.
+ // Let them dismiss without closing the surrounding editing surface.
+ if (
+ popoverContainer?.contains(
+ popoverContainer.ownerDocument.activeElement,
+ )
+ ) {
+ event.preventDefault();
+ }
+ }}
+ >
+ <LayerContainerProvider name="modal" container={modalContainer}>
+ <PopoverLayerProvider container={popoverContainer}>
+ {children}
+ </PopoverLayerProvider>
+ </LayerContainerProvider>
+ <SheetPrimitive.Close className="ring-offset-background focus:ring-ring data-[state=open]:bg-secondary absolute top-2 right-2 inline-flex size-8 items-center justify-center rounded-sm opacity-70 transition-opacity hover:opacity-100 focus:ring-2 focus:ring-offset-2 focus:outline-hidden disabled:pointer-events-none">
+ <X className="icon-base" />
+ <span className="sr-only">Close</span>
+ </SheetPrimitive.Close>
+ <div
+ className="absolute isolate"
+ data-layer="modal"
+ ref={setModalContainer}
+ />
+ <div
+ className="absolute isolate"
+ data-layer="popover"
+ ref={setPopoverContainer}
+ />
+ </SheetPrimitive.Content>
+ </SheetPortal>
+ );
+});
SheetContent.displayName = SheetPrimitive.Content.displayName;
const SheetHeader = ({
diff --git a/web/src/components/ui/side-panel.stories.tsx b/web/src/components/ui/side-panel.stories.tsx
new file mode 100644
index 0000000000..8b297bdf32
--- /dev/null
+++ b/web/src/components/ui/side-panel.stories.tsx
@@ -0,0 +1,24 @@
+import preview from "../../../.storybook/preview";
+import { SidePanel, SidePanelContent, SidePanelHeader } from "./side-panel";
+
+const meta = preview.meta({ component: SidePanel });
+
+export const SetupSidebar = meta.story({
+ render: () => (
+ <div className="flex h-[480px] justify-end border">
+ <SidePanel
+ id="setup-sidebar-story"
+ mobileTitle="Queue setup"
+ variant="sidebar"
+ scrollable={false}
+ >
+ <SidePanelHeader>
+ <span className="text-sm font-bold">Queue setup</span>
+ </SidePanelHeader>
+ <SidePanelContent className="p-3">
+ <p className="text-sm">Queue setup content</p>
+ </SidePanelContent>
+ </SidePanel>
+ </div>
+ ),
+});
diff --git a/web/src/components/ui/side-panel.tsx b/web/src/components/ui/side-panel.tsx
index 213864ecd9..87806c7437 100644
--- a/web/src/components/ui/side-panel.tsx
+++ b/web/src/components/ui/side-panel.tsx
@@ -1,7 +1,14 @@
/* eslint-disable @repo/no-style-props, @repo/no-abstracted-overlay-trigger, @repo/no-null-render */
import { default as React, useState, useCallback, type ReactNode } from "react";
import { Button } from "@/src/components/ui/button";
-import { ChevronLeft, ChevronRight } from "lucide-react";
+import {
+ ChevronLeft,
+ ChevronRight,
+ PanelRightOpen,
+ PanelRightClose,
+} from "lucide-react";
+import { IconButton } from "@/src/components/design-system/IconButton/IconButton";
+import { Tooltip } from "@/src/components/design-system/Tooltip/Tooltip";
import { SubHeader } from "@/src/components/layouts/header";
import { cn } from "@/src/utils/tailwind";
import {
@@ -20,6 +27,8 @@ const SidePanelContext = React.createContext<{
setShowPanel: (value: boolean) => void;
isMobile: boolean;
isControlled: boolean;
+ variant: "default" | "sidebar";
+ label: string;
} | null>(null);
/**
@@ -43,22 +52,31 @@ const SidePanel = ({
className,
mobileTitle,
scrollable = true,
+ variant = "default",
openState,
+ mobileOpenState,
}: {
id: string;
children: ReactNode;
className?: string;
mobileTitle?: string;
scrollable?: boolean;
+ variant?: "default" | "sidebar";
/** Controlled mode: provide to control panel state externally */
openState?: {
open: boolean;
onOpenChange: (open: boolean) => void;
};
+ /** Persist the mobile sheet state externally while retaining built-in toggles. */
+ mobileOpenState?: {
+ open: boolean;
+ onOpenChange: (open: boolean) => void;
+ };
}) => {
const isControlled = openState !== undefined;
const controlledOpen = openState?.open;
const onOpenChange = openState?.onOpenChange;
+ const onMobileOpenChange = mobileOpenState?.onOpenChange;
const [uncontrolledOpen, setUncontrolledOpen] = useSessionStorage<boolean>(
`${id}-showPanel`,
true,
@@ -67,6 +85,7 @@ const SidePanel = ({
const isMobile = useIsMobile();
const showPanel = isControlled ? (controlledOpen ?? false) : uncontrolledOpen;
+ const collapsedWidth = variant === "sidebar" ? "w-10" : "w-fit";
const setShowPanel = useCallback(
(value: boolean) => {
@@ -83,11 +102,13 @@ const SidePanel = ({
(open: boolean) => {
if (isControlled) {
onOpenChange?.(open);
+ } else if (onMobileOpenChange) {
+ onMobileOpenChange(open);
} else {
setMobileSheetOpen(open);
}
},
- [isControlled, onOpenChange],
+ [isControlled, onOpenChange, onMobileOpenChange],
);
const contextValue = React.useMemo(
@@ -96,14 +117,18 @@ const SidePanel = ({
setShowPanel,
isMobile,
isControlled,
+ variant,
+ label: mobileTitle?.toLowerCase() ?? "details",
}),
- [showPanel, setShowPanel, isMobile, isControlled],
+ [showPanel, setShowPanel, isMobile, isControlled, variant, mobileTitle],
);
if (isMobile) {
return (
<Sheet
- open={isControlled ? showPanel : mobileSheetOpen}
+ open={
+ isControlled ? showPanel : (mobileOpenState?.open ?? mobileSheetOpen)
+ }
onOpenChange={handleMobileOpenChange}
>
{!isControlled && (
@@ -113,9 +138,17 @@ const SidePanel = ({
variant="ghost"
size="icon"
className="h-8 w-8"
- title="Show details"
+ title={
+ variant === "sidebar"
+ ? `Show ${mobileTitle?.toLowerCase() ?? "details"}`
+ : "Show details"
+ }
>
- <ChevronLeft className="icon-base text-icon-foreground" />
+ {variant === "sidebar" ? (
+ <PanelRightOpen className="icon-base text-icon-foreground" />
+ ) : (
+ <ChevronLeft className="icon-base text-icon-foreground" />
+ )}
</Button>
</SheetTrigger>
</div>
@@ -142,14 +175,16 @@ const SidePanel = ({
<div
className={cn(
"flex h-full flex-row overflow-hidden border-l",
- showPanel ? "w-[25vw]" : "w-fit",
+ showPanel ? "w-[25vw]" : collapsedWidth,
+ variant === "sidebar" && "bg-surface",
className,
)}
>
<div className="grid h-full w-full items-start gap-2 overflow-hidden">
<div
className={cn(
- "flex h-full w-full flex-col gap-2",
+ "flex h-full w-full flex-col",
+ variant === "sidebar" ? "gap-0" : "gap-2",
scrollable ? "overflow-y-auto" : "overflow-hidden",
)}
>
@@ -166,7 +201,33 @@ const SidePanelHeader = ({ children }: { children: ReactNode }) => {
if (!context) return null;
- const { showPanel, setShowPanel, isControlled } = context;
+ const { showPanel, setShowPanel, isControlled, variant, label } = context;
+
+ if (variant === "sidebar" && !isControlled) {
+ const toggleLabel = `${showPanel ? "Hide" : "Show"} ${label}`;
+ return (
+ <div
+ className={cn(
+ "bg-surface flex h-10 shrink-0 items-center",
+ showPanel ? "px-2" : "justify-center border-b",
+ )}
+ >
+ <Tooltip label={toggleLabel}>
+ {({ getTriggerProps }) => (
+ <IconButton
+ {...getTriggerProps()}
+ icon={showPanel ? PanelRightClose : PanelRightOpen}
+ label={toggleLabel}
+ size="sm"
+ aria-expanded={showPanel}
+ onClick={() => setShowPanel(!showPanel)}
+ />
+ )}
+ </Tooltip>
+ {showPanel && children}
+ </div>
+ );
+ }
if (isControlled) {
if (!showPanel) return null;
diff --git a/web/src/context/LayerContext/LayerContext.tsx b/web/src/context/LayerContext/LayerContext.tsx
index 7068b36068..905fac434e 100644
--- a/web/src/context/LayerContext/LayerContext.tsx
+++ b/web/src/context/LayerContext/LayerContext.tsx
@@ -1,10 +1,30 @@
import * as React from "react";
import { LAYER_ORDER, type LayerName } from "./layers";
+import { PopoverLayerContext } from "./PopoverLayerProvider";
-const LayerContext = React.createContext<Map<LayerName, HTMLElement> | null>(
- null,
-);
+const LayerContext = React.createContext<Map<
+ LayerName,
+ HTMLElement | null
+> | null>(null);
+
+/** Keeps nested overlays within the accessibility subtree of their parent. */
+export function LayerContainerProvider({
+ name,
+ container,
+ children,
+}: {
+ name: LayerName;
+ container: HTMLElement | null;
+ children: React.ReactNode;
+}) {
+ const inheritedLayers = React.useContext(LayerContext);
+ const layers = new Map(inheritedLayers);
+ layers.set(name, container);
+ return (
+ <LayerContext.Provider value={layers}>{children}</LayerContext.Provider>
+ );
+}
function ensureLayerContainers(): Map<LayerName, HTMLElement> {
let root = document.querySelector<HTMLElement>("[data-overlay-root]");
@@ -17,7 +37,7 @@ function ensureLayerContainers(): Map<LayerName, HTMLElement> {
const layerContainers = new Map<LayerName, HTMLElement>();
for (const name of LAYER_ORDER) {
let layer = root.querySelector<HTMLElement>(
- `[data-layer="${CSS.escape(name)}"]`,
+ `:scope > [data-layer="${CSS.escape(name)}"]`,
);
if (!layer) {
layer = document.createElement("div");
@@ -51,7 +71,11 @@ export function LayerProvider({ children }: { children: React.ReactNode }) {
);
}
-/** Returns the body-level container used by Radix/Vaul portals. */
+/** Returns the scoped layer container used by Radix/Vaul portals. */
export function useLayerContainer(name: LayerName) {
- return React.useContext(LayerContext)?.get(name) ?? null;
+ const layers = React.useContext(LayerContext);
+ const popoverContainer = React.useContext(PopoverLayerContext);
+ return name === "popover" && popoverContainer !== undefined
+ ? popoverContainer
+ : (layers?.get(name) ?? null);
}
diff --git a/web/src/context/LayerContext/PopoverLayerProvider.tsx b/web/src/context/LayerContext/PopoverLayerProvider.tsx
new file mode 100644
index 0000000000..fb24ef8f4e
--- /dev/null
+++ b/web/src/context/LayerContext/PopoverLayerProvider.tsx
@@ -0,0 +1,22 @@
+import * as React from "react";
+
+export const PopoverLayerContext = React.createContext<
+ HTMLElement | null | undefined
+>(undefined);
+
+/** Keeps a modal's interactive popovers inside its accessibility subtree. */
+export function PopoverLayerProvider({
+ container,
+ children,
+}: PopoverLayerProviderProps) {
+ return (
+ <PopoverLayerContext.Provider value={container}>
+ {children}
+ </PopoverLayerContext.Provider>
+ );
+}
+
+type PopoverLayerProviderProps = {
+ container: HTMLElement | null;
+ children: React.ReactNode;
+};
diff --git a/web/src/features/annotation-queues/README.md b/web/src/features/annotation-queues/README.md
index 28a98483a2..df374609c0 100644
--- a/web/src/features/annotation-queues/README.md
+++ b/web/src/features/annotation-queues/README.md
@@ -27,10 +27,34 @@ each project and queue; changing the current item keeps the same run history.
- `processors/TraceAnnotationProcessor.tsx` and `SessionAnnotationProcessor.tsx`
prepare their object views. They do not initialize the route's selected span.
- Score forms own their field state; the comments feature owns discussion drafts.
- Shared processing layout components own resize preferences.
+ Queue fields wait for the opening refresh, then remain stable for the active
+ item so later query updates cannot discard unsaved drafts. Reopening an item
+ uses the current queue configuration. Shared processing layout components own
+ resize preferences.
## Structure and consumers
+Queue setup keeps server data in tRPC. `AnnotationQueueSetupController` gates
+initial loading and adapts mutations to `AnnotationQueueSetupForm`. Its React
+Hook Form draft lives above the responsive sidebar, so collapsing the panel or
+closing its mobile sheet preserves unsaved changes. `AnnotationQueueItems` owns
+the mobile open state across initial loading. The Settings tab contains name,
+description, and user assignments; the Annotation panel tab contains ordered
+field previews. Both share one draft and a fixed Save/Discard footer.
+Saves patch only dirty queue fields.
+Background refreshes and project-wide config edits do not reset the draft.
+
+`QueueScoreConfigFields` adapts the existing project-wide create/edit dialogs to
+the controlled `AnnotationQueueScoreConfigEditor`. Creating a config appends its
+ID without saving the queue. The same searchable dropdown used by tracing’s
+Add score action lists available configs and offers config creation. Choosing a
+config appends its ID; the design-system
+`SortableList` owns pointer/touch/keyboard dragging and reports a single reorder
+on drop. Row removal confirms for populated queues.
+`AnnotationQueueFormDialogController` handles queue creation and retains
+creation/assignment retry coordination. The queue list links to each queue's
+inline setup instead of exposing a separate edit action.
+
`components/` contains queue forms, tables, and processing views. `components/shared`
contains object/query adapters and layout primitives; `server/` contains queue and
assignment procedures. Trace/session headers and bulk tables reuse queue-add
diff --git a/web/src/features/annotation-queues/components/AddTracesToAnnotationQueueSelectDialogContent.stories.tsx b/web/src/features/annotation-queues/components/AddTracesToAnnotationQueueSelectDialogContent.stories.tsx
index 66cb4a5b5f..a973fc5998 100644
--- a/web/src/features/annotation-queues/components/AddTracesToAnnotationQueueSelectDialogContent.stories.tsx
+++ b/web/src/features/annotation-queues/components/AddTracesToAnnotationQueueSelectDialogContent.stories.tsx
@@ -1,5 +1,7 @@
import preview from "../../../../.storybook/preview";
import { Dialog, DialogContent } from "@/src/components/ui/dialog";
+import { DialogController } from "@/src/components/design-system/DialogController/DialogController";
+import { AnnotationQueueScoreConfigEditor } from "./AnnotationQueueScoreConfigEditor";
import { AnnotationQueueFormDialogContent } from "@/src/features/annotation-queues/components/AnnotationQueueFormDialogContent";
import { type CreateQueueWithAssignments } from "@langfuse/shared";
import { useState } from "react";
@@ -18,7 +20,6 @@ const meta = preview.meta({
export const CreateQueueInStackedDialog = meta.story({
name: "(Test) Create Queue In Stacked Dialog",
render: () => {
- const [createOpen, setCreateOpen] = useState(false);
const [selectedQueueId, setSelectedQueueId] = useState("queue-1");
const [queueOptions, setQueueOptions] = useState([
{ id: "queue-1", name: "Support review" },
@@ -28,37 +29,32 @@ export const CreateQueueInStackedDialog = meta.story({
const queue = { id: `queue-${queueOptions.length + 1}`, name: data.name };
flushSync(() => setQueueOptions((current) => current.concat(queue)));
setSelectedQueueId(queue.id);
- setCreateOpen(false);
};
return (
- <>
- <Dialog open onOpenChange={fn()}>
- <DialogContent className="sm:max-w-md">
- <AddTracesToAnnotationQueueSelectDialogContent
- key={selectedQueueId}
- description="Add 12 selected traces to an annotation queue."
- initialTargetId={selectedQueueId}
- queueOptionsState={{ status: "ready", options: queueOptions }}
- onSubmit={fn()}
- onCreateNewQueue={() => setCreateOpen(true)}
- createQueueState={{ status: "enabled" }}
- hasAccess={true}
- batchActionState={{ status: "ready" }}
- />
- </DialogContent>
- </Dialog>
- <Dialog open={createOpen} onOpenChange={setCreateOpen}>
- <DialogContent className="max-h-[90vh] overflow-y-auto sm:max-w-lg">
- {createOpen ? (
- <AnnotationQueueFormDialogContent
- mode="create"
- initialValues={{
- name: "",
- description: "",
- scoreConfigIds: ["config-1"],
- newAssignmentUserIds: [],
- }}
+ <DialogController
+ renderDialog={({ closeDialog }) => (
+ <AnnotationQueueFormDialogContent
+ mode="create"
+ initialValues={{
+ name: "",
+ description: "",
+ scoreConfigIds: ["config-1"],
+ newAssignmentUserIds: [],
+ }}
+ projectId="project-1"
+ queueNames={queueOptions.map((queue) => queue.name)}
+ hasQueueAssignmentsReadAccess={false}
+ isSubmitting={false}
+ onSubmit={(data) => {
+ handleCreate(data);
+ closeDialog();
+ }}
+ submitLabel="Create queue"
+ >
+ {(control) => (
+ <AnnotationQueueScoreConfigEditor
+ {...control}
scoreConfigs={[
{
id: "config-1",
@@ -73,18 +69,33 @@ export const CreateQueueInStackedDialog = meta.story({
isArchived: false,
},
]}
- projectId="project-1"
- queueNames={queueOptions.map((queue) => queue.name)}
- onManageScoreConfigsClick={fn()}
- hasQueueAssignmentsReadAccess={false}
- isSubmitting={false}
- onSubmit={handleCreate}
- submitLabel="Create queue"
+ createAction={{ disabled: undefined, onClick: fn() }}
+ editAction={{ disabled: undefined, onClick: fn() }}
+ warnOnRemoval={false}
+ disabled={false}
+ />
+ )}
+ </AnnotationQueueFormDialogContent>
+ )}
+ >
+ {({ openDialog }) => (
+ <Dialog open onOpenChange={fn()}>
+ <DialogContent className="sm:max-w-md">
+ <AddTracesToAnnotationQueueSelectDialogContent
+ key={selectedQueueId}
+ description="Add 12 selected traces to an annotation queue."
+ initialTargetId={selectedQueueId}
+ queueOptionsState={{ status: "ready", options: queueOptions }}
+ onSubmit={fn()}
+ onCreateNewQueue={openDialog}
+ createQueueState={{ status: "enabled" }}
+ hasAccess={true}
+ batchActionState={{ status: "ready" }}
/>
- ) : null}
- </DialogContent>
- </Dialog>
- </>
+ </DialogContent>
+ </Dialog>
+ )}
+ </DialogController>
);
},
play: async ({ canvasElement }) => {
diff --git a/web/src/features/annotation-queues/components/AnnotationQueueDetails.stories.tsx b/web/src/features/annotation-queues/components/AnnotationQueueDetails.stories.tsx
index d6d22c0274..eb5cef3df4 100644
--- a/web/src/features/annotation-queues/components/AnnotationQueueDetails.stories.tsx
+++ b/web/src/features/annotation-queues/components/AnnotationQueueDetails.stories.tsx
@@ -1,3 +1,5 @@
+import { Button } from "@/src/components/design-system/Button/Button";
+import { fn } from "storybook/test";
import { type ComponentProps } from "react";
import preview from "../../../../.storybook/preview";
import { AnnotationQueueDetails } from "./AnnotationQueueDetails";
@@ -47,8 +49,18 @@ const scoreConfigs = [
const meta = preview.meta({
component: AnnotationQueueDetails,
+ args: {
+ editAction: (
+ <Button
+ text="Edit score configs"
+ onClick={fn()}
+ variant="secondary"
+ size="sm"
+ />
+ ),
+ },
render: (args) => (
- <div className="w-80">
+ <div className="h-[600px] w-80">
<AnnotationQueueDetails {...args} />
</div>
),
diff --git a/web/src/features/annotation-queues/components/AnnotationQueueDetails.tsx b/web/src/features/annotation-queues/components/AnnotationQueueDetails.tsx
index 5a4e8f7e9c..098dff1bea 100644
--- a/web/src/features/annotation-queues/components/AnnotationQueueDetails.tsx
+++ b/web/src/features/annotation-queues/components/AnnotationQueueDetails.tsx
@@ -1,3 +1,6 @@
+import { type ReactNode, useState } from "react";
+import { SearchInput } from "@/src/components/design-system/SearchInput/SearchInput";
+import { getScoreDataTypeIcon } from "@/src/features/scores";
import { Badge } from "@/src/components/design-system/Badge/Badge";
import { type ScoreConfigDomain } from "@langfuse/shared";
@@ -36,53 +39,87 @@ function describeScoreField(config: ScoreField): string {
export function AnnotationQueueDetails({
description,
scoreConfigs,
+ editAction,
}: {
description: string | null;
scoreConfigs: ScoreField[];
+ editAction: ReactNode;
}) {
+ const [search, setSearch] = useState("");
+ const visibleConfigs = scoreConfigs.filter((config) =>
+ config.name.toLocaleLowerCase().includes(search.trim().toLocaleLowerCase()),
+ );
return (
- <div className="flex w-full min-w-0 flex-col gap-6 p-4">
+ <div className="flex h-full min-h-0 w-full flex-col">
{description && (
- <section className="flex flex-col gap-2">
- <h3 className="text-sm font-bold">Description</h3>
- <p className="text-muted-foreground text-sm leading-relaxed wrap-anywhere whitespace-pre-wrap">
+ <section className="shrink-0 border-b p-4">
+ <h3 className="text-xs font-bold">Description</h3>
+ <p
+ className="text-muted-foreground mt-2 line-clamp-4 text-xs leading-relaxed wrap-anywhere whitespace-pre-wrap"
+ title={description}
+ >
{description}
</p>
</section>
)}
- <section className="flex min-w-0 flex-col gap-2">
- <div className="flex items-center gap-2">
- <h3 className="text-sm font-bold">Score fields</h3>
- <Badge text={String(scoreConfigs.length)} />
+ <section
+ className="flex min-h-0 flex-1 flex-col"
+ aria-label="Queue score configs"
+ >
+ <div className="flex shrink-0 flex-col gap-3 p-4">
+ <div className="flex items-center justify-between gap-2">
+ <div className="flex min-w-0 items-center gap-2">
+ <h3 className="text-xs font-bold">Score configs</h3>
+ <Badge text={String(scoreConfigs.length)} color="ghost" />
+ </div>
+ </div>
+ {editAction}
+ {scoreConfigs.length > 8 && (
+ <SearchInput
+ value={search}
+ onChange={setSearch}
+ onSubmit={setSearch}
+ placeholder="Find a score config..."
+ />
+ )}
</div>
- {scoreConfigs.length === 0 ? (
- <p className="text-muted-foreground text-sm">
- No score fields configured.
- </p>
- ) : (
- <ul className="divide-y" aria-label="Score fields">
- {[...scoreConfigs]
- .sort((a, b) => a.name.localeCompare(b.name))
- .map((config) => (
- <li key={config.id} className="flex flex-col gap-1 py-3">
- <div className="flex flex-wrap items-center gap-2">
- <span className="min-w-0 text-sm font-bold wrap-anywhere">
- {config.name}
- </span>
- {config.isArchived && <Badge text="Archived" />}
- </div>
- <p className="text-muted-foreground text-xs">
- {describeScoreField(config)}
- </p>
- {config.description && (
- <p className="text-muted-foreground text-xs leading-relaxed wrap-anywhere whitespace-pre-wrap">
- {config.description}
- </p>
- )}
- </li>
- ))}
- </ul>
- )}
+ <ul
+ className="min-h-0 flex-1 overflow-y-auto px-4 pb-4"
+ aria-label="Queue score config order"
+ >
+ {visibleConfigs.map((config) => (
+ <li
+ key={config.id}
+ className="flex items-center gap-2 py-1.5"
+ title={[
+ config.name,
+ describeScoreField(config),
+ config.description,
+ ]
+ .filter(Boolean)
+ .join(" · ")}
+ >
+ <span
+ className="text-muted-foreground w-4 shrink-0 text-center text-xs"
+ aria-hidden
+ >
+ {getScoreDataTypeIcon(config.dataType)}
+ </span>
+ <span
+ className="min-w-0 flex-1 truncate text-xs"
+ title={config.name}
+ >
+ {config.name}
+ </span>
+ {config.isArchived && <Badge text="Archived" size="sm" />}
+ </li>
+ ))}
+ {search && visibleConfigs.length === 0 && (
+ <li className="text-muted-foreground py-2 text-xs">
+ No matching score configs.
+ </li>
+ )}
+ </ul>
</section>
</div>
);
diff --git a/web/src/features/annotation-queues/components/AnnotationQueueFormDialogContent.stories.tsx b/web/src/features/annotation-queues/components/AnnotationQueueFormDialogContent.stories.tsx
index 8bfee41186..01f2ca2c5a 100644
--- a/web/src/features/annotation-queues/components/AnnotationQueueFormDialogContent.stories.tsx
+++ b/web/src/features/annotation-queues/components/AnnotationQueueFormDialogContent.stories.tsx
@@ -1,8 +1,10 @@
import preview from "../../../../.storybook/preview";
-import { Dialog, DialogContent } from "@/src/components/ui/dialog";
+import { DialogController } from "@/src/components/design-system/DialogController/DialogController";
import { type ScoreConfigDomain } from "@langfuse/shared";
import { expect, fn, userEvent, waitFor, within } from "storybook/test";
+import { AnnotationQueueScoreConfigEditor } from "./AnnotationQueueScoreConfigEditor";
+import { type ComponentProps } from "react";
import { AnnotationQueueFormDialogContent } from "./AnnotationQueueFormDialogContent";
const scoreConfigs = [
@@ -29,6 +31,19 @@ const scoreConfigs = [
"id" | "name" | "dataType" | "isArchived"
>[];
+const children: ComponentProps<
+ typeof AnnotationQueueFormDialogContent
+>["children"] = (control) => (
+ <AnnotationQueueScoreConfigEditor
+ {...control}
+ scoreConfigs={scoreConfigs}
+ createAction={{ disabled: undefined, onClick: fn() }}
+ editAction={{ disabled: undefined, onClick: fn() }}
+ warnOnRemoval={false}
+ disabled={false}
+ />
+);
+
const meta = preview.meta({
component: AnnotationQueueFormDialogContent,
parameters: {
@@ -40,8 +55,9 @@ export const Create = meta.story({
name: "(Test) Create",
render: () => {
return (
- <Dialog open onOpenChange={fn()}>
- <DialogContent className="max-h-[90vh] overflow-y-auto">
+ <DialogController<boolean>
+ initialState={() => true}
+ renderDialog={() => (
<AnnotationQueueFormDialogContent
mode="create"
initialValues={{
@@ -49,36 +65,73 @@ export const Create = meta.story({
scoreConfigIds: ["config-1"],
newAssignmentUserIds: [],
}}
- scoreConfigs={scoreConfigs}
projectId="project-1"
queueNames={[]}
- onManageScoreConfigsClick={fn()}
hasQueueAssignmentsReadAccess={false}
isSubmitting={false}
onSubmit={fn()}
submitLabel="Create queue"
- />
- </DialogContent>
- </Dialog>
+ >
+ {children}
+ </AnnotationQueueFormDialogContent>
+ )}
+ >
+ {() => null}
+ </DialogController>
);
},
play: async ({ canvasElement }) => {
const body = within(canvasElement.ownerDocument.body);
+ await userEvent.type(
+ await body.findByRole("textbox", { name: "Name" }),
+ "Review queue draft",
+ );
+ await userEvent.type(
+ body.getByRole("textbox", { name: "Description (optional)" }),
+ "Keep these review instructions",
+ );
await userEvent.click(
- await body.findByRole("combobox", { name: "Score configs" }),
+ await body.findByRole("button", { name: "Add score" }),
+ );
+ await userEvent.type(
+ await body.findByPlaceholderText("Search scores..."),
+ "Correctness",
);
+ await userEvent.click(
+ await body.findByRole("menuitem", { name: /Correctness/ }),
+ );
+ await expect(
+ body.getByRole("list", { name: "Selected score configs" }),
+ ).toHaveTextContent("Correctness");
- await waitFor(() =>
+ await userEvent.click(body.getByRole("button", { name: "Add score" }));
+ await userEvent.type(
+ await body.findByPlaceholderText("Search scores..."),
+ "Another score",
+ );
+ await userEvent.keyboard("{Escape}");
+ await waitFor(() => {
+ expect(body.queryByRole("menu")).not.toBeInTheDocument();
+ expect(body.getByRole("textbox", { name: "Name" })).toHaveValue(
+ "Review queue draft",
+ );
expect(
- body.getByPlaceholderText("Search score configs..."),
- ).toBeVisible(),
+ body.getByRole("textbox", { name: "Description (optional)" }),
+ ).toHaveValue("Keep these review instructions");
+ expect(
+ body.getByRole("list", { name: "Selected score configs" }),
+ ).toHaveTextContent("Correctness");
+ });
+ await userEvent.keyboard("{Escape}");
+ await waitFor(() =>
+ expect(body.queryByRole("dialog")).not.toBeInTheDocument(),
);
},
});
export const Edit = meta.story({
- name: "(Test) Edit with archived config",
+ name: "(Test) Queue settings preserve score configs",
args: {
mode: "edit",
initialValues: {
@@ -87,36 +140,40 @@ export const Edit = meta.story({
scoreConfigIds: ["config-1", "config-2", "config-3"],
newAssignmentUserIds: [],
},
- scoreConfigs,
+ children,
projectId: "project-1",
queueId: "queue-1",
queueNames: ["Support review queue", "Other queue"],
- onManageScoreConfigsClick: fn(),
hasQueueAssignmentsReadAccess: false,
isSubmitting: false,
onSubmit: fn(),
submitLabel: "Save queue",
},
render: (args) => (
- <Dialog open onOpenChange={fn()}>
- <DialogContent className="max-h-[90vh] overflow-y-auto">
- <AnnotationQueueFormDialogContent {...args} />
- </DialogContent>
- </Dialog>
+ <DialogController<boolean>
+ initialState={() => true}
+ renderDialog={() => <AnnotationQueueFormDialogContent {...args} />}
+ >
+ {() => null}
+ </DialogController>
),
play: async ({ canvasElement, args }) => {
const body = within(canvasElement.ownerDocument.body);
- const picker = await body.findByRole("combobox", { name: "Score configs" });
- await expect(picker).toHaveTextContent("Legacy quality");
- await userEvent.click(picker);
- await userEvent.click(
- body.getByRole("option", { name: /Legacy quality/, hidden: true }),
+ await expect(
+ body.queryByRole("button", { name: "Add score" }),
+ ).not.toBeInTheDocument();
+ await userEvent.clear(body.getByRole("textbox", { name: "Name" }));
+ await userEvent.type(
+ body.getByRole("textbox", { name: "Name" }),
+ "Renamed queue",
);
- await userEvent.keyboard("{Escape}");
await userEvent.click(body.getByRole("button", { name: "Save queue" }));
await waitFor(() =>
expect(args.onSubmit).toHaveBeenCalledWith(
- expect.objectContaining({ scoreConfigIds: ["config-1", "config-2"] }),
+ expect.objectContaining({
+ name: "Renamed queue",
+ scoreConfigIds: ["config-1", "config-2", "config-3"],
+ }),
expect.anything(),
),
);
@@ -126,8 +183,9 @@ export const Edit = meta.story({
export const Submitting = meta.story({
render: () => {
return (
- <Dialog open onOpenChange={fn()}>
- <DialogContent className="max-h-[90vh] overflow-y-auto">
+ <DialogController<boolean>
+ initialState={() => true}
+ renderDialog={() => (
<AnnotationQueueFormDialogContent
mode="create"
initialValues={{
@@ -135,17 +193,19 @@ export const Submitting = meta.story({
scoreConfigIds: ["config-1"],
newAssignmentUserIds: [],
}}
- scoreConfigs={scoreConfigs}
projectId="project-1"
queueNames={[]}
- onManageScoreConfigsClick={fn()}
hasQueueAssignmentsReadAccess={false}
isSubmitting={true}
onSubmit={fn()}
submitLabel="Create queue"
- />
- </DialogContent>
- </Dialog>
+ >
+ {children}
+ </AnnotationQueueFormDialogContent>
+ )}
+ >
+ {() => null}
+ </DialogController>
);
},
});
@@ -153,8 +213,9 @@ export const Submitting = meta.story({
export const WithAdvancedOpen = meta.story({
render: () => {
return (
- <Dialog open onOpenChange={fn()}>
- <DialogContent className="max-h-[90vh] overflow-y-auto">
+ <DialogController<boolean>
+ initialState={() => true}
+ renderDialog={() => (
<AnnotationQueueFormDialogContent
mode="create"
initialValues={{
@@ -162,17 +223,19 @@ export const WithAdvancedOpen = meta.story({
scoreConfigIds: ["config-1"],
newAssignmentUserIds: ["user-1"],
}}
- scoreConfigs={scoreConfigs}
projectId="project-1"
queueNames={[]}
- onManageScoreConfigsClick={fn()}
hasQueueAssignmentsReadAccess={true}
isSubmitting={false}
onSubmit={fn()}
submitLabel="Create queue"
- />
- </DialogContent>
- </Dialog>
+ >
+ {children}
+ </AnnotationQueueFormDialogContent>
+ )}
+ >
+ {() => null}
+ </DialogController>
);
},
});
diff --git a/web/src/features/annotation-queues/components/AnnotationQueueFormDialogContent.tsx b/web/src/features/annotation-queues/components/AnnotationQueueFormDialogContent.tsx
index 32dd63f26f..d570f13a62 100644
--- a/web/src/features/annotation-queues/components/AnnotationQueueFormDialogContent.tsx
+++ b/web/src/features/annotation-queues/components/AnnotationQueueFormDialogContent.tsx
@@ -1,288 +1,183 @@
-import { Button } from "@/src/components/ui/button";
-import {
- DialogBody,
- DialogDescription,
- DialogFooter,
- DialogHeader,
- DialogTitle,
-} from "@/src/components/ui/dialog";
-import {
- FormControl,
- FormDescription,
- FormField,
- FormItem,
- FormLabel,
- FormMessage,
- Form,
-} from "@/src/components/ui/form";
-import { Input } from "@/src/components/ui/input";
-import { Textarea } from "@/src/components/ui/textarea";
-import {
- Collapsible,
- CollapsibleContent,
- CollapsibleTrigger,
-} from "@/src/components/ui/collapsible";
-import { Settings2 } from "lucide-react";
-import { DropdownIndicator } from "@/src/components/design-system/DropdownIndicator/DropdownIndicator";
-import Link from "next/link";
-import { getScoreDataTypeIcon } from "@/src/features/scores";
-import { MultiSelectTagInput } from "@/src/components/design-system/MultiSelectTagInput/MultiSelectTagInput";
+import { type ReactNode, useId } from "react";
+import { zodResolver } from "@hookform/resolvers/zod";
+import { useForm } from "react-hook-form";
+import { z } from "zod";
import {
CreateQueueWithAssignmentsData,
type CreateQueueWithAssignments,
- type ScoreConfigDomain,
} from "@langfuse/shared";
-import { zodResolver } from "@hookform/resolvers/zod";
-import { useMemo, useState } from "react";
-import { useForm } from "react-hook-form";
-import { UserAssignmentSection } from "@/src/features/annotation-queues/components/UserAssignmentSection";
-import { useUniqueNameValidation } from "@/src/hooks/useUniqueNameValidation";
-type AnnotationQueueScoreConfigOption = Pick<
- ScoreConfigDomain,
- "id" | "name" | "dataType" | "isArchived"
->;
+import { Accordion } from "@/src/components/design-system/Accordion/Accordion";
+import { Dialog } from "@/src/components/design-system/Dialog/Dialog";
+import { Form } from "@/src/components/design-system/Form/Form";
+import { Input } from "@/src/components/design-system/Input/Input";
+import { Textarea } from "@/src/components/ui/textarea";
+import { UserAssignmentSection } from "@/src/features/annotation-queues/components/UserAssignmentSection";
type AnnotationQueueFormDialogContentProps = {
mode: "create" | "edit";
initialValues: CreateQueueWithAssignments;
- scoreConfigs: AnnotationQueueScoreConfigOption[];
+ children: (control: {
+ value: string[];
+ onChange: (ids: string[]) => void;
+ }) => ReactNode;
projectId: string;
queueId?: string;
queueNames: string[];
- onManageScoreConfigsClick: () => void;
hasQueueAssignmentsReadAccess: boolean;
isSubmitting: boolean;
onSubmit: (data: CreateQueueWithAssignments) => void;
submitLabel: string;
};
+const queueFormSchema = CreateQueueWithAssignmentsData.extend({
+ scoreConfigIds: z
+ .array(z.string())
+ .min(1, "Add or create at least one score config."),
+});
+
export function AnnotationQueueFormDialogContent({
mode,
initialValues,
- scoreConfigs,
+ children,
projectId,
queueId,
queueNames,
- onManageScoreConfigsClick,
hasQueueAssignmentsReadAccess,
isSubmitting,
onSubmit,
submitLabel,
}: AnnotationQueueFormDialogContentProps) {
- const [isAdvancedOpen, setIsAdvancedOpen] = useState(false);
+ const formId = useId();
const form = useForm<CreateQueueWithAssignments>({
- resolver: zodResolver(CreateQueueWithAssignmentsData),
+ resolver: zodResolver(
+ (mode === "create"
+ ? queueFormSchema
+ : CreateQueueWithAssignmentsData.extend({
+ scoreConfigIds: z.array(z.string()),
+ })
+ ).superRefine((values, context) => {
+ if (
+ queueNames.includes(values.name) &&
+ !(mode === "edit" && values.name === initialValues.name)
+ ) {
+ context.addIssue({
+ code: "custom",
+ path: ["name"],
+ message: "Queue name already exists.",
+ });
+ }
+ }),
+ ),
defaultValues: initialValues,
+ mode: "onChange",
});
- const queueNameOptions = useMemo(
- () => queueNames.map((name) => ({ value: name })),
- [queueNames],
- );
-
- useUniqueNameValidation({
- currentName: form.watch("name"),
- allNames: queueNameOptions,
- form,
- errorMessage: "Queue name already exists.",
- whitelistedName: mode === "edit" ? initialValues.name : undefined,
- });
-
const handleScoreConfigValueChange = (values: string[]) => {
- form.setValue("scoreConfigIds", values);
-
- if (values.length === 0) {
- form.setError("scoreConfigIds", {
- type: "manual",
- message: "At least 1 score config must be selected",
- });
- } else {
- form.clearErrors("scoreConfigIds");
- }
+ form.setValue("scoreConfigIds", values, {
+ shouldDirty: true,
+ shouldValidate: true,
+ });
};
-
return (
- <>
- <DialogHeader>
- <DialogTitle>
- {mode === "edit" ? "Edit" : "New"} annotation queue
- </DialogTitle>
- <DialogDescription>
- {mode === "edit" ? "Edit" : "Create a new"} queue to manage your
- annotation workflows.
- </DialogDescription>
- </DialogHeader>
- <Form {...form}>
- <form className="space-y-6" onSubmit={form.handleSubmit(onSubmit)}>
- <DialogBody>
- <FormField
- control={form.control}
- name="name"
- render={({ field }) => (
- <FormItem>
- <FormLabel>Name</FormLabel>
- <FormControl>
- <Input
- {...field}
- type="text"
- className="text-xs"
- onBlur={(e) => field.onChange(e.target.value.trimEnd())}
- />
- </FormControl>
- <FormMessage />
- </FormItem>
- )}
- />
- <FormField
- control={form.control}
- name="description"
- render={({ field }) => (
- <FormItem>
- <FormLabel>Description (optional)</FormLabel>
- <FormControl>
- <Textarea
- {...field}
- placeholder="Add description..."
- className="text-xs focus:ring-0 focus:outline-hidden focus-visible:ring-0 focus-visible:ring-offset-0 active:ring-0"
- />
- </FormControl>
- <FormMessage />
- </FormItem>
- )}
- />
- <FormField
+ <Dialog
+ stopPropagationOnEnterSpace={false}
+ title={mode === "edit" ? "Queue settings" : "New annotation queue"}
+ actions={[
+ {
+ label: submitLabel,
+ type: "submit",
+ form: formId,
+ disabled: !!form.formState.errors.name,
+ loading: isSubmitting,
+ },
+ ]}
+ >
+ <Dialog.Body>
+ <Form id={formId} actions={[]} onSubmit={form.handleSubmit(onSubmit)}>
+ <Form.Field control={form.control} name="name" label="Name">
+ {(field) => (
+ <Input
+ id={field.id}
+ name={field.name}
+ value={field.value}
+ onChange={field.onChange}
+ onBlur={(event) => {
+ field.onChange(event.target.value.trimEnd());
+ field.onBlur();
+ }}
+ ref={field.ref}
+ type="text"
+ aria-describedby={field.inputDescribedById}
+ aria-invalid={!!field.error}
+ error={!!field.error}
+ />
+ )}
+ </Form.Field>
+ <Form.Field
+ control={form.control}
+ name="description"
+ label="Description (optional)"
+ >
+ {(field) => (
+ <Textarea
+ id={field.id}
+ name={field.name}
+ value={field.value ?? ""}
+ onChange={field.onChange}
+ onBlur={field.onBlur}
+ ref={field.ref}
+ placeholder="Add description..."
+ aria-describedby={field.inputDescribedById}
+ aria-invalid={!!field.error}
+ />
+ )}
+ </Form.Field>
+ {mode === "create" && (
+ <Form.Field
control={form.control}
name="scoreConfigIds"
- render={({ field }) => (
- <FormItem>
- <div className="flex flex-wrap items-center justify-between gap-2">
- <FormLabel>Score configs</FormLabel>
- <Button
- variant="outline"
- size="sm"
- className="bg-accent gap-1.5 text-xs"
- asChild
- >
- <Link
- href={`/project/${projectId}/settings/scores`}
- target="_blank"
- onClick={onManageScoreConfigsClick}
- onAuxClick={(event) => {
- if (event.button === 1) onManageScoreConfigsClick();
- }}
- >
- <Settings2
- className="icon-base text-icon-foreground"
- aria-hidden="true"
- />
- Manage score configs
- </Link>
- </Button>
- </div>
- <FormDescription>
- Define which dimensions annotators should score for the
- given queue.
- </FormDescription>
- <FormControl>
- <MultiSelectTagInput
- aria-label="Score configs"
- placeholder="Choose score configs"
- searchPlaceholder="Search score configs..."
- emptyMessage="No score configs found."
- onValueChange={handleScoreConfigValueChange}
- options={scoreConfigs
- .filter(
- (config) =>
- !config.isArchived ||
- field.value.includes(config.id),
- )
- .map((config) => ({
- value: config.id,
- label: `${getScoreDataTypeIcon(config.dataType)} ${config.name}`,
- secondaryLabel: config.isArchived
- ? "Archived"
- : undefined,
- showSecondaryLabelInTag: true,
- }))}
- value={field.value}
- />
- </FormControl>
- <FormMessage />
- </FormItem>
+ label="Score configs"
+ description="Choose the fields annotators will fill in."
+ >
+ {(field) => (
+ <div>
+ {children({
+ value: field.value,
+ onChange: handleScoreConfigValueChange,
+ })}
+ </div>
)}
- />
-
- <FormField
+ </Form.Field>
+ )}
+ {hasQueueAssignmentsReadAccess ? (
+ <Form.Field
control={form.control}
name="newAssignmentUserIds"
- render={() => (
- <FormItem>
- <FormLabel>Advanced Settings</FormLabel>
- <div className="mt-1 rounded-md border">
- <Collapsible
- open={isAdvancedOpen && hasQueueAssignmentsReadAccess}
- onOpenChange={(open) => {
- if (!hasQueueAssignmentsReadAccess) {
- setIsAdvancedOpen(false);
- } else {
- setIsAdvancedOpen(open);
- }
- }}
- >
- <CollapsibleTrigger asChild>
- <Button
- type="button"
- variant="ghost"
- className="group flex w-full items-center justify-between px-3 py-2.5 text-left hover:bg-transparent"
- >
- <div className="flex items-center gap-2">
- {isAdvancedOpen ? (
- <DropdownIndicator nudge />
- ) : (
- <DropdownIndicator direction="right" nudge />
- )}
- <span className="text-sm font-bold">
- User Assignment
- </span>
- </div>
- </Button>
- </CollapsibleTrigger>
- <CollapsibleContent className="border-border/20 border-t px-3 pt-1 pb-3">
- {hasQueueAssignmentsReadAccess ? (
- <>
- <FormControl>
- <UserAssignmentSection
- projectId={projectId}
- queueId={queueId}
- selectedUserIds={form.watch(
- "newAssignmentUserIds",
- )}
- onChange={(userIds) =>
- form.setValue("newAssignmentUserIds", userIds)
- }
- />
- </FormControl>
- <FormMessage />
- </>
- ) : null}
- </CollapsibleContent>
- </Collapsible>
- </div>
- </FormItem>
- )}
- />
- </DialogBody>
- <DialogFooter>
- <Button
- type="submit"
- className="text-xs"
- disabled={!!form.formState.errors.name || isSubmitting}
+ label="Advanced Settings"
>
- {isSubmitting ? "Processing..." : submitLabel}
- </Button>
- </DialogFooter>
- </form>
- </Form>
- </>
+ {(field) => (
+ <Accordion type="single" collapsible>
+ <Accordion.Item value="assignments">
+ <Accordion.Trigger size="sm">
+ User Assignment
+ </Accordion.Trigger>
+ <Accordion.Content>
+ <div className="py-3">
+ <UserAssignmentSection
+ projectId={projectId}
+ queueId={queueId}
+ selectedUserIds={field.value}
+ onChange={field.onChange}
+ />
+ </div>
+ </Accordion.Content>
+ </Accordion.Item>
+ </Accordion>
+ )}
+ </Form.Field>
+ ) : null}
+ </Form>
+ </Dialog.Body>
+ </Dialog>
);
}
diff --git a/web/src/features/annotation-queues/components/AnnotationQueueFormDialogController.clienttest.tsx b/web/src/features/annotation-queues/components/AnnotationQueueFormDialogController.clienttest.tsx
index a250168a8b..ccb43e4bf8 100644
--- a/web/src/features/annotation-queues/components/AnnotationQueueFormDialogController.clienttest.tsx
+++ b/web/src/features/annotation-queues/components/AnnotationQueueFormDialogController.clienttest.tsx
@@ -1,5 +1,4 @@
import { fireEvent, render, screen, waitFor } from "@testing-library/react";
-import { type ReactNode } from "react";
import { vi } from "vitest";
import { AnnotationQueueFormDialogController } from "./AnnotationQueueFormDialogController";
@@ -21,10 +20,6 @@ vi.mock("@/src/features/notifications", () => ({ showErrorToast: notify }));
vi.mock("@/src/features/posthog-analytics", () => ({
usePostHogClientCapture: () => vi.fn(),
}));
-vi.mock("@/src/components/ui/dialog", () => ({
- Dialog: ({ children }: { children: ReactNode }) => children,
- DialogContent: ({ children }: { children: ReactNode }) => children,
-}));
vi.mock("./AnnotationQueueFormDialogContent", () => ({
AnnotationQueueFormDialogContent: ({
onSubmit,
@@ -50,6 +45,9 @@ vi.mock("@/src/utils/api", () => ({
create: { useMutation: () => ({ mutateAsync: create }) },
update: { useMutation: () => ({ mutateAsync: update }) },
},
+ annotationQueueItems: {
+ itemsByQueueId: { useQuery: () => ({ data: { totalItems: 0 } }) },
+ },
annotationQueueAssignments: {
createMany: { useMutation: () => ({ mutateAsync: assign }) },
},
diff --git a/web/src/features/annotation-queues/components/AnnotationQueueFormDialogController.tsx b/web/src/features/annotation-queues/components/AnnotationQueueFormDialogController.tsx
index 4a499ffb5a..b1f64ba460 100644
--- a/web/src/features/annotation-queues/components/AnnotationQueueFormDialogController.tsx
+++ b/web/src/features/annotation-queues/components/AnnotationQueueFormDialogController.tsx
@@ -1,126 +1,106 @@
-/* eslint-disable no-nested-ternary */
-import { showErrorToast } from "@/src/features/notifications";
-import { useHasProjectAccess } from "@/src/features/rbac";
-import {
- type CreateQueueWithAssignments,
- type ScoreConfigDomain,
-} from "@langfuse/shared";
import { type ReactNode, useState } from "react";
+import { type CreateQueueWithAssignments } from "@langfuse/shared";
-import { Dialog, DialogContent } from "@/src/components/ui/dialog";
+import { DialogController } from "@/src/components/design-system/DialogController/DialogController";
+import { Skeleton } from "@/src/components/ui/skeleton";
+import { Dialog } from "@/src/components/design-system/Dialog/Dialog";
+import { showErrorToast } from "@/src/features/notifications";
+import { useHasProjectAccess } from "@/src/features/rbac";
import { AnnotationQueueFormDialogContent } from "@/src/features/annotation-queues/components/AnnotationQueueFormDialogContent";
-import { usePostHogClientCapture } from "@/src/features/posthog-analytics";
+import { QueueScoreConfigFields } from "./QueueScoreConfigFields";
import { useWatchedPromiseCallback } from "@/src/hooks/useWatchedPromiseCallback";
-import { api } from "@/src/utils/api";
+import { api, type RouterInputs, type RouterOutputs } from "@/src/utils/api";
import { classifyTrpcToastError } from "@/src/utils/trpcErrorClassification";
-type AnnotationQueueFormDialogControllerProps = {
- projectId: string;
- onSuccess: (queueId: string) => void;
- children: (control: {
- disabled: { reason: string } | undefined;
- openDialog: () => void;
- }) => ReactNode;
-} & (
- | { mode: "create" }
- | {
- mode: "edit";
- queueId: string;
- }
-);
-
export function AnnotationQueueFormDialogController(
props: AnnotationQueueFormDialogControllerProps,
) {
const { projectId, onSuccess, mode, children } = props;
- const queueId = mode === "edit" ? props.queueId : undefined;
-
- const [open, setOpen] = useState(false);
- const [createdQueueId, setCreatedQueueId] = useState<string>();
-
const hasQueueAccess = useHasProjectAccess({
projectId,
scope: "annotationQueues:CUD",
});
-
const disabled = hasQueueAccess
? undefined
- : {
- reason: `You don't have permission to ${mode} annotation queues.`,
- };
-
- const openDialog = () => {
- if (!hasQueueAccess) return;
- setCreatedQueueId(undefined);
- setOpen(true);
- };
+ : { reason: `You don't have permission to ${mode} annotation queues.` };
+ return (
+ <DialogController
+ renderDialog={({ closeDialog, isOpen }) =>
+ hasQueueAccess ? (
+ <AnnotationQueueDialogData
+ projectId={projectId}
+ queueId={mode === "edit" ? props.queueId : undefined}
+ mode={mode}
+ isOpen={isOpen}
+ onSuccess={onSuccess}
+ closeDialog={closeDialog}
+ />
+ ) : null
+ }
+ >
+ {({ openDialog }) =>
+ children({
+ disabled,
+ openDialog: () => {
+ if (hasQueueAccess) openDialog();
+ },
+ })
+ }
+ </DialogController>
+ );
+}
+function AnnotationQueueDialogData({
+ projectId,
+ queueId,
+ mode,
+ isOpen,
+ onSuccess,
+ closeDialog,
+}: {
+ projectId: string;
+ queueId: string | undefined;
+ mode: "create" | "edit";
+ isOpen: boolean;
+ onSuccess: (queueId: string) => void;
+ closeDialog: () => void;
+}) {
+ const [createdQueueId, setCreatedQueueId] = useState<string>();
const hasQueueAssignmentsReadAccess = useHasProjectAccess({
projectId,
scope: "annotationQueueAssignments:read",
});
-
- const capture = usePostHogClientCapture();
-
- const queueQuery = api.annotationQueues.byId.useQuery(
- { projectId, queueId: queueId ?? "" },
- { enabled: mode === "edit" && hasQueueAccess && open },
- );
- const configsQuery = api.scoreConfigs.all.useQuery(
- { projectId },
- { enabled: hasQueueAccess && open },
- );
- const allQueueNamesAndIds = api.annotationQueues.allNamesAndIds.useQuery(
- { projectId },
- { enabled: hasQueueAccess && open },
- );
-
+ const { queueQuery, configsQuery, allQueueNamesAndIds } = useQueueFormData({
+ projectId,
+ queueId,
+ mode,
+ isOpen,
+ });
const utils = api.useUtils();
const createQueueMutation = api.annotationQueues.create.useMutation();
const editQueueMutation = api.annotationQueues.update.useMutation();
const createQueueAssignmentsMutation =
api.annotationQueueAssignments.createMany.useMutation();
-
const [handleSubmit, isSubmitting] = useWatchedPromiseCallback(
async (data: CreateQueueWithAssignments) => {
try {
- let targetQueueId: string;
- const existingQueueId = queueId ?? createdQueueId;
- if (existingQueueId) {
- await editQueueMutation.mutateAsync({
- name: data.name,
- description: data.description,
- scoreConfigIds: data.scoreConfigIds,
- projectId,
- queueId: existingQueueId,
- });
- targetQueueId = existingQueueId;
- } else {
- const queueResponse = await createQueueMutation.mutateAsync({
- name: data.name,
- description: data.description,
- scoreConfigIds: data.scoreConfigIds,
- projectId,
- });
- targetQueueId = queueResponse.id;
- setCreatedQueueId(targetQueueId);
- }
-
- if (data.newAssignmentUserIds.length > 0) {
- await createQueueAssignmentsMutation.mutateAsync({
- projectId,
- queueId: targetQueueId,
- userIds: data.newAssignmentUserIds,
- });
- }
-
+ const targetQueueId = await saveQueueWithAssignments({
+ data,
+ projectId,
+ existingQueueId: queueId ?? createdQueueId,
+ includeScoreConfigs: mode === "create",
+ createQueue: createQueueMutation.mutateAsync,
+ updateQueue: editQueueMutation.mutateAsync,
+ assignUsers: createQueueAssignmentsMutation.mutateAsync,
+ onQueueCreated: setCreatedQueueId,
+ });
await Promise.all([
utils.annotationQueues.invalidate(),
utils.annotationQueueAssignments.invalidate(),
]);
setCreatedQueueId(undefined);
onSuccess(targetQueueId);
- setOpen(false);
+ closeDialog();
} catch (error) {
showErrorToast(
"Operation failed",
@@ -134,60 +114,162 @@ export function AnnotationQueueFormDialogController(
createQueueMutation,
editQueueMutation,
createdQueueId,
+ closeDialog,
onSuccess,
projectId,
queueId,
+ mode,
utils.annotationQueueAssignments,
utils.annotationQueues,
],
);
-
- const initialValues: CreateQueueWithAssignments | undefined =
- mode === "edit"
- ? queueQuery.data
- ? {
- name: queueQuery.data.name ?? "",
- description: queueQuery.data.description || undefined,
- scoreConfigIds: queueQuery.data.scoreConfigs.map(
- (config: ScoreConfigDomain) => config.id,
- ),
- newAssignmentUserIds: [],
- }
- : undefined
- : {
- name: "",
- scoreConfigIds: [],
- newAssignmentUserIds: [],
- };
-
+ const initialValues = prepareQueueFormValues(mode, queueQuery.data);
+ const configs = configsQuery.data?.configs;
+ const queues = allQueueNamesAndIds.data;
+ if ((mode === "create" && !configs) || !initialValues || !queues) {
+ return (
+ <Dialog
+ title={mode === "edit" ? "Queue settings" : "New annotation queue"}
+ >
+ <Dialog.Body>
+ <Skeleton className="h-64 w-full" />
+ </Dialog.Body>
+ </Dialog>
+ );
+ }
return (
- <Dialog open={hasQueueAccess && open} onOpenChange={setOpen}>
- {children({ disabled, openDialog })}
- {open && configsQuery.data && initialValues ? (
- <DialogContent className="max-h-[90vh] overflow-y-auto">
- <AnnotationQueueFormDialogContent
- mode={mode}
- initialValues={initialValues}
- scoreConfigs={configsQuery.data.configs}
- projectId={projectId}
- queueId={queueId ?? createdQueueId}
- queueNames={
- allQueueNamesAndIds.data
- ?.filter((queue) => queue.id !== (queueId ?? createdQueueId))
- .map((queue) => queue.name) ?? []
- }
- onManageScoreConfigsClick={() => {
- capture("score_configs:manage_configs_item_click", {
- source: "AnnotationQueue",
- });
- }}
- hasQueueAssignmentsReadAccess={hasQueueAssignmentsReadAccess}
- isSubmitting={isSubmitting}
- onSubmit={handleSubmit}
- submitLabel={mode === "edit" ? "Save queue" : "Create queue"}
- />
- </DialogContent>
- ) : null}
- </Dialog>
+ <AnnotationQueueFormDialogContent
+ mode={mode}
+ initialValues={initialValues}
+ projectId={projectId}
+ queueId={queueId ?? createdQueueId}
+ queueNames={queues
+ .filter((queue) => queue.id !== (queueId ?? createdQueueId))
+ .map((queue) => queue.name)}
+ hasQueueAssignmentsReadAccess={hasQueueAssignmentsReadAccess}
+ isSubmitting={isSubmitting}
+ onSubmit={handleSubmit}
+ submitLabel={mode === "edit" ? "Save queue" : "Create queue"}
+ >
+ {({ value, onChange }) => (
+ <QueueScoreConfigFields
+ projectId={projectId}
+ configs={configs ?? []}
+ value={value}
+ onChange={onChange}
+ disabled={isSubmitting}
+ warnOnRemoval={false}
+ />
+ )}
+ </AnnotationQueueFormDialogContent>
+ );
+}
+
+function useQueueFormData({
+ projectId,
+ queueId,
+ mode,
+ isOpen,
+}: {
+ projectId: string;
+ queueId: string | undefined;
+ mode: "create" | "edit";
+ isOpen: boolean;
+}) {
+ const queueQuery = api.annotationQueues.byId.useQuery(
+ { projectId, queueId: queueId ?? "" },
+ { enabled: mode === "edit" && isOpen },
+ );
+ const configsQuery = api.scoreConfigs.all.useQuery(
+ { projectId },
+ { enabled: isOpen && mode === "create" },
+ );
+ const allQueueNamesAndIds = api.annotationQueues.allNamesAndIds.useQuery(
+ { projectId },
+ { enabled: isOpen },
);
+ return { queueQuery, configsQuery, allQueueNamesAndIds };
}
+function prepareQueueFormValues(
+ mode: "create" | "edit",
+ queue: RouterOutputs["annotationQueues"]["byId"] | undefined,
+): CreateQueueWithAssignments | undefined {
+ if (mode === "create") {
+ return { name: "", scoreConfigIds: [], newAssignmentUserIds: [] };
+ }
+ if (!queue) return undefined;
+ return {
+ name: queue.name ?? "",
+ description: queue.description || undefined,
+ scoreConfigIds: queue.scoreConfigs.map((config) => config.id),
+ newAssignmentUserIds: [],
+ };
+}
+
+async function saveQueueWithAssignments({
+ data,
+ projectId,
+ existingQueueId,
+ includeScoreConfigs,
+ createQueue,
+ updateQueue,
+ assignUsers,
+ onQueueCreated,
+}: {
+ data: CreateQueueWithAssignments;
+ projectId: string;
+ existingQueueId: string | undefined;
+ includeScoreConfigs: boolean;
+ createQueue: (
+ input: RouterInputs["annotationQueues"]["create"],
+ ) => Promise<{ id: string }>;
+ updateQueue: (
+ input: RouterInputs["annotationQueues"]["update"],
+ ) => Promise<unknown>;
+ assignUsers: (
+ input: RouterInputs["annotationQueueAssignments"]["createMany"],
+ ) => Promise<unknown>;
+ onQueueCreated: (queueId: string) => void;
+}) {
+ const queueData = {
+ name: data.name,
+ description: data.description,
+ scoreConfigIds: data.scoreConfigIds,
+ projectId,
+ };
+ let targetQueueId = existingQueueId;
+ if (targetQueueId) {
+ await updateQueue({
+ ...queueData,
+ scoreConfigIds: includeScoreConfigs ? data.scoreConfigIds : undefined,
+ queueId: targetQueueId,
+ });
+ } else {
+ const queue = await createQueue(queueData);
+ targetQueueId = queue.id;
+ onQueueCreated(targetQueueId);
+ }
+ if (data.newAssignmentUserIds.length > 0) {
+ await assignUsers({
+ projectId,
+ queueId: targetQueueId,
+ userIds: data.newAssignmentUserIds,
+ });
+ }
+ return targetQueueId;
+}
+
+type AnnotationQueueFormDialogControllerProps = {
+ projectId: string;
+ onSuccess: (queueId: string) => void;
+ children: (control: {
+ disabled: { reason: string } | undefined;
+ openDialog: () => void;
+ }) => ReactNode;
+} & (
+ | { mode: "create" }
+ | {
+ mode: "edit";
+ queueId: string;
+ }
+);
diff --git a/web/src/features/annotation-queues/components/AnnotationQueueItemPage.tsx b/web/src/features/annotation-queues/components/AnnotationQueueItemPage.tsx
index 9257d49ef6..1f5a7e8196 100644
--- a/web/src/features/annotation-queues/components/AnnotationQueueItemPage.tsx
+++ b/web/src/features/annotation-queues/components/AnnotationQueueItemPage.tsx
@@ -202,7 +202,10 @@ function AnnotationQueueRunContent({
{ projectId, itemId: itemId as string },
{ enabled: !!itemId, refetchOnMount: false },
);
- const { configs } = useAnnotationQueueData({ annotationQueueId, projectId });
+ const { configs, isLoading: queueConfigsLoading } = useAnnotationQueueData({
+ annotationQueueId,
+ projectId,
+ });
const unseenPendingItemCount =
api.annotationQueueItems.unseenPendingItemCountByQueueId.useQuery(
{ queueId: annotationQueueId, projectId, seenItemIds },
@@ -260,7 +263,8 @@ function AnnotationQueueRunContent({
// not complete or skip an item the annotator hasn't actually seen yet
// (e.g. a quick → between ⌘/Ctrl+Enter and onSuccess advancing would skip
// the next item, which only flashed as a Skeleton).
- if (objectData.isLoading || isTransitioning) return;
+ if (queueConfigsLoading || objectData.isLoading || isTransitioning)
+ return;
// Complete + next — the Cmd/Ctrl+Enter submit chord. Handled first and
// *before* the typing guard so it works even while the annotator is in the
@@ -358,6 +362,7 @@ function AnnotationQueueRunContent({
isTransitioning,
objectData.isError,
objectData.isLoading,
+ queueConfigsLoading,
handleComplete,
handleNavigateNext,
handleNavigateBack,
@@ -368,6 +373,7 @@ function AnnotationQueueRunContent({
(seenItemData.isPending && itemId) ||
(isTransitioning && !itemId) ||
unseenPendingItemCount.isPending ||
+ queueConfigsLoading ||
objectData.isLoading
) {
return <Skeleton className="h-full w-full" />;
diff --git a/web/src/features/annotation-queues/components/AnnotationQueueScoreConfigEditor.stories.tsx b/web/src/features/annotation-queues/components/AnnotationQueueScoreConfigEditor.stories.tsx
new file mode 100644
index 0000000000..6c0fdb4a26
--- /dev/null
+++ b/web/src/features/annotation-queues/components/AnnotationQueueScoreConfigEditor.stories.tsx
@@ -0,0 +1,269 @@
+import { type ComponentProps, useState } from "react";
+import { expect, fn, userEvent, waitFor, within } from "storybook/test";
+import preview from "../../../../.storybook/preview";
+import { AnnotationQueueScoreConfigEditor } from "./AnnotationQueueScoreConfigEditor";
+
+type EditorProps = ComponentProps<typeof AnnotationQueueScoreConfigEditor>;
+const configs: EditorProps["scoreConfigs"] = [
+ {
+ id: "accuracy",
+ name: "Accuracy",
+ dataType: "NUMERIC",
+ isArchived: false,
+ minValue: 1,
+ maxValue: 5,
+ },
+ {
+ id: "helpfulness",
+ name: "Helpfulness",
+ dataType: "BOOLEAN",
+ isArchived: false,
+ categories: [
+ { label: "True", value: 1 },
+ { label: "False", value: 0 },
+ ],
+ },
+ { id: "feedback", name: "Feedback", dataType: "TEXT", isArchived: false },
+ {
+ id: "legacy",
+ name: "Legacy quality",
+ dataType: "NUMERIC",
+ isArchived: true,
+ },
+];
+const largeConfigs: EditorProps["scoreConfigs"] = Array.from(
+ { length: 120 },
+ (_, index) => ({
+ id: `config-${index}`,
+ name: `Review dimension ${String(index + 1).padStart(3, "0")} — response quality and grounding`,
+ dataType: "NUMERIC",
+ isArchived: false,
+ }),
+);
+const baseArgs: EditorProps = {
+ scoreConfigs: configs,
+ value: ["helpfulness", "accuracy"],
+ onChange: fn(),
+ createAction: { disabled: undefined, onClick: fn() },
+ editAction: { disabled: undefined, onClick: fn() },
+ warnOnRemoval: false,
+ disabled: false,
+};
+function StatefulEditor(args: EditorProps) {
+ const [value, setValue] = useState(args.value);
+ return (
+ <div className="w-[620px] max-w-full">
+ <AnnotationQueueScoreConfigEditor
+ {...args}
+ value={value}
+ onChange={(ids) => {
+ setValue(ids);
+ args.onChange(ids);
+ }}
+ />
+ </div>
+ );
+}
+const meta = preview.meta({ component: AnnotationQueueScoreConfigEditor });
+
+export const Default = meta.story({ args: baseArgs, render: StatefulEditor });
+export const FieldPreviews = meta.story({
+ args: {
+ ...baseArgs,
+ scoreConfigs: [
+ ...configs,
+ {
+ id: "quality",
+ name: "Quality",
+ dataType: "CATEGORICAL",
+ isArchived: false,
+ description:
+ "How well does the answer address the request, including all the relevant details?\nConsider clarity and completeness.",
+ categories: [
+ { label: "Poor", value: 0 },
+ { label: "Good", value: 1 },
+ { label: "Great", value: 2 },
+ ],
+ },
+ ],
+ value: ["quality", "helpfulness", "accuracy", "feedback"],
+ },
+ render: (args) => (
+ <div className="w-80">
+ <StatefulEditor {...args} />
+ </div>
+ ),
+});
+export const ManyScoreConfigs = meta.story({
+ args: {
+ ...baseArgs,
+ scoreConfigs: largeConfigs,
+ value: largeConfigs.slice(0, 80).map((config) => config.id),
+ },
+ render: StatefulEditor,
+});
+export const Empty = meta.story({
+ args: { ...baseArgs, scoreConfigs: [], value: [] },
+ render: StatefulEditor,
+});
+
+export const TestKeyboardReorderAndAdd = meta.story({
+ name: "(Test) Keyboard Reorder and Add Preserve Order",
+ args: { ...baseArgs, onChange: fn() },
+ render: StatefulEditor,
+ play: async ({ args, canvasElement }) => {
+ const canvas = within(canvasElement);
+ const body = within(canvasElement.ownerDocument.body);
+ const handle = canvas.getByRole("button", { name: "Reorder Helpfulness" });
+ handle.focus();
+ await userEvent.keyboard(" ");
+ await userEvent.keyboard("{ArrowDown}");
+ await userEvent.keyboard(" ");
+ await waitFor(() =>
+ expect(args.onChange).toHaveBeenLastCalledWith([
+ "accuracy",
+ "helpfulness",
+ ]),
+ );
+ const rows = within(
+ canvas.getByRole("list", { name: "Selected score configs" }),
+ ).getAllByRole("listitem");
+ await expect(within(rows[0]!).getByText("Accuracy")).toBeVisible();
+ await userEvent.click(canvas.getByRole("button", { name: "Add score" }));
+ await expect(
+ body.queryByRole("menuitem", { name: /^Accuracy/ }),
+ ).not.toBeInTheDocument();
+ await expect(
+ body.queryByRole("menuitem", { name: /^Helpfulness/ }),
+ ).not.toBeInTheDocument();
+ await expect(
+ body.queryByRole("menuitem", { name: /^Legacy quality/ }),
+ ).not.toBeInTheDocument();
+ await userEvent.click(body.getByRole("menuitem", { name: /^Feedback \(/ }));
+ await expect(args.onChange).toHaveBeenLastCalledWith([
+ "accuracy",
+ "helpfulness",
+ "feedback",
+ ]);
+ await expect(body.queryByRole("menu")).not.toBeInTheDocument();
+ await expect(
+ canvas.getAllByRole("button", { name: /^Reorder / }),
+ ).toHaveLength(3);
+ await userEvent.click(canvas.getByRole("button", { name: "Add score" }));
+ await expect(
+ body.queryByRole("menuitem", { name: /^Feedback/ }),
+ ).not.toBeInTheDocument();
+ await userEvent.keyboard("{Escape}");
+ },
+});
+
+export const TestCreateAfterNoSearchResults = meta.story({
+ name: "(Test) Create Score Config After No Search Results",
+ args: {
+ ...baseArgs,
+ createAction: { disabled: undefined, onClick: fn() },
+ },
+ render: StatefulEditor,
+ play: async ({ args, canvasElement }) => {
+ const canvas = within(canvasElement);
+ const body = within(canvasElement.ownerDocument.body);
+ await userEvent.click(canvas.getByRole("button", { name: "Add score" }));
+ await userEvent.type(
+ body.getByPlaceholderText("Search scores..."),
+ "No matching score config",
+ );
+ await expect(
+ body.queryByRole("menuitem", { name: /^Feedback/ }),
+ ).not.toBeInTheDocument();
+ await userEvent.click(
+ body.getByRole("menuitem", { name: "Add score config" }),
+ );
+ await expect(args.createAction.onClick).toHaveBeenCalledOnce();
+ await expect(body.queryByRole("menu")).not.toBeInTheDocument();
+ },
+});
+
+export const TestCancelRemoval = meta.story({
+ name: "(Test) Cancel Removal Keeps Selection",
+ args: { ...baseArgs, warnOnRemoval: true, onChange: fn() },
+ render: StatefulEditor,
+ play: async ({ args, canvasElement }) => {
+ const canvas = within(canvasElement);
+ const body = within(canvasElement.ownerDocument.body);
+ await userEvent.click(
+ canvas.getByRole("button", { name: "Remove Accuracy from queue" }),
+ );
+ const confirmation = await body.findByRole("dialog", {
+ name: "Remove score configs from queue?",
+ });
+ await userEvent.click(
+ within(confirmation).getByRole("button", { name: "Cancel" }),
+ );
+ await expect(args.onChange).not.toHaveBeenCalled();
+ await waitFor(() =>
+ expect(
+ canvas.getByRole("button", { name: "Reorder Accuracy" }),
+ ).toBeInTheDocument(),
+ );
+ await userEvent.click(
+ canvas.getByRole("button", { name: "Remove Accuracy from queue" }),
+ );
+ const secondConfirmation = await body.findByRole("dialog", {
+ name: "Remove score configs from queue?",
+ });
+ await userEvent.click(
+ within(secondConfirmation).getByRole("button", {
+ name: "Remove from queue",
+ }),
+ );
+ await expect(args.onChange).toHaveBeenLastCalledWith(["helpfulness"]);
+ await expect(
+ canvas.queryByRole("button", { name: "Reorder Accuracy" }),
+ ).not.toBeInTheDocument();
+ },
+});
+
+export const TestDescriptionOverflow = meta.story({
+ name: "(Test) Description Disclosure Follows Available Width",
+ args: {
+ ...baseArgs,
+ scoreConfigs: [
+ {
+ ...configs[0]!,
+ id: "responsive-description",
+ description: "Check correctness and completeness of the answer.",
+ },
+ ],
+ value: ["responsive-description"],
+ },
+ render: (args) => (
+ <div data-testid="responsive-editor" style={{ width: 620 }}>
+ <StatefulEditor {...args} />
+ </div>
+ ),
+ play: async ({ canvasElement }) => {
+ const canvas = within(canvasElement);
+ const container = canvas.getByTestId("responsive-editor");
+ const toggle = () =>
+ canvas.queryByRole("button", { name: /description for Accuracy/ });
+ await waitFor(() => expect(toggle()).not.toBeInTheDocument());
+ container.style.width = "200px";
+ await waitFor(() => expect(toggle()).toBeInTheDocument());
+ toggle()?.focus();
+ await waitFor(() => expect(toggle()).toBeVisible());
+ const collapse = canvas.queryByRole("button", {
+ name: "Hide description for Accuracy",
+ });
+ if (collapse) await userEvent.click(collapse);
+ container.style.width = "620px";
+ await waitFor(() => expect(toggle()).not.toBeInTheDocument());
+ container.style.width = "200px";
+ await waitFor(() => expect(toggle()).toBeInTheDocument());
+ toggle()?.focus();
+ await waitFor(() =>
+ expect(
+ canvas.getByRole("button", { name: "Show description for Accuracy" }),
+ ).toBeVisible(),
+ );
+ },
+});
diff --git a/web/src/features/annotation-queues/components/AnnotationQueueScoreConfigEditor.tsx b/web/src/features/annotation-queues/components/AnnotationQueueScoreConfigEditor.tsx
new file mode 100644
index 0000000000..c8fd5c82a4
--- /dev/null
+++ b/web/src/features/annotation-queues/components/AnnotationQueueScoreConfigEditor.tsx
@@ -0,0 +1,277 @@
+import { Info, Pencil, Plus, X } from "lucide-react";
+import { arrayMove } from "@dnd-kit/sortable";
+import { type ScoreConfigDomain } from "@langfuse/shared";
+import { formatScoreRange } from "@/src/features/scores";
+import { Badge } from "@/src/components/design-system/Badge/Badge";
+import { IconButton } from "@/src/components/design-system/IconButton/IconButton";
+import { DropdownMenu } from "@/src/components/design-system/DropdownMenu/DropdownMenu";
+import { Button } from "@/src/components/ui/button";
+import { SortableList } from "@/src/components/design-system/SortableList/SortableList";
+import { ConfirmationDialogController } from "@/src/components/design-system/ConfirmationDialogController/ConfirmationDialogController";
+import { Input } from "@/src/components/design-system/Input/Input";
+import { SelectInput } from "@/src/components/design-system/SelectInput/SelectInput";
+import { Tabs } from "@/src/components/design-system/Tabs/Tabs";
+import { Textarea } from "@/src/components/ui/textarea";
+import { shouldUseCombobox } from "@/src/features/scores/components/CategoricalScoreInput";
+import { ScoreConfigDescription } from "@/src/features/scores/components/ScoreConfigDescription";
+import { Tooltip } from "@/src/components/design-system/Tooltip/Tooltip";
+
+export function AnnotationQueueScoreConfigEditor({
+ scoreConfigs,
+ value,
+ onChange,
+ createAction,
+ editAction,
+ warnOnRemoval,
+ disabled,
+}: AnnotationQueueScoreConfigEditorProps) {
+ const configsById = new Map(
+ scoreConfigs.map((config) => [config.id, config]),
+ );
+ const selectedIds = new Set(value);
+ const availableConfigs = scoreConfigs.filter(
+ (config) => !config.isArchived && !selectedIds.has(config.id),
+ );
+
+ return (
+ <ConfirmationDialogController<string[]>
+ title="Remove score configs from queue?"
+ text="After saving, reopened items, including completed items, will no longer show these fields. Historical scores and completion status stay unchanged. The project-wide score configs are kept."
+ confirmLabel="Remove from queue"
+ variant="default"
+ onConfirm={onChange}
+ >
+ {({ openDialog }) => {
+ const changeSelection = (nextValue: string[]) => {
+ if (disabled) return;
+ if (warnOnRemoval && value.some((id) => !nextValue.includes(id))) {
+ openDialog(nextValue);
+ } else {
+ onChange(nextValue);
+ }
+ };
+ return (
+ <div className="flex min-h-0 flex-col gap-3">
+ <div className="flex shrink-0 items-center gap-1">
+ <DropdownMenu
+ disabled={disabled}
+ maxHeight="18rem"
+ placement="bottom-start"
+ search={{ placeholder: "Search scores..." }}
+ items={[
+ ...availableConfigs.map((config) => ({
+ type: "item" as const,
+ id: config.id,
+ title: `${config.name} (${scoreDataTypeLabels[config.dataType]})`,
+ onClick: () => {
+ if (!disabled) onChange([...value, config.id]);
+ },
+ })),
+ { type: "separator", id: "score-config-actions" },
+ {
+ type: "item",
+ id: "create-score-config",
+ title: "Add score config",
+ icon: Plus,
+ disabled: createAction.disabled,
+ searchBehavior: "show-when-no-results",
+ onClick: createAction.onClick,
+ },
+ ]}
+ >
+ {({ getTriggerProps }) => (
+ <Button
+ type="button"
+ variant="outline"
+ size="sm"
+ className="gap-1.5 text-xs"
+ disabled={disabled}
+ {...getTriggerProps()}
+ >
+ <Plus
+ className="icon-base text-icon-foreground"
+ aria-hidden
+ />
+ Add score
+ </Button>
+ )}
+ </DropdownMenu>
+ <Tooltip label="Score configs define the fields annotators see. Adding or removing fields applies when items are reopened, including completed items. Historical scores and completion status stay unchanged. Editing a score config changes its project-wide definition for future scoring wherever it is used.">
+ {({ getTriggerProps }) => (
+ <IconButton
+ {...getTriggerProps()}
+ icon={Info}
+ label="About queue score configs"
+ size="sm"
+ />
+ )}
+ </Tooltip>
+ </div>
+ {value.length > 0 && (
+ <div className="min-h-0">
+ <SortableList
+ gap="md"
+ variant="hover-card"
+ items={value}
+ getId={(id) => id}
+ getLabel={(id) => configsById.get(id)?.name ?? id}
+ disabled={disabled}
+ aria-label="Selected score configs"
+ onReorder={(from, to) => onChange(arrayMove(value, from, to))}
+ renderItem={(id) => {
+ const config = configsById.get(id);
+ const name = config?.name ?? id;
+ return (
+ <div className="flex flex-col gap-2">
+ <div className="flex min-h-6 items-center gap-2 pr-6">
+ <span
+ className="min-w-0 flex-1 text-xs font-bold wrap-break-word"
+ title={name}
+ >
+ {name}
+ </span>
+ {config?.isArchived && <Badge text="Archived" />}
+ <div className="flex shrink-0 items-center gap-1 opacity-100 group-focus-within/sortable:opacity-100 group-hover/sortable:opacity-100 [@media(hover:hover)]:opacity-0">
+ <IconButton
+ icon={Pencil}
+ label={`Edit score config ${name}`}
+ title={
+ editAction.disabled?.reason ??
+ "Edit project-wide score config"
+ }
+ size="sm"
+ disabled={
+ disabled || !!editAction.disabled || !config
+ }
+ onClick={() => editAction.onClick(id)}
+ />
+ <IconButton
+ icon={X}
+ label={`Remove ${name} from queue`}
+ title="Remove from queue"
+ size="sm"
+ disabled={disabled}
+ onClick={() =>
+ changeSelection(
+ value.filter((configId) => configId !== id),
+ )
+ }
+ />
+ </div>
+ </div>
+ {config && <ScoreConfigFieldPreview config={config} />}
+ </div>
+ );
+ }}
+ />
+ </div>
+ )}
+ </div>
+ );
+ }}
+ </ConfirmationDialogController>
+ );
+}
+
+type AnnotationQueueScoreConfigEditorProps = {
+ scoreConfigs: Pick<
+ ScoreConfigDomain,
+ | "id"
+ | "name"
+ | "dataType"
+ | "isArchived"
+ | "description"
+ | "minValue"
+ | "maxValue"
+ | "categories"
+ >[];
+ value: string[];
+ onChange: (ids: string[]) => void;
+ createAction: {
+ disabled: { reason: string } | undefined;
+ onClick: () => void;
+ };
+ editAction: {
+ disabled: { reason: string } | undefined;
+ onClick: (configId: string) => void;
+ };
+ warnOnRemoval: boolean;
+ disabled: boolean;
+};
+
+function ScoreConfigFieldPreview({
+ config,
+}: {
+ config: AnnotationQueueScoreConfigEditorProps["scoreConfigs"][number];
+}) {
+ const categories = config.categories ?? [];
+ const rangeLabel = formatScoreRange(config.minValue, config.maxValue);
+ const categorical =
+ config.dataType === "CATEGORICAL" || config.dataType === "BOOLEAN";
+ const useDropdown = shouldUseCombobox(categories);
+
+ return (
+ <div className="flex flex-col gap-1.5">
+ {config.description?.trim() && (
+ <ScoreConfigDescription
+ key={config.id}
+ configId={config.id}
+ name={config.name}
+ description={config.description}
+ />
+ )}
+ <div inert aria-hidden="true">
+ {config.dataType === "TEXT" && (
+ <Textarea
+ readOnly
+ rows={2}
+ className="min-h-16 resize-none text-xs"
+ placeholder="Enter free form text..."
+ />
+ )}
+ {config.dataType === "NUMERIC" && (
+ <Input readOnly type="number" placeholder={rangeLabel} />
+ )}
+ {categorical && useDropdown && (
+ <SelectInput
+ value=""
+ onValueChange={() => {}}
+ placeholder="Select category"
+ options={categories.map((category) => ({
+ value: category.label,
+ label: category.label,
+ }))}
+ />
+ )}
+ {categorical && !useDropdown && (
+ <Tabs value="">
+ <Tabs.List
+ variant="inset"
+ size="md"
+ layout="full"
+ aria-label={config.name}
+ >
+ {categories.map((category) => (
+ <Tabs.Trigger key={category.value} value={category.label}>
+ {category.label}
+ </Tabs.Trigger>
+ ))}
+ </Tabs.List>
+ </Tabs>
+ )}
+ </div>
+ {categories.length > 0 && useDropdown && (
+ <p className="text-muted-foreground line-clamp-2 text-xs">
+ {categories.map((category) => category.label).join(" · ")}
+ </p>
+ )}
+ </div>
+ );
+}
+
+const scoreDataTypeLabels = {
+ NUMERIC: "Numeric",
+ CATEGORICAL: "Categorical",
+ BOOLEAN: "Boolean",
+ TEXT: "Text",
+};
diff --git a/web/src/features/annotation-queues/components/AnnotationQueueSetupController.clienttest.tsx b/web/src/features/annotation-queues/components/AnnotationQueueSetupController.clienttest.tsx
new file mode 100644
index 0000000000..a8138964ac
--- /dev/null
+++ b/web/src/features/annotation-queues/components/AnnotationQueueSetupController.clienttest.tsx
@@ -0,0 +1,318 @@
+import { useState } from "react";
+import { fireEvent, render, screen, waitFor } from "@testing-library/react";
+import { beforeEach, expect, it, vi } from "vitest";
+import { TooltipProvider } from "@/src/components/ui/tooltip";
+import { SidePanel } from "@/src/components/ui/side-panel";
+import { LayerProvider } from "@/src/context/LayerContext/LayerContext";
+import { AnnotationQueueSetupController } from "./AnnotationQueueSetupController";
+
+const mocks = vi.hoisted(() => ({
+ update: vi.fn(),
+ assign: vi.fn(),
+ invalidateQueues: vi.fn(),
+ invalidateAssignments: vi.fn(),
+ capture: vi.fn(),
+ refetch: vi.fn(),
+ fetchedAfterMount: false,
+ refetchFailed: false,
+ isMobile: false,
+ queue: {
+ id: "queue",
+ name: "Initial name",
+ description: "Initial description",
+ scoreConfigs: [{ id: "first-config" }] as { id: string }[],
+ },
+}));
+
+vi.mock("@/src/features/rbac", () => ({ useHasProjectAccess: () => true }));
+vi.mock("@/src/hooks/use-mobile", () => ({
+ useIsMobile: () => mocks.isMobile,
+}));
+vi.mock("@/src/features/posthog-analytics", () => ({
+ usePostHogClientCapture: () => mocks.capture,
+}));
+vi.mock("./QueueScoreConfigFields", () => ({
+ QueueScoreConfigFields: ({
+ value,
+ onChange,
+ }: {
+ value: string[];
+ onChange: (ids: string[]) => void;
+ }) => (
+ <>
+ <output aria-label="Selected config IDs">{value.join(",")}</output>
+ <button
+ type="button"
+ onClick={() => onChange([...value, "second-config"])}
+ >
+ Add existing config
+ </button>
+ </>
+ ),
+}));
+vi.mock("./UserAssignmentSection", () => ({
+ UserAssignmentSection: () => null,
+}));
+vi.mock("@/src/utils/api", () => ({
+ api: {
+ useUtils: () => ({
+ annotationQueues: { invalidate: mocks.invalidateQueues },
+ annotationQueueAssignments: { invalidate: mocks.invalidateAssignments },
+ }),
+ annotationQueues: {
+ byId: {
+ useQuery: () => ({
+ data: mocks.queue,
+ isError: mocks.refetchFailed,
+ isFetchedAfterMount: mocks.fetchedAfterMount,
+ refetch: mocks.refetch,
+ }),
+ },
+ allNamesAndIds: {
+ useQuery: () => ({
+ data: [{ id: "queue", name: "Initial name" }],
+ isError: mocks.refetchFailed,
+ refetch: mocks.refetch,
+ }),
+ },
+ update: {
+ useMutation: () => ({
+ mutateAsync: mocks.update,
+ isPending: false,
+ }),
+ },
+ },
+ scoreConfigs: {
+ all: {
+ useQuery: () => ({
+ data: { configs: [] },
+ isError: mocks.refetchFailed,
+ isFetchedAfterMount: mocks.fetchedAfterMount,
+ refetch: mocks.refetch,
+ }),
+ },
+ },
+ annotationQueueItems: {
+ itemsByQueueId: { useQuery: () => ({ data: { totalItems: 0 } }) },
+ },
+ annotationQueueAssignments: {
+ createMany: {
+ useMutation: () => ({
+ mutateAsync: mocks.assign,
+ isPending: false,
+ }),
+ },
+ },
+ },
+}));
+
+beforeEach(() => {
+ vi.clearAllMocks();
+ mocks.update.mockResolvedValue(undefined);
+ mocks.assign.mockResolvedValue(undefined);
+ mocks.fetchedAfterMount = false;
+ mocks.refetchFailed = false;
+ mocks.isMobile = false;
+ mocks.queue = {
+ id: "queue",
+ name: "Initial name",
+ description: "Initial description",
+ scoreConfigs: [{ id: "first-config" }],
+ };
+});
+
+it("waits for initial refresh and retains drafts across panel unmounts and background query failures", async () => {
+ const view = render(<SetupPanel />);
+ expect(
+ screen.queryByRole("button", { name: "Add existing config" }),
+ ).not.toBeInTheDocument();
+
+ mocks.fetchedAfterMount = true;
+ view.rerender(<SetupPanel />);
+ fireEvent.change(screen.getByRole("textbox", { name: "Name" }), {
+ target: { value: "Draft name" },
+ });
+ fireEvent.change(
+ screen.getByRole("textbox", { name: "Description (optional)" }),
+ { target: { value: "Draft description" } },
+ );
+ fireEvent.mouseDown(screen.getByRole("tab", { name: "Annotation panel" }));
+ fireEvent.click(screen.getByRole("button", { name: "Add existing config" }));
+ fireEvent.click(screen.getByRole("button", { name: "Hide setup" }));
+ expect(
+ screen.queryByRole("textbox", { name: "Name" }),
+ ).not.toBeInTheDocument();
+
+ mocks.queue = {
+ ...mocks.queue,
+ name: "Concurrent name",
+ description: "Concurrent description",
+ };
+ mocks.refetchFailed = true;
+ view.rerender(<SetupPanel />);
+ fireEvent.click(screen.getByRole("button", { name: "Show setup" }));
+ expect(screen.getByRole("tab", { name: "Annotation panel" })).toHaveAttribute(
+ "aria-selected",
+ "true",
+ );
+ expect(screen.getByLabelText("Selected config IDs")).toHaveTextContent(
+ "first-config,second-config",
+ );
+ fireEvent.mouseDown(screen.getByRole("tab", { name: "Settings" }));
+ expect(screen.getByRole("textbox", { name: "Name" })).toHaveValue(
+ "Draft name",
+ );
+ expect(
+ screen.getByRole("textbox", { name: "Description (optional)" }),
+ ).toHaveValue("Draft description");
+ expect(screen.queryByRole("alert")).not.toBeInTheDocument();
+ await waitFor(() =>
+ expect(screen.getByRole("button", { name: "Discard" })).toBeEnabled(),
+ );
+
+ await waitFor(() =>
+ expect(screen.getByRole("button", { name: "Save changes" })).toBeEnabled(),
+ );
+ fireEvent.click(screen.getByRole("button", { name: "Save changes" }));
+ await waitFor(() =>
+ expect(mocks.update).toHaveBeenCalledExactlyOnceWith({
+ projectId: "project",
+ queueId: "queue",
+ name: "Draft name",
+ description: "Draft description",
+ scoreConfigIds: ["first-config", "second-config"],
+ }),
+ );
+});
+
+it("saves only changed config IDs without overwriting concurrent queue metadata", async () => {
+ mocks.fetchedAfterMount = true;
+ const view = render(<SetupPanel />);
+ fireEvent.mouseDown(screen.getByRole("tab", { name: "Annotation panel" }));
+ fireEvent.click(screen.getByRole("button", { name: "Add existing config" }));
+ mocks.queue = {
+ ...mocks.queue,
+ name: "Concurrent name",
+ description: "Concurrent description",
+ };
+ view.rerender(<SetupPanel />);
+
+ await waitFor(() =>
+ expect(screen.getByRole("button", { name: "Save changes" })).toBeEnabled(),
+ );
+ fireEvent.click(screen.getByRole("button", { name: "Save changes" }));
+ await waitFor(() =>
+ expect(mocks.update).toHaveBeenCalledExactlyOnceWith({
+ projectId: "project",
+ queueId: "queue",
+ scoreConfigIds: ["first-config", "second-config"],
+ }),
+ );
+ expect(mocks.assign).not.toHaveBeenCalled();
+ expect(mocks.capture).toHaveBeenCalledWith(
+ "annotation_queues:score_configs_saved",
+ {
+ scoreConfigCount: 2,
+ },
+ );
+});
+
+it("allows a metadata-only save for an initially unconfigured queue", async () => {
+ mocks.fetchedAfterMount = true;
+ mocks.queue.scoreConfigs = [];
+ render(<SetupPanel />);
+ fireEvent.change(screen.getByRole("textbox", { name: "Name" }), {
+ target: { value: "Renamed empty queue" },
+ });
+ await waitFor(() =>
+ expect(screen.getByRole("button", { name: "Discard" })).toBeEnabled(),
+ );
+
+ await waitFor(() =>
+ expect(screen.getByRole("button", { name: "Save changes" })).toBeEnabled(),
+ );
+ fireEvent.click(screen.getByRole("button", { name: "Save changes" }));
+ await waitFor(() =>
+ expect(mocks.update).toHaveBeenCalledExactlyOnceWith({
+ projectId: "project",
+ queueId: "queue",
+ name: "Renamed empty queue",
+ }),
+ );
+ expect(mocks.capture).not.toHaveBeenCalled();
+});
+
+it("keeps the mobile setup sheet open when initial data arrives and retains drafts when reopened", async () => {
+ mocks.isMobile = true;
+ const view = render(<MobileSetupPanel />);
+ fireEvent.click(screen.getByTitle("Show queue setup"));
+ expect(screen.getByRole("dialog")).toBeVisible();
+
+ mocks.fetchedAfterMount = true;
+ view.rerender(<MobileSetupPanel />);
+ expect(screen.getByRole("dialog")).toBeVisible();
+ fireEvent.change(screen.getByRole("textbox", { name: "Name" }), {
+ target: { value: "Mobile draft" },
+ });
+ fireEvent.mouseDown(screen.getByRole("tab", { name: "Annotation panel" }));
+ fireEvent.click(screen.getByRole("button", { name: "Add existing config" }));
+ fireEvent.click(screen.getByRole("button", { name: "Close" }));
+ expect(screen.queryByRole("dialog")).not.toBeInTheDocument();
+ fireEvent.click(screen.getByTitle("Show queue setup"));
+ expect(screen.getByRole("tab", { name: "Annotation panel" })).toHaveAttribute(
+ "aria-selected",
+ "true",
+ );
+ expect(screen.getByLabelText("Selected config IDs")).toHaveTextContent(
+ "first-config,second-config",
+ );
+ fireEvent.mouseDown(screen.getByRole("tab", { name: "Settings" }));
+ expect(screen.getByRole("textbox", { name: "Name" })).toHaveValue(
+ "Mobile draft",
+ );
+ await waitFor(() =>
+ expect(screen.getByRole("button", { name: "Save changes" })).toBeEnabled(),
+ );
+});
+
+function MobileSetupPanel() {
+ const [open, onOpenChange] = useState(false);
+ return (
+ <LayerProvider>
+ <TooltipProvider>
+ <AnnotationQueueSetupController
+ projectId="project"
+ queueId="queue"
+ renderLayout={(content) => (
+ <SidePanel
+ id="mobile-queue-setup"
+ mobileTitle="Queue setup"
+ variant="sidebar"
+ mobileOpenState={{ open, onOpenChange }}
+ >
+ {content}
+ </SidePanel>
+ )}
+ />
+ </TooltipProvider>
+ </LayerProvider>
+ );
+}
+
+function SetupPanel() {
+ const [visible, setVisible] = useState(true);
+ return (
+ <LayerProvider>
+ <TooltipProvider>
+ <button onClick={() => setVisible(!visible)}>
+ {visible ? "Hide setup" : "Show setup"}
+ </button>
+ <AnnotationQueueSetupController
+ projectId="project"
+ queueId="queue"
+ renderLayout={(content) => (visible ? content : null)}
+ />
+ </TooltipProvider>
+ </LayerProvider>
+ );
+}
diff --git a/web/src/features/annotation-queues/components/AnnotationQueueSetupController.tsx b/web/src/features/annotation-queues/components/AnnotationQueueSetupController.tsx
new file mode 100644
index 0000000000..3a02c7f12b
--- /dev/null
+++ b/web/src/features/annotation-queues/components/AnnotationQueueSetupController.tsx
@@ -0,0 +1,192 @@
+import { type ReactNode } from "react";
+import { Button } from "@/src/components/design-system/Button/Button";
+import { Skeleton } from "@/src/components/ui/skeleton";
+import { useHasProjectAccess } from "@/src/features/rbac";
+import { usePostHogClientCapture } from "@/src/features/posthog-analytics";
+import { api, type RouterInputs } from "@/src/utils/api";
+import { AnnotationQueueDetails } from "./AnnotationQueueDetails";
+import {
+ AnnotationQueueSetupForm,
+ type AnnotationQueueSetupValues,
+ type AnnotationQueueSetupDirtyFields,
+} from "./AnnotationQueueSetupForm";
+import { QueueScoreConfigFields } from "./QueueScoreConfigFields";
+import { UserAssignmentSection } from "./UserAssignmentSection";
+
+export function AnnotationQueueSetupController({
+ projectId,
+ queueId,
+ renderLayout,
+}: {
+ projectId: string;
+ queueId: string;
+ renderLayout: (content: ReactNode) => ReactNode;
+}) {
+ const canEdit = useHasProjectAccess({
+ projectId,
+ scope: "annotationQueues:CUD",
+ });
+ const hasAssignments = useHasProjectAccess({
+ projectId,
+ scope: "annotationQueueAssignments:read",
+ });
+ const queue = api.annotationQueues.byId.useQuery(
+ { projectId, queueId },
+ { refetchOnMount: "always" },
+ );
+ const configs = api.scoreConfigs.all.useQuery(
+ { projectId },
+ { enabled: canEdit, refetchOnMount: "always" },
+ );
+ const queues = api.annotationQueues.allNamesAndIds.useQuery(
+ { projectId },
+ { enabled: canEdit },
+ );
+ const items = api.annotationQueueItems.itemsByQueueId.useQuery(
+ { projectId, queueId, page: 0, limit: 1 },
+ { enabled: canEdit },
+ );
+ const utils = api.useUtils();
+ const capture = usePostHogClientCapture();
+ const update = api.annotationQueues.update.useMutation();
+ const assign = api.annotationQueueAssignments.createMany.useMutation();
+
+ if (
+ (!queue.data && queue.isError) ||
+ (canEdit &&
+ ((!configs.data && configs.isError) || (!queues.data && queues.isError)))
+ ) {
+ return renderLayout(
+ <div className="flex flex-col gap-2 p-3">
+ <p role="alert">Could not load queue setup.</p>
+ <Button
+ text="Retry"
+ variant="secondary"
+ onClick={() => {
+ queue.refetch();
+ configs.refetch();
+ queues.refetch();
+ }}
+ />
+ </div>,
+ );
+ }
+ if (
+ !queue.data ||
+ !queue.isFetchedAfterMount ||
+ (canEdit && (!configs.data || !configs.isFetchedAfterMount || !queues.data))
+ ) {
+ return renderLayout(
+ <div className="w-full p-3">
+ <Skeleton className="h-64 w-full" />
+ </div>,
+ );
+ }
+ if (!canEdit) {
+ return renderLayout(
+ <AnnotationQueueDetails
+ description={queue.data.description ?? null}
+ scoreConfigs={queue.data.scoreConfigs}
+ editAction={null}
+ />,
+ );
+ }
+ return (
+ <AnnotationQueueSetupForm
+ initialValues={{
+ name: queue.data.name ?? "",
+ description: queue.data.description ?? "",
+ scoreConfigIds: queue.data.scoreConfigs.map((config) => config.id),
+ newAssignmentUserIds: [],
+ }}
+ queueNames={(queues.data ?? [])
+ .filter((candidate) => candidate.id !== queueId)
+ .map((candidate) => candidate.name)}
+ canEdit={canEdit}
+ isSubmitting={update.isPending || assign.isPending}
+ hasAssignments={hasAssignments}
+ renderLayout={renderLayout}
+ onSubmit={(values, dirty) =>
+ saveQueueSetup({
+ projectId,
+ queueId,
+ values,
+ dirty,
+ updateQueue: update.mutateAsync,
+ assignUsers: assign.mutateAsync,
+ onSaved: async () => {
+ await Promise.all([
+ utils.annotationQueues.invalidate(),
+ utils.annotationQueueAssignments.invalidate(),
+ ]);
+ if (dirty.scoreConfigIds)
+ capture("annotation_queues:score_configs_saved", {
+ scoreConfigCount: values.scoreConfigIds.length,
+ });
+ },
+ })
+ }
+ renderAssignments={({ value, onChange }) => (
+ <UserAssignmentSection
+ projectId={projectId}
+ queueId={queueId}
+ selectedUserIds={value}
+ onChange={onChange}
+ />
+ )}
+ >
+ {({ value, onChange, disabled }) => (
+ <QueueScoreConfigFields
+ projectId={projectId}
+ configs={configs.data?.configs ?? []}
+ value={value}
+ onChange={onChange}
+ disabled={disabled}
+ warnOnRemoval={items.data?.totalItems !== 0}
+ />
+ )}
+ </AnnotationQueueSetupForm>
+ );
+}
+
+async function saveQueueSetup({
+ projectId,
+ queueId,
+ values,
+ dirty,
+ updateQueue,
+ assignUsers,
+ onSaved,
+}: {
+ projectId: string;
+ queueId: string;
+ values: AnnotationQueueSetupValues;
+ dirty: AnnotationQueueSetupDirtyFields;
+ updateQueue: (
+ input: RouterInputs["annotationQueues"]["update"],
+ ) => Promise<unknown>;
+ assignUsers: (
+ input: RouterInputs["annotationQueueAssignments"]["createMany"],
+ ) => Promise<unknown>;
+ onSaved: () => Promise<void>;
+}) {
+ if (dirty.name || dirty.description || dirty.scoreConfigIds) {
+ await updateQueue({
+ projectId,
+ queueId,
+ ...(dirty.name ? { name: values.name } : {}),
+ ...(dirty.description ? { description: values.description } : {}),
+ ...(dirty.scoreConfigIds
+ ? { scoreConfigIds: values.scoreConfigIds }
+ : {}),
+ });
+ }
+ if (values.newAssignmentUserIds.length > 0) {
+ await assignUsers({
+ projectId,
+ queueId,
+ userIds: values.newAssignmentUserIds,
+ });
+ }
+ await onSaved();
+}
diff --git a/web/src/features/annotation-queues/components/AnnotationQueueSetupForm.stories.tsx b/web/src/features/annotation-queues/components/AnnotationQueueSetupForm.stories.tsx
new file mode 100644
index 0000000000..72c135b8dd
--- /dev/null
+++ b/web/src/features/annotation-queues/components/AnnotationQueueSetupForm.stories.tsx
@@ -0,0 +1,222 @@
+import { type ComponentProps } from "react";
+import { expect, fn, mocked, userEvent, waitFor, within } from "storybook/test";
+import preview from "../../../../.storybook/preview";
+import { AnnotationQueueScoreConfigEditor } from "./AnnotationQueueScoreConfigEditor";
+import { AnnotationQueueSetupForm } from "./AnnotationQueueSetupForm";
+
+type SetupProps = ComponentProps<typeof AnnotationQueueSetupForm>;
+const configs: ComponentProps<
+ typeof AnnotationQueueScoreConfigEditor
+>["scoreConfigs"] = [
+ { id: "accuracy", name: "Accuracy", dataType: "NUMERIC", isArchived: false },
+ {
+ id: "helpfulness",
+ name: "Helpfulness",
+ dataType: "BOOLEAN",
+ isArchived: false,
+ },
+ { id: "feedback", name: "Feedback", dataType: "TEXT", isArchived: false },
+];
+
+const baseArgs: SetupProps = {
+ initialValues: {
+ name: "Review queue",
+ description: "Review response quality.",
+ scoreConfigIds: ["helpfulness", "accuracy"],
+ newAssignmentUserIds: [],
+ },
+ queueNames: ["Another queue"],
+ canEdit: true,
+ isSubmitting: false,
+ hasAssignments: false,
+ onSubmit: fn().mockResolvedValue(undefined),
+ renderLayout: (content) => (
+ <div className="bg-background h-[640px] w-[380px] max-w-full border">
+ {content}
+ </div>
+ ),
+ renderAssignments: () => null,
+ children: ({ value, onChange, disabled }) => (
+ <AnnotationQueueScoreConfigEditor
+ scoreConfigs={configs}
+ value={value}
+ onChange={onChange}
+ disabled={disabled}
+ warnOnRemoval={false}
+ createAction={{ disabled: undefined, onClick: fn() }}
+ editAction={{ disabled: undefined, onClick: fn() }}
+ />
+ ),
+};
+
+const meta = preview.meta({ component: AnnotationQueueSetupForm });
+
+export const Default = meta.story({ args: baseArgs });
+
+export const NoScoreConfigs = meta.story({
+ args: {
+ ...baseArgs,
+ initialValues: { ...baseArgs.initialValues, scoreConfigIds: [] },
+ },
+});
+
+export const Saving = meta.story({
+ args: { ...baseArgs, isSubmitting: true },
+});
+
+export const TestLegacyMetadata = meta.story({
+ name: "(Test) Legacy Metadata Does Not Block Score Config Changes",
+ args: {
+ ...baseArgs,
+ initialValues: {
+ ...baseArgs.initialValues,
+ name: "An API-created queue name longer than thirty-five characters",
+ description: "<p>Legacy description</p>",
+ },
+ onSubmit: fn().mockResolvedValue(undefined),
+ },
+ play: async ({ args, canvasElement }) => {
+ const canvas = within(canvasElement);
+ const body = within(canvasElement.ownerDocument.body);
+ await userEvent.click(
+ canvas.getByRole("tab", { name: "Annotation panel" }),
+ );
+ await userEvent.click(canvas.getByRole("button", { name: "Add score" }));
+ await userEvent.click(body.getByRole("menuitem", { name: /^Feedback \(/ }));
+ const save = canvas.getByRole("button", { name: "Save changes" });
+ await waitFor(() => expect(save).toBeEnabled());
+ await userEvent.click(save);
+ await waitFor(() =>
+ expect(args.onSubmit).toHaveBeenCalledWith(
+ {
+ ...args.initialValues,
+ scoreConfigIds: ["helpfulness", "accuracy", "feedback"],
+ },
+ {
+ name: false,
+ description: false,
+ scoreConfigIds: true,
+ newAssignmentUserIds: false,
+ },
+ ),
+ );
+ await userEvent.click(canvas.getByRole("tab", { name: "Settings" }));
+ const name = canvas.getByRole("textbox", { name: "Name" });
+ await userEvent.type(name, " too long");
+ await expect(save).toBeDisabled();
+ await userEvent.clear(name);
+ await userEvent.type(name, "Valid name");
+ await waitFor(() => expect(save).toBeEnabled());
+ await userEvent.type(
+ canvas.getByRole("textbox", { name: "Description (optional)" }),
+ " edited",
+ );
+ await expect(save).toBeDisabled();
+ },
+});
+
+export const TestSaveAndDiscard = meta.story({
+ name: "(Test) Save and Discard Queue Setup",
+ args: { ...baseArgs, onSubmit: fn().mockResolvedValue(undefined) },
+ play: async ({ args, canvasElement }) => {
+ const canvas = within(canvasElement);
+ const name = canvas.getByRole("textbox", { name: "Name" });
+ const save = canvas.getByRole("button", { name: "Save changes" });
+ await expect(save).toBeDisabled();
+ await userEvent.clear(name);
+ await userEvent.type(name, "Draft queue");
+ await userEvent.click(canvas.getByRole("button", { name: "Discard" }));
+ await expect(name).toHaveValue("Review queue");
+ await expect(save).toBeDisabled();
+ await userEvent.clear(name);
+ await userEvent.type(name, "Saved queue");
+ await waitFor(() => expect(save).toBeEnabled());
+ await userEvent.click(save);
+ await waitFor(() =>
+ expect(args.onSubmit).toHaveBeenCalledWith(
+ { ...args.initialValues, name: "Saved queue" },
+ {
+ name: true,
+ description: false,
+ scoreConfigIds: false,
+ newAssignmentUserIds: false,
+ },
+ ),
+ );
+ await expect(name).toHaveValue("Saved queue");
+ await waitFor(() => expect(save).toBeDisabled());
+ },
+});
+
+export const TestFailedSaveKeepsDraft = meta.story({
+ name: "(Test) Failed Save Keeps Queue Setup Draft",
+ args: {
+ ...baseArgs,
+ onSubmit: fn(),
+ },
+ play: async ({ args, canvasElement }) => {
+ mocked(args.onSubmit).mockRejectedValueOnce(new Error("Save failed"));
+ const canvas = within(canvasElement);
+ const name = canvas.getByRole("textbox", { name: "Name" });
+ await userEvent.clear(name);
+ await userEvent.type(name, "Unsaved queue");
+ await waitFor(() =>
+ expect(
+ canvas.getByRole("button", { name: "Save changes" }),
+ ).toBeEnabled(),
+ );
+ await userEvent.click(canvas.getByRole("button", { name: "Save changes" }));
+ await waitFor(() => expect(args.onSubmit).toHaveBeenCalled());
+ await expect(await canvas.findByRole("alert")).toHaveTextContent(
+ "Your changes are still here.",
+ );
+ await expect(name).toHaveValue("Unsaved queue");
+ await expect(
+ canvas.getByRole("button", { name: "Save changes" }),
+ ).toBeEnabled();
+ await userEvent.click(canvas.getByRole("button", { name: "Discard" }));
+ await expect(name).toHaveValue("Review queue");
+ await expect(canvas.queryByRole("alert")).not.toBeInTheDocument();
+ },
+});
+
+export const TestTabsKeepDraft = meta.story({
+ name: "(Test) Tabs Share Queue Setup Draft",
+ args: { ...baseArgs, onSubmit: fn().mockResolvedValue(undefined) },
+ play: async ({ args, canvasElement }) => {
+ const canvas = within(canvasElement);
+ const body = within(canvasElement.ownerDocument.body);
+ const name = canvas.getByRole("textbox", { name: "Name" });
+ await userEvent.clear(name);
+ await userEvent.type(name, "Draft across tabs");
+ await userEvent.click(
+ canvas.getByRole("tab", { name: "Annotation panel" }),
+ );
+ await userEvent.click(canvas.getByRole("button", { name: "Add score" }));
+ await userEvent.click(body.getByRole("menuitem", { name: /^Feedback \(/ }));
+ await userEvent.click(canvas.getByRole("tab", { name: "Settings" }));
+ await expect(canvas.getByRole("textbox", { name: "Name" })).toHaveValue(
+ "Draft across tabs",
+ );
+ await userEvent.click(
+ canvas.getByRole("tab", { name: "Annotation panel" }),
+ );
+ await expect(canvas.getByText("Feedback", { exact: true })).toBeVisible();
+ await userEvent.click(canvas.getByRole("button", { name: "Save changes" }));
+ await waitFor(() =>
+ expect(args.onSubmit).toHaveBeenCalledWith(
+ {
+ ...args.initialValues,
+ name: "Draft across tabs",
+ scoreConfigIds: ["helpfulness", "accuracy", "feedback"],
+ },
+ {
+ name: true,
+ description: false,
+ scoreConfigIds: true,
+ newAssignmentUserIds: false,
+ },
+ ),
+ );
+ },
+});
diff --git a/web/src/features/annotation-queues/components/AnnotationQueueSetupForm.tsx b/web/src/features/annotation-queues/components/AnnotationQueueSetupForm.tsx
new file mode 100644
index 0000000000..27b47fb643
--- /dev/null
+++ b/web/src/features/annotation-queues/components/AnnotationQueueSetupForm.tsx
@@ -0,0 +1,291 @@
+import { type ReactNode, useId, useState } from "react";
+import { zodResolver } from "@hookform/resolvers/zod";
+import { Controller, useForm, useWatch } from "react-hook-form";
+import { z } from "zod";
+import { CreateQueueWithAssignmentsData } from "@langfuse/shared";
+
+import { Tabs } from "@/src/components/design-system/Tabs/Tabs";
+import { Button } from "@/src/components/design-system/Button/Button";
+import { Form } from "@/src/components/design-system/Form/Form";
+import { Input } from "@/src/components/design-system/Input/Input";
+import { Textarea } from "@/src/components/ui/textarea";
+
+const setupSchema = CreateQueueWithAssignmentsData.extend({
+ name: z.string(),
+ description: z.string(),
+ scoreConfigIds: z.array(z.string()),
+});
+
+export type AnnotationQueueSetupValues = z.infer<typeof setupSchema>;
+export type AnnotationQueueSetupDirtyFields = Record<
+ keyof AnnotationQueueSetupValues,
+ boolean
+>;
+
+type SelectionControl = {
+ value: string[];
+ onChange: (ids: string[]) => void;
+ disabled: boolean;
+};
+
+type AnnotationQueueSetupFormProps = {
+ initialValues: AnnotationQueueSetupValues;
+ queueNames: string[];
+ canEdit: boolean;
+ isSubmitting: boolean;
+ hasAssignments: boolean;
+ onSubmit: (
+ values: AnnotationQueueSetupValues,
+ dirtyFields: AnnotationQueueSetupDirtyFields,
+ ) => Promise<void>;
+ children: (control: SelectionControl) => ReactNode;
+ renderAssignments: (control: SelectionControl) => ReactNode;
+ renderLayout: (content: ReactNode) => ReactNode;
+};
+
+function sameIds(
+ left: readonly string[],
+ right: readonly (string | undefined)[],
+) {
+ return (
+ left.length === right.length &&
+ left.every((id, index) => id === right[index])
+ );
+}
+
+export function AnnotationQueueSetupForm({
+ initialValues,
+ queueNames,
+ canEdit,
+ isSubmitting,
+ hasAssignments,
+ onSubmit,
+ children,
+ renderAssignments,
+ renderLayout,
+}: AnnotationQueueSetupFormProps) {
+ const formId = useId();
+ const [saveFailed, setSaveFailed] = useState(false);
+ const [activeTab, setActiveTab] = useState("settings");
+ const form = useForm<AnnotationQueueSetupValues>({
+ defaultValues: initialValues,
+ resolver: zodResolver(
+ setupSchema.superRefine((values, context) => {
+ // Existing API-created metadata can exceed the UI creation limits.
+ for (const key of ["name", "description"] as const) {
+ if (values[key] === initialValues[key]) continue;
+ const result = CreateQueueWithAssignmentsData.shape[key].safeParse(
+ values[key],
+ );
+ if (!result.success) {
+ for (const issue of result.error.issues) {
+ context.addIssue({ ...issue, path: [key, ...issue.path] });
+ }
+ }
+ }
+ if (
+ values.name !== initialValues.name &&
+ queueNames.includes(values.name)
+ ) {
+ context.addIssue({
+ code: "custom",
+ path: ["name"],
+ message: "Queue name already exists.",
+ });
+ }
+ }),
+ ),
+ mode: "onChange",
+ });
+ const selectedIds = useWatch({
+ control: form.control,
+ name: "scoreConfigIds",
+ });
+ const {
+ defaultValues: defaults,
+ isDirty,
+ isValid,
+ isSubmitting: formIsSubmitting,
+ } = form.formState;
+ const missingScoreConfigs =
+ selectedIds.length === 0 &&
+ !sameIds(selectedIds, defaults?.scoreConfigIds ?? []);
+ const submitting = isSubmitting || formIsSubmitting;
+ const disabled = !canEdit || submitting;
+
+ const submit = form.handleSubmit(async (values) => {
+ if (!canEdit || missingScoreConfigs) return;
+ setSaveFailed(false);
+ const dirtyFields: AnnotationQueueSetupDirtyFields = {
+ name: values.name !== defaults?.name,
+ description: values.description !== defaults?.description,
+ scoreConfigIds: !sameIds(
+ values.scoreConfigIds,
+ defaults?.scoreConfigIds ?? [],
+ ),
+ newAssignmentUserIds: !sameIds(
+ values.newAssignmentUserIds,
+ defaults?.newAssignmentUserIds ?? [],
+ ),
+ };
+ try {
+ await onSubmit(values, dirtyFields);
+ form.reset({ ...values, newAssignmentUserIds: [] });
+ } catch {
+ setSaveFailed(true);
+ }
+ });
+
+ return renderLayout(
+ <div className="isolate flex h-full min-h-0 w-full flex-col">
+ <Tabs value={activeTab} onValueChange={setActiveTab} layout="fill">
+ <div className="shrink-0 px-3">
+ <Tabs.List variant="underline" layout="full" aria-label="Queue setup">
+ <Tabs.Trigger value="settings">Settings</Tabs.Trigger>
+ <Tabs.Trigger value="annotation">Annotation panel</Tabs.Trigger>
+ </Tabs.List>
+ </div>
+ <div className="min-h-0 flex-1 overflow-y-auto p-3">
+ <Form
+ id={formId}
+ density="compact"
+ actions={[]}
+ onSubmit={(event) => {
+ if (event.target !== event.currentTarget) return;
+ event.stopPropagation();
+ return submit(event);
+ }}
+ >
+ <Tabs.Content value="settings">
+ <div className="flex flex-col gap-4">
+ <Form.Field control={form.control} name="name" label="Name">
+ {(field) => (
+ <Input
+ id={field.id}
+ name={field.name}
+ value={field.value}
+ onChange={field.onChange}
+ onBlur={(event) => {
+ field.onChange(event.target.value.trimEnd());
+ field.onBlur();
+ }}
+ ref={field.ref}
+ disabled={disabled}
+ aria-describedby={field.inputDescribedById}
+ aria-invalid={!!field.error}
+ error={!!field.error}
+ />
+ )}
+ </Form.Field>
+ <Form.Field
+ control={form.control}
+ name="description"
+ label="Description (optional)"
+ >
+ {(field) => (
+ <Textarea
+ id={field.id}
+ name={field.name}
+ value={field.value}
+ onChange={field.onChange}
+ onBlur={field.onBlur}
+ ref={field.ref}
+ disabled={disabled}
+ rows={3}
+ aria-describedby={field.inputDescribedById}
+ aria-invalid={!!field.error}
+ />
+ )}
+ </Form.Field>
+ {hasAssignments && (
+ <section
+ className="flex flex-col gap-3"
+ aria-labelledby={`${formId}-assignments`}
+ >
+ <h3
+ id={`${formId}-assignments`}
+ className="text-sm font-bold"
+ >
+ User assignment
+ </h3>
+ <Controller
+ control={form.control}
+ name="newAssignmentUserIds"
+ render={({ field }) => (
+ <fieldset disabled={disabled} className="min-w-0">
+ {renderAssignments({
+ value: field.value,
+ onChange: field.onChange,
+ disabled,
+ })}
+ </fieldset>
+ )}
+ />
+ </section>
+ )}
+ </div>
+ </Tabs.Content>
+ <Tabs.Content value="annotation">
+ <section
+ className="flex flex-col gap-3"
+ aria-label="Queue score configs"
+ >
+ <p className="text-muted-foreground text-xs leading-relaxed">
+ Choose and arrange the score configs annotators fill out for
+ items in this queue.
+ </p>
+ <Controller
+ control={form.control}
+ name="scoreConfigIds"
+ render={({ field }) => (
+ <div>
+ {children({
+ value: field.value,
+ onChange: (ids) =>
+ form.setValue("scoreConfigIds", ids, {
+ shouldDirty: true,
+ shouldValidate: true,
+ }),
+ disabled,
+ })}
+ </div>
+ )}
+ />
+ {missingScoreConfigs ? (
+ <p role="alert" className="text-destructive text-sm">
+ Add at least one score config before saving.
+ </p>
+ ) : null}
+ </section>
+ </Tabs.Content>
+ </Form>
+ </div>
+ </Tabs>
+ <div className="flex shrink-0 flex-col gap-2 border-t p-3">
+ {saveFailed ? (
+ <p role="alert" className="text-destructive text-sm">
+ Could not save queue setup. Your changes are still here. Try again.
+ </p>
+ ) : null}
+ <div className="flex gap-2">
+ <Button
+ text="Save changes"
+ type="submit"
+ form={formId}
+ loading={submitting}
+ disabled={disabled || !isDirty || !isValid || missingScoreConfigs}
+ />
+ <Button
+ text="Discard"
+ variant="secondary"
+ disabled={disabled || !isDirty}
+ onClick={() => {
+ form.reset(initialValues);
+ setSaveFailed(false);
+ }}
+ />
+ </div>
+ </div>
+ </div>,
+ );
+}
diff --git a/web/src/features/annotation-queues/components/AnnotationQueuesItem.clienttest.tsx b/web/src/features/annotation-queues/components/AnnotationQueuesItem.clienttest.tsx
index 0c6efac1da..fdfc7824c6 100644
--- a/web/src/features/annotation-queues/components/AnnotationQueuesItem.clienttest.tsx
+++ b/web/src/features/annotation-queues/components/AnnotationQueuesItem.clienttest.tsx
@@ -14,14 +14,23 @@ function render(ui: ReactNode) {
const client = new QueryClient({
defaultOptions: { queries: { retry: false } },
});
- return renderWithTestingLibrary(ui, {
+ const rendered = renderWithTestingLibrary(ui, {
wrapper: ({ children }) => (
<QueryClientProvider client={client}>{children}</QueryClientProvider>
),
});
+ return { ...rendered, queryClient: client };
}
-const { router, fetchNext, complete, refreshItems, items } = vi.hoisted(() => ({
+const {
+ router,
+ fetchNext,
+ complete,
+ refreshItems,
+ refreshQueue,
+ queueFields,
+ items,
+} = vi.hoisted(() => ({
router: {
isReady: true,
pathname: "/project/[projectId]/annotation-queues/[queueId]/items/[itemId]",
@@ -32,6 +41,8 @@ const { router, fetchNext, complete, refreshItems, items } = vi.hoisted(() => ({
fetchNext: vi.fn(),
complete: vi.fn(),
refreshItems: vi.fn(),
+ refreshQueue: vi.fn(),
+ queueFields: { configs: [] as { id: string; name: string }[] },
items: new Map<string, Record<string, unknown>>(),
}));
@@ -58,49 +69,68 @@ vi.mock("@/src/features/scores", () => ({
isOpenDialogPresent: () => false,
isTypingTarget: () => false,
}));
-vi.mock("@/src/utils/api", () => ({
- api: {
- useUtils: () => ({
- annotationQueueItems: {
- invalidate: refreshItems,
+vi.mock("@/src/utils/api", async () => {
+ const { useQuery } = await import("@tanstack/react-query");
+ return {
+ api: {
+ useUtils: () => ({
+ annotationQueueItems: {
+ invalidate: refreshItems,
+ byId: {
+ fetch: ({ itemId }: { itemId: string }) =>
+ Promise.resolve(items.get(itemId) ?? null),
+ setData: (
+ { itemId }: { itemId: string },
+ item: Record<string, unknown>,
+ ) => items.set(itemId, item),
+ },
+ },
+ }),
+ annotationQueues: {
byId: {
- fetch: ({ itemId }: { itemId: string }) =>
- Promise.resolve(items.get(itemId) ?? null),
- setData: (
- { itemId }: { itemId: string },
- item: Record<string, unknown>,
- ) => items.set(itemId, item),
+ useQuery: (
+ input: { projectId: string; queueId: string },
+ options: {
+ enabled?: boolean;
+ refetchOnMount?: boolean | "always";
+ refetchOnWindowFocus?: boolean;
+ refetchOnReconnect?: boolean;
+ },
+ ) =>
+ useQuery({
+ queryKey: ["annotation-queues", input.projectId, input.queueId],
+ queryFn: () => refreshQueue(input),
+ initialData: {
+ name: "Review queue",
+ scoreConfigs: [...queueFields.configs],
+ },
+ ...options,
+ }),
+ },
+ fetchAndLockNext: {
+ useMutation: () => ({ mutateAsync: fetchNext, isPending: false }),
},
},
- }),
- annotationQueues: {
- byId: { useQuery: () => ({ data: { name: "Review queue" } }) },
- fetchAndLockNext: {
- useMutation: () => ({ mutateAsync: fetchNext, isPending: false }),
- },
- },
- annotationQueueItems: {
- byId: {
- useQuery: ({ itemId }: { itemId: string }) => ({
- data: items.get(itemId),
- isPending: false,
- }),
- },
- unseenPendingItemCountByQueueId: {
- useQuery: ({ seenItemIds }: { seenItemIds: string[] }) => ({
- data: Math.max(2 - seenItemIds.length, 0),
- isPending: false,
- }),
- },
- complete: {
- useMutation: () => ({ mutateAsync: complete, isPending: false }),
+ annotationQueueItems: {
+ byId: {
+ useQuery: ({ itemId }: { itemId: string }) => ({
+ data: items.get(itemId),
+ isPending: false,
+ }),
+ },
+ unseenPendingItemCountByQueueId: {
+ useQuery: ({ seenItemIds }: { seenItemIds: string[] }) => ({
+ data: Math.max(2 - seenItemIds.length, 0),
+ isPending: false,
+ }),
+ },
+ complete: {
+ useMutation: () => ({ mutateAsync: complete, isPending: false }),
+ },
},
},
- },
-}));
-vi.mock("./shared/hooks/useAnnotationQueueData", () => ({
- useAnnotationQueueData: () => ({ configs: [] }),
-}));
+ };
+});
vi.mock("./shared/hooks/useAnnotationObjectData", () => ({
useAnnotationObjectData: (item: { objectId: string } | null) => ({
data: item ? { id: item.objectId } : undefined,
@@ -111,9 +141,18 @@ vi.mock("@/src/features/traces", () => ({
Trace: () => <p>Trace loaded</p>,
}));
vi.mock("./shared/AnnotationDrawerSection", () => ({
- AnnotationDrawerSection: ({ item }: { item: { objectId: string } }) => (
+ AnnotationDrawerSection: ({
+ item,
+ configs,
+ }: {
+ item: { objectId: string };
+ configs: { name: string }[];
+ }) => (
<>
<p>{item.objectId}</p>
+ <output aria-label="Queue fields">
+ {configs.map((config) => config.name).join(", ")}
+ </output>
<input aria-label="Comment draft" />
</>
),
@@ -142,6 +181,14 @@ beforeEach(() => {
fetchNext.mockReset();
complete.mockReset();
refreshItems.mockReset();
+ refreshQueue.mockReset();
+ queueFields.configs = [];
+ refreshQueue.mockImplementation(() =>
+ Promise.resolve({
+ name: "Review queue",
+ scoreConfigs: [...queueFields.configs],
+ }),
+ );
items.clear();
for (const queueId of ["first", "second"]) {
items.set(`${queueId}-item`, {
@@ -182,6 +229,54 @@ beforeEach(() => {
refreshItems.mockResolvedValue(undefined);
});
+it.each(["success", "failure"])(
+ "waits for cached queue mount refresh %s before editing and preserves drafts on later invalidation",
+ async (result) => {
+ queueFields.configs = [{ id: "cached", name: "Cached field" }];
+ let resolveRefresh!: (value: unknown) => void;
+ let rejectRefresh!: (error: Error) => void;
+ refreshQueue.mockImplementationOnce(
+ () =>
+ new Promise((resolve, reject) => {
+ resolveRefresh = resolve;
+ rejectRefresh = reject;
+ }),
+ );
+ const { queryClient } = render(
+ <AnnotationQueuesItem projectId="project" annotationQueueId="first" />,
+ );
+ await waitFor(() => expect(refreshQueue).toHaveBeenCalledTimes(1));
+ expect(
+ screen.queryByRole("textbox", { name: "Comment draft" }),
+ ).not.toBeInTheDocument();
+ await act(async () => {
+ if (result === "failure")
+ rejectRefresh(new Error("Queue refresh unavailable"));
+ else
+ resolveRefresh({
+ name: "Review queue",
+ scoreConfigs: [{ id: "refreshed", name: "Refreshed field" }],
+ });
+ });
+ const input = await screen.findByRole("textbox", { name: "Comment draft" });
+ expect(screen.getByLabelText("Queue fields")).toHaveTextContent(
+ result === "failure" ? "Cached field" : "Refreshed field",
+ );
+ fireEvent.change(input, { target: { value: "Unsent review" } });
+ refreshQueue.mockResolvedValueOnce({
+ name: "Review queue",
+ scoreConfigs: [{ id: "later", name: "Later field" }],
+ });
+ await act(async () => {
+ await queryClient.invalidateQueries({
+ queryKey: ["annotation-queues", "project", "first"],
+ });
+ });
+ expect(screen.getByRole("textbox", { name: "Comment draft" })).toBe(input);
+ expect(input).toHaveValue("Unsent review");
+ },
+);
+
it.each(["refresh", "advance", "navigation"])(
"retains completion and retries a failed %s without completing twice",
async (phase) => {
diff --git a/web/src/features/annotation-queues/components/AnnotationQueuesTable.tsx b/web/src/features/annotation-queues/components/AnnotationQueuesTable.tsx
index 9a02b15b2c..e2dbfedf7f 100644
--- a/web/src/features/annotation-queues/components/AnnotationQueuesTable.tsx
+++ b/web/src/features/annotation-queues/components/AnnotationQueuesTable.tsx
@@ -16,7 +16,6 @@ import {
useCompactRows,
useRowHeightLocalStorage,
} from "@/src/components/table/data-table-row-height-switch";
-import { CreateOrEditAnnotationQueueButton } from "@/src/features/annotation-queues/components/CreateOrEditAnnotationQueueButton";
import { ClipboardPen, Lock } from "lucide-react";
import { Button } from "@/src/components/ui/button";
import { cn } from "@/src/utils/tailwind";
@@ -193,13 +192,6 @@ export function AnnotationQueuesTable({ projectId }: { projectId: string }) {
const key: RowData["key"] = row.getValue("key");
return (
<div className="flex items-center gap-1">
- <CreateOrEditAnnotationQueueButton
- projectId={projectId}
- queueId={key.id}
- variant="ghost"
- size="icon-xs"
- isTableAction
- />
<DeleteAnnotationQueueButton
projectId={projectId}
queueId={key.id}
diff --git a/web/src/features/annotation-queues/components/QueueScoreConfigFields.tsx b/web/src/features/annotation-queues/components/QueueScoreConfigFields.tsx
new file mode 100644
index 0000000000..c029e0d366
--- /dev/null
+++ b/web/src/features/annotation-queues/components/QueueScoreConfigFields.tsx
@@ -0,0 +1,65 @@
+import { type ScoreConfigDomain } from "@langfuse/shared";
+import {
+ CreateScoreConfigDialogController,
+ EditScoreConfigDialogController,
+} from "@/src/features/score-configs";
+import { AnnotationQueueScoreConfigEditor } from "./AnnotationQueueScoreConfigEditor";
+
+export function QueueScoreConfigFields({
+ projectId,
+ configs,
+ value,
+ onChange,
+ disabled,
+ warnOnRemoval,
+}: {
+ projectId: string;
+ configs: ScoreConfigDomain[];
+ value: string[];
+ onChange: (ids: string[]) => void;
+ disabled: boolean;
+ warnOnRemoval: boolean;
+}) {
+ return (
+ <CreateScoreConfigDialogController
+ projectId={projectId}
+ onAfterCreate={(config) => onChange([...value, config.id])}
+ >
+ {(create) => (
+ <EditScoreConfigDialogController projectId={projectId}>
+ {(edit) => (
+ <AnnotationQueueScoreConfigEditor
+ scoreConfigs={configs}
+ value={value}
+ onChange={onChange}
+ disabled={disabled}
+ warnOnRemoval={warnOnRemoval}
+ createAction={{
+ disabled: create.disabled,
+ onClick: create.openDialog,
+ }}
+ editAction={{
+ disabled: edit.disabled,
+ onClick: (configId) => {
+ const config = configs.find(
+ (candidate) => candidate.id === configId,
+ );
+ if (!config) return;
+ edit.openDialog({
+ id: config.id,
+ name: config.name,
+ dataType: config.dataType,
+ description: config.description ?? undefined,
+ minValue: config.minValue ?? undefined,
+ maxValue: config.maxValue ?? undefined,
+ categories: config.categories ?? undefined,
+ });
+ },
+ }}
+ />
+ )}
+ </EditScoreConfigDialogController>
+ )}
+ </CreateScoreConfigDialogController>
+ );
+}
diff --git a/web/src/features/annotation-queues/components/shared/hooks/useAnnotationQueueData.ts b/web/src/features/annotation-queues/components/shared/hooks/useAnnotationQueueData.ts
index 8d034b743e..1353861f16 100644
--- a/web/src/features/annotation-queues/components/shared/hooks/useAnnotationQueueData.ts
+++ b/web/src/features/annotation-queues/components/shared/hooks/useAnnotationQueueData.ts
@@ -12,7 +12,7 @@ export const useAnnotationQueueData = (config: AnnotationQueueDataConfig) => {
projectId: config.projectId,
},
{
- refetchOnMount: false,
+ refetchOnMount: "always",
refetchOnWindowFocus: false,
refetchOnReconnect: false,
},
@@ -20,6 +20,7 @@ export const useAnnotationQueueData = (config: AnnotationQueueDataConfig) => {
return {
queueData,
+ isLoading: queueData.isPending || !queueData.isFetchedAfterMount,
configs: queueData.data?.scoreConfigs ?? [],
};
};
diff --git a/web/src/features/annotation-queues/pages/AnnotationQueueItems.tsx b/web/src/features/annotation-queues/pages/AnnotationQueueItems.tsx
index c626bc92a0..c7da782123 100644
--- a/web/src/features/annotation-queues/pages/AnnotationQueueItems.tsx
+++ b/web/src/features/annotation-queues/pages/AnnotationQueueItems.tsx
@@ -1,20 +1,17 @@
import { api } from "@/src/utils/api";
+import { useState } from "react";
import { AnnotationQueueItemsTable } from "@/src/features/annotation-queues/components/AnnotationQueueItemsTable";
-import { Button } from "@/src/components/ui/button";
+import { Button } from "@/src/components/design-system/Button/Button";
import { ClipboardPen, Lock } from "lucide-react";
-import Link from "next/link";
-import { CreateOrEditAnnotationQueueButton } from "@/src/features/annotation-queues/components/CreateOrEditAnnotationQueueButton";
import { useHasProjectAccess } from "@/src/features/rbac";
import { SupportOrUpgradePage } from "@/src/ee/features/billing";
-import { Skeleton } from "@/src/components/ui/skeleton";
import Page from "@/src/components/layouts/page";
import {
SidePanel,
SidePanelContent,
SidePanelHeader,
- SidePanelTitle,
} from "@/src/components/ui/side-panel";
-import { AnnotationQueueDetails } from "@/src/features/annotation-queues/components/AnnotationQueueDetails";
+import { AnnotationQueueSetupController } from "@/src/features/annotation-queues/components/AnnotationQueueSetupController";
export default function QueueItems({
projectId,
@@ -23,6 +20,7 @@ export default function QueueItems({
projectId: string;
queueId: string;
}) {
+ const [mobileSetupOpen, setMobileSetupOpen] = useState(false);
const queue = api.annotationQueues.byId.useQuery(
{
queueId,
@@ -54,59 +52,44 @@ export default function QueueItems({
},
],
actionButtonsRight: !hasWriteAccess ? (
- <Button disabled>
- <Lock className="icon-base text-icon-foreground mr-1" />
- <span className="text-sm">Process queue</span>
- </Button>
+ <Button disabled icon={Lock} text="Process queue" />
) : (
- <Button asChild>
- <Link
- href={`/project/${projectId}/annotation-queues/${queueId}/items`}
- >
- <ClipboardPen className="icon-base text-icon-foreground mr-1" />
- <span className="text-sm">Process queue</span>
- </Link>
- </Button>
+ <Button
+ href={`/project/${projectId}/annotation-queues/${queueId}/items`}
+ icon={ClipboardPen}
+ text="Process queue"
+ />
),
}}
>
- <div className="grid flex-1 grid-cols-[1fr_auto] overflow-hidden">
- <div className="flex h-full flex-col overflow-hidden">
+ <div className="grid flex-1 grid-cols-[minmax(0,1fr)_auto] overflow-hidden">
+ <div className="flex h-full min-w-0 flex-col overflow-hidden">
<AnnotationQueueItemsTable projectId={projectId} queueId={queueId} />
</div>
- <SidePanel mobileTitle="Queue details" id="queue-details">
- <SidePanelHeader>
- <SidePanelTitle>Details</SidePanelTitle>
- <CreateOrEditAnnotationQueueButton
- projectId={projectId}
- queueId={queueId}
- variant="ghost"
- size="sm"
- />
- </SidePanelHeader>
- <SidePanelContent>
- <div className="w-full min-w-0">
- <div className="flex justify-end px-4 md:hidden">
- <CreateOrEditAnnotationQueueButton
- projectId={projectId}
- queueId={queueId}
- variant="ghost"
- size="sm"
- />
- </div>
- {queue.isLoading ? (
- <div className="p-4">
- <Skeleton className="h-40 w-full" />
- </div>
- ) : (
- <AnnotationQueueDetails
- description={queue.data?.description ?? null}
- scoreConfigs={queue.data?.scoreConfigs ?? []}
- />
- )}
- </div>
- </SidePanelContent>
- </SidePanel>
+ <AnnotationQueueSetupController
+ key={`${projectId}:${queueId}`}
+ projectId={projectId}
+ queueId={queueId}
+ renderLayout={(content) => (
+ <SidePanel
+ mobileTitle="Queue setup"
+ id="queue-details"
+ scrollable={false}
+ variant="sidebar"
+ mobileOpenState={{
+ open: mobileSetupOpen,
+ onOpenChange: setMobileSetupOpen,
+ }}
+ >
+ <SidePanelHeader>
+ <span className="text-sm font-bold">Queue setup</span>
+ </SidePanelHeader>
+ <SidePanelContent className="isolate min-h-0 flex-1 gap-0">
+ {content}
+ </SidePanelContent>
+ </SidePanel>
+ )}
+ />
</div>
</Page>
);
diff --git a/web/src/features/annotation-queues/server/annotationQueuesRouter.ts b/web/src/features/annotation-queues/server/annotationQueuesRouter.ts
index 2223884834..b34d6d1703 100644
--- a/web/src/features/annotation-queues/server/annotationQueuesRouter.ts
+++ b/web/src/features/annotation-queues/server/annotationQueuesRouter.ts
@@ -11,6 +11,7 @@ import {
CreateQueueData,
filterAndValidateDbScoreConfigList,
isBaseError,
+ isPresent,
LangfuseNotFoundError,
optionalPaginationZod,
Prisma,
@@ -24,6 +25,13 @@ import {
import { TRPCError } from "@trpc/server";
import { z } from "zod";
+const QueueData = CreateQueueData.extend({
+ scoreConfigIds: CreateQueueData.shape.scoreConfigIds.refine(
+ (ids) => new Set(ids).size === ids.length,
+ { message: "Score config IDs must be unique" },
+ ),
+});
+
export const queueRouter = createTRPCRouter({
hasAny: protectedProjectProcedure
.input(
@@ -123,13 +131,17 @@ export const queueRouter = createTRPCRouter({
},
});
+ const scoreConfigsById = new Map(
+ scoreConfigs.map((config) => [config.id, config]),
+ );
+
return {
totalCount,
queues: queues.map((queue) => ({
...queue,
- scoreConfigs: scoreConfigs.filter((config) =>
- queue.scoreConfigIds.includes(config.id),
- ),
+ scoreConfigs: Array.from(new Set(queue.scoreConfigIds))
+ .map((id) => scoreConfigsById.get(id))
+ .filter(isPresent),
isCurrentUserAssigned: userQueueAssignments.some(
(assignment) => assignment.queueId === queue.id,
),
@@ -217,9 +229,18 @@ export const queueRouter = createTRPCRouter({
},
});
+ const scoreConfigsById = new Map(
+ filterAndValidateDbScoreConfigList(configs).map((config) => [
+ config.id,
+ config,
+ ]),
+ );
+
return {
...queue,
- scoreConfigs: filterAndValidateDbScoreConfigList(configs),
+ scoreConfigs: Array.from(new Set(queue?.scoreConfigIds ?? []))
+ .map((id) => scoreConfigsById.get(id))
+ .filter(isPresent),
};
} catch (error) {
logger.error(error);
@@ -299,7 +320,7 @@ export const queueRouter = createTRPCRouter({
}),
create: protectedProjectProcedure
.input(
- CreateQueueData.extend({
+ QueueData.extend({
projectId: z.string(),
}),
)
@@ -323,6 +344,12 @@ export const queueRouter = createTRPCRouter({
// unique index cannot prevent.
const queue = await ctx.prisma.$transaction(
async (tx) => {
+ await validateProjectScoreConfigs(
+ tx,
+ input.projectId,
+ input.scoreConfigIds,
+ );
+
if (plan === "cloud:hobby") {
const queueCount = await tx.annotationQueue.count({
where: { projectId: input.projectId },
@@ -403,7 +430,7 @@ export const queueRouter = createTRPCRouter({
}),
update: protectedProjectProcedure
.input(
- CreateQueueData.extend({
+ QueueData.partial().extend({
projectId: z.string(),
queueId: z.string(),
}),
@@ -427,21 +454,31 @@ export const queueRouter = createTRPCRouter({
throw new LangfuseNotFoundError("Queue not found in project");
}
- const existingQueueWithName =
- await ctx.prisma.annotationQueue.findFirst({
- where: {
- projectId: input.projectId,
- name: input.name,
- id: { not: input.queueId },
- },
- select: { id: true },
- });
+ if (input.scoreConfigIds !== undefined) {
+ await validateProjectScoreConfigs(
+ ctx.prisma,
+ input.projectId,
+ input.scoreConfigIds,
+ );
+ }
- if (existingQueueWithName) {
- throw new TRPCError({
- code: "CONFLICT",
- message: "A queue with this name already exists in the project",
- });
+ if (input.name !== undefined) {
+ const existingQueueWithName =
+ await ctx.prisma.annotationQueue.findFirst({
+ where: {
+ projectId: input.projectId,
+ name: input.name,
+ id: { not: input.queueId },
+ },
+ select: { id: true },
+ });
+
+ if (existingQueueWithName) {
+ throw new TRPCError({
+ code: "CONFLICT",
+ message: "A queue with this name already exists in the project",
+ });
+ }
}
const updatedQueue = await ctx.prisma.annotationQueue.update({
@@ -601,3 +638,19 @@ export const queueRouter = createTRPCRouter({
return inflatedUpdatedItem;
}),
});
+
+async function validateProjectScoreConfigs(
+ prisma: Pick<Prisma.TransactionClient, "scoreConfig">,
+ projectId: string,
+ scoreConfigIds: string[],
+) {
+ const configCount = await prisma.scoreConfig.count({
+ where: { projectId, id: { in: scoreConfigIds } },
+ });
+ if (configCount !== scoreConfigIds.length) {
+ throw new TRPCError({
+ code: "BAD_REQUEST",
+ message: "All score configs must belong to this project.",
+ });
+ }
+}
diff --git a/web/src/features/annotation-queues/server/publicAnnotationQueueService.ts b/web/src/features/annotation-queues/server/publicAnnotationQueueService.ts
index 07553c6305..ea48c3e344 100644
--- a/web/src/features/annotation-queues/server/publicAnnotationQueueService.ts
+++ b/web/src/features/annotation-queues/server/publicAnnotationQueueService.ts
@@ -171,6 +171,10 @@ export const createAnnotationQueueForApi = async ({
plan?: string;
input: CreateAnnotationQueueInput;
} & OptionalAuditScope) => {
+ if (new Set(input.scoreConfigIds).size !== input.scoreConfigIds.length) {
+ throw new InvalidRequestError("Score config IDs must be unique.");
+ }
+
const queue = await prisma
.$transaction(
async (tx) => {
diff --git a/web/src/features/evals/v2/components/Evaluators/DecisionModel/ChoiceOptionsEditor/ChoiceOptionsEditor.tsx b/web/src/features/evals/v2/components/Evaluators/DecisionModel/ChoiceOptionsEditor/ChoiceOptionsEditor.tsx
index 4887861b63..e5fc6ea347 100644
--- a/web/src/features/evals/v2/components/Evaluators/DecisionModel/ChoiceOptionsEditor/ChoiceOptionsEditor.tsx
+++ b/web/src/features/evals/v2/components/Evaluators/DecisionModel/ChoiceOptionsEditor/ChoiceOptionsEditor.tsx
@@ -6,7 +6,7 @@ import { Button } from "@/src/components/ui/button";
import { InfoTooltip } from "@/src/components/ui/InfoTooltip/InfoTooltip";
import { Input } from "@/src/components/ui/input";
import { Label } from "@/src/components/ui/label";
-import { SortableList } from "@/src/features/evals/v2/components/SortableList/SortableList";
+import { SortableList } from "@/src/components/design-system/SortableList/SortableList";
import { moveItem } from "@/src/features/evals/v2/fns/moveItem";
export type ChoiceOptionDraft = { value: string; description: string };
diff --git a/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelQuestionList/DecisionModelQuestionList.tsx b/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelQuestionList/DecisionModelQuestionList.tsx
index 10d18d0287..5b980f1791 100644
--- a/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelQuestionList/DecisionModelQuestionList.tsx
+++ b/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelQuestionList/DecisionModelQuestionList.tsx
@@ -8,7 +8,7 @@ import { TextActionButton } from "@/src/components/TextActionButton/TextActionBu
import { Button } from "@/src/components/ui/button";
import { InfoTooltip } from "@/src/components/ui/InfoTooltip/InfoTooltip";
import { Label } from "@/src/components/ui/label";
-import { SortableList } from "@/src/features/evals/v2/components/SortableList/SortableList";
+import { SortableList } from "@/src/components/design-system/SortableList/SortableList";
import { DecisionModelQuestionCard } from "@/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelQuestionCard/DecisionModelQuestionCard";
import { QUESTION_TYPE_COPY } from "@/src/features/evals/v2/components/Evaluators/DecisionModel/QuestionTypeSelector/QuestionTypeSelector";
import type {
diff --git a/web/src/features/evals/v2/components/Evaluators/DecisionModel/ScoreLevelsEditor/ScoreLevelsEditor.tsx b/web/src/features/evals/v2/components/Evaluators/DecisionModel/ScoreLevelsEditor/ScoreLevelsEditor.tsx
index 74114e8e55..01077b2f3d 100644
--- a/web/src/features/evals/v2/components/Evaluators/DecisionModel/ScoreLevelsEditor/ScoreLevelsEditor.tsx
+++ b/web/src/features/evals/v2/components/Evaluators/DecisionModel/ScoreLevelsEditor/ScoreLevelsEditor.tsx
@@ -6,7 +6,7 @@ import { Button } from "@/src/components/ui/button";
import { InfoTooltip } from "@/src/components/ui/InfoTooltip/InfoTooltip";
import { Input } from "@/src/components/ui/input";
import { Label } from "@/src/components/ui/label";
-import { SortableList } from "@/src/features/evals/v2/components/SortableList/SortableList";
+import { SortableList } from "@/src/components/design-system/SortableList/SortableList";
import { moveItem } from "@/src/features/evals/v2/fns/moveItem";
export type ScoreLevelDraft = { description: string };
diff --git a/web/src/features/posthog-analytics/usePostHogClientCapture.ts b/web/src/features/posthog-analytics/usePostHogClientCapture.ts
index f9aa25cfea..4876f660e6 100644
--- a/web/src/features/posthog-analytics/usePostHogClientCapture.ts
+++ b/web/src/features/posthog-analytics/usePostHogClientCapture.ts
@@ -149,7 +149,12 @@ const events = {
"level_added",
],
annotation: ["entry_click"],
- annotation_queues: ["item_added", "item_removed", "manage_click"],
+ annotation_queues: [
+ "item_added",
+ "item_removed",
+ "manage_click",
+ "score_configs_saved",
+ ],
score_configs: [
"create_form_submit",
"update_form_submit",
diff --git a/web/src/features/score-configs/ScoreConfigsTable/ScoreConfigsTable.tsx b/web/src/features/score-configs/ScoreConfigsTable/ScoreConfigsTable.tsx
index cf0d0e6f12..9a8aa90b9e 100644
--- a/web/src/features/score-configs/ScoreConfigsTable/ScoreConfigsTable.tsx
+++ b/web/src/features/score-configs/ScoreConfigsTable/ScoreConfigsTable.tsx
@@ -1,5 +1,5 @@
import { useCallback, useMemo } from "react";
-import { Archive, Edit, PlusIcon } from "lucide-react";
+import { Archive, PlusIcon } from "lucide-react";
import {
type Prisma,
type ScoreConfigCategoryDomain,
@@ -11,6 +11,7 @@ import {
type AsyncTableData,
type TableProps,
} from "@/src/components/design-system/table/Table";
+import { createButtonTableColumn } from "@/src/components/design-system/table/columns/createButtonTableColumn";
import { createBadgeTableColumn } from "@/src/components/design-system/table/columns/createBadgeTableColumn";
import { createDateTableColumn } from "@/src/components/design-system/table/columns/createDateTableColumn";
import { createIdTableColumn } from "@/src/components/design-system/table/columns/createIdTableColumn";
@@ -131,6 +132,18 @@ export function ScoreConfigsTable({
enableHiding: true,
defaultHidden: true,
}),
+ createButtonTableColumn<ScoreConfigTableRow, string>({
+ id: "edit",
+ accessorFn: (config) => config.id,
+ header: "",
+ size: 80,
+ getButton: ({ row }) => ({
+ text: "Edit",
+ disabled: !!editAction.disabled,
+ disabledReason: editAction.disabled?.reason,
+ onClick: () => editAction.openDialog(row.original),
+ }),
+ }),
createBadgeTableColumn<ScoreConfigTableRow>({
id: "status",
accessorFn: (config) => (config.isArchived ? "Archived" : "Active"),
@@ -139,21 +152,13 @@ export function ScoreConfigsTable({
enableHiding: true,
}),
],
- [],
+ [editAction],
);
const actions = useCallback<
NonNullable<TableProps<ScoreConfigTableRow>["actions"]>
>(
(config) => [
- {
- id: "edit",
- type: "item",
- title: "Edit",
- icon: Edit,
- disabled: editAction.disabled,
- onClick: () => editAction.openDialog(config),
- },
{
id: "archive",
type: "item",
@@ -164,7 +169,7 @@ export function ScoreConfigsTable({
onClick: () => archiveAction.openDialog(config),
},
],
- [archiveAction, editAction],
+ [archiveAction],
);
const toolbarActions: SettingsTableProps<ScoreConfigTableRow>["toolbarActions"] =
diff --git a/web/src/features/score-configs/components/ScoreConfigDetails.tsx b/web/src/features/score-configs/components/ScoreConfigDetails.tsx
deleted file mode 100644
index 2dbeac035b..0000000000
--- a/web/src/features/score-configs/components/ScoreConfigDetails.tsx
+++ /dev/null
@@ -1,18 +0,0 @@
-import { isNumericDataType } from "@/src/features/scores/lib/helpers";
-import { isPresent, type ScoreConfigDomain } from "@langfuse/shared";
-
-export function ScoreConfigDetails({ config }: { config: ScoreConfigDomain }) {
- const { name, description, minValue, maxValue, dataType } = config;
- const isNameTruncated = name.length > 20;
-
- return (
- <div className="bg-background p-2 text-xs text-wrap">
- {!!description && <p>{`Description: ${description}`}</p>}
- {isNumericDataType(dataType) &&
- (isPresent(minValue) || isPresent(maxValue)) ? (
- <p>{`Range: [${minValue ?? "-∞"}, ${maxValue ?? "∞"}]`}</p>
- ) : null}
- {isNameTruncated && <p>{`Full name: ${name}`}</p>}
- </div>
- );
-}
diff --git a/web/src/features/score-configs/components/ScoreConfigSettings.tsx b/web/src/features/score-configs/components/ScoreConfigSettings.tsx
index 4c783c473b..04dc0b5a62 100644
--- a/web/src/features/score-configs/components/ScoreConfigSettings.tsx
+++ b/web/src/features/score-configs/components/ScoreConfigSettings.tsx
@@ -15,7 +15,10 @@ export function ScoreConfigSettings({ projectId }: { projectId: string }) {
>
annotation
</a>{" "}
- in your project. Please note that all score configs are immutable.
+ and API/SDK scoring in your project. Data type cannot be changed after
+ creation. You can edit the name, description, range, and categories
+ where supported. Changes affect future scoring wherever the score config
+ is used.
</p>
<ConnectedScoreConfigsTable projectId={projectId} />
</div>
diff --git a/web/src/features/score-configs/components/UpsertScoreConfigDialogContent.stories.tsx b/web/src/features/score-configs/components/UpsertScoreConfigDialogContent.stories.tsx
index a4fdcdd74f..93daed1d2d 100644
--- a/web/src/features/score-configs/components/UpsertScoreConfigDialogContent.stories.tsx
+++ b/web/src/features/score-configs/components/UpsertScoreConfigDialogContent.stories.tsx
@@ -2,7 +2,7 @@ import { type ComponentProps } from "react";
import { expect, fn, userEvent, within } from "storybook/test";
import preview from "../../../../.storybook/preview";
-import { Dialog, DialogContent } from "@/src/components/ui/dialog";
+import { DialogController } from "@/src/components/design-system/DialogController/DialogController";
import { UpsertScoreConfigDialogContent } from "./UpsertScoreConfigDialogContent";
@@ -26,11 +26,12 @@ const meta = preview.meta({
},
decorators: [
(Story) => (
- <Dialog open onOpenChange={fn()}>
- <DialogContent className="max-h-[90vh] overflow-y-auto">
- <Story />
- </DialogContent>
- </Dialog>
+ <DialogController<boolean>
+ initialState={() => true}
+ renderDialog={() => <Story />}
+ >
+ {() => null}
+ </DialogController>
),
],
});
diff --git a/web/src/features/score-configs/components/UpsertScoreConfigDialogContent.tsx b/web/src/features/score-configs/components/UpsertScoreConfigDialogContent.tsx
index 38ec6719f2..493a5e5fca 100644
--- a/web/src/features/score-configs/components/UpsertScoreConfigDialogContent.tsx
+++ b/web/src/features/score-configs/components/UpsertScoreConfigDialogContent.tsx
@@ -1,17 +1,13 @@
/* eslint-disable no-nested-ternary */
import { zodResolver } from "@hookform/resolvers/zod";
import { ScoreDataTypeEnum, type ScoreConfigDataType } from "@langfuse/shared";
-import { useState } from "react";
+import { useId, useState } from "react";
import { type UseFormReturn, useFieldArray, useForm } from "react-hook-form";
-import { Button } from "@/src/components/ui/button";
+import { Button } from "@/src/components/design-system/Button/Button";
+import { IconButton } from "@/src/components/design-system/IconButton/IconButton";
import { SelectInput } from "@/src/components/design-system/SelectInput/SelectInput";
-import {
- DialogBody,
- DialogFooter,
- DialogHeader,
- DialogTitle,
-} from "@/src/components/ui/dialog";
+import { Dialog } from "@/src/components/design-system/Dialog/Dialog";
import {
Form,
FormControl,
@@ -20,7 +16,7 @@ import {
FormLabel,
FormMessage,
} from "@/src/components/ui/form";
-import { Input } from "@/src/components/ui/input";
+import { Input } from "@/src/components/design-system/Input/Input";
import { Textarea } from "@/src/components/ui/textarea";
import DocPopup from "@/src/components/layouts/doc-popup";
import {
@@ -61,6 +57,7 @@ export function UpsertScoreConfigDialogContent({
onFormSuccess,
isSubmitting,
}: UpsertScoreConfigDialogContentProps) {
+ const formId = useId();
const [formError, setFormError] = useState<string | null>(null);
const form = useForm({
resolver: zodResolver(
@@ -94,15 +91,32 @@ export function UpsertScoreConfigDialogContent({
}
return (
- <>
- <DialogHeader>
- <DialogTitle>
- {mode === "edit" ? "Update score config" : "Add new score config"}
- </DialogTitle>
- </DialogHeader>
+ <Dialog
+ title={mode === "edit" ? "Update score config" : "Add new score config"}
+ actions={[
+ {
+ label: "Submit",
+ type: "submit",
+ form: formId,
+ loading: isSubmitting,
+ },
+ ]}
+ >
<Form {...form}>
- <form className="space-y-6" onSubmit={form.handleSubmit(handleSubmit)}>
- <DialogBody>
+ <form
+ id={formId}
+ className="flex min-h-0 flex-1 flex-col overflow-hidden"
+ onSubmit={(event) => {
+ event.stopPropagation();
+ return form.handleSubmit(handleSubmit)(event);
+ }}
+ >
+ <Dialog.Body>
+ <p className="text-muted-foreground">
+ {mode === "edit"
+ ? "This score config is project-wide. Changes affect future scoring wherever it is used, including other queues and API/SDK scores. Data type cannot be changed after creation."
+ : "Create a project-wide score config for annotation and API/SDK scoring."}
+ </p>
<FormField
control={form.control}
name="name"
@@ -250,7 +264,6 @@ export function UpsertScoreConfigDialogContent({
readOnly
disabled
inputMode="numeric"
- className="text-center"
/>
</FormControl>
<FormMessage />
@@ -278,17 +291,14 @@ export function UpsertScoreConfigDialogContent({
)}
/>
{isCategoricalDataType(dataType) && (
- <Button
- type="button"
+ <IconButton
+ icon={Trash}
+ label={`Remove category ${index + 1}`}
onClick={() => remove(index)}
- variant="outline"
- size="icon"
disabled={
index === 0 || index !== fields.length - 1
}
- >
- <Trash className="icon-base text-icon-foreground" />
- </Button>
+ />
)}
</div>
</div>
@@ -297,6 +307,7 @@ export function UpsertScoreConfigDialogContent({
<div className="grid">
<Button
type="button"
+ text="Add category"
variant="secondary"
disabled={
isBooleanDataType(dataType) && fields.length === 2
@@ -314,9 +325,7 @@ export function UpsertScoreConfigDialogContent({
: 0,
})
}
- >
- Add category
- </Button>
+ />
</div>
)}
</>
@@ -341,21 +350,14 @@ export function UpsertScoreConfigDialogContent({
</FormItem>
)}
/>
- </DialogBody>
- <DialogFooter>
- <div className="flex w-full flex-col items-end gap-4">
- {formError ? (
- <p className="w-full text-center">
- <span className="font-bold">Error:</span> {formError}
- </p>
- ) : null}
- <Button type="submit" loading={isSubmitting}>
- Submit
- </Button>
- </div>
- </DialogFooter>
+ {formError && (
+ <p role="alert">
+ <span className="font-bold">Error:</span> {formError}
+ </p>
+ )}
+ </Dialog.Body>
</form>
</Form>
- </>
+ </Dialog>
);
}
diff --git a/web/src/features/score-configs/components/UpsertScoreConfigDialogController.tsx b/web/src/features/score-configs/components/UpsertScoreConfigDialogController.tsx
index dac9463e35..97031ebd84 100644
--- a/web/src/features/score-configs/components/UpsertScoreConfigDialogController.tsx
+++ b/web/src/features/score-configs/components/UpsertScoreConfigDialogController.tsx
@@ -2,7 +2,6 @@ import { ScoreDataTypeEnum } from "@langfuse/shared";
import { type ReactNode } from "react";
import { DialogController } from "@/src/components/design-system/DialogController/DialogController";
-import { DialogContent } from "@/src/components/ui/dialog";
import { usePostHogClientCapture } from "@/src/features/posthog-analytics";
import { useHasProjectAccess } from "@/src/features/rbac";
import { UpsertScoreConfigDialogContent } from "@/src/features/score-configs/components/UpsertScoreConfigDialogContent";
@@ -48,28 +47,26 @@ export function CreateScoreConfigDialogController({
return (
<DialogController
renderDialog={({ closeDialog }) => (
- <DialogContent className="max-h-[90vh] overflow-y-auto">
- <UpsertScoreConfigDialogContent
- mode="create"
- defaultValues={createDefaultValues}
- onSubmit={async (values) => {
- const config = await mutation.mutateAsync({
- projectId,
- ...values,
- description: values.description ?? null,
- categories: values.categories?.length
- ? values.categories
- : undefined,
- });
- capture("score_configs:create_form_submit", {
- dataType: values.dataType,
- });
- onAfterCreate?.(config);
- }}
- onFormSuccess={closeDialog}
- isSubmitting={mutation.isPending}
- />
- </DialogContent>
+ <UpsertScoreConfigDialogContent
+ mode="create"
+ defaultValues={createDefaultValues}
+ onSubmit={async (values) => {
+ const config = await mutation.mutateAsync({
+ projectId,
+ ...values,
+ description: values.description ?? null,
+ categories: values.categories?.length
+ ? values.categories
+ : undefined,
+ });
+ capture("score_configs:create_form_submit", {
+ dataType: values.dataType,
+ });
+ onAfterCreate?.(config);
+ }}
+ onFormSuccess={closeDialog}
+ isSubmitting={mutation.isPending}
+ />
)}
>
{({ openDialog }) =>
@@ -96,7 +93,11 @@ export function EditScoreConfigDialogController({
});
const utils = api.useUtils();
const mutation = api.scoreConfigs.update.useMutation({
- onSuccess: () => utils.scoreConfigs.invalidate(),
+ onSuccess: () =>
+ Promise.all([
+ utils.scoreConfigs.invalidate(),
+ utils.annotationQueues.invalidate(),
+ ]),
});
const disabled = hasAccess
? undefined
@@ -105,29 +106,27 @@ export function EditScoreConfigDialogController({
return (
<DialogController<UpdateConfig>
renderDialog={({ state: defaultValues, closeDialog }) => (
- <DialogContent className="max-h-[90vh] overflow-y-auto">
- <UpsertScoreConfigDialogContent
- key={defaultValues.id}
- mode="edit"
- defaultValues={defaultValues}
- onSubmit={async (values) => {
- await mutation.mutateAsync({
- ...values,
- projectId,
- id: defaultValues.id,
- description: values.description ?? null,
- categories: values.categories?.length
- ? values.categories
- : undefined,
- });
- capture("score_configs:update_form_submit", {
- dataType: values.dataType,
- });
- }}
- onFormSuccess={closeDialog}
- isSubmitting={mutation.isPending}
- />
- </DialogContent>
+ <UpsertScoreConfigDialogContent
+ key={defaultValues.id}
+ mode="edit"
+ defaultValues={defaultValues}
+ onSubmit={async (values) => {
+ await mutation.mutateAsync({
+ ...values,
+ projectId,
+ id: defaultValues.id,
+ description: values.description ?? null,
+ categories: values.categories?.length
+ ? values.categories
+ : undefined,
+ });
+ capture("score_configs:update_form_submit", {
+ dataType: values.dataType,
+ });
+ }}
+ onFormSuccess={closeDialog}
+ isSubmitting={mutation.isPending}
+ />
)}
>
{({ openDialog }) => children({ disabled, openDialog })}
diff --git a/web/src/features/score-configs/index.ts b/web/src/features/score-configs/index.ts
index a050804f9a..bbfe67c86d 100644
--- a/web/src/features/score-configs/index.ts
+++ b/web/src/features/score-configs/index.ts
@@ -1 +1,5 @@
export { ScoreConfigSettings } from "@/src/features/score-configs/components/ScoreConfigSettings";
+export {
+ CreateScoreConfigDialogController,
+ EditScoreConfigDialogController,
+} from "@/src/features/score-configs/components/UpsertScoreConfigDialogController";
diff --git a/web/src/features/scores/README.md b/web/src/features/scores/README.md
index 8cab33d5be..f2c2371a29 100644
--- a/web/src/features/scores/README.md
+++ b/web/src/features/scores/README.md
@@ -9,6 +9,11 @@ trace and observation. Trace/session detail, annotation queues, and dataset
comparison supply their targets and scores. The adapters prepare data and gate
the form on score-config readiness. Target identity keys the mounted form;
refetches of the same target preserve its draft.
+Queue editing waits for the initial queue refresh. A fixed form retains its
+fields and order for that item while using refreshed score config definitions,
+including categories added inline. Removed queue fields keep their initial
+definition until reopen, so queue setup changes cannot discard open drafts.
+Reopening the item applies the latest queue fields and their order.
| Owner | Responsibility |
| -------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
@@ -20,6 +25,7 @@ refetches of the same target preserve its draft.
| `hooks/useScoreMutations.ts` | tRPC mutation bridge and optimistic shared score-cache writes/rollback. Server/query state remains in tRPC. |
| `lib/annotationConfigSelection.ts`, `hooks/useScoreConfigs.ts` | Config query data, explicit selection changes, and remembered empty-config choices for each target type. |
| `components/AnnotationScoreRow.tsx` | Value-control presentation, mount-local score-comment draft, and DOM focus deferral while opening its clear menu. |
+| `components/ScoreConfigDescription.tsx` | Inline descriptions shared by field previews and annotation rows; per-config collapse preference in local storage and overflow measured in the browser. |
| `hooks/useAnnotationKeyboard.ts` | Browser keyboard integration, reading current rows only when a key is pressed. |
Actions resolve a row by target + config both at invocation and async completion.
@@ -35,6 +41,8 @@ default. Draft-only level changes and adding a blank field for the other level
belong to the form actions; neither changes an existing score's attachment.
Saved scores, pending writes, invalid drafts, and occupied destinations prevent a
level change. Single-target and fixed queue forms retain their supplied targets.
+Fixed queue fields follow the supplied score config order, including fields with
+saved scores. Selectable trace/session fields remain alphabetical.
The row menu removes only empty fields and clears their remembered choices so
hidden preferences cannot restore them on the next open. Saved and nonempty
fields keep Clear score; pending clears cannot remove their row before rollback
@@ -62,6 +70,7 @@ explicitly. Autosave and query refreshes never request focus.
| `useAnnotationKeyboard` capture listener | Escape focus handling before the parent drawer's capture listener. | Retained; symmetric registration/cleanup. |
| `useAnnotationKeyboard` activation focus | DOM focus when the ready form becomes active, after the opening menu closes. | Retained; scheduled frame cancelled on inactivity or unmount. |
| Score-comment editor | No external system: draft was mirrored from `savedComment`. | Removed; popover mount seeds the editor. |
+| `ScoreConfigDescription` | ResizeObserver measures whether the description needs disclosure. | Retained; disconnects on description change or unmount. |
No other direct effects exist in this feature. Query, cache, and local-storage
hooks keep their existing external ownership. The shared `ScoreCacheProvider`
diff --git a/web/src/features/scores/components/AnnotationForm.tsx b/web/src/features/scores/components/AnnotationForm.tsx
index b919f03251..024f725a7e 100644
--- a/web/src/features/scores/components/AnnotationForm.tsx
+++ b/web/src/features/scores/components/AnnotationForm.tsx
@@ -95,7 +95,7 @@ const getEmptySelectedConfigIdsStorageKey = (scoreTarget: ScoreTarget) => {
};
export function AnnotationFormContent({
- targets,
+ targets: suppliedTargets,
actionButtons,
isActive = true,
refreshRef,
@@ -106,19 +106,46 @@ export function AnnotationFormContent({
refreshRef?: React.Ref<AnnotationRefreshHandle>;
}) {
const capture = usePostHogClientCapture();
+ const [initialTargets] = useState(() => suppliedTargets);
+ const allowManualSelection = suppliedTargets.some(
+ (target) => target.configControl.allowManualSelection,
+ );
+ const targets = allowManualSelection
+ ? suppliedTargets
+ : initialTargets.map((initialTarget) => {
+ const liveTarget = suppliedTargets.find(
+ (target) => target.key === initialTarget.key,
+ );
+ const liveConfigs = new Map(
+ liveTarget?.configControl.configs.map((config) => [
+ config.id,
+ config,
+ ]),
+ );
+ return {
+ ...initialTarget,
+ configControl: {
+ ...initialTarget.configControl,
+ // Keep this item's fields and order, but accept definition updates
+ // such as a category added while annotating. Removed queue fields
+ // retain their definition until the item is reopened.
+ configs: initialTarget.configControl.configs.map(
+ (config) => liveConfigs.get(config.id) ?? config,
+ ),
+ },
+ };
+ });
const primaryTarget = targets[0]!;
const { scoreMetadata, analyticsData } = primaryTarget;
- const allowManualSelection = targets.some(
- (target) => target.configControl.allowManualSelection,
+ const initialFormData = targets.flatMap((target) =>
+ target.initialFormData.map((field) => ({
+ ...field,
+ targetKey: target.key,
+ })),
);
- const initialFormData = targets
- .flatMap((target) =>
- target.initialFormData.map((field) => ({
- ...field,
- targetKey: target.key,
- })),
- )
- .sort((a, b) => a.name.localeCompare(b.name));
+ if (allowManualSelection) {
+ initialFormData.sort((a, b) => a.name.localeCompare(b.name));
+ }
const form = useForm<AnnotateFormSchemaType>({
resolver: zodResolver(AnnotateFormSchema),
defaultValues: { scoreData: initialFormData },
@@ -169,6 +196,7 @@ export function AnnotationFormContent({
transformToAnnotationScores(scores, target.configControl.configs),
target.configControl.configs,
selected,
+ target.configControl.allowManualSelection ? "name" : "selected",
).map((field) => ({ ...field, targetKey: target.key }));
});
actions.reconcileServerFields(refreshed);
@@ -351,7 +379,7 @@ export function AnnotationFormContent({
{description}
</p>
) : null}
- <div className="grid grid-flow-row gap-2.5">
+ <div className="grid grid-flow-row gap-4">
{fields.map((field, index) => {
const target = targetFor(field);
const config = configFor(field);
@@ -476,6 +504,7 @@ export function usePreparedAnnotationFormTarget<Target extends ScoreTarget>({
annotationScores,
availableConfigs,
selectedConfigIds,
+ configSelection.mode === "fixed" ? "selected" : "name",
);
return {
diff --git a/web/src/features/scores/components/AnnotationScoreRow.tsx b/web/src/features/scores/components/AnnotationScoreRow.tsx
index 15df021a17..efd7073490 100644
--- a/web/src/features/scores/components/AnnotationScoreRow.tsx
+++ b/web/src/features/scores/components/AnnotationScoreRow.tsx
@@ -29,7 +29,6 @@ import {
PopoverContent,
PopoverTrigger,
} from "@/src/components/ui/popover";
-import { ControlledHoverCard } from "@/src/components/design-system/ControlledHoverCard/ControlledHoverCard";
import { Badge } from "@/src/components/ui/badge";
import {
DropdownMenuController,
@@ -37,9 +36,10 @@ import {
DropdownMenuLabel,
DropdownMenuSeparator,
} from "@/src/components/ui/dropdown-menu";
-import { ScoreConfigDetails } from "@/src/features/score-configs/components/ScoreConfigDetails";
+import { ScoreConfigDescription } from "./ScoreConfigDescription";
import { CategoricalScoreInput } from "@/src/features/scores/components/CategoricalScoreInput";
import {
+ formatScoreRange,
isTextDataType,
isNumericDataType,
isScoreUnsaved,
@@ -167,7 +167,6 @@ export function AnnotationScoreRow({
}) {
const [commentOpen, setCommentOpen] = useState(false);
const [commentDraft, setCommentDraft] = useState<string | null>(null);
- const [detailsOpen, setDetailsOpen] = useState(false);
const score = useWatch({ control: form.control, name: `scoreData.${index}` });
const fieldState = useFormState({
control: form.control,
@@ -235,115 +234,97 @@ export function AnnotationScoreRow({
role="group"
aria-label={showTarget ? `${score.name} (${target.label})` : score.name}
className={cn(
- "group grid w-full grid-cols-[minmax(0,1fr)_minmax(0,2fr)] items-center gap-3 rounded-md px-3 py-1 text-left transition-colors outline-none",
+ "group relative grid w-full grid-cols-1 items-start gap-2 rounded-md px-3 py-2 text-left transition-colors outline-none",
"focus-within:ring-primary/30 focus-within:bg-accent/40 focus-within:ring-1 focus-within:ring-inset",
)}
>
- <div className="flex h-full min-w-0 flex-col items-start justify-center gap-1">
+ <div className="flex min-w-0 flex-col items-start gap-1 pr-8">
<div className="flex max-w-full min-w-0 items-center gap-1">
- {config.description ||
- isPresent(config.maxValue) ||
- isPresent(config.minValue) ? (
- <ControlledHoverCard
- open={isActive && detailsOpen}
- onOpenChange={setDetailsOpen}
- content={
- <div className="max-h-[60vh] w-64 max-w-64 overflow-y-auto p-3">
- <ScoreConfigDetails config={config} />
- </div>
- }
- >
- {({ getTriggerProps }) => (
- <span
- tabIndex={0}
- {...getTriggerProps()}
- className={cn(
- "decoration-muted-gray line-clamp-2 min-w-0 text-xs font-bold wrap-break-word underline decoration-dashed underline-offset-2",
- config.isArchived ? "text-foreground/40" : "",
- )}
- >
- {score.name}
- </span>
- )}
- </ControlledHoverCard>
- ) : (
- <span
- className={cn(
- "line-clamp-2 min-w-0 text-xs font-bold wrap-break-word",
- config.isArchived ? "text-foreground/40" : "",
- )}
- title={score.name}
- >
- {score.name}
- </span>
- )}
- <Popover
- open={isActive && commentOpen}
- onOpenChange={setCommentOpen}
+ <span
+ className={cn(
+ "line-clamp-2 min-w-0 text-xs font-bold wrap-break-word",
+ config.isArchived ? "text-foreground/40" : "",
+ )}
+ title={score.name}
>
- <PopoverTrigger asChild>
- <Button
- variant="link"
- type="button"
- size="xs"
- title="Add or view score comment"
- // Center the comment icon vertically
- // against the score label instead of stretching
- // to the full (possibly multi-line) row height,
- // and keep it hugging the label (shrink-0) rather
- // than floating in the middle of the row.
- className="disabled:text-primary/50 flex h-auto shrink-0 items-center self-center px-0 pl-1 disabled:opacity-100"
- disabled={
- isScoreUnsaved(score.id) ||
- (config.isArchived && !score.comment)
- }
+ {score.name}
+ </span>
+ {!isScoreUnsaved(score.id) &&
+ (!config.isArchived || score.comment) && (
+ <Popover
+ open={isActive && commentOpen}
+ onOpenChange={setCommentOpen}
>
- {score.comment ? (
- <MessageCircleMore className="icon-sm" />
- ) : (
- <MessageCircle className="icon-sm" />
- )}
- </Button>
- </PopoverTrigger>
- <PopoverContent
- onCloseAutoFocus={(event) => {
- if (!isActive) event.preventDefault();
- }}
- >
- <FormField
- control={form.control}
- name={`scoreData.${index}.comment`}
- render={() => (
- <FormItem className="space-y-4">
- <FormControl>
- <CommentField
- savedComment={score.comment ?? null}
- localValue={commentDraft ?? score.comment ?? ""}
- onValueChange={setCommentDraft}
- disabled={config.isArchived}
- loading={commentSaving}
- onSave={(newComment) => {
- const trimmed = newComment?.trim();
- actions.saveComment(
- fieldKey,
- target,
- trimmed || null,
- );
- }}
- />
- </FormControl>
- <FormMessage className="text-xs" />
- </FormItem>
- )}
- />
- </PopoverContent>
- </Popover>
+ <PopoverTrigger asChild>
+ <Button
+ variant="link"
+ type="button"
+ size="xs"
+ title="Add or view score comment"
+ // Center the comment icon vertically
+ // against the score label instead of stretching
+ // to the full (possibly multi-line) row height,
+ // and keep it hugging the label (shrink-0) rather
+ // than floating in the middle of the row.
+ className="flex h-auto shrink-0 items-center self-center px-0 pl-1"
+ >
+ {score.comment ? (
+ <MessageCircleMore className="icon-sm" />
+ ) : (
+ <MessageCircle className="icon-sm" />
+ )}
+ </Button>
+ </PopoverTrigger>
+ <PopoverContent
+ onCloseAutoFocus={(event) => {
+ if (!isActive) event.preventDefault();
+ }}
+ >
+ <FormField
+ control={form.control}
+ name={`scoreData.${index}.comment`}
+ render={() => (
+ <FormItem className="space-y-4">
+ <FormControl>
+ <CommentField
+ savedComment={score.comment ?? null}
+ localValue={commentDraft ?? score.comment ?? ""}
+ onValueChange={setCommentDraft}
+ disabled={config.isArchived}
+ loading={commentSaving}
+ onSave={(newComment) => {
+ const trimmed = newComment?.trim();
+ actions.saveComment(
+ fieldKey,
+ target,
+ trimmed || null,
+ );
+ }}
+ />
+ </FormControl>
+ <FormMessage className="text-xs" />
+ </FormItem>
+ )}
+ />
+ </PopoverContent>
+ </Popover>
+ )}
</div>
{showTarget ? (
<Badge variant="outline-solid">{target.label}</Badge>
) : null}
</div>
- <div className="grid grid-cols-[minmax(0,1fr)_1.5rem] items-center gap-1 py-1">
+ {config.description?.trim() && (
+ <div className="min-w-0">
+ <ScoreConfigDescription
+ key={config.id}
+ configId={config.id}
+ name={config.name}
+ description={config.description}
+ />
+ </div>
+ )}
+ <div className="min-w-0">
{/* data-score-control wraps only the value control so
keyboard 1-9 scoring targets it, not the in-row
comment or score actions. */}
@@ -396,6 +377,10 @@ export function AnnotationScoreRow({
)
}
type="number"
+ placeholder={formatScoreRange(
+ config.minValue,
+ config.maxValue,
+ )}
// Mirror the config range as native
// constraints so out-of-range values are
// catchable via the ⌘/Ctrl+Enter complete
@@ -579,6 +564,7 @@ export function AnnotationScoreRow({
type="button"
size="icon-xs"
data-score-actions
+ className="absolute top-2 right-2"
onBlur={(event) => {
if (event.currentTarget.dataset.state !== "open")
commitDeferredScore();
diff --git a/web/src/features/scores/components/DualAnnotationContent.clienttest.tsx b/web/src/features/scores/components/DualAnnotationContent.clienttest.tsx
index b0caa23d2e..87ff953767 100644
--- a/web/src/features/scores/components/DualAnnotationContent.clienttest.tsx
+++ b/web/src/features/scores/components/DualAnnotationContent.clienttest.tsx
@@ -802,6 +802,108 @@ describe("unified annotation targets", () => {
);
});
+ it.each([
+ ["fixed", "trace"],
+ ["fixed", "session"],
+ ["selectable", "trace"],
+ ["selectable", "session"],
+ ] as const)(
+ "keeps %s %s fields in their configured order across refresh without writing scores",
+ (mode, targetType) => {
+ const accuracy = {
+ ...defaultConfig,
+ id: "accuracy",
+ name: "Accuracy",
+ };
+ configs.push(accuracy);
+ localStorage.setItem(
+ `emptySelectedConfigIds:${targetType}`,
+ JSON.stringify(["quality", "accuracy"]),
+ );
+ const score = {
+ ...serverQualityScore(),
+ id: "saved-accuracy",
+ configId: "accuracy",
+ name: "Accuracy",
+ observationId: null,
+ traceId: targetType === "trace" ? "trace" : null,
+ sessionId: targetType === "session" ? "session" : null,
+ metadata: "{}",
+ };
+ const refreshRef = createRef<AnnotationRefreshHandle>();
+ const scoreTarget =
+ targetType === "trace"
+ ? { type: "trace" as const, traceId: "trace" }
+ : { type: "session" as const, sessionId: "session" };
+ const sourceByTargetType = {
+ trace: "TraceDetail",
+ session: "SessionDetail",
+ } as const;
+ const analyticsData = {
+ type: targetType,
+ source:
+ mode === "fixed"
+ ? ("AnnotationQueue" as const)
+ : sourceByTargetType[targetType],
+ isV4: true,
+ };
+ const contentFor = (orderedConfigs: ScoreConfigDomain[]) => (
+ <AnnotationForm
+ refreshRef={refreshRef}
+ scoreTarget={scoreTarget}
+ serverScores={[score]}
+ scoreMetadata={{ projectId: "project" }}
+ analyticsData={analyticsData}
+ configSelection={
+ mode === "fixed" ? { mode, configs: orderedConfigs } : { mode }
+ }
+ />
+ );
+ const rendered = renderContent(contentFor([defaultConfig, accuracy]));
+ const fieldOrder = () =>
+ screen
+ .getAllByRole("group")
+ .map((row) => row.getAttribute("aria-label"));
+ const expectedOrder =
+ mode === "fixed" ? ["Quality", "Accuracy"] : ["Accuracy", "Quality"];
+ expect(fieldOrder()).toEqual(expectedOrder);
+ act(() =>
+ refreshRef.current?.refresh({
+ scoreTarget,
+ scoreMetadata: { projectId: "project" },
+ analyticsData,
+ scores: [{ ...score, value: 1, stringValue: "True" }],
+ }),
+ );
+ expect(fieldOrder()).toEqual(expectedOrder);
+ expect(
+ within(screen.getByRole("group", { name: "Accuracy" })).getByRole(
+ "tab",
+ { name: /True/ },
+ ),
+ ).toHaveAttribute("aria-selected", "true");
+ if (mode === "fixed") {
+ expect(
+ screen.queryByRole("button", { name: "Add score" }),
+ ).not.toBeInTheDocument();
+ rendered.rerenderContent(contentFor([accuracy, defaultConfig]));
+ expect(fieldOrder()).toEqual(expectedOrder);
+ rendered.unmount();
+ renderContent(contentFor([accuracy, defaultConfig]));
+ expect(fieldOrder()).toEqual(["Accuracy", "Quality"]);
+ expect(
+ within(screen.getByRole("group", { name: "Accuracy" })).getByRole(
+ "tab",
+ { name: /False/ },
+ ),
+ ).toHaveAttribute("aria-selected", "true");
+ }
+ expect(mocks.create).not.toHaveBeenCalled();
+ expect(mocks.update).not.toHaveBeenCalled();
+ expect(mocks.remove).not.toHaveBeenCalled();
+ },
+ );
+
it("keeps draft edits local, preserves them through refetch, and resets only for another target", () => {
const text = {
...defaultConfig,
@@ -857,6 +959,206 @@ describe("unified annotation targets", () => {
expect(mocks.update).not.toHaveBeenCalled();
});
+ it("remembers queue description visibility per score config across reopened items", async () => {
+ const configs = [
+ {
+ ...defaultConfig,
+ id: "description-a",
+ name: "Quality",
+ description: "Judge answer quality.\nConsider relevance and clarity.",
+ },
+ {
+ ...defaultConfig,
+ id: "description-b",
+ name: "Accuracy",
+ description: "Check factual accuracy.",
+ },
+ ];
+ const contentFor = (sessionId: string) => (
+ <AnnotationForm
+ scoreTarget={{ type: "session", sessionId }}
+ serverScores={[]}
+ scoreMetadata={{ projectId: "project", queueId: "queue" }}
+ analyticsData={{
+ type: "session",
+ source: "AnnotationQueue",
+ isV4: true,
+ }}
+ configSelection={{ mode: "fixed", configs }}
+ />
+ );
+ const view = renderContent(contentFor("first-item"));
+ expect(
+ screen.getByText(
+ /Judge answer quality\.\s+Consider relevance and clarity\./,
+ ),
+ ).toBeVisible();
+ fireEvent.click(
+ screen.getByRole("button", { name: "Hide description for Quality" }),
+ );
+ expect(
+ screen.getByRole("button", { name: "Show description for Quality" }),
+ ).toHaveAttribute("aria-expanded", "false");
+ expect(screen.getByText("Check factual accuracy.")).toBeVisible();
+ await waitFor(() =>
+ expect(
+ localStorage.getItem(
+ "annotation-score-config-description-description-a",
+ ),
+ ).toBe("false"),
+ );
+ view.rerenderContent(contentFor("next-item"));
+ expect(
+ screen.getByRole("button", { name: "Show description for Quality" }),
+ ).toHaveAttribute("aria-expanded", "false");
+ view.unmount();
+ renderContent(contentFor("reopened-item"));
+ expect(
+ screen.getByRole("button", { name: "Show description for Quality" }),
+ ).toHaveAttribute("aria-expanded", "false");
+ expect(screen.getByText("Check factual accuracy.")).toBeVisible();
+ fireEvent.click(
+ screen.getByRole("button", { name: "Show description for Quality" }),
+ );
+ expect(
+ screen.getByText(
+ /Judge answer quality\.\s+Consider relevance and clarity\./,
+ ),
+ ).toBeVisible();
+ expect(mocks.create).not.toHaveBeenCalled();
+ expect(mocks.update).not.toHaveBeenCalled();
+ });
+
+ it("uses refreshed category definitions without changing an open queue's fields or order", async () => {
+ mocks.create.mockResolvedValue({});
+ const category: ScoreConfigDomain = {
+ ...defaultConfig,
+ dataType: "CATEGORICAL",
+ categories: [{ label: "Old", value: 1 }],
+ };
+ const feedback: ScoreConfigDomain = {
+ ...defaultConfig,
+ id: "feedback",
+ name: "Feedback",
+ dataType: "TEXT",
+ categories: null,
+ };
+ const contentFor = (orderedConfigs: ScoreConfigDomain[]) => (
+ <AnnotationForm
+ scoreTarget={{ type: "session", sessionId: "session" }}
+ serverScores={[]}
+ scoreMetadata={{ projectId: "project", queueId: "queue" }}
+ analyticsData={{
+ type: "session",
+ source: "AnnotationQueue",
+ isV4: true,
+ }}
+ configSelection={{ mode: "fixed", configs: orderedConfigs }}
+ />
+ );
+ const view = renderContent(contentFor([feedback, category]));
+ const text = within(
+ screen.getByRole("group", { name: "Feedback" }),
+ ).getByRole("textbox");
+ fireEvent.change(text, { target: { value: "Keep my draft" } });
+ view.rerenderContent(
+ contentFor([
+ {
+ ...category,
+ description: "Updated guidance",
+ categories: [...category.categories!, { label: "New", value: 7 }],
+ },
+ { ...defaultConfig, id: "extra", name: "Extra" },
+ feedback,
+ ]),
+ );
+ expect(screen.getByText("Updated guidance")).toBeVisible();
+ expect(text).toHaveValue("Keep my draft");
+ expect(
+ screen.getAllByRole("group").map((row) => row.getAttribute("aria-label")),
+ ).toEqual(["Feedback", "Quality"]);
+ const newCategory = within(
+ screen.getByRole("group", { name: "Quality" }),
+ ).getByRole("tab", { name: /New/ });
+ expect(newCategory).toBeEnabled();
+ fireEvent.mouseDown(newCategory);
+ await waitFor(() =>
+ expect(mocks.create).toHaveBeenCalledWith(
+ expect.objectContaining({
+ configId: "quality",
+ value: 7,
+ stringValue: "New",
+ }),
+ ),
+ );
+ });
+
+ it("preserves fixed queue text and comment drafts through config refresh and applies fields on reopen", async () => {
+ const feedback = {
+ ...defaultConfig,
+ id: "feedback",
+ name: "Feedback",
+ dataType: "TEXT" as const,
+ categories: null,
+ };
+ const accuracy = { ...defaultConfig, id: "accuracy", name: "Accuracy" };
+ const score = {
+ ...serverQualityScore(),
+ metadata: "{}",
+ traceId: null,
+ observationId: null,
+ sessionId: "session",
+ };
+ const contentFor = (orderedConfigs: ScoreConfigDomain[]) => (
+ <AnnotationForm
+ scoreTarget={{ type: "session", sessionId: "session" }}
+ serverScores={[score]}
+ scoreMetadata={{ projectId: "project", queueId: "queue" }}
+ analyticsData={{
+ type: "session",
+ source: "AnnotationQueue",
+ isV4: true,
+ }}
+ configSelection={{ mode: "fixed", configs: orderedConfigs }}
+ />
+ );
+ const view = renderContent(contentFor([feedback, defaultConfig]));
+ const text = within(
+ screen.getByRole("group", { name: "Feedback" }),
+ ).getByRole("textbox");
+ fireEvent.change(text, { target: { value: "Unsent feedback" } });
+ fireEvent.click(
+ within(screen.getByRole("group", { name: "Quality" })).getByTitle(
+ "Add or view score comment",
+ ),
+ );
+ const comment = await within(screen.getByRole("dialog")).findByRole(
+ "textbox",
+ );
+ fireEvent.change(comment, { target: { value: "Unsent score comment" } });
+ view.rerenderContent(contentFor([defaultConfig, accuracy]));
+ expect(text).toBeInTheDocument();
+ expect(text).toHaveValue("Unsent feedback");
+ expect(comment).toHaveValue("Unsent score comment");
+ expect(
+ screen.queryByRole("group", { name: "Accuracy" }),
+ ).not.toBeInTheDocument();
+ expect(
+ screen.getAllByRole("group").map((row) => row.getAttribute("aria-label")),
+ ).toEqual(["Feedback", "Quality"]);
+ expect(mocks.create).not.toHaveBeenCalled();
+ expect(mocks.update).not.toHaveBeenCalled();
+ expect(mocks.remove).not.toHaveBeenCalled();
+ view.unmount();
+ renderContent(contentFor([defaultConfig, accuracy]));
+ expect(
+ screen.queryByRole("group", { name: "Feedback" }),
+ ).not.toBeInTheDocument();
+ expect(
+ screen.getAllByRole("group").map((row) => row.getAttribute("aria-label")),
+ ).toEqual(["Quality", "Accuracy"]);
+ });
+
it("navigates one form and saves the same config independently for observation and trace", async () => {
mocks.create.mockResolvedValue({});
await renderBothTargets();
diff --git a/web/src/features/scores/components/ScoreConfigDescription.tsx b/web/src/features/scores/components/ScoreConfigDescription.tsx
new file mode 100644
index 0000000000..371501f3ca
--- /dev/null
+++ b/web/src/features/scores/components/ScoreConfigDescription.tsx
@@ -0,0 +1,72 @@
+import { useEffect, useRef, useState } from "react";
+import { ChevronDown, ChevronRight } from "lucide-react";
+import { IconButton } from "@/src/components/design-system/IconButton/IconButton";
+import {
+ Collapsible,
+ CollapsibleContent,
+ CollapsibleTrigger,
+} from "@/src/components/ui/collapsible";
+import useLocalStorage from "@/src/components/useLocalStorage";
+import { cn } from "@/src/utils/tailwind";
+
+export function ScoreConfigDescription({
+ configId,
+ name,
+ description,
+}: {
+ configId: string;
+ name: string;
+ description: string;
+}) {
+ const [expanded, setExpanded] = useLocalStorage(
+ `annotation-score-config-description-${configId}`,
+ true,
+ );
+ const textRef = useRef<HTMLParagraphElement>(null);
+ const [overflows, setOverflows] = useState(false);
+ const canExpand = overflows || description.trim().includes("\n");
+
+ useEffect(() => {
+ const text = textRef.current;
+ if (!text) return;
+ // Clamping preserves scrollHeight, so disclosure follows the rendered
+ // line count even when the panel is resized while collapsed.
+ const measure = () => {
+ const lineHeight = Number.parseFloat(getComputedStyle(text).lineHeight);
+ setOverflows(text.scrollHeight > lineHeight + 1);
+ };
+ const observer = new ResizeObserver(measure);
+ observer.observe(text);
+ measure();
+ return () => observer.disconnect();
+ }, [description]);
+
+ return (
+ <Collapsible open={canExpand && expanded} onOpenChange={setExpanded}>
+ <div className="group/description grid grid-cols-[minmax(0,1fr)_1rem] items-start gap-1.5">
+ <CollapsibleContent forceMount className="min-w-0">
+ <p
+ ref={textRef}
+ className={cn(
+ "text-foreground text-xs leading-4 wrap-break-word whitespace-pre-wrap",
+ !(canExpand && expanded) && "line-clamp-1",
+ )}
+ >
+ {description.trim()}
+ </p>
+ </CollapsibleContent>
+ {canExpand && (
+ <span className="opacity-100 group-focus-within/description:opacity-100 group-hover/description:opacity-100 [@media(hover:hover)]:opacity-0">
+ <CollapsibleTrigger asChild>
+ <IconButton
+ icon={expanded ? ChevronDown : ChevronRight}
+ label={`${expanded ? "Hide" : "Show"} description for ${name}`}
+ size="xs"
+ />
+ </CollapsibleTrigger>
+ </span>
+ )}
+ </div>
+ </Collapsible>
+ );
+}
diff --git a/web/src/features/scores/index.ts b/web/src/features/scores/index.ts
index 69642b6f6a..566ad248b3 100644
--- a/web/src/features/scores/index.ts
+++ b/web/src/features/scores/index.ts
@@ -2,9 +2,6 @@
// only — the list is exactly what other features already imported, nothing
// added for the future.
//
-// score-configs/ScoreConfigDetails stays on the deep path: routing it through
-// this index would close a runtime cycle (scores -> score-configs -> scores).
-//
// Dual-use modules (aggregateScores, scoreColumns, helpers, scoresSearchRegistry)
// stay a deep import for server callers. This door already carries React UI
// (AnnotationForm, ScoresTable), and routing those files through here would
@@ -40,6 +37,7 @@ export {
type ScoreToAggregate,
} from "@/src/features/scores/lib/aggregateScores";
export {
+ formatScoreRange,
isBooleanDataType,
isCategoricalDataType,
isNumericDataType,
diff --git a/web/src/features/scores/lib/helpers.ts b/web/src/features/scores/lib/helpers.ts
index 3f04b8ecb1..6fd2f9acb7 100644
--- a/web/src/features/scores/lib/helpers.ts
+++ b/web/src/features/scores/lib/helpers.ts
@@ -41,3 +41,13 @@ export const formatAnnotateDescription = <Target extends ScoreTarget>(
}
return `Annotate ${sourceEntity} with scores to capture human evaluation across different dimensions.`;
};
+
+export function formatScoreRange(
+ min: number | null | undefined,
+ max: number | null | undefined,
+) {
+ if (min != null && max != null) return `${min} – ${max}`;
+ if (min != null) return `Min ${min}`;
+ if (max != null) return `Max ${max}`;
+ return "Enter a number";
+}
diff --git a/web/src/features/scores/lib/prepareAnnotationFormData.ts b/web/src/features/scores/lib/prepareAnnotationFormData.ts
index 094142d22e..2968b94c79 100644
--- a/web/src/features/scores/lib/prepareAnnotationFormData.ts
+++ b/web/src/features/scores/lib/prepareAnnotationFormData.ts
@@ -42,6 +42,7 @@ export function prepareAnnotationFormData(
scores: AnnotationScore[],
configs: ScoreConfigDomain[],
selectedConfigIds: string[],
+ order: "name" | "selected" = "name",
): AnnotationScoreFormData[] {
const fields = scores.map(
({
@@ -80,5 +81,15 @@ export function prepareAnnotationFormData(
timestamp: null,
});
}
+ if (order === "selected") {
+ const configOrder = new Map(
+ selectedConfigIds.map((configId, index) => [configId, index]),
+ );
+ return fields.sort(
+ (left, right) =>
+ (configOrder.get(left.configId) ?? selectedConfigIds.length) -
+ (configOrder.get(right.configId) ?? selectedConfigIds.length),
+ );
+ }
return fields.sort((left, right) => left.name.localeCompare(right.name));
}