Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(annotations): unify score editing and review state (#17740)

Nikita Kabardinea43e543b4b5
diff --git a/web/src/features/annotation-queues/README.md b/web/src/features/annotation-queues/README.md
new file mode 100644
index 0000000000..28a98483a2
--- /dev/null
+++ b/web/src/features/annotation-queues/README.md
@@ -0,0 +1,49 @@
+# Annotation queues
+
+## Surface and entry points
+
+Queue routes under `pages/project/[projectId]/annotation-queues` mount
+`components/AnnotationQueuesItem.tsx`. It creates a fresh processing run for
+each project and queue; changing the current item keeps the same run history.
+
+## State ownership
+
+- `state/annotationQueueRun.ts` owns one run's visited item locations and progress.
+ Its named start, next, back, and complete actions serialize transitions and
+ commit navigation and history together. They can run independently of React.
+ The current editor stays mounted while transitions are pending or fail;
+ navigation controls stay disabled until the action finishes. Failed transitions
+ expose a retry action. Confirmed completion IDs survive a failed refresh or
+ advance, and an acquired next-item lock survives failed navigation, so retries
+ neither complete twice nor acquire a different item.
+- `AnnotationQueueItemPage.tsx` adapts tRPC and routing to those actions. A query
+ gates initial loading, shares the initial lock during Strict Mode replay, and
+ owns cancellation. Actions check that this query still has an active observer
+ before navigating, so late responses cannot leave the user's new queue.
+- tRPC owns queue/item/object server data. Mutation results populate the item
+ cache; the run stores only IDs, initial observation locations, and transition outcomes. Initial
+ direct links preserve an explicit observation and single-item mode. Refetches
+ do not reset the reviewer's chosen span or trace.
+- `processors/TraceAnnotationProcessor.tsx` and `SessionAnnotationProcessor.tsx`
+ prepare their object views. They do not initialize the route's selected span.
+- Score forms own their field state; the comments feature owns discussion drafts.
+ Shared processing layout components own resize preferences.
+
+## Structure and consumers
+
+`components/` contains queue forms, tables, and processing views. `components/shared`
+contains object/query adapters and layout primitives; `server/` contains queue and
+assignment procedures. Trace/session headers and bulk tables reuse queue-add
+controllers. Dataset review reuses the shared comments section.
+
+## Stability and migration
+
+Item query updates may rerender processors, but navigation belongs only to run
+actions. The page's remaining effects are keyboard listeners and shortcut timer
+cleanup. Queue creation and assignment retry coordination belongs to its dialog
+controller: after creation succeeds, retries update that queue with the latest
+form values before assigning users. The form keeps its draft across retries and
+excludes its own queue from duplicate-name validation.
+
+Read `frontend-large-feature-architecture` and `refactor-react-effects` before
+extending processing lifecycle or query-to-form behavior.
diff --git a/web/src/features/annotation-queues/components/AnnotationQueueDetails.stories.tsx b/web/src/features/annotation-queues/components/AnnotationQueueDetails.stories.tsx
new file mode 100644
index 0000000000..d6d22c0274
--- /dev/null
+++ b/web/src/features/annotation-queues/components/AnnotationQueueDetails.stories.tsx
@@ -0,0 +1,81 @@
+import { type ComponentProps } from "react";
+import preview from "../../../../.storybook/preview";
+import { AnnotationQueueDetails } from "./AnnotationQueueDetails";
+
+const scoreConfigs = [
+ {
+ id: "accuracy",
+ name: "Accuracy",
+ dataType: "CATEGORICAL",
+ description: "How closely does the answer match the expected result?",
+ categories: [
+ { label: "Correct", value: 1 },
+ { label: "Incorrect", value: 0 },
+ ],
+ isArchived: false,
+ },
+ {
+ id: "confidence",
+ name: "Confidence",
+ dataType: "NUMERIC",
+ minValue: 0,
+ maxValue: 1,
+ isArchived: false,
+ },
+ {
+ id: "feedback",
+ name: "Feedback",
+ dataType: "TEXT",
+ description: "Explain anything the reviewer should follow up on.",
+ isArchived: false,
+ },
+ {
+ id: "toxicity",
+ name: "Toxicity",
+ dataType: "BOOLEAN",
+ isArchived: false,
+ },
+ {
+ id: "legacy",
+ name: "Legacy rating",
+ dataType: "NUMERIC",
+ minValue: 1,
+ maxValue: 5,
+ isArchived: true,
+ },
+] satisfies ComponentProps<typeof AnnotationQueueDetails>["scoreConfigs"];
+
+const meta = preview.meta({
+ component: AnnotationQueueDetails,
+ render: (args) => (
+ <div className="w-80">
+ <AnnotationQueueDetails {...args} />
+ </div>
+ ),
+});
+
+export const Default = meta.story({
+ args: {
+ description: "Review answer quality and leave feedback for the team.",
+ scoreConfigs,
+ },
+});
+
+export const Empty = meta.story({
+ args: { description: null, scoreConfigs: [] },
+});
+
+export const WithLongText = meta.story({
+ args: {
+ description:
+ "Review answers that require additional context.\nInclude the reason for any score that needs follow-up.",
+ scoreConfigs: [
+ {
+ ...scoreConfigs[0],
+ name: "Answer quality for complex requests",
+ description:
+ "Consider whether the answer is complete, supported by the provided context, and clear enough for someone unfamiliar with the conversation.",
+ },
+ ],
+ },
+});
diff --git a/web/src/features/annotation-queues/components/AnnotationQueueDetails.tsx b/web/src/features/annotation-queues/components/AnnotationQueueDetails.tsx
new file mode 100644
index 0000000000..51b6e37287
--- /dev/null
+++ b/web/src/features/annotation-queues/components/AnnotationQueueDetails.tsx
@@ -0,0 +1,89 @@
+import { Badge } from "@/src/components/design-system/Badge/Badge";
+import { type ScoreConfigDomain } from "@langfuse/shared";
+
+type ScoreField = Pick<
+ ScoreConfigDomain,
+ | "id"
+ | "name"
+ | "description"
+ | "dataType"
+ | "minValue"
+ | "maxValue"
+ | "categories"
+ | "isArchived"
+>;
+
+function describeScoreField(config: ScoreField): string {
+ switch (config.dataType) {
+ case "NUMERIC":
+ if (config.minValue != null && config.maxValue != null) {
+ return `Numeric · ${config.minValue} to ${config.maxValue}`;
+ }
+ if (config.minValue != null) return `Numeric · ≥ ${config.minValue}`;
+ if (config.maxValue != null) return `Numeric · ≤ ${config.maxValue}`;
+ return "Numeric";
+ case "CATEGORICAL": {
+ const count = config.categories?.length ?? 0;
+ return `Categorical · ${count} option${count === 1 ? "" : "s"}`;
+ }
+ case "BOOLEAN":
+ return "Boolean · True / False";
+ case "TEXT":
+ return "Text";
+ }
+}
+
+export function AnnotationQueueDetails({
+ description,
+ scoreConfigs,
+}: {
+ description: string | null;
+ scoreConfigs: ScoreField[];
+}) {
+ return (
+ <div className="flex w-full min-w-0 flex-col gap-6 p-4">
+ {description && (
+ <section className="flex flex-col gap-2">
+ <h3 className="text-sm font-bold">Description</h3>
+ <p className="text-muted-foreground text-sm leading-relaxed wrap-anywhere whitespace-pre-wrap">
+ {description}
+ </p>
+ </section>
+ )}
+ <section className="flex min-w-0 flex-col gap-2">
+ <div className="flex items-center gap-2">
+ <h3 className="text-sm font-bold">Score fields</h3>
+ <Badge text={String(scoreConfigs.length)} size="sm" />
+ </div>
+ {scoreConfigs.length === 0 ? (
+ <p className="text-muted-foreground text-sm">
+ No score fields configured.
+ </p>
+ ) : (
+ <ul className="divide-y" aria-label="Score fields">
+ {[...scoreConfigs]
+ .sort((a, b) => a.name.localeCompare(b.name))
+ .map((config) => (
+ <li key={config.id} className="flex flex-col gap-1 py-3">
+ <div className="flex flex-wrap items-center gap-2">
+ <span className="min-w-0 text-sm font-bold wrap-anywhere">
+ {config.name}
+ </span>
+ {config.isArchived && <Badge text="Archived" size="sm" />}
+ </div>
+ <p className="text-muted-foreground text-xs">
+ {describeScoreField(config)}
+ </p>
+ {config.description && (
+ <p className="text-muted-foreground text-xs leading-relaxed wrap-anywhere whitespace-pre-wrap">
+ {config.description}
+ </p>
+ )}
+ </li>
+ ))}
+ </ul>
+ )}
+ </section>
+ </div>
+ );
+}
diff --git a/web/src/features/annotation-queues/components/AnnotationQueueFormDialogContent.stories.tsx b/web/src/features/annotation-queues/components/AnnotationQueueFormDialogContent.stories.tsx
index 13ec0c66f8..8bfee41186 100644
--- a/web/src/features/annotation-queues/components/AnnotationQueueFormDialogContent.stories.tsx
+++ b/web/src/features/annotation-queues/components/AnnotationQueueFormDialogContent.stories.tsx
@@ -66,40 +66,59 @@ export const Create = meta.story({
const body = within(canvasElement.ownerDocument.body);
await userEvent.click(
- await body.findByRole("button", { name: /Select.*Helpfulness/ }),
+ await body.findByRole("combobox", { name: "Score configs" }),
);
await waitFor(() =>
- expect(body.getByPlaceholderText("Value")).toBeVisible(),
+ expect(
+ body.getByPlaceholderText("Search score configs..."),
+ ).toBeVisible(),
);
},
});
export const Edit = meta.story({
- render: () => {
- return (
- <Dialog open onOpenChange={fn()}>
- <DialogContent className="max-h-[90vh] overflow-y-auto">
- <AnnotationQueueFormDialogContent
- mode="edit"
- initialValues={{
- name: "Support review queue",
- description: "Weekly support trace review",
- scoreConfigIds: ["config-1", "config-2"],
- newAssignmentUserIds: [],
- }}
- scoreConfigs={scoreConfigs}
- projectId="project-1"
- queueId="queue-1"
- queueNames={["Support review queue", "Other queue"]}
- onManageScoreConfigsClick={fn()}
- hasQueueAssignmentsReadAccess={false}
- isSubmitting={false}
- onSubmit={fn()}
- submitLabel="Save queue"
- />
- </DialogContent>
- </Dialog>
+ name: "(Test) Edit with archived config",
+ args: {
+ mode: "edit",
+ initialValues: {
+ name: "Support review queue",
+ description: "Weekly support trace review",
+ scoreConfigIds: ["config-1", "config-2", "config-3"],
+ newAssignmentUserIds: [],
+ },
+ scoreConfigs,
+ projectId: "project-1",
+ queueId: "queue-1",
+ queueNames: ["Support review queue", "Other queue"],
+ onManageScoreConfigsClick: fn(),
+ hasQueueAssignmentsReadAccess: false,
+ isSubmitting: false,
+ onSubmit: fn(),
+ submitLabel: "Save queue",
+ },
+ render: (args) => (
+ <Dialog open onOpenChange={fn()}>
+ <DialogContent className="max-h-[90vh] overflow-y-auto">
+ <AnnotationQueueFormDialogContent {...args} />
+ </DialogContent>
+ </Dialog>
+ ),
+ play: async ({ canvasElement, args }) => {
+ const body = within(canvasElement.ownerDocument.body);
+ const picker = await body.findByRole("combobox", { name: "Score configs" });
+ await expect(picker).toHaveTextContent("Legacy quality");
+ await userEvent.click(picker);
+ await userEvent.click(
+ body.getByRole("option", { name: /Legacy quality/, hidden: true }),
+ );
+ await userEvent.keyboard("{Escape}");
+ await userEvent.click(body.getByRole("button", { name: "Save queue" }));
+ await waitFor(() =>
+ expect(args.onSubmit).toHaveBeenCalledWith(
+ expect.objectContaining({ scoreConfigIds: ["config-1", "config-2"] }),
+ expect.anything(),
+ ),
);
},
});
diff --git a/web/src/features/annotation-queues/components/AnnotationQueueFormDialogContent.tsx b/web/src/features/annotation-queues/components/AnnotationQueueFormDialogContent.tsx
index e0101b1d20..9b8e9236de 100644
--- a/web/src/features/annotation-queues/components/AnnotationQueueFormDialogContent.tsx
+++ b/web/src/features/annotation-queues/components/AnnotationQueueFormDialogContent.tsx
@@ -22,12 +22,10 @@ import {
CollapsibleContent,
CollapsibleTrigger,
} from "@/src/components/ui/collapsible";
-import { ChevronDown, ChevronRight } from "lucide-react";
-import {
- getScoreDataTypeIcon,
- MultiSelectKeyValues,
-} from "@/src/features/scores";
-import { DropdownMenuItemWithSecondaryAction } from "@/src/components/ui/dropdown-menu";
+import { ChevronDown, ChevronRight, Settings2 } from "lucide-react";
+import Link from "next/link";
+import { getScoreDataTypeIcon } from "@/src/features/scores";
+import { MultiSelectTagInput } from "@/src/components/design-system/MultiSelectTagInput/MultiSelectTagInput";
import {
CreateQueueWithAssignmentsData,
type CreateQueueWithAssignments,
@@ -89,11 +87,8 @@ export function AnnotationQueueFormDialogContent({
whitelistedName: mode === "edit" ? initialValues.name : undefined,
});
- const handleScoreConfigValueChange = (values: Record<string, string>[]) => {
- form.setValue(
- "scoreConfigIds",
- values.map((value) => value.key),
- );
+ const handleScoreConfigValueChange = (values: string[]) => {
+ form.setValue("scoreConfigIds", values);
if (values.length === 0) {
form.setError("scoreConfigIds", {
@@ -105,10 +100,6 @@ export function AnnotationQueueFormDialogContent({
}
};
- const activeScoreConfigs = scoreConfigs.filter(
- (config) => !config.isArchived,
- );
-
return (
<>
<DialogHeader>
@@ -163,42 +154,53 @@ export function AnnotationQueueFormDialogContent({
name="scoreConfigIds"
render={({ field }) => (
<FormItem>
- <FormLabel>Score Configs</FormLabel>
+ <div className="flex flex-wrap items-center justify-between gap-2">
+ <FormLabel>Score configs</FormLabel>
+ <Button
+ variant="outline"
+ size="sm"
+ className="bg-accent gap-1.5 text-xs"
+ asChild
+ >
+ <Link
+ href={`/project/${projectId}/settings/scores`}
+ target="_blank"
+ onClick={onManageScoreConfigsClick}
+ onAuxClick={(event) => {
+ if (event.button === 1) onManageScoreConfigsClick();
+ }}
+ >
+ <Settings2 className="size-3" aria-hidden="true" />
+ Manage score configs
+ </Link>
+ </Button>
+ </div>
<FormDescription>
Define which dimensions annotators should score for the
given queue.
</FormDescription>
<FormControl>
- <MultiSelectKeyValues
- placeholder="Value"
- align="end"
- variant="outline"
- className="grid grid-cols-[auto_1fr_auto_auto] gap-2"
+ <MultiSelectTagInput
+ aria-label="Score configs"
+ placeholder="Choose score configs"
+ searchPlaceholder="Search score configs..."
+ emptyMessage="No score configs found."
onValueChange={handleScoreConfigValueChange}
- options={activeScoreConfigs.map((config) => ({
- key: config.id,
- value: `${getScoreDataTypeIcon(config.dataType)} ${config.name}`,
- isArchived: config.isArchived,
- }))}
- values={field.value.map((configId) => {
- const config = scoreConfigs.find(
- (scoreConfig) => scoreConfig.id === configId,
- );
- return {
- value: config
- ? `${getScoreDataTypeIcon(config.dataType)} ${config.name}`
- : `${configId}`,
- key: configId,
- };
- })}
- controlButtons={
- <DropdownMenuItemWithSecondaryAction
- onBeforeAction={onManageScoreConfigsClick}
- href={`/project/${projectId}/settings/scores`}
- target="_blank"
- title="Manage score configs"
- />
- }
+ options={scoreConfigs
+ .filter(
+ (config) =>
+ !config.isArchived ||
+ field.value.includes(config.id),
+ )
+ .map((config) => ({
+ value: config.id,
+ label: `${getScoreDataTypeIcon(config.dataType)} ${config.name}`,
+ secondaryLabel: config.isArchived
+ ? "Archived"
+ : undefined,
+ showSecondaryLabelInTag: true,
+ }))}
+ value={field.value}
/>
</FormControl>
<FormMessage />
diff --git a/web/src/features/annotation-queues/components/AnnotationQueueFormDialogController.clienttest.tsx b/web/src/features/annotation-queues/components/AnnotationQueueFormDialogController.clienttest.tsx
new file mode 100644
index 0000000000..a250168a8b
--- /dev/null
+++ b/web/src/features/annotation-queues/components/AnnotationQueueFormDialogController.clienttest.tsx
@@ -0,0 +1,98 @@
+import { fireEvent, render, screen, waitFor } from "@testing-library/react";
+import { type ReactNode } from "react";
+import { vi } from "vitest";
+import { AnnotationQueueFormDialogController } from "./AnnotationQueueFormDialogController";
+
+const { create, update, assign, notify, draft } = vi.hoisted(() => ({
+ create: vi.fn(),
+ update: vi.fn(),
+ assign: vi.fn(),
+ notify: vi.fn(),
+ draft: {
+ name: "Initial name",
+ description: "Initial description",
+ scoreConfigIds: ["first-config"],
+ newAssignmentUserIds: ["reviewer"],
+ },
+}));
+
+vi.mock("@/src/features/rbac", () => ({ useHasProjectAccess: () => true }));
+vi.mock("@/src/features/notifications", () => ({ showErrorToast: notify }));
+vi.mock("@/src/features/posthog-analytics", () => ({
+ usePostHogClientCapture: () => vi.fn(),
+}));
+vi.mock("@/src/components/ui/dialog", () => ({
+ Dialog: ({ children }: { children: ReactNode }) => children,
+ DialogContent: ({ children }: { children: ReactNode }) => children,
+}));
+vi.mock("./AnnotationQueueFormDialogContent", () => ({
+ AnnotationQueueFormDialogContent: ({
+ onSubmit,
+ isSubmitting,
+ }: {
+ onSubmit: (data: typeof draft) => void;
+ isSubmitting: boolean;
+ }) => (
+ <button disabled={isSubmitting} onClick={() => onSubmit({ ...draft })}>
+ Save
+ </button>
+ ),
+}));
+vi.mock("@/src/utils/api", () => ({
+ api: {
+ useUtils: () => ({
+ annotationQueues: { invalidate: vi.fn() },
+ annotationQueueAssignments: { invalidate: vi.fn() },
+ }),
+ annotationQueues: {
+ byId: { useQuery: () => ({ data: undefined }) },
+ allNamesAndIds: { useQuery: () => ({ data: [] }) },
+ create: { useMutation: () => ({ mutateAsync: create }) },
+ update: { useMutation: () => ({ mutateAsync: update }) },
+ },
+ annotationQueueAssignments: {
+ createMany: { useMutation: () => ({ mutateAsync: assign }) },
+ },
+ scoreConfigs: { all: { useQuery: () => ({ data: { configs: [] } }) } },
+ },
+}));
+
+it("saves edited queue fields when retrying a failed assignment without duplicating the queue", async () => {
+ create.mockResolvedValue({ id: "created-queue" });
+ update.mockResolvedValue(undefined);
+ assign
+ .mockRejectedValueOnce(new Error("Assignment unavailable"))
+ .mockResolvedValue(undefined);
+ const onSuccess = vi.fn();
+ render(
+ <AnnotationQueueFormDialogController
+ projectId="project"
+ mode="create"
+ onSuccess={onSuccess}
+ >
+ {({ openDialog }) => <button onClick={openDialog}>Open</button>}
+ </AnnotationQueueFormDialogController>,
+ );
+ fireEvent.click(screen.getByRole("button", { name: "Open" }));
+ fireEvent.click(screen.getByRole("button", { name: "Save" }));
+ await waitFor(() => expect(notify).toHaveBeenCalledTimes(1));
+ await waitFor(() =>
+ expect(screen.getByRole("button", { name: "Save" })).toBeEnabled(),
+ );
+ Object.assign(draft, {
+ name: "Edited name",
+ description: "Edited description",
+ scoreConfigIds: ["second-config"],
+ });
+ fireEvent.click(screen.getByRole("button", { name: "Save" }));
+ await waitFor(() => expect(onSuccess).toHaveBeenCalledWith("created-queue"));
+ expect(create).toHaveBeenCalledTimes(1);
+ expect(update).toHaveBeenCalledWith({
+ projectId: "project",
+ queueId: "created-queue",
+ name: "Edited name",
+ description: "Edited description",
+ scoreConfigIds: ["second-config"],
+ });
+ expect(assign).toHaveBeenCalledTimes(2);
+});
diff --git a/web/src/features/annotation-queues/components/AnnotationQueueFormDialogController.tsx b/web/src/features/annotation-queues/components/AnnotationQueueFormDialogController.tsx
index 647bbd4ffa..fd2df3a805 100644
--- a/web/src/features/annotation-queues/components/AnnotationQueueFormDialogController.tsx
+++ b/web/src/features/annotation-queues/components/AnnotationQueueFormDialogController.tsx
@@ -5,7 +5,7 @@ import {
type CreateQueueWithAssignments,
type ScoreConfigDomain,
} from "@langfuse/shared";
-import { type ReactNode, useRef, useState } from "react";
+import { type ReactNode, useState } from "react";
import { Dialog, DialogContent } from "@/src/components/ui/dialog";
import { AnnotationQueueFormDialogContent } from "@/src/features/annotation-queues/components/AnnotationQueueFormDialogContent";
@@ -35,7 +35,7 @@ export function AnnotationQueueFormDialogController(
const queueId = mode === "edit" ? props.queueId : undefined;
const [open, setOpen] = useState(false);
- const createdQueueIdRef = useRef<string | undefined>(undefined);
+ const [createdQueueId, setCreatedQueueId] = useState<string>();
const hasQueueAccess = useHasProjectAccess({
projectId,
@@ -50,7 +50,7 @@ export function AnnotationQueueFormDialogController(
const openDialog = () => {
if (!hasQueueAccess) return;
- createdQueueIdRef.current = undefined;
+ setCreatedQueueId(undefined);
setOpen(true);
};
@@ -84,17 +84,16 @@ export function AnnotationQueueFormDialogController(
async (data: CreateQueueWithAssignments) => {
try {
let targetQueueId: string;
- if (mode === "edit") {
+ const existingQueueId = queueId ?? createdQueueId;
+ if (existingQueueId) {
await editQueueMutation.mutateAsync({
name: data.name,
description: data.description,
scoreConfigIds: data.scoreConfigIds,
projectId,
- queueId: props.queueId,
+ queueId: existingQueueId,
});
- targetQueueId = props.queueId;
- } else if (createdQueueIdRef.current) {
- targetQueueId = createdQueueIdRef.current;
+ targetQueueId = existingQueueId;
} else {
const queueResponse = await createQueueMutation.mutateAsync({
name: data.name,
@@ -103,7 +102,7 @@ export function AnnotationQueueFormDialogController(
projectId,
});
targetQueueId = queueResponse.id;
- createdQueueIdRef.current = targetQueueId;
+ setCreatedQueueId(targetQueueId);
}
if (data.newAssignmentUserIds.length > 0) {
@@ -118,7 +117,7 @@ export function AnnotationQueueFormDialogController(
utils.annotationQueues.invalidate(),
utils.annotationQueueAssignments.invalidate(),
]);
- createdQueueIdRef.current = undefined;
+ setCreatedQueueId(undefined);
onSuccess(targetQueueId);
setOpen(false);
} catch {
@@ -132,10 +131,10 @@ export function AnnotationQueueFormDialogController(
createQueueAssignmentsMutation,
createQueueMutation,
editQueueMutation,
- mode,
+ createdQueueId,
onSuccess,
projectId,
- props,
+ queueId,
utils.annotationQueueAssignments,
utils.annotationQueues,
],
@@ -169,9 +168,11 @@ export function AnnotationQueueFormDialogController(
initialValues={initialValues}
scoreConfigs={configsQuery.data.configs}
projectId={projectId}
- queueId={queueId}
+ queueId={queueId ?? createdQueueId}
queueNames={
- allQueueNamesAndIds.data?.map((queue) => queue.name) ?? []
+ allQueueNamesAndIds.data
+ ?.filter((queue) => queue.id !== (queueId ?? createdQueueId))
+ .map((queue) => queue.name) ?? []
}
onManageScoreConfigsClick={() => {
capture("score_configs:manage_configs_item_click", {
diff --git a/web/src/features/annotation-queues/components/AnnotationQueueItemDropdownMenuController.tsx b/web/src/features/annotation-queues/components/AnnotationQueueItemDropdownMenuController.tsx
index 480d132546..edfe0f4b14 100644
--- a/web/src/features/annotation-queues/components/AnnotationQueueItemDropdownMenuController.tsx
+++ b/web/src/features/annotation-queues/components/AnnotationQueueItemDropdownMenuController.tsx
@@ -11,11 +11,14 @@ import { api, reportNonTrpcError } from "@/src/utils/api";
import { type AnnotationQueueObjectType } from "@langfuse/shared";
import { type ReactNode, useCallback, useState } from "react";
import { useSession } from "next-auth/react";
+import { usePostHogClientCapture } from "@/src/features/posthog-analytics";
+import { type AnalyticsData } from "@/src/features/scores/types";
type AnnotationQueueItemDropdownMenuControllerProps = {
projectId: string;
objectId: string;
objectType: AnnotationQueueObjectType;
+ analyticsData: Pick<AnalyticsData, "source" | "isV4">;
children: (control: {
disabled: { reason: string } | undefined;
totalCount: number;
@@ -27,8 +30,10 @@ export function AnnotationQueueItemDropdownMenuController({
projectId,
objectId,
objectType,
+ analyticsData,
children,
}: AnnotationQueueItemDropdownMenuControllerProps) {
+ const capture = usePostHogClientCapture();
const [isDropdownOpen, setIsDropdownOpen] = useState(false);
const session = useSession();
const hasAccess = useHasProjectAccess({
@@ -48,26 +53,53 @@ export function AnnotationQueueItemDropdownMenuController({
const addToQueueMutation = api.annotationQueueItems.createMany.useMutation();
const removeFromQueueMutation =
api.annotationQueueItems.deleteMany.useMutation();
+ const targetType = (
+ {
+ TRACE: "trace",
+ OBSERVATION: "observation",
+ SESSION: "session",
+ } as const
+ )[objectType];
+ const type = objectType === "SESSION" ? "session" : "trace";
+ const queueCount = queues.data?.totalCount ?? 0;
const handleQueueItemToggle = useCallback(
async (queueId: string, queueName: string, itemId?: string) => {
try {
if (!itemId) {
- await addToQueueMutation.mutateAsync({
+ const result = await addToQueueMutation.mutateAsync({
projectId,
objectIds: [objectId],
objectType,
queueId,
});
+ if (result.createdCount > 0) {
+ capture("annotation_queues:item_added", {
+ ...analyticsData,
+ type,
+ targetType,
+ objectType,
+ queueCount: 1,
+ });
+ }
} else if (
confirm(
`Are you sure you want to remove this item from the queue "${queueName}"?`,
)
) {
- await removeFromQueueMutation.mutateAsync({
+ const result = await removeFromQueueMutation.mutateAsync({
projectId,
itemIds: [itemId],
});
+ if (result.deletedCount > 0) {
+ capture("annotation_queues:item_removed", {
+ ...analyticsData,
+ type,
+ targetType,
+ objectType,
+ queueCount: 1,
+ });
+ }
}
await utils.annotationQueues.byObjectId.invalidate({
@@ -81,10 +113,14 @@ export function AnnotationQueueItemDropdownMenuController({
},
[
addToQueueMutation,
+ analyticsData,
+ capture,
objectId,
objectType,
projectId,
removeFromQueueMutation,
+ targetType,
+ type,
utils.annotationQueues,
],
);
@@ -104,7 +140,17 @@ export function AnnotationQueueItemDropdownMenuController({
<DropdownMenu
open={hasAccess && isDropdownOpen}
onOpenChange={(open) => {
- if (hasAccess) setIsDropdownOpen(open);
+ if (hasAccess) {
+ if (open && !isDropdownOpen) {
+ capture("annotation:entry_click", {
+ ...analyticsData,
+ type,
+ targetType,
+ entryPoint: "queue_button",
+ });
+ }
+ setIsDropdownOpen(open);
+ }
}}
>
{children({
@@ -119,6 +165,15 @@ export function AnnotationQueueItemDropdownMenuController({
(queues.data?.queues ?? []) satisfies AnnotationQueueItemMenuQueue[]
}
onQueueItemToggle={handleQueueItemToggle}
+ onManageClick={() =>
+ capture("annotation_queues:manage_click", {
+ ...analyticsData,
+ type,
+ targetType,
+ objectType,
+ queueCount,
+ })
+ }
/>
) : null}
</DropdownMenu>
diff --git a/web/src/features/annotation-queues/components/AnnotationQueueItemMenuContent.tsx b/web/src/features/annotation-queues/components/AnnotationQueueItemMenuContent.tsx
index b026b66f9e..74e900da23 100644
--- a/web/src/features/annotation-queues/components/AnnotationQueueItemMenuContent.tsx
+++ b/web/src/features/annotation-queues/components/AnnotationQueueItemMenuContent.tsx
@@ -25,12 +25,14 @@ type AnnotationQueueItemMenuContentProps = {
queueName: string,
itemId?: string,
) => void;
+ onManageClick?: () => void;
};
export function AnnotationQueueItemMenuContent({
projectId,
queues,
onQueueItemToggle,
+ onManageClick,
}: AnnotationQueueItemMenuContentProps) {
const preventMenuItemAction = (event: MouseEvent) => {
event.preventDefault();
@@ -39,7 +41,7 @@ export function AnnotationQueueItemMenuContent({
return (
<DropdownMenuContent className="max-h-[min(300px,var(--radix-dropdown-menu-content-available-height))] overflow-y-auto">
- <DropdownMenuLabel>In queue(s)</DropdownMenuLabel>
+ <DropdownMenuLabel>Add to human annotation queue</DropdownMenuLabel>
{queues.length ? (
queues.map((queue) => (
<DropdownMenuCheckboxItem
@@ -69,7 +71,10 @@ export function AnnotationQueueItemMenuContent({
<DropdownMenuItem className="hover:bg-accent" asChild>
<div>
<ExternalLink className="mr-2 h-4 w-4" />
- <Link href={`/project/${projectId}/annotation-queues`}>
+ <Link
+ href={`/project/${projectId}/annotation-queues`}
+ onClick={onManageClick}
+ >
Manage queues
</Link>
</div>
diff --git a/web/src/features/annotation-queues/components/AnnotationQueueItemPage.tsx b/web/src/features/annotation-queues/components/AnnotationQueueItemPage.tsx
index 3bd1cef4b7..08365eb753 100644
--- a/web/src/features/annotation-queues/components/AnnotationQueueItemPage.tsx
+++ b/web/src/features/annotation-queues/components/AnnotationQueueItemPage.tsx
@@ -2,14 +2,20 @@ import { Card } from "@/src/components/ui/card";
import { Skeleton } from "@/src/components/ui/skeleton";
import { useHasProjectAccess } from "@/src/features/rbac";
import { api } from "@/src/utils/api";
-import { type RouterOutput } from "@/src/utils/types";
import {
AnnotationQueueStatus,
AnnotationQueueObjectType,
} from "@langfuse/shared";
import { ArrowLeft, ArrowRight, Keyboard, SearchXIcon } from "lucide-react";
import { useRouter } from "next/router";
-import { useCallback, useEffect, useMemo, useRef, useState } from "react";
+import { useCallback, useEffect, useId, useRef, useState } from "react";
+import { useQuery, useQueryClient } from "@tanstack/react-query";
+import { useStore } from "zustand";
+import {
+ createAnnotationQueueRun,
+ type AnnotationQueueRun,
+ type QueueRunDependencies,
+} from "../state/annotationQueueRun";
import { Button } from "@/src/components/ui/button";
import { KeyboardShortcut } from "@/src/components/design-system/KeyboardShortcut/KeyboardShortcut";
import {
@@ -52,161 +58,179 @@ const ShortcutRow: React.FC<{
</div>
);
-export const AnnotationQueueItemPage: React.FC<{
+type QueuePageProps = {
annotationQueueId: string;
projectId: string;
queryItemId?: string;
-}> = ({ annotationQueueId, projectId, queryItemId }) => {
- const router = useRouter();
- const { status: sessionStatus } = useSession();
- const sessionLoaded = sessionStatus !== "loading";
- const isSingleItem = router.query.singleItem === "true";
- const [nextItemData, setNextItemData] = useState<
- RouterOutput["annotationQueues"]["fetchAndLockNext"] | null
- >(null);
- const [seenItemIds, setSeenItemIds] = useState<string[]>([]);
- const [progressIndex, setProgressIndex] = useState(0);
+};
+export function AnnotationQueueItemPage(props: QueuePageProps) {
+ const router = useRouter();
+ const { status } = useSession();
+ if (!router.isReady) return <Skeleton className="h-full w-full" />;
+ const singleItem = router.query.singleItem === "true";
+ return (
+ <AnnotationQueueRunLoader
+ key={singleItem ? props.queryItemId : "run"}
+ {...props}
+ singleItem={singleItem}
+ sessionReady={status !== "loading"}
+ />
+ );
+}
+
+function AnnotationQueueRunLoader({
+ annotationQueueId,
+ projectId,
+ queryItemId,
+ singleItem,
+ sessionReady,
+}: QueuePageProps & { singleItem: boolean; sessionReady: boolean }) {
+ const router = useRouter();
+ const runId = useId();
+ const queryClient = useQueryClient();
+ const queryKey = ["annotation-queue-run", runId];
+ const [run] = useState(() =>
+ createAnnotationQueueRun({ initialItemId: queryItemId, singleItem }),
+ );
+ const utils = api.useUtils();
+ const fetchNext = api.annotationQueues.fetchAndLockNext.useMutation();
+ const complete = api.annotationQueueItems.complete.useMutation();
+ const dependencies: QueueRunDependencies = {
+ isActive: () =>
+ queryClient.getQueryCache().find({ queryKey })?.isActive() ?? false,
+ loadItem: (itemId) =>
+ utils.annotationQueueItems.byId.fetch({ projectId, itemId }),
+ loadNext: async (seenItemIds) => {
+ const item = await fetchNext.mutateAsync({
+ queueId: annotationQueueId,
+ projectId,
+ seenItemIds,
+ });
+ return item
+ ? {
+ ...item,
+ lockedByUser: { name: item.lockedByUser.name ?? null },
+ }
+ : null;
+ },
+ cacheItem: (item) =>
+ utils.annotationQueueItems.byId.setData(
+ { projectId, itemId: item.id },
+ item,
+ ),
+ completeItem: (itemId) => complete.mutateAsync({ projectId, itemId }),
+ refreshItems: () => utils.annotationQueueItems.invalidate(),
+ navigate: async (item, initialize) => {
+ if (initialize && router.query.itemId === item.id) {
+ if (item.observationId && !router.query.observation) {
+ return router.replace(
+ {
+ pathname: router.pathname,
+ query: { ...router.query, observation: item.observationId },
+ },
+ undefined,
+ { shallow: true },
+ );
+ }
+ return;
+ }
+ return router.push({
+ pathname: `/project/${projectId}/annotation-queues/${annotationQueueId}/items/${item.id}`,
+ query: item.observationId
+ ? { observation: item.observationId }
+ : undefined,
+ });
+ },
+ };
+ const bootstrap = useQuery({
+ queryKey,
+ queryFn: ({ signal }) => run.actions.start(dependencies, signal),
+ enabled: sessionReady,
+ staleTime: Infinity,
+ gcTime: 0,
+ retry: false,
+ refetchOnMount: false,
+ refetchOnWindowFocus: false,
+ refetchOnReconnect: false,
+ });
+ if (bootstrap.isPending) return <Skeleton className="h-full w-full" />;
+ if (bootstrap.isError) {
+ return (
+ <div className="flex h-full flex-col items-center justify-center gap-3">
+ <p>Unable to load the annotation queue.</p>
+ <Button onClick={() => bootstrap.refetch()}>Try again</Button>
+ </div>
+ );
+ }
+ return (
+ <AnnotationQueueRunContent
+ annotationQueueId={annotationQueueId}
+ projectId={projectId}
+ isSingleItem={singleItem}
+ run={run}
+ dependencies={dependencies}
+ />
+ );
+}
+
+function AnnotationQueueRunContent({
+ annotationQueueId,
+ projectId,
+ isSingleItem,
+ run,
+ dependencies,
+}: {
+ annotationQueueId: string;
+ projectId: string;
+ isSingleItem: boolean;
+ run: AnnotationQueueRun;
+ dependencies: QueueRunDependencies;
+}) {
+ const {
+ history,
+ progressIndex,
+ isTransitioning,
+ exhausted,
+ completedItemIds,
+ error,
+ } = useStore(run.store);
+ const seenItemIds = history.map((item) => item.id);
+ const itemId = history[progressIndex]?.id;
const hasAccess = useHasProjectAccess({
projectId,
scope: "annotationQueues:CUD",
});
-
- const itemId = isSingleItem ? queryItemId : seenItemIds[progressIndex];
-
const seenItemData = api.annotationQueueItems.byId.useQuery(
{ projectId, itemId: itemId as string },
- { enabled: !!itemId && sessionLoaded, refetchOnMount: false },
+ { enabled: !!itemId, refetchOnMount: false },
);
-
- const fetchAndLockNextMutation =
- api.annotationQueues.fetchAndLockNext.useMutation();
-
- // Effects
- useEffect(() => {
- async function fetchNextItem() {
- if (!itemId && !isSingleItem && sessionLoaded) {
- const nextItem = await fetchAndLockNextMutation.mutateAsync({
- queueId: annotationQueueId,
- projectId,
- seenItemIds,
- });
- setNextItemData(nextItem);
- }
- }
- fetchNextItem();
- // eslint-disable-next-line react-hooks/exhaustive-deps
- }, [sessionLoaded]);
const { configs } = useAnnotationQueueData({ annotationQueueId, projectId });
-
const unseenPendingItemCount =
api.annotationQueueItems.unseenPendingItemCountByQueueId.useQuery(
- {
- queueId: annotationQueueId,
- projectId,
- seenItemIds,
- },
+ { queueId: annotationQueueId, projectId, seenItemIds },
{ refetchOnWindowFocus: false },
);
-
- const utils = api.useUtils();
- const completeMutation = api.annotationQueueItems.complete.useMutation({
- onSuccess: async () => {
- utils.annotationQueueItems.invalidate();
- if (isSingleItem) {
- return;
- }
-
- if (progressIndex >= seenItemIds.length - 1) {
- const nextItem = await fetchAndLockNextMutation.mutateAsync({
- queueId: annotationQueueId,
- projectId,
- seenItemIds,
- });
- setNextItemData(nextItem);
- }
-
- if (progressIndex + 1 < totalItems) {
- setProgressIndex(Math.max(progressIndex + 1, 0));
- }
- },
- });
-
- const totalItems = useMemo(() => {
- return seenItemIds.length + (unseenPendingItemCount.data ?? 0);
- }, [unseenPendingItemCount.data, seenItemIds.length]);
-
- const relevantItem = useMemo(() => {
- if (isSingleItem) return seenItemData.data;
- return progressIndex < seenItemIds.length
- ? seenItemData.data
- : nextItemData;
- }, [
- progressIndex,
- seenItemIds.length,
- seenItemData.data,
- nextItemData,
- isSingleItem,
- ]);
-
+ const totalItems =
+ seenItemIds.length + (exhausted ? 0 : (unseenPendingItemCount.data ?? 0));
+ const relevantItem = seenItemData.data;
const objectData = useAnnotationObjectData(relevantItem ?? null, projectId);
-
- useEffect(() => {
- if (relevantItem?.id && router.query.itemId !== relevantItem.id) {
- const observation =
- relevantItem.objectType === AnnotationQueueObjectType.OBSERVATION
- ? relevantItem.objectId
- : undefined;
- router.push(
- {
- pathname: `/project/${projectId}/annotation-queues/${annotationQueueId}/items/${relevantItem.id}`,
- query: observation ? { observation } : undefined,
- },
- undefined,
- );
- }
- }, [relevantItem, router, projectId, annotationQueueId]);
-
- useEffect(() => {
- if (
- relevantItem &&
- !seenItemIds.includes(relevantItem.id) &&
- !isSingleItem
- ) {
- setSeenItemIds((prev) => [...prev, relevantItem.id]);
- }
- // eslint-disable-next-line react-hooks/exhaustive-deps
- }, [relevantItem]);
-
const isNextItemAvailable = totalItems > progressIndex + 1;
- const isPending = relevantItem?.status === AnnotationQueueStatus.PENDING;
-
- // LFE-7628 — keyboard-first navigation/completion.
- const handleNavigateBack = useCallback(() => {
- setProgressIndex((prev) => prev - 1);
- }, []);
-
- const handleNavigateNext = useCallback(async () => {
- if (progressIndex >= seenItemIds.length - 1) {
- const nextItem = await fetchAndLockNextMutation.mutateAsync({
- queueId: annotationQueueId,
- projectId,
- seenItemIds,
- });
- setNextItemData(nextItem);
- }
- setProgressIndex(Math.max(progressIndex + 1, 0));
- // eslint-disable-next-line react-hooks/exhaustive-deps
- }, [progressIndex, seenItemIds, annotationQueueId, projectId]);
-
- const handleComplete = useCallback(async () => {
- if (!relevantItem) return;
- await completeMutation.mutateAsync({
- itemId: relevantItem.id,
- projectId,
- });
- // eslint-disable-next-line react-hooks/exhaustive-deps
- }, [relevantItem?.id, projectId]);
+ const isPending =
+ relevantItem?.status === AnnotationQueueStatus.PENDING &&
+ !completedItemIds.has(itemId);
+ const handleNavigateBack = useCallback(
+ () => run.actions.back(dependencies),
+ [run, dependencies],
+ );
+ const handleNavigateNext = useCallback(
+ () => run.actions.next(dependencies),
+ [run, dependencies],
+ );
+ const handleComplete = useCallback(
+ () => run.actions.complete(dependencies),
+ [run, dependencies],
+ );
// Brief highlight on the button when its shortcut fires.
const [shortcutPulse, setShortcutPulse] = useState<
@@ -239,12 +263,7 @@ export const AnnotationQueueItemPage: React.FC<{
// not complete or skip an item the annotator hasn't actually seen yet
// (e.g. a quick → between ⌘/Ctrl+Enter and onSuccess advancing would skip
// the next item, which only flashed as a Skeleton).
- if (
- objectData.isLoading ||
- fetchAndLockNextMutation.isPending ||
- completeMutation.isPending
- )
- return;
+ if (objectData.isLoading || isTransitioning) return;
// Complete + next — the Cmd/Ctrl+Enter submit chord. Handled first and
// *before* the typing guard so it works even while the annotator is in the
@@ -252,7 +271,7 @@ export const AnnotationQueueItemPage: React.FC<{
// open drawer/dialog so it never steals that surface's own submit.
if (isCompleteShortcut(event)) {
if (isOpenDialogPresent()) return;
- if (isPending && !completeMutation.isPending && !objectData.isError) {
+ if (isPending && !isTransitioning && !objectData.isError) {
event.preventDefault();
// An out-of-range numeric score is vetoed on blur (no mutation fires),
// so completing now would silently drop it. Scan *all* numeric score
@@ -278,7 +297,7 @@ export const AnnotationQueueItemPage: React.FC<{
active.blur();
}
pulse("complete");
- handleComplete().catch(() => {});
+ handleComplete();
}
return;
}
@@ -317,7 +336,7 @@ export const AnnotationQueueItemPage: React.FC<{
if (isNextItemAvailable) {
event.preventDefault();
pulse("next");
- handleNavigateNext().catch(() => {});
+ handleNavigateNext();
}
return;
}
@@ -339,10 +358,9 @@ export const AnnotationQueueItemPage: React.FC<{
isPending,
isNextItemAvailable,
progressIndex,
- completeMutation.isPending,
+ isTransitioning,
objectData.isError,
objectData.isLoading,
- fetchAndLockNextMutation.isPending,
handleComplete,
handleNavigateNext,
handleNavigateBack,
@@ -351,10 +369,9 @@ export const AnnotationQueueItemPage: React.FC<{
if (
(seenItemData.isPending && itemId) ||
- (fetchAndLockNextMutation.isPending && !itemId) ||
+ (isTransitioning && !itemId) ||
unseenPendingItemCount.isPending ||
- objectData.isLoading ||
- (!sessionLoaded && !isSingleItem)
+ objectData.isLoading
) {
return <Skeleton className="h-full w-full" />;
}
@@ -422,6 +439,22 @@ export const AnnotationQueueItemPage: React.FC<{
<div className="flex min-h-0 flex-1 overflow-hidden">
{renderContent()}
</div>
+ {error && (
+ <div
+ role="alert"
+ className="flex shrink-0 items-center justify-between gap-3 border-t px-3 py-2 text-sm"
+ >
+ <p>{error.message}</p>
+ <Button
+ variant="outline"
+ size="sm"
+ disabled={isTransitioning || !hasAccess}
+ onClick={() => run.actions.retry(dependencies)}
+ >
+ Try again
+ </Button>
+ </div>
+ )}
<div className="grid w-full shrink-0 grid-cols-1 justify-end gap-2 py-2 sm:grid-cols-[auto_min-content]">
{!isSingleItem && (
<div className="flex max-h-10 flex-row items-center gap-2">
@@ -433,7 +466,9 @@ export const AnnotationQueueItemPage: React.FC<{
<Button
onClick={handleNavigateBack}
variant="outline"
- disabled={progressIndex === 0 || !hasAccess}
+ disabled={
+ progressIndex === 0 || !hasAccess || isTransitioning
+ }
size="lg"
className={cn(
"gap-1.5 px-4 transition-colors duration-150",
@@ -479,7 +514,9 @@ export const AnnotationQueueItemPage: React.FC<{
<TooltipTrigger asChild>
<Button
onClick={handleNavigateNext}
- disabled={!isNextItemAvailable || !hasAccess}
+ disabled={
+ !isNextItemAvailable || !hasAccess || isTransitioning
+ }
size="lg"
className={cn(
"gap-1.5 px-4 transition-colors duration-150",
@@ -505,7 +542,7 @@ export const AnnotationQueueItemPage: React.FC<{
</Tooltip>
)}
{!!relevantItem &&
- (relevantItem.status === AnnotationQueueStatus.PENDING ? (
+ (isPending ? (
<Tooltip>
<TooltipTrigger asChild>
<Button
@@ -516,9 +553,7 @@ export const AnnotationQueueItemPage: React.FC<{
shortcutPulse === "complete" && "ring-primary/40 ring-2",
)}
disabled={
- completeMutation.isPending ||
- !hasAccess ||
- objectData.isError
+ isTransitioning || !hasAccess || objectData.isError
}
>
<span>Mark Completed</span>
@@ -613,4 +648,4 @@ export const AnnotationQueueItemPage: React.FC<{
</Dialog>
</div>
);
-};
+}
diff --git a/web/src/features/annotation-queues/components/AnnotationQueueItemsTable.tsx b/web/src/features/annotation-queues/components/AnnotationQueueItemsTable.tsx
index ede747f2f1..db9beaf16e 100644
--- a/web/src/features/annotation-queues/components/AnnotationQueueItemsTable.tsx
+++ b/web/src/features/annotation-queues/components/AnnotationQueueItemsTable.tsx
@@ -32,6 +32,8 @@ import {
} from "@/src/components/ui/dialog";
import { Checkbox } from "@/src/components/design-system/Checkbox/Checkbox";
import { useHasProjectAccess } from "@/src/features/rbac";
+import { useReadPath } from "@/src/features/events";
+import { usePostHogClientCapture } from "@/src/features/posthog-analytics";
import { createStatusTableColumn } from "@/src/components/design-system/table/columns/createStatusTableColumn";
import { type Status } from "@/src/components/ui/StatusBadge/StatusBadge";
import { createIdTableColumn } from "@/src/components/design-system/table/columns/createIdTableColumn";
@@ -39,23 +41,56 @@ import { createLinkTableColumn } from "@/src/components/design-system/table/colu
import { createUserTableColumn } from "@/src/components/design-system/table/columns/createUserTableColumn";
const QueueItemTableMultiSelectAction = ({
- selectedItemIds,
+ selectedItems,
projectId,
+ isV4,
onDeleteSuccess,
}: {
- selectedItemIds: string[];
+ selectedItems: Pick<QueueItemRowData, "id" | "objectType">[];
projectId: string;
+ isV4: boolean;
onDeleteSuccess: () => void;
}) => {
const utils = api.useUtils();
+ const capture = usePostHogClientCapture();
const [open, setOpen] = useState(false);
+ const selectedItemIds = selectedItems.map((item) => item.id);
const hasDeleteAccess = useHasProjectAccess({
projectId,
scope: "annotationQueues:CUD",
});
const mutDeleteItems = api.annotationQueueItems.deleteMany.useMutation({
- onSuccess: () => {
+ onMutate: (variables) => {
+ const itemIds = new Set(variables.itemIds);
+ return {
+ isV4,
+ objectTypes: new Set(
+ selectedItems
+ .filter((item) => itemIds.has(item.id))
+ .map((item) => item.objectType),
+ ),
+ };
+ },
+ onSuccess: (data, _variables, context) => {
+ if (context && data.deletedCount > 0) {
+ for (const objectType of context.objectTypes) {
+ capture("annotation_queues:item_removed", {
+ type: objectType === "SESSION" ? "session" : "trace",
+ source: "AnnotationQueue",
+ targetType: (
+ {
+ TRACE: "trace",
+ OBSERVATION: "observation",
+ SESSION: "session",
+ } as const
+ )[objectType],
+ objectType,
+ queueCount: 1,
+ isV4: context.isV4,
+ });
+ }
+ }
onDeleteSuccess();
utils.annotationQueueItems.itemsByQueueId.invalidate();
},
@@ -164,6 +199,7 @@ export function AnnotationQueueItemsTable({
projectId: string;
queueId: string;
}) {
+ const { isV4 } = useReadPath();
const [paginationState, setPaginationState] = useQueryParams({
pageIndex: withDefault(NumberParam, 0),
pageSize: withDefault(NumberParam, 50),
@@ -177,6 +213,8 @@ export function AnnotationQueueItemsTable({
page: paginationState.pageIndex,
limit: paginationState.pageSize,
});
+ const selectedItems =
+ items.data?.queueItems.filter((item) => selectedRows[item.id]) ?? [];
const columns: LangfuseColumnDef<QueueItemRowData>[] = [
{
@@ -413,15 +451,11 @@ export function AnnotationQueueItemsTable({
rowHeight={rowHeight}
setRowHeight={setRowHeight}
actionButtons={[
- Object.keys(selectedRows).filter((itemId) =>
- items.data?.queueItems.map((item) => item.id).includes(itemId),
- ).length > 0 ? (
+ selectedItems.length > 0 ? (
<QueueItemTableMultiSelectAction
- // Exclude items that are not in the current page
- selectedItemIds={Object.keys(selectedRows).filter((itemId) =>
- items.data?.queueItems.map((item) => item.id).includes(itemId),
- )}
+ selectedItems={selectedItems}
projectId={projectId}
+ isV4={isV4}
onDeleteSuccess={() => {
setSelectedRows({});
}}
diff --git a/web/src/features/annotation-queues/components/AnnotationQueuesItem.clienttest.tsx b/web/src/features/annotation-queues/components/AnnotationQueuesItem.clienttest.tsx
new file mode 100644
index 0000000000..0c6efac1da
--- /dev/null
+++ b/web/src/features/annotation-queues/components/AnnotationQueuesItem.clienttest.tsx
@@ -0,0 +1,363 @@
+import {
+ act,
+ fireEvent,
+ render as renderWithTestingLibrary,
+ screen,
+ waitFor,
+} from "@testing-library/react";
+import { StrictMode, type ReactNode } from "react";
+import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
+import { vi } from "vitest";
+import { AnnotationQueuesItem } from "./AnnotationQueuesItem";
+
+function render(ui: ReactNode) {
+ const client = new QueryClient({
+ defaultOptions: { queries: { retry: false } },
+ });
+ return renderWithTestingLibrary(ui, {
+ wrapper: ({ children }) => (
+ <QueryClientProvider client={client}>{children}</QueryClientProvider>
+ ),
+ });
+}
+
+const { router, fetchNext, complete, refreshItems, items } = vi.hoisted(() => ({
+ router: {
+ isReady: true,
+ pathname: "/project/[projectId]/annotation-queues/[queueId]/items/[itemId]",
+ query: {} as Record<string, string>,
+ push: vi.fn(),
+ replace: vi.fn(),
+ },
+ fetchNext: vi.fn(),
+ complete: vi.fn(),
+ refreshItems: vi.fn(),
+ items: new Map<string, Record<string, unknown>>(),
+}));
+
+vi.mock("next/router", () => ({ useRouter: () => router }));
+vi.mock("next-auth/react", () => ({
+ useSession: () => ({ status: "authenticated" }),
+}));
+vi.mock("@/src/features/rbac", () => ({ useHasProjectAccess: () => true }));
+vi.mock("@/src/features/events", () => ({
+ useReadPath: () => ({ isV4: true }),
+}));
+vi.mock("@/src/components/layouts/page", () => ({
+ default: ({ children }: { children: ReactNode }) => children,
+}));
+vi.mock("@/src/components/ui/tooltip", () => ({
+ Tooltip: ({ children }: { children: ReactNode }) => children,
+ TooltipTrigger: ({ children }: { children: ReactNode }) => children,
+ TooltipContent: () => null,
+}));
+vi.mock("@/src/features/scores", () => ({
+ hasModifier: () => false,
+ isCompleteShortcut: () => false,
+ isInteractiveTarget: () => false,
+ isOpenDialogPresent: () => false,
+ isTypingTarget: () => false,
+}));
+vi.mock("@/src/utils/api", () => ({
+ api: {
+ useUtils: () => ({
+ annotationQueueItems: {
+ invalidate: refreshItems,
+ byId: {
+ fetch: ({ itemId }: { itemId: string }) =>
+ Promise.resolve(items.get(itemId) ?? null),
+ setData: (
+ { itemId }: { itemId: string },
+ item: Record<string, unknown>,
+ ) => items.set(itemId, item),
+ },
+ },
+ }),
+ annotationQueues: {
+ byId: { useQuery: () => ({ data: { name: "Review queue" } }) },
+ fetchAndLockNext: {
+ useMutation: () => ({ mutateAsync: fetchNext, isPending: false }),
+ },
+ },
+ annotationQueueItems: {
+ byId: {
+ useQuery: ({ itemId }: { itemId: string }) => ({
+ data: items.get(itemId),
+ isPending: false,
+ }),
+ },
+ unseenPendingItemCountByQueueId: {
+ useQuery: ({ seenItemIds }: { seenItemIds: string[] }) => ({
+ data: Math.max(2 - seenItemIds.length, 0),
+ isPending: false,
+ }),
+ },
+ complete: {
+ useMutation: () => ({ mutateAsync: complete, isPending: false }),
+ },
+ },
+ },
+}));
+vi.mock("./shared/hooks/useAnnotationQueueData", () => ({
+ useAnnotationQueueData: () => ({ configs: [] }),
+}));
+vi.mock("./shared/hooks/useAnnotationObjectData", () => ({
+ useAnnotationObjectData: (item: { objectId: string } | null) => ({
+ data: item ? { id: item.objectId } : undefined,
+ isLoading: false,
+ }),
+}));
+vi.mock("@/src/features/traces", () => ({
+ Trace: () => <p>Trace loaded</p>,
+}));
+vi.mock("./shared/AnnotationDrawerSection", () => ({
+ AnnotationDrawerSection: ({ item }: { item: { objectId: string } }) => (
+ <>
+ <p>{item.objectId}</p>
+ <input aria-label="Comment draft" />
+ </>
+ ),
+}));
+vi.mock("./shared/AnnotationProcessingLayout", () => ({
+ AnnotationProcessingLayout: ({
+ leftPanel,
+ rightPanel,
+ }: {
+ leftPanel: ReactNode;
+ rightPanel: ReactNode;
+ }) => (
+ <>
+ {leftPanel}
+ {rightPanel}
+ </>
+ ),
+}));
+vi.mock("./processors/SessionAnnotationProcessor", () => ({
+ SessionAnnotationProcessor: () => null,
+}));
+beforeEach(() => {
+ vi.clearAllMocks();
+ router.push.mockReset();
+ router.replace.mockReset();
+ fetchNext.mockReset();
+ complete.mockReset();
+ refreshItems.mockReset();
+ items.clear();
+ for (const queueId of ["first", "second"]) {
+ items.set(`${queueId}-item`, {
+ id: `${queueId}-item`,
+ queueId,
+ projectId: "project",
+ objectId: `${queueId}-observation`,
+ objectType: "OBSERVATION",
+ parentTraceId: `${queueId}-trace`,
+ status: "PENDING",
+ lockedByUser: { name: "Reviewer" },
+ });
+ }
+ router.query = {};
+ router.isReady = true;
+ router.push.mockImplementation(
+ ({
+ pathname,
+ query,
+ }: {
+ pathname: string;
+ query?: Record<string, string>;
+ }) => {
+ router.query = { ...query, itemId: pathname.split("/").at(-1)! };
+ return Promise.resolve(true);
+ },
+ );
+ router.replace.mockImplementation(
+ ({ query }: { query: Record<string, string> }) => {
+ router.query = query;
+ return Promise.resolve(true);
+ },
+ );
+ fetchNext.mockImplementation(({ queueId }: { queueId: string }) =>
+ Promise.resolve(items.get(`${queueId}-item`)),
+ );
+ complete.mockResolvedValue(undefined);
+ refreshItems.mockResolvedValue(undefined);
+});
+
+it.each(["refresh", "advance", "navigation"])(
+ "retains completion and retries a failed %s without completing twice",
+ async (phase) => {
+ fetchNext.mockResolvedValueOnce(items.get("first-item"));
+ if (phase === "refresh")
+ refreshItems.mockRejectedValueOnce(new Error("Refresh failed"));
+ else if (phase === "advance")
+ fetchNext.mockRejectedValueOnce(new Error("Next item failed"));
+ fetchNext.mockResolvedValueOnce({
+ ...items.get("second-item"),
+ queueId: "first",
+ });
+ render(
+ <AnnotationQueuesItem projectId="project" annotationQueueId="first" />,
+ );
+ expect(await screen.findByText("first-observation")).toBeVisible();
+ if (phase === "navigation")
+ router.push.mockRejectedValueOnce(new Error("Navigation failed"));
+ fireEvent.click(screen.getByRole("button", { name: /Mark Completed/ }));
+ expect(await screen.findByRole("alert")).toHaveTextContent(
+ "Item completed",
+ );
+ expect(screen.getByText("Completed", { exact: true })).toBeVisible();
+ expect(
+ screen.queryByRole("button", { name: /Mark Completed/ }),
+ ).not.toBeInTheDocument();
+ fireEvent.click(screen.getByRole("button", { name: "Try again" }));
+ expect(await screen.findByText("second-observation")).toBeVisible();
+ expect(complete).toHaveBeenCalledTimes(1);
+ expect(fetchNext).toHaveBeenCalledTimes(phase === "advance" ? 3 : 2);
+ expect(screen.queryByRole("alert")).not.toBeInTheDocument();
+ },
+);
+
+it("locks only one starting item during Strict Mode replay", async () => {
+ render(
+ <StrictMode>
+ <AnnotationQueuesItem projectId="project" annotationQueueId="first" />
+ </StrictMode>,
+ );
+ expect(await screen.findByText("first-observation")).toBeVisible();
+ expect(fetchNext).toHaveBeenCalledTimes(1);
+});
+
+it("serializes skip and completion transitions and keeps back navigation in the run", async () => {
+ let resolveNext!: (value: Record<string, unknown>) => void;
+ const nextItem = { ...items.get("second-item"), queueId: "first" };
+ fetchNext
+ .mockResolvedValueOnce(items.get("first-item"))
+ .mockImplementationOnce(
+ () =>
+ new Promise((resolve) => {
+ resolveNext = resolve;
+ }),
+ );
+ render(
+ <AnnotationQueuesItem projectId="project" annotationQueueId="first" />,
+ );
+ expect(await screen.findByText("first-observation")).toBeVisible();
+ const next = screen.getByRole("button", { name: "Skip to next item" });
+ fireEvent.click(next);
+ fireEvent.click(next);
+ expect(fetchNext).toHaveBeenCalledTimes(2);
+ resolveNext(nextItem);
+ expect(await screen.findByText("second-observation")).toBeVisible();
+ expect(router.query.itemId).toBe("second-item");
+ fireEvent.click(screen.getByRole("button", { name: "Previous item" }));
+ expect(await screen.findByText("first-observation")).toBeVisible();
+ expect(router.query.itemId).toBe("first-item");
+ fireEvent.click(screen.getByRole("button", { name: /Mark Completed/ }));
+ expect(await screen.findByText("second-observation")).toBeVisible();
+ expect(complete).toHaveBeenCalledWith({
+ itemId: "first-item",
+ projectId: "project",
+ });
+ expect(fetchNext).toHaveBeenCalledTimes(2);
+});
+
+it("starts a fresh processing run when the queue changes", async () => {
+ const { rerender } = render(
+ <AnnotationQueuesItem projectId="project" annotationQueueId="first" />,
+ );
+ expect(await screen.findByText("first-observation")).toBeVisible();
+ rerender(
+ <AnnotationQueuesItem projectId="project" annotationQueueId="second" />,
+ );
+ expect(await screen.findByText("second-observation")).toBeVisible();
+ expect(router.query.observation).toBe("second-observation");
+});
+
+it("preserves the current draft during a failed next-item transition", async () => {
+ let rejectNext!: (error: Error) => void;
+ fetchNext
+ .mockResolvedValueOnce(items.get("first-item"))
+ .mockImplementationOnce(
+ () =>
+ new Promise((_resolve, reject) => {
+ rejectNext = reject;
+ }),
+ );
+ render(
+ <AnnotationQueuesItem projectId="project" annotationQueueId="first" />,
+ );
+ expect(await screen.findByText("first-observation")).toBeVisible();
+ fireEvent.change(screen.getByRole("textbox", { name: "Comment draft" }), {
+ target: { value: "Unsent review" },
+ });
+ fireEvent.click(screen.getByRole("button", { name: "Skip to next item" }));
+ expect(screen.getByRole("textbox", { name: "Comment draft" })).toHaveValue(
+ "Unsent review",
+ );
+ expect(
+ screen.getByRole("button", { name: "Skip to next item" }),
+ ).toBeDisabled();
+ expect(screen.getByRole("button", { name: /Mark Completed/ })).toBeDisabled();
+ await act(async () => {
+ rejectNext(new Error("Queue unavailable"));
+ });
+ expect(screen.getByRole("textbox", { name: "Comment draft" })).toHaveValue(
+ "Unsent review",
+ );
+ expect(
+ screen.getByRole("button", { name: "Skip to next item" }),
+ ).toBeEnabled();
+});
+
+it("ignores a next-item response after leaving its queue", async () => {
+ let resolveNext!: (value: Record<string, unknown>) => void;
+ fetchNext
+ .mockResolvedValueOnce(items.get("first-item"))
+ .mockImplementationOnce(
+ () =>
+ new Promise((resolve) => {
+ resolveNext = resolve;
+ }),
+ );
+ const { rerender } = render(
+ <AnnotationQueuesItem projectId="project" annotationQueueId="first" />,
+ );
+ expect(await screen.findByText("first-observation")).toBeVisible();
+ fireEvent.click(screen.getByRole("button", { name: "Skip to next item" }));
+ rerender(
+ <AnnotationQueuesItem projectId="project" annotationQueueId="second" />,
+ );
+ expect(await screen.findByText("second-observation")).toBeVisible();
+ router.push.mockClear();
+ await act(async () => {
+ resolveNext({ ...items.get("first-item"), id: "late-first-item" });
+ });
+ expect(router.push).not.toHaveBeenCalled();
+});
+
+it("selects a direct observation item once and preserves reviewer selection on refetch", async () => {
+ router.isReady = false;
+ const view = () => (
+ <AnnotationQueuesItem
+ projectId="project"
+ annotationQueueId="first"
+ itemId="first-item"
+ />
+ );
+ const { rerender } = render(view());
+ await act(() => Promise.resolve());
+ expect(router.push).not.toHaveBeenCalled();
+ router.isReady = true;
+ router.query = { itemId: "first-item", singleItem: "true" };
+ rerender(view());
+ await waitFor(() =>
+ expect(router.query.observation).toBe("first-observation"),
+ );
+ router.query.observation = "reviewer-selected-sibling";
+ items.set("first-item", { ...items.get("first-item") });
+ rerender(view());
+ expect(router.query.observation).toBe("reviewer-selected-sibling");
+ delete router.query.observation;
+ items.set("first-item", { ...items.get("first-item") });
+ rerender(view());
+ expect(router.query.observation).toBeUndefined();
+});
diff --git a/web/src/features/annotation-queues/components/AnnotationQueuesItem.tsx b/web/src/features/annotation-queues/components/AnnotationQueuesItem.tsx
index 8022e25bb9..e362adc5fa 100644
--- a/web/src/features/annotation-queues/components/AnnotationQueuesItem.tsx
+++ b/web/src/features/annotation-queues/components/AnnotationQueuesItem.tsx
@@ -56,6 +56,7 @@ export const AnnotationQueuesItem = ({
}}
>
<AnnotationQueueItemPage
+ key={JSON.stringify([projectId, annotationQueueId])}
projectId={projectId}
annotationQueueId={annotationQueueId}
queryItemId={itemId}
diff --git a/web/src/features/annotation-queues/components/processors/SessionAnnotationProcessor.tsx b/web/src/features/annotation-queues/components/processors/SessionAnnotationProcessor.tsx
index 893b13c59b..b496c08975 100644
--- a/web/src/features/annotation-queues/components/processors/SessionAnnotationProcessor.tsx
+++ b/web/src/features/annotation-queues/components/processors/SessionAnnotationProcessor.tsx
@@ -285,6 +285,7 @@ export const SessionAnnotationProcessor: React.FC<
const rightPanel = (
<AnnotationDrawerSection
item={item}
+ isV4={isV4}
scoreTarget={{
type: "session",
sessionId: item.objectId,
diff --git a/web/src/features/annotation-queues/components/processors/TraceAnnotationProcessor.tsx b/web/src/features/annotation-queues/components/processors/TraceAnnotationProcessor.tsx
index 7a4d2a3c64..a8fcef7467 100644
--- a/web/src/features/annotation-queues/components/processors/TraceAnnotationProcessor.tsx
+++ b/web/src/features/annotation-queues/components/processors/TraceAnnotationProcessor.tsx
@@ -1,11 +1,9 @@
+import { useReadPath } from "@/src/features/events";
import { Trace } from "@/src/features/traces";
import {
type AnnotationQueueItem,
- AnnotationQueueObjectType,
type ScoreConfigDomain,
} from "@langfuse/shared";
-import { useEffect } from "react";
-import { StringParam, useQueryParam } from "use-query-params";
import { AnnotationDrawerSection } from "../shared/AnnotationDrawerSection";
import { AnnotationProcessingLayout } from "../shared/AnnotationProcessingLayout";
@@ -22,19 +20,9 @@ interface TraceAnnotationProcessorProps {
export const TraceAnnotationProcessor: React.FC<
TraceAnnotationProcessorProps
> = ({ item, data, configs, projectId }) => {
+ const { isV4 } = useReadPath();
const traceId = item.parentTraceId ?? item.objectId;
- const [, setCurrentObservationId] = useQueryParam("observation", StringParam);
-
- // If annotating an observation, set it as selected so the tree highlights it
- useEffect(() => {
- if (item.objectType === AnnotationQueueObjectType.OBSERVATION) {
- setCurrentObservationId(item.objectId);
- } else {
- setCurrentObservationId(undefined);
- }
- }, [item, setCurrentObservationId]);
-
if (!data) return <div className="p-3">Loading...</div>;
const leftPanel = (
@@ -52,6 +40,7 @@ export const TraceAnnotationProcessor: React.FC<
const rightPanel = (
<AnnotationDrawerSection
item={item}
+ isV4={isV4}
scoreTarget={{
type: "trace",
traceId: traceId,
diff --git a/web/src/features/annotation-queues/components/shared/AnnotationDrawerSection.tsx b/web/src/features/annotation-queues/components/shared/AnnotationDrawerSection.tsx
index 9d9345f0b6..b95627dc87 100644
--- a/web/src/features/annotation-queues/components/shared/AnnotationDrawerSection.tsx
+++ b/web/src/features/annotation-queues/components/shared/AnnotationDrawerSection.tsx
@@ -19,11 +19,12 @@ interface AnnotationDrawerSectionProps {
scores: WithStringifiedMetadata<ScoreDomain>[];
configs: ScoreConfigDomain[];
environment?: string;
+ isV4: boolean;
}
export const AnnotationDrawerSection: React.FC<
AnnotationDrawerSectionProps
-> = ({ item, scoreTarget, scores, configs, environment }) => {
+> = ({ item, scoreTarget, scores, configs, environment, isV4 }) => {
const session = useSession();
const isLockedByOtherUser = item.lockedByUserId !== session.data?.user?.id;
@@ -33,7 +34,7 @@ export const AnnotationDrawerSection: React.FC<
);
return (
- <Card className="col-span-2 flex h-full flex-col overflow-y-auto border-none p-3">
+ <Card className="col-span-2 flex h-full flex-col overflow-y-auto border-none p-3 [--annotation-surface:var(--card)]">
<AnnotationForm
key={"annotation-drawer-content" + item.objectId}
scoreTarget={scoreTarget}
@@ -47,6 +48,7 @@ export const AnnotationDrawerSection: React.FC<
analyticsData={{
type: scoreTarget.type,
source: "AnnotationQueue",
+ isV4,
}}
actionButtons={
isLockedByOtherUser && isPresent(item.lockedByUser?.name) ? (
diff --git a/web/src/features/annotation-queues/pages/AnnotationQueueItems.tsx b/web/src/features/annotation-queues/pages/AnnotationQueueItems.tsx
index 29e91cbd18..ffe6460445 100644
--- a/web/src/features/annotation-queues/pages/AnnotationQueueItems.tsx
+++ b/web/src/features/annotation-queues/pages/AnnotationQueueItems.tsx
@@ -1,9 +1,7 @@
import { api } from "@/src/utils/api";
import { AnnotationQueueItemsTable } from "@/src/features/annotation-queues/components/AnnotationQueueItemsTable";
-import { CardDescription } from "@/src/components/ui/card";
import { Button } from "@/src/components/ui/button";
import { ClipboardPen, Lock } from "lucide-react";
-import { Badge } from "@/src/components/ui/badge";
import Link from "next/link";
import { CreateOrEditAnnotationQueueButton } from "@/src/features/annotation-queues/components/CreateOrEditAnnotationQueueButton";
import { useHasProjectAccess } from "@/src/features/rbac";
@@ -16,8 +14,7 @@ import {
SidePanelHeader,
SidePanelTitle,
} from "@/src/components/ui/side-panel";
-import { SubHeaderLabel } from "@/src/components/layouts/header";
-import { getScoreDataTypeIcon } from "@/src/features/scores";
+import { AnnotationQueueDetails } from "@/src/features/annotation-queues/components/AnnotationQueueDetails";
export default function QueueItems({
projectId,
@@ -77,42 +74,37 @@ export default function QueueItems({
<div className="flex h-full flex-col overflow-hidden">
<AnnotationQueueItemsTable projectId={projectId} queueId={queueId} />
</div>
- <SidePanel
- mobileTitle={queue.data?.name ?? "Queue details"}
- id="queue-details"
- >
+ <SidePanel mobileTitle="Queue details" id="queue-details">
<SidePanelHeader>
- <SidePanelTitle>
- {queue.data?.name ?? "Queue details"}
- </SidePanelTitle>
+ <SidePanelTitle>Details</SidePanelTitle>
<CreateOrEditAnnotationQueueButton
projectId={projectId}
queueId={queueId}
+ variant="ghost"
+ size="sm"
/>
</SidePanelHeader>
<SidePanelContent>
- {queue.isLoading ? (
- <Skeleton className="h-full w-full" />
- ) : (
- <>
- {queue.data?.description && (
- <CardDescription className="text-sm">
- {queue.data?.description}
- </CardDescription>
- )}
- <div className="flex flex-col gap-2">
- <SubHeaderLabel title="Score Configs" />
- <div className="flex flex-wrap gap-2">
- {queue.data?.scoreConfigs.map((scoreConfig) => (
- <Badge key={scoreConfig.id} variant="outline">
- {getScoreDataTypeIcon(scoreConfig.dataType)}
- <span className="ml-0.5">{scoreConfig.name}</span>
- </Badge>
- ))}
- </div>
+ <div className="w-full min-w-0">
+ <div className="flex justify-end px-4 md:hidden">
+ <CreateOrEditAnnotationQueueButton
+ projectId={projectId}
+ queueId={queueId}
+ variant="ghost"
+ size="sm"
+ />
+ </div>
+ {queue.isLoading ? (
+ <div className="p-4">
+ <Skeleton className="h-40 w-full" />
</div>
- </>
- )}
+ ) : (
+ <AnnotationQueueDetails
+ description={queue.data?.description ?? null}
+ scoreConfigs={queue.data?.scoreConfigs ?? []}
+ />
+ )}
+ </div>
</SidePanelContent>
</SidePanel>
</div>
diff --git a/web/src/features/annotation-queues/state/annotationQueueRun.ts b/web/src/features/annotation-queues/state/annotationQueueRun.ts
new file mode 100644
index 0000000000..659b619498
--- /dev/null
+++ b/web/src/features/annotation-queues/state/annotationQueueRun.ts
@@ -0,0 +1,171 @@
+import { createStore } from "zustand/vanilla";
+import { type RouterOutput } from "@/src/utils/types";
+
+type QueueItem = NonNullable<RouterOutput["annotationQueueItems"]["byId"]>;
+type Transition = "next" | "back" | "complete";
+
+export type QueueItemLocation = { id: string; observationId?: string };
+
+export type QueueRunDependencies = {
+ isActive: () => boolean;
+ loadItem: (itemId: string) => Promise<QueueItem | null>;
+ loadNext: (seenItemIds: string[]) => Promise<QueueItem | null>;
+ cacheItem: (item: QueueItem) => void;
+ completeItem: (itemId: string) => Promise<unknown>;
+ refreshItems: () => Promise<unknown>;
+ navigate: (item: QueueItemLocation, initialize: boolean) => Promise<unknown>;
+};
+
+function locationFor(item: QueueItem): QueueItemLocation {
+ return {
+ id: item.id,
+ observationId:
+ item.objectType === "OBSERVATION" ? item.objectId : undefined,
+ };
+}
+
+export function createAnnotationQueueRun({
+ initialItemId,
+ singleItem,
+}: {
+ initialItemId?: string;
+ singleItem: boolean;
+}) {
+ let initialRequest: Promise<QueueItem | null> | undefined;
+ let pendingNext: QueueItem | undefined;
+ const store = createStore<{
+ history: QueueItemLocation[];
+ progressIndex: number;
+ isTransitioning: boolean;
+ exhausted: boolean;
+ completedItemIds: Set<string>;
+ error: { action: Transition; message: string } | null;
+ }>(() => ({
+ history: [],
+ progressIndex: 0,
+ isTransitioning: false,
+ exhausted: false,
+ completedItemIds: new Set(),
+ error: null,
+ }));
+
+ async function enter(
+ dependencies: QueueRunDependencies,
+ history: QueueItemLocation[],
+ progressIndex: number,
+ initialize = false,
+ ) {
+ if (!dependencies.isActive()) return;
+ const navigated = await dependencies.navigate(
+ history[progressIndex],
+ initialize,
+ );
+ if (navigated === false) throw new Error("Queue navigation was cancelled");
+ store.setState({ history, progressIndex });
+ }
+
+ async function advance(dependencies: QueueRunDependencies) {
+ if (!dependencies.isActive()) return;
+ const { history, progressIndex } = store.getState();
+ if (progressIndex + 1 < history.length) {
+ await enter(dependencies, history, progressIndex + 1);
+ return;
+ }
+ const next =
+ pendingNext ??
+ (await dependencies.loadNext(history.map((item) => item.id)));
+ if (!dependencies.isActive()) return;
+ if (!next) {
+ store.setState({ exhausted: true });
+ return;
+ }
+ // Retain an acquired lock if navigation fails, so retry opens the same item.
+ pendingNext = next;
+ dependencies.cacheItem(next);
+ await enter(dependencies, [...history, locationFor(next)], history.length);
+ pendingNext = undefined;
+ store.setState({ exhausted: false });
+ }
+
+ async function transition(name: Transition, action: () => Promise<void>) {
+ if (store.getState().isTransitioning) return;
+ store.setState({ isTransitioning: true, error: null });
+ try {
+ await action();
+ } catch {
+ const { history, progressIndex, completedItemIds } = store.getState();
+ const completed = completedItemIds.has(history[progressIndex]?.id);
+ let message = "Could not open the queue item. Try again.";
+ if (name === "complete")
+ message = completed
+ ? "Item completed, but the queue could not refresh. Try again to continue."
+ : "Could not complete this item. Try again.";
+ store.setState({
+ error: {
+ action: name,
+ message,
+ },
+ });
+ } finally {
+ store.setState({ isTransitioning: false });
+ }
+ }
+
+ const actions = {
+ async start(dependencies: QueueRunDependencies, signal: AbortSignal) {
+ if (!initialRequest) {
+ initialRequest = initialItemId
+ ? dependencies.loadItem(initialItemId)
+ : dependencies.loadNext([]);
+ }
+ let item: QueueItem | null;
+ try {
+ item = await initialRequest;
+ } catch (error) {
+ initialRequest = undefined;
+ throw error;
+ }
+ signal.throwIfAborted();
+ if (item) {
+ dependencies.cacheItem(item);
+ await enter(dependencies, [locationFor(item)], 0, true);
+ } else {
+ store.setState({ exhausted: true });
+ }
+ return true;
+ },
+ next(dependencies: QueueRunDependencies) {
+ return transition("next", () => advance(dependencies));
+ },
+ back(dependencies: QueueRunDependencies) {
+ return transition("back", async () => {
+ const { history, progressIndex } = store.getState();
+ if (progressIndex > 0)
+ await enter(dependencies, history, progressIndex - 1);
+ });
+ },
+ complete(dependencies: QueueRunDependencies) {
+ return transition("complete", async () => {
+ const { history, progressIndex, completedItemIds } = store.getState();
+ const item = history[progressIndex];
+ if (!item) return;
+ if (!completedItemIds.has(item.id)) {
+ await dependencies.completeItem(item.id);
+ store.setState({
+ completedItemIds: new Set([...completedItemIds, item.id]),
+ });
+ }
+ await dependencies.refreshItems();
+ if (!singleItem) await advance(dependencies);
+ });
+ },
+ retry(dependencies: QueueRunDependencies): Promise<void> {
+ const action = store.getState().error?.action;
+ return action ? actions[action](dependencies) : Promise.resolve();
+ },
+ };
+
+ return { store, actions };
+}
+
+export type AnnotationQueueRun = ReturnType<typeof createAnnotationQueueRun>;
diff --git a/web/src/features/datasets/DatasetComparePage.tsx b/web/src/features/datasets/DatasetComparePage.tsx
index 4c59e8207a..97e8dcb656 100644
--- a/web/src/features/datasets/DatasetComparePage.tsx
+++ b/web/src/features/datasets/DatasetComparePage.tsx
@@ -38,7 +38,6 @@ function DatasetCompareLegacy() {
const [isCreateExperimentDialogOpen, setIsCreateExperimentDialogOpen] =
useState(false);
- const [isAnnotationPanelOpen, setIsAnnotationPanelOpen] = useState(false);
const hasExperimentWriteAccess = useHasProjectAccess({
projectId,
@@ -55,7 +54,7 @@ function DatasetCompareLegacy() {
setLocalRuns,
} = useDatasetRunsCompare(projectId, datasetId);
- const { activeCell, clearActiveCell } = useActiveCell();
+ const { activeCell, clearActiveCell, closeRunAnnotation } = useActiveCell();
const handleExperimentSettled = async (data?: {
success: boolean;
@@ -67,32 +66,12 @@ function DatasetCompareLegacy() {
await handleExperimentSettledBase(data);
};
- // Clear annotation state on URL change (filters, navigation, etc.)
- useEffect(() => {
- clearActiveCell();
- }, [router.query, clearActiveCell]);
-
- // Open panel when cell becomes active, close when cleared
- useEffect(() => {
- setIsAnnotationPanelOpen(!!activeCell);
- }, [activeCell]);
-
- // Clear active cell when panel manually closed
const handlePanelOpenChange = (open: boolean) => {
if (!open) {
clearActiveCell();
}
- setIsAnnotationPanelOpen(open);
};
- if (!runsData.data || runs.length === 0) {
- return (
- <div className="flex h-full items-center justify-center">
- <Spinner size="xl" variant="muted" />
- </div>
- );
- }
-
return (
<Page
headerProps={{
@@ -173,6 +152,7 @@ function DatasetCompareLegacy() {
});
setLocalRuns([]);
} else {
+ if (!closeRunAnnotation(changedValueId)) return;
capture("dataset_run:compare_run_removed");
const newRunIds =
runIds?.filter((id) => id !== changedValueId) ?? [];
@@ -201,17 +181,23 @@ function DatasetCompareLegacy() {
>
<div className="grid flex-1 grid-cols-[1fr_auto] overflow-hidden">
<div className="flex h-full flex-col overflow-hidden">
- <DatasetCompareRunsTable
- key={runIds?.join(",") ?? "empty"}
- projectId={projectId}
- datasetId={datasetId}
- runIds={runIds ?? []}
- />
+ {!runsData.data || runs.length === 0 ? (
+ <div className="flex h-full items-center justify-center">
+ <Spinner size="xl" variant="muted" />
+ </div>
+ ) : (
+ <DatasetCompareRunsTable
+ key={runIds?.join(",") ?? "empty"}
+ projectId={projectId}
+ datasetId={datasetId}
+ runIds={runIds ?? []}
+ />
+ )}
</div>
<SidePanel
id="annotation-panel"
openState={{
- open: isAnnotationPanelOpen,
+ open: activeCell !== null,
onOpenChange: handlePanelOpenChange,
}}
mobileTitle="Annotate"
@@ -262,7 +248,9 @@ export default function DatasetComparePage() {
}
return (
- <ActiveCellProvider>
+ <ActiveCellProvider
+ key={JSON.stringify([projectId, router.query.datasetId])}
+ >
<DatasetCompareLegacy />
</ActiveCellProvider>
);
diff --git a/web/src/features/datasets/README.md b/web/src/features/datasets/README.md
new file mode 100644
index 0000000000..c5a9695b40
--- /dev/null
+++ b/web/src/features/datasets/README.md
@@ -0,0 +1,43 @@
+# Datasets
+
+## Surface and entry points
+
+This feature owns dataset items, experiments, and the legacy comparison view.
+Routes live under `pages/project/[projectId]/datasets`. The compare route creates
+one `ActiveCellProvider` per project and dataset.
+
+## State ownership
+
+- tRPC owns dataset, run, item, and score query data. Router hooks own filters and
+ selected runs.
+- `contexts/ActiveCellContext.tsx` owns the active review target and its comment
+ draft guard in a per-view store. Its select and clear actions are the only
+ way to change review targets; both reject leaving an unfinished comment.
+- `components/AnnotationPanel.tsx` owns the mounted review surface. Target identity
+ controls remounting; refreshed scores and table changes preserve the draft.
+ Panel visibility derives from the active target. Loading table data does not
+ unmount the panel.
+- The shared comments composer owns editable comment content and reports draft
+ presence through `onDraftChange`. Reports are scoped to their target so a late
+ callback cannot change another target's guard.
+- `components/NewDatasetItemForm.tsx` prepares initial schema examples before
+ mounting editable fields. Later dataset selections fill only empty, unedited
+ fields; metadata refetches and obsolete generation results preserve drafts.
+
+## Structure and consumers
+
+`components/` contains dataset forms, tables, and review views. `hooks/` owns
+query adapters; `contexts/` owns compare-view state; `lib/` and `utils/` contain
+data preparation. `server/` owns tRPC procedures and persistence workflows.
+Trace, observation, and session surfaces reuse the dataset-item dialog. Only the
+legacy compare page and its cells consume the active-cell provider.
+
+## Stability and migration
+
+The provider exposes a stable store; reporting a comment draft does not rerender
+comparison cells. All close paths, including the mobile sheet, use the same
+guarded clear action. Query-string changes do not reset review state.
+
+Run selection workflows remain in the compare page as a separate migration
+boundary. Follow `frontend-large-feature-architecture` and
+`refactor-react-effects` before extending their state ownership.
diff --git a/web/src/features/datasets/components/AnnotationPanel.clienttest.tsx b/web/src/features/datasets/components/AnnotationPanel.clienttest.tsx
new file mode 100644
index 0000000000..ce3991044d
--- /dev/null
+++ b/web/src/features/datasets/components/AnnotationPanel.clienttest.tsx
@@ -0,0 +1,150 @@
+import { fireEvent, render, screen } from "@testing-library/react";
+import { type ReactNode, useState } from "react";
+import { vi } from "vitest";
+import { AnnotationPanel } from "./AnnotationPanel";
+import {
+ ActiveCellProvider,
+ useActiveCell,
+} from "../contexts/ActiveCellContext";
+
+vi.mock("sonner", () => ({ toast: { error: vi.fn() } }));
+vi.mock("@/src/components/useSessionStorage", () => ({
+ default: function useSessionStorage(_key: string, initialValue: number) {
+ return useState(initialValue);
+ },
+}));
+vi.mock("@/src/components/ui/resizable", () => ({
+ ResizablePanelGroup: ({ children }: { children: ReactNode }) => children,
+ ResizablePanel: ({ children }: { children: ReactNode }) => children,
+ ResizableHandle: () => <hr />,
+}));
+vi.mock("@/src/features/scores", () => ({
+ AnnotationForm: ({ actionButtons }: { actionButtons: ReactNode }) =>
+ actionButtons,
+ decomposeAggregateScoreKey: () => ({ source: "ANNOTATION" }),
+}));
+vi.mock(
+ "@/src/features/annotation-queues/components/shared/CommentsSection",
+ () => ({
+ CommentsSection: ({
+ onDraftChange,
+ }: {
+ onDraftChange: (hasDraft: boolean) => void;
+ }) => {
+ const [draft, setDraft] = useState("");
+ return (
+ <textarea
+ aria-label="Comment draft"
+ value={draft}
+ onChange={(event) => {
+ setDraft(event.target.value);
+ onDraftChange(Boolean(event.target.value));
+ }}
+ />
+ );
+ },
+ }),
+);
+
+function ReviewHarness() {
+ const { activeCell, setActiveCell, clearActiveCell, closeRunAnnotation } =
+ useActiveCell();
+ const [runs, setRuns] = useState(["first", "second"]);
+ return (
+ <>
+ {["first", "second"].map((traceId) => (
+ <button
+ key={traceId}
+ onClick={() =>
+ setActiveCell({
+ traceId,
+ datasetRunId: traceId,
+ scoreAggregates: {},
+ })
+ }
+ >
+ Review {traceId}
+ </button>
+ ))}
+ {["first", "second"].map((runId) => (
+ <button
+ key={runId}
+ onClick={() => {
+ if (closeRunAnnotation?.(runId))
+ setRuns((current) => current.filter((id) => id !== runId));
+ }}
+ >
+ Remove {runId}
+ </button>
+ ))}
+ <output aria-label="Compared runs">{runs.join(",")}</output>
+ <button onClick={clearActiveCell}>Close side panel</button>
+ <span data-testid="active-cell">{activeCell?.traceId ?? "closed"}</span>
+ {activeCell && <AnnotationPanel projectId="project" />}
+ </>
+ );
+}
+
+it("guards every cell switch and close while preserving the current draft on rerenders", () => {
+ const { rerender } = render(
+ <ActiveCellProvider>
+ <ReviewHarness />
+ </ActiveCellProvider>,
+ );
+ fireEvent.click(screen.getByRole("button", { name: "Review first" }));
+ fireEvent.change(screen.getByLabelText("Comment draft"), {
+ target: { value: "Unfinished review" },
+ });
+
+ fireEvent.click(screen.getByRole("button", { name: "Review second" }));
+ expect(screen.getByTestId("active-cell")).toHaveTextContent("first");
+ fireEvent.click(screen.getByRole("button", { name: "Close side panel" }));
+ expect(screen.getByTestId("active-cell")).toHaveTextContent("first");
+ fireEvent.click(
+ screen.getByRole("button", { name: "Close annotation panel" }),
+ );
+ expect(screen.getByTestId("active-cell")).toHaveTextContent("first");
+ fireEvent.click(screen.getByRole("button", { name: "Review first" }));
+ rerender(
+ <ActiveCellProvider>
+ <ReviewHarness />
+ </ActiveCellProvider>,
+ );
+ expect(screen.getByLabelText("Comment draft")).toHaveValue(
+ "Unfinished review",
+ );
+
+ fireEvent.change(screen.getByLabelText("Comment draft"), {
+ target: { value: "" },
+ });
+ fireEvent.click(screen.getByRole("button", { name: "Review second" }));
+ expect(screen.getByTestId("active-cell")).toHaveTextContent("second");
+ fireEvent.click(screen.getByRole("button", { name: "Close side panel" }));
+ expect(screen.getByTestId("active-cell")).toHaveTextContent("closed");
+});
+
+it("guards removal of the reviewed run and leaves unrelated run changes alone", () => {
+ render(
+ <ActiveCellProvider>
+ <ReviewHarness />
+ </ActiveCellProvider>,
+ );
+ fireEvent.click(screen.getByRole("button", { name: "Review first" }));
+ fireEvent.change(screen.getByLabelText("Comment draft"), {
+ target: { value: "Keep this comment" },
+ });
+ fireEvent.click(screen.getByRole("button", { name: "Remove second" }));
+ expect(screen.getByLabelText("Compared runs")).toHaveTextContent(/^first$/);
+ expect(screen.getByLabelText("Comment draft")).toHaveValue(
+ "Keep this comment",
+ );
+ fireEvent.click(screen.getByRole("button", { name: "Remove first" }));
+ expect(screen.getByLabelText("Compared runs")).toHaveTextContent(/^first$/);
+ expect(screen.getByTestId("active-cell")).toHaveTextContent("first");
+ fireEvent.change(screen.getByLabelText("Comment draft"), {
+ target: { value: "" },
+ });
+ fireEvent.click(screen.getByRole("button", { name: "Remove first" }));
+ expect(screen.getByLabelText("Compared runs")).toBeEmptyDOMElement();
+ expect(screen.getByTestId("active-cell")).toHaveTextContent("closed");
+});
diff --git a/web/src/features/datasets/components/AnnotationPanel.tsx b/web/src/features/datasets/components/AnnotationPanel.tsx
index 77bc138c13..28948fd498 100644
--- a/web/src/features/datasets/components/AnnotationPanel.tsx
+++ b/web/src/features/datasets/components/AnnotationPanel.tsx
@@ -13,22 +13,48 @@ import {
decomposeAggregateScoreKey,
} from "@/src/features/scores";
import { ChevronRight } from "lucide-react";
-import { useState } from "react";
-import { toast } from "sonner";
+import { useCallback } from "react";
export const AnnotationPanel = ({ projectId }: { projectId: string }) => {
- const [hasCommentDraft, setHasCommentDraft] = useState(false);
- const { activeCell, clearActiveCell } = useActiveCell();
+ const { activeCell, clearActiveCell, setCommentDraft } = useActiveCell();
+ if (!activeCell) return <Skeleton className="h-full w-full" />;
+ return (
+ <ActiveAnnotationPanel
+ key={JSON.stringify([
+ projectId,
+ activeCell.traceId,
+ activeCell.observationId,
+ ])}
+ projectId={projectId}
+ activeCell={activeCell}
+ clearActiveCell={clearActiveCell}
+ setCommentDraft={setCommentDraft}
+ />
+ );
+};
+function ActiveAnnotationPanel({
+ projectId,
+ activeCell,
+ clearActiveCell,
+ setCommentDraft,
+}: {
+ projectId: string;
+ activeCell: NonNullable<ReturnType<typeof useActiveCell>["activeCell"]>;
+ clearActiveCell: ReturnType<typeof useActiveCell>["clearActiveCell"];
+ setCommentDraft: ReturnType<typeof useActiveCell>["setCommentDraft"];
+}) {
+ const { traceId, observationId } = activeCell;
+ const handleDraftChange = useCallback(
+ (hasDraft: boolean) =>
+ setCommentDraft({ traceId, observationId }, hasDraft),
+ [traceId, observationId, setCommentDraft],
+ );
const [verticalSize, setVerticalSize] = useSessionStorage(
`annotationQueueDrawerVertical-compare-${projectId}`,
60,
);
- if (!activeCell) {
- return <Skeleton className="h-full w-full" />;
- }
-
const hasNonAnnotationScores = Object.keys(activeCell.scoreAggregates).some(
(key) => {
const { source } = decomposeAggregateScoreKey(key);
@@ -47,7 +73,7 @@ export const AnnotationPanel = ({ projectId }: { projectId: string }) => {
>
<ResizablePanel
id="annotation-top"
- className="w-full overflow-y-auto p-2"
+ className="w-full overflow-y-auto p-2 [--annotation-surface:var(--modal)] md:[--annotation-surface:var(--background)]"
minSize="30%"
defaultSize={`${verticalSize}%`}
>
@@ -64,6 +90,7 @@ export const AnnotationPanel = ({ projectId }: { projectId: string }) => {
analyticsData={{
type: "trace",
source: "DatasetCompare",
+ isV4: false,
}}
scoreMetadata={{
projectId,
@@ -73,13 +100,8 @@ export const AnnotationPanel = ({ projectId }: { projectId: string }) => {
<Button
variant="outline"
size="icon"
- onClick={() => {
- if (hasCommentDraft)
- toast.error(
- "Please save or discard your comment before proceeding",
- );
- else clearActiveCell();
- }}
+ aria-label="Close annotation panel"
+ onClick={clearActiveCell}
>
<ChevronRight className="h-4 w-4" />
</Button>
@@ -102,11 +124,9 @@ export const AnnotationPanel = ({ projectId }: { projectId: string }) => {
projectId={projectId}
objectId={activeCell.observationId ?? activeCell.traceId}
objectType={activeCell.observationId ? "OBSERVATION" : "TRACE"}
- onDraftChange={(draft) => {
- setHasCommentDraft(draft);
- }}
+ onDraftChange={handleDraftChange}
/>
</ResizablePanel>
</ResizablePanelGroup>
);
-};
+}
diff --git a/web/src/features/datasets/components/DatasetAggregateTableCell.tsx b/web/src/features/datasets/components/DatasetAggregateTableCell.tsx
index 5fc8969005..1bb14ea526 100644
--- a/web/src/features/datasets/components/DatasetAggregateTableCell.tsx
+++ b/web/src/features/datasets/components/DatasetAggregateTableCell.tsx
@@ -142,12 +142,22 @@ const DatasetAggregateCellContent = ({
};
const handleOpenReview = () => {
- setActiveCell({
+ const opened = setActiveCell({
+ datasetRunId: value.datasetRunId,
traceId: value.trace.id,
observationId: value.observation?.id,
scoreAggregates: scores,
environment: data?.environment,
});
+ if (opened && !isActiveCell) {
+ capture("annotation:entry_click", {
+ type: "trace",
+ entryPoint: "annotate_button",
+ source: "DatasetCompare",
+ targetType: value.observation ? "observation" : "trace",
+ isV4: false,
+ });
+ }
};
const isActiveCell =
diff --git a/web/src/features/datasets/contexts/ActiveCellContext.tsx b/web/src/features/datasets/contexts/ActiveCellContext.tsx
index 429f7c3904..467701989f 100644
--- a/web/src/features/datasets/contexts/ActiveCellContext.tsx
+++ b/web/src/features/datasets/contexts/ActiveCellContext.tsx
@@ -1,63 +1,84 @@
import { type ScoreAggregate } from "@langfuse/shared";
-import {
- createContext,
- useContext,
- useState,
- useCallback,
- useMemo,
- type ReactNode,
-} from "react";
+import { createContext, useContext, useState, type ReactNode } from "react";
+import { toast } from "sonner";
+import { useStore } from "zustand";
+import { createStore } from "zustand/vanilla";
type ActiveCell = {
+ datasetRunId: string;
traceId: string;
observationId?: string;
scoreAggregates: ScoreAggregate;
environment?: string;
};
-type ActiveCellContextValue = {
+type CellTarget = Pick<ActiveCell, "traceId" | "observationId">;
+
+type ActiveCellState = {
activeCell: ActiveCell | null;
- setActiveCell: (cell: ActiveCell | null) => void;
- clearActiveCell: () => void;
+ hasCommentDraft: boolean;
+ actions: {
+ setActiveCell: (cell: ActiveCell | null) => boolean;
+ clearActiveCell: () => boolean;
+ closeRunAnnotation: (datasetRunId: string) => boolean;
+ setCommentDraft: (target: CellTarget, hasDraft: boolean) => void;
+ };
};
-/**
- * Tracks active dataset run item cell for annotation UI.
- *
- * Single cell can be active at a time. Used for cell highlighting
- * and side panel state management.
- */
-const ActiveCellContext = createContext<ActiveCellContextValue | undefined>(
- undefined,
-);
-
-export function ActiveCellProvider({ children }: { children: ReactNode }) {
- const [activeCell, setActiveCellState] = useState<ActiveCell | null>(null);
+const isSameTarget = (left: CellTarget | null, right: CellTarget | null) =>
+ left?.traceId === right?.traceId &&
+ left?.observationId === right?.observationId;
- const setActiveCell = useCallback((cell: ActiveCell | null) => {
- setActiveCellState(cell);
- }, []);
+function createActiveCellStore() {
+ return createStore<ActiveCellState>((set, get) => ({
+ activeCell: null,
+ hasCommentDraft: false,
+ actions: {
+ setActiveCell: (cell) => {
+ const { activeCell, hasCommentDraft } = get();
+ const sameTarget = isSameTarget(activeCell, cell);
+ if (!sameTarget && hasCommentDraft) {
+ toast.error("Please save or discard your comment before proceeding");
+ return false;
+ }
+ set({
+ activeCell: cell,
+ hasCommentDraft: sameTarget && hasCommentDraft,
+ });
+ return true;
+ },
+ clearActiveCell: () => get().actions.setActiveCell(null),
+ closeRunAnnotation: (datasetRunId) =>
+ get().activeCell?.datasetRunId !== datasetRunId ||
+ get().actions.clearActiveCell(),
+ setCommentDraft: (target, hasDraft) => {
+ if (isSameTarget(get().activeCell, target)) {
+ set({ hasCommentDraft: hasDraft });
+ }
+ },
+ },
+ }));
+}
- const clearActiveCell = useCallback(() => {
- setActiveCellState(null);
- }, []);
+const ActiveCellContext = createContext<
+ ReturnType<typeof createActiveCellStore> | undefined
+>(undefined);
- const value = useMemo(
- () => ({ activeCell, setActiveCell, clearActiveCell }),
- [activeCell, setActiveCell, clearActiveCell],
- );
+export function ActiveCellProvider({ children }: { children: ReactNode }) {
+ const [store] = useState(createActiveCellStore);
return (
- <ActiveCellContext.Provider value={value}>
+ <ActiveCellContext.Provider value={store}>
{children}
</ActiveCellContext.Provider>
);
}
export function useActiveCell() {
- const context = useContext(ActiveCellContext);
- if (!context) {
+ const store = useContext(ActiveCellContext);
+ if (!store) {
throw new Error("useActiveCell must be used within ActiveCellProvider");
}
- return context;
+ const activeCell = useStore(store, (state) => state.activeCell);
+ return { activeCell, ...store.getState().actions };
}
diff --git a/web/src/features/events/hooks/useEventsTableData.ts b/web/src/features/events/hooks/useEventsTableData.ts
index e459b43acf..654363af76 100644
--- a/web/src/features/events/hooks/useEventsTableData.ts
+++ b/web/src/features/events/hooks/useEventsTableData.ts
@@ -8,6 +8,7 @@ import {
} from "@langfuse/shared";
import { type FullEventsObservations } from "@langfuse/shared/src/server";
import { showSuccessToast } from "@/src/features/notifications";
+import { usePostHogClientCapture } from "@/src/features/posthog-analytics";
import { joinTableCoreAndMetrics } from "@/src/components/table/utils/joinTableCoreAndMetrics";
import { usePendingRowIds } from "@/src/components/table/hooks/usePendingRowIds";
import {
@@ -71,6 +72,7 @@ export function useEventsTableData({
rowsEnabled = true,
ioCharLimit,
}: UseEventsTableDataParams) {
+ const capture = usePostHogClientCapture();
// Prepare query payloads
const getCountPayload = useMemo(
() => ({
@@ -265,7 +267,18 @@ export function useEventsTableData({
// Add to queue mutation
const addToQueueMutation = api.annotationQueueItems.createMany.useMutation({
- onSuccess: (data) => {
+ onSuccess: (data, variables) => {
+ if (variables.isBatchAction || data.createdCount > 0) {
+ capture("annotation_queues:item_added", {
+ type: "trace",
+ source: "ObservationTable",
+ targetType: "observation",
+ objectType: "OBSERVATION",
+ queueCount: 1,
+ isV4: true,
+ ...(variables.isBatchAction ? {} : { itemCount: data.createdCount }),
+ });
+ }
showSuccessToast({
title: "Observations added to queue",
description: `Selected observations will be added to queue "${data.queueName}". This may take a minute.`,
diff --git a/web/src/features/posthog-analytics/usePostHogClientCapture.ts b/web/src/features/posthog-analytics/usePostHogClientCapture.ts
index ab221d797f..33cc7abd4d 100644
--- a/web/src/features/posthog-analytics/usePostHogClientCapture.ts
+++ b/web/src/features/posthog-analytics/usePostHogClientCapture.ts
@@ -1,6 +1,7 @@
import { type CaptureResult, type CaptureOptions } from "posthog-js";
import { usePostHog } from "posthog-js/react";
import { useCallback } from "react";
+import type { AnnotationEventMap } from "@/src/features/scores/lib/annotationAnalytics";
export const V4_BETA_ENABLED_POSTHOG_PROPERTY = "v4BetaEnabled";
@@ -138,7 +139,13 @@ const events = {
"create_form_open",
"update_comment",
"delete_comment",
+ "form_abandoned",
+ "value_set",
+ "level_changed",
+ "level_added",
],
+ annotation: ["entry_click"],
+ annotation_queues: ["item_added", "item_removed", "manage_click"],
score_configs: [
"create_form_submit",
"update_form_submit",
@@ -467,15 +474,22 @@ type EventName = {
[Resource in keyof typeof events]: `${Resource}:${(typeof events)[Resource][number]}`;
}[keyof typeof events];
+type EventProperties = AnnotationEventMap & {
+ [E in Exclude<EventName, keyof AnnotationEventMap>]: Record<
+ string,
+ any
+ > | null;
+};
+
export const usePostHogClientCapture = () => {
const posthog = usePostHog();
// wrapped posthog.capture function that only allows events that are in the
// allowlist; stable identity so it is safe in useCallback/useMemo deps
return useCallback(
- function capture(
- eventName: EventName,
- properties?: Record<string, any> | null,
+ function capture<E extends EventName>(
+ eventName: E,
+ properties?: EventProperties[E],
options?: CaptureOptions,
): CaptureResult | void {
return posthog.capture(eventName, properties, options);
diff --git a/web/src/features/scores/README.md b/web/src/features/scores/README.md
index faea719e2c..3a05cc875a 100644
--- a/web/src/features/scores/README.md
+++ b/web/src/features/scores/README.md
@@ -2,6 +2,75 @@
Score rendering, annotation forms, mutations, and the project Scores search bar.
+## Annotation ownership
+
+`AnnotationForm` mounts single-target forms; `DualAnnotationContent` combines a
+trace and observation. Trace/session detail, annotation queues, and dataset
+comparison supply their targets and scores. The adapters prepare data and gate
+the form on score-config readiness. Target identity keys the mounted form;
+refetches of the same target preserve its draft.
+
+| Owner | Responsibility |
+| -------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `components/AnnotationForm.tsx` | Loaded-data boundary, form lifetime, selected row structure, header and add-score composition. |
+| `lib/prepareAnnotationFormData.ts` | Pure construction of initial scored and remembered empty fields. |
+| React Hook Form | The mounted form's draft values and validation. Each `AnnotationScoreRow` subscribes to its own row and provides that scoped form state to input primitives. |
+| `actions/annotationFormActions.ts` | Create/update/clear/comment workflows, current draft reads, target-specific analytics, optimistic form updates, and operation-owned rollback. |
+| `state/annotationSaveStore.ts` | Per-mount pending/error/confirmed-save state. `AnnotationSaveStatus` subscribes independently of the header; row actions subscribe only to whether a save is pending. Drafts are not duplicated in this store. |
+| `hooks/useScoreMutations.ts` | tRPC mutation bridge and optimistic shared score-cache writes/rollback. Server/query state remains in tRPC. |
+| `lib/annotationConfigSelection.ts`, `hooks/useScoreConfigs.ts` | Config query data, explicit selection changes, and remembered empty-config choices for each target type. |
+| `components/AnnotationScoreRow.tsx` | Value-control presentation, mount-local score-comment draft, and DOM focus deferral while opening its clear menu. |
+| `hooks/useAnnotationKeyboard.ts` | Browser keyboard integration, reading current rows only when a key is pressed. |
+
+Actions resolve a row by target + config both at invocation and async completion.
+The client-only target key never enters score API payloads. Mutation promise
+completion owns form rollback, since a later mutation replaces per-call observer
+callbacks. Each form owns one action instance and one save store; unrelated forms
+cannot share drafts or save status. Existing score cache scope and API payloads
+remain owned by the mutation bridge.
+
+The Add score picker lists each available config once and excludes fields already in the form. Initial preparation
+keeps every saved score and places remembered empty fields on the observation by
+default. Draft-only level changes and adding a blank field for the other level
+belong to the form actions; neither changes an existing score's attachment.
+Saved scores, pending writes, invalid drafts, and occupied destinations prevent a
+level change. Single-target and fixed queue forms retain their supplied targets.
+The row menu removes only empty fields and clears their remembered choices so
+hidden preferences cannot restore them on the next open. Saved and nonempty
+fields keep Clear score; pending clears cannot remove their row before rollback
+is settled. Fixed queue fields cannot be removed. The picker shows configs
+without level badges. Mixed-level forms show badges beside score fields;
+each row's menu always identifies its current level.
+
+Typing updates the owning row and save-status subscriber. It must not rebuild the
+header, picker, other rows, or keyboard listeners. Changing field selection may
+rerender the row structure. Score-comment popovers seed drafts on mount; closing
+discards that local draft. Clear cancels deferred autosave, while dismissing the
+clear menu commits the deferred edit. Numeric validation reads native `badInput`
+on every input event before interpreting an empty value as a deliberate clear.
+
+## Effect inventory
+
+| Location | External system / lifecycle | Status |
+| ---------------------------------------- | ----------------------------------------------------------------------------------- | ---------------------------------------------------------------------------- |
+| `AnnotationFormContent` | Analytics session opens on mount and closes on unmount, using current field counts. | Retained; stable action-owner dependency. |
+| `useAnnotationKeyboard` keydown listener | Window keyboard events for row navigation and score selection. | Retained; symmetric registration/cleanup, current values read at event time. |
+| `useAnnotationKeyboard` capture listener | Escape focus handling before the parent drawer's capture listener. | Retained; symmetric registration/cleanup. |
+| Score-comment editor | No external system: draft was mirrored from `savedComment`. | Removed; popover mount seeds the editor. |
+
+No other direct effects exist in this feature. Query, cache, and local-storage
+hooks keep their existing external ownership. The shared `ScoreCacheProvider`
+still exposes changing cache snapshots to its consumers; narrowing subscriptions
+there is a separate migration across score-rendering surfaces. Table column and
+aggregate hooks are outside the annotation form's lifetime.
+
+Before extending this boundary, read the `frontend-large-feature-architecture`,
+`refactor-react-effects`, and `posthog-instrumentation` skills. Keep new workflows
+in actions and pure preparation in `lib`; do not add a query-to-draft sync effect
+or a second store for the same input value.
+
+## Scores search
+
- `components/ScoresSearchBar.tsx` owns the shared composer's lifecycle. It
edits the table's existing sidebar filter state; the only local state is the
shared editor's draft. It uses the table's filter-option response.
diff --git a/web/src/features/scores/actions/annotationFormActions.ts b/web/src/features/scores/actions/annotationFormActions.ts
new file mode 100644
index 0000000000..026c8b51a9
--- /dev/null
+++ b/web/src/features/scores/actions/annotationFormActions.ts
@@ -0,0 +1,470 @@
+import type {
+ UseFieldArrayInsert,
+ UseFieldArrayUpdate,
+ UseFormReturn,
+} from "react-hook-form";
+import { v4 as uuid } from "uuid";
+import {
+ isPresent,
+ type CreateAnnotationScoreData,
+ type UpdateAnnotationScoreData,
+} from "@langfuse/shared";
+import type { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
+import type {
+ AnnotateFormSchemaType,
+ AnnotationScoreFormData,
+ PreparedAnnotationTarget,
+} from "@/src/features/scores/types";
+import { shouldUseCombobox } from "@/src/features/scores/components/CategoricalScoreInput";
+import {
+ createAnnotationAnalytics,
+ getAnnotationTargetType,
+} from "@/src/features/scores/lib/annotationAnalytics";
+import { annotationFieldKey } from "@/src/features/scores/lib/annotationConfigSelection";
+import {
+ enrichCategoryOptionsWithStaleScoreValue,
+ resolveCategoricalNumericValue,
+ validateNumericScore,
+} from "@/src/features/scores/lib/annotationFormHelpers";
+import {
+ isNumericDataType,
+ isTextDataType,
+} from "@/src/features/scores/lib/helpers";
+import {
+ createAnnotationSaveStore,
+ hasChangedAnnotationValue,
+} from "@/src/features/scores/state/annotationSaveStore";
+
+type Dependencies = {
+ form: UseFormReturn<AnnotateFormSchemaType>;
+ replaceField: UseFieldArrayUpdate<AnnotateFormSchemaType, "scoreData">;
+ insertField: UseFieldArrayInsert<AnnotateFormSchemaType, "scoreData">;
+ initialTargets: PreparedAnnotationTarget[];
+ capture: ReturnType<typeof usePostHogClientCapture>;
+ createScore: (data: CreateAnnotationScoreData) => Promise<unknown>;
+ updateScore: (data: UpdateAnnotationScoreData) => Promise<unknown>;
+ deleteScore: (data: { id: string; projectId: string }) => Promise<unknown>;
+};
+
+// One owner per mounted form. Actions read drafts at invocation and resolve rows
+// by target + config again when asynchronous operations finish.
+export function createAnnotationFormActions({
+ form,
+ replaceField,
+ insertField,
+ initialTargets,
+ capture,
+ createScore,
+ updateScore,
+ deleteScore,
+}: Dependencies) {
+ const fields = (): AnnotationScoreFormData[] =>
+ form
+ .getValues("scoreData")
+ .map((field) => ({ ...field, id: field.id ?? null }));
+ const indexOf = (key: string) =>
+ fields().findIndex((field) => annotationFieldKey(field) === key);
+ const find = (key: string) =>
+ fields().find((field) => annotationFieldKey(field) === key);
+ const saveStore = createAnnotationSaveStore(fields());
+ const analytics = new Map(
+ initialTargets.map((target) => [
+ target.key,
+ createAnnotationAnalytics(
+ capture,
+ {
+ ...target.analyticsData,
+ targetType: getAnnotationTargetType(target.scoreTarget),
+ },
+ target.initialFormData,
+ ),
+ ]),
+ );
+ let sequence = 0;
+ const latestSaves = new Map<string, number>();
+ const pendingFields = new Map<string, number>();
+ const failedFields = new Set<string>();
+
+ const finishPendingField = (key: string) => {
+ const remaining = (pendingFields.get(key) ?? 1) - 1;
+ if (remaining) pendingFields.set(key, remaining);
+ else pendingFields.delete(key);
+ };
+
+ const beginSave = (
+ kind: Parameters<
+ ReturnType<typeof createAnnotationAnalytics>["beginSave"]
+ >[0],
+ next: AnnotationScoreFormData,
+ control?: Parameters<
+ ReturnType<typeof createAnnotationAnalytics>["beginSave"]
+ >[3],
+ ) =>
+ analytics.get(next.targetKey!)!.beginSave(
+ kind,
+ next,
+ fields()
+ .filter((field) => field.targetKey === next.targetKey)
+ .map((field) =>
+ annotationFieldKey(field) === annotationFieldKey(next) ? next : field,
+ ),
+ control,
+ );
+
+ const trackSave = (
+ operation: Promise<unknown>,
+ tracked: ReturnType<typeof beginSave>,
+ field: AnnotationScoreFormData,
+ onFailure: () => void,
+ ) => {
+ const operationSequence = ++sequence;
+ const key = annotationFieldKey(field);
+ const confirmed = saveStore.getState().confirmedFields.get(key)?.field;
+ const changed =
+ (field.id ?? null) !== (confirmed?.id ?? null) ||
+ hasChangedAnnotationValue(field, confirmed);
+ latestSaves.set(key, operationSequence);
+ pendingFields.set(key, (pendingFields.get(key) ?? 0) + 1);
+ failedFields.delete(key);
+ saveStore.setState((state) => ({
+ pending: state.pending + 1,
+ failed: failedFields.size > 0,
+ }));
+ // Promise-owned completion survives overlapping mutation observers.
+ operation.then(
+ () => {
+ finishPendingField(key);
+ saveStore.setState((state) => {
+ const confirmedFields = new Map(state.confirmedFields);
+ if ((confirmedFields.get(key)?.sequence ?? 0) <= operationSequence)
+ confirmedFields.set(key, {
+ field: { ...field },
+ sequence: operationSequence,
+ });
+ return {
+ confirmedFields,
+ pending: state.pending - 1,
+ saved: state.saved || changed,
+ };
+ });
+ tracked?.success();
+ },
+ () => {
+ finishPendingField(key);
+ const isLatest = latestSaves.get(key) === operationSequence;
+ if (isLatest) {
+ failedFields.add(key);
+ onFailure();
+ }
+ tracked?.failure();
+ saveStore.setState((state) => ({
+ pending: state.pending - 1,
+ failed: failedFields.size > 0,
+ }));
+ },
+ );
+ };
+
+ const reportFailure = (field: AnnotationScoreFormData, message: string) => {
+ const index = indexOf(annotationFieldKey(field));
+ if (index < 0) return;
+ form.setError(
+ `scoreData.${index}.${isNumericDataType(field.dataType) ? "value" : "stringValue"}`,
+ { type: "server", message },
+ );
+ };
+
+ const clear = (key: string, target: PreparedAnnotationTarget) => {
+ const field = find(key);
+ if (!field) return;
+ const cleared = {
+ ...field,
+ id: null,
+ value: null,
+ stringValue: null,
+ comment: null,
+ timestamp: null,
+ };
+ const tracked = field.id ? beginSave("delete", cleared) : undefined;
+ const index = indexOf(key);
+ form.clearErrors(`scoreData.${index}`);
+ replaceField(index, cleared);
+ if (field.id)
+ trackSave(
+ deleteScore({
+ id: field.id,
+ projectId: target.scoreMetadata.projectId,
+ }),
+ tracked,
+ cleared,
+ () => {
+ const currentIndex = indexOf(key);
+ if (currentIndex < 0) return;
+ replaceField(currentIndex, field);
+ reportFailure(field, "Failed to clear score");
+ },
+ );
+ };
+
+ const upsert = (
+ key: string,
+ target: PreparedAnnotationTarget,
+ value: number | null,
+ stringValue: string | null,
+ ) => {
+ const field = find(key);
+ if (!field) return;
+ const config = target.configControl.configs.find(
+ (config) => config.id === field.configId,
+ );
+ const next = {
+ ...field,
+ value,
+ stringValue,
+ id: field.id ?? uuid(),
+ timestamp: field.timestamp ?? new Date(),
+ };
+ const categories = enrichCategoryOptionsWithStaleScoreValue(
+ config?.categories ?? [],
+ field.stringValue,
+ );
+ let control: "text" | "number" | "select" | "segmented" = "segmented";
+ if (isTextDataType(field.dataType)) control = "text";
+ else if (isNumericDataType(field.dataType)) control = "number";
+ else if (shouldUseCombobox(categories)) control = "select";
+ const tracked = beginSave(field.id ? "update" : "create", next, {
+ control,
+ optionCount: config?.categories?.length ?? 0,
+ });
+ const index = indexOf(key);
+ form.clearErrors([
+ `scoreData.${index}.value`,
+ `scoreData.${index}.stringValue`,
+ ]);
+ form.setValue(`scoreData.${index}.value`, value);
+ form.setValue(`scoreData.${index}.stringValue`, stringValue);
+ const { id, timestamp, targetKey: _targetKey, ...score } = field;
+ const data = {
+ ...score,
+ ...target.scoreMetadata,
+ value,
+ stringValue,
+ scoreTarget: target.scoreTarget,
+ };
+ if (!id) {
+ form.setValue(`scoreData.${index}.id`, next.id);
+ form.setValue(`scoreData.${index}.timestamp`, next.timestamp);
+ }
+ trackSave(
+ id
+ ? updateScore({
+ ...data,
+ id,
+ timestamp: timestamp ?? undefined,
+ } as UpdateAnnotationScoreData)
+ : createScore({
+ ...data,
+ id: next.id,
+ timestamp: next.timestamp,
+ } as CreateAnnotationScoreData),
+ tracked,
+ next,
+ () => {
+ const currentIndex = indexOf(key);
+ if (currentIndex < 0) return;
+ form.setValue(`scoreData.${currentIndex}.value`, field.value);
+ form.setValue(
+ `scoreData.${currentIndex}.stringValue`,
+ field.stringValue,
+ );
+ if (!id) {
+ form.setValue(`scoreData.${currentIndex}.id`, field.id);
+ form.setValue(`scoreData.${currentIndex}.timestamp`, field.timestamp);
+ }
+ reportFailure(
+ field,
+ id ? "Failed to update score" : "Failed to create score",
+ );
+ },
+ );
+ };
+
+ const validateNumericInput = (
+ key: string,
+ target: PreparedAnnotationTarget,
+ input: HTMLInputElement,
+ ) => {
+ const field = find(key);
+ const config = target.configControl.configs.find(
+ (config) => config.id === field?.configId,
+ );
+ if (!field || !config) return undefined;
+ const value = input.value === "" ? null : input.valueAsNumber;
+ const error = validateNumericScore({
+ value,
+ badInput: input.validity.badInput,
+ minValue: config.minValue,
+ maxValue: config.maxValue,
+ });
+ const index = indexOf(key);
+ const name = `scoreData.${index}.value` as const;
+ const currentError = form.getFieldState(name).error;
+ if (error) {
+ if (currentError?.message !== error)
+ form.setError(name, { type: "validate", message: error });
+ return undefined;
+ }
+ if (currentError) form.clearErrors(name);
+ return value;
+ };
+
+ return {
+ saveStore,
+ indexOf,
+ isSaving: (key: string) => pendingFields.has(key),
+ clear,
+ validateNumericInput,
+ addDraftTarget(key: string, destination: PreparedAnnotationTarget) {
+ const field = find(key);
+ const config = destination.configControl.configs.find(
+ (config) => config.id === field?.configId,
+ );
+ if (
+ !field ||
+ !destination.configControl.allowManualSelection ||
+ !config ||
+ config.isArchived
+ )
+ return undefined;
+ const next = {
+ ...field,
+ targetKey: destination.key,
+ id: null,
+ timestamp: null,
+ value: null,
+ stringValue: null,
+ comment: null,
+ };
+ const nextKey = annotationFieldKey(next);
+ if (find(nextKey)) return nextKey;
+ insertField(indexOf(key) + 1, next, { shouldFocus: false });
+ capture("score:level_added", {
+ ...destination.analyticsData,
+ targetType: getAnnotationTargetType(destination.scoreTarget),
+ dataType: field.dataType,
+ });
+ return nextKey;
+ },
+ changeDraftTarget(key: string, destination: PreparedAnnotationTarget) {
+ const field = find(key);
+ if (!field || field.id || saveStore.getState().pending) return undefined;
+ const source = initialTargets.find(
+ (target) => target.key === field.targetKey,
+ );
+ const config = destination.configControl.configs.find(
+ (config) => config.id === field.configId,
+ );
+ const next = { ...field, targetKey: destination.key };
+ const index = indexOf(key);
+ if (
+ !source?.configControl.allowManualSelection ||
+ !destination.configControl.allowManualSelection ||
+ !config ||
+ config.isArchived ||
+ find(annotationFieldKey(next)) ||
+ form.getFieldState(`scoreData.${index}.value`).invalid ||
+ form.getFieldState(`scoreData.${index}.stringValue`).invalid
+ )
+ return undefined;
+ replaceField(index, next);
+ capture("score:level_changed", {
+ ...source.analyticsData,
+ targetType: getAnnotationTargetType(destination.scoreTarget),
+ previousTargetType: getAnnotationTargetType(source.scoreTarget),
+ dataType: field.dataType,
+ });
+ return annotationFieldKey(next);
+ },
+ open() {
+ analytics.forEach((tracker) => tracker.open());
+ },
+ close() {
+ analytics.forEach((tracker, key) =>
+ tracker.close(fields().filter((field) => field.targetKey === key)),
+ );
+ },
+ saveNumeric(
+ key: string,
+ target: PreparedAnnotationTarget,
+ input: HTMLInputElement,
+ ) {
+ const value = validateNumericInput(key, target, input);
+ if (value === undefined) return;
+ if (value === null) {
+ if (find(key)?.id) clear(key, target);
+ } else upsert(key, target, value, null);
+ },
+ saveText(key: string, target: PreparedAnnotationTarget) {
+ const field = find(key);
+ if (!field) return;
+ if (field.stringValue) upsert(key, target, 0, field.stringValue);
+ else if (field.id) clear(key, target);
+ },
+ saveCategory(
+ key: string,
+ target: PreparedAnnotationTarget,
+ stringValue: string,
+ numericValue?: number,
+ ) {
+ const field = find(key);
+ const config = target.configControl.configs.find(
+ (config) => config.id === field?.configId,
+ );
+ if (!config) return;
+ const value = resolveCategoricalNumericValue({
+ categories: config.categories,
+ stringValue,
+ numericValue,
+ });
+ if (isPresent(value)) upsert(key, target, value, stringValue);
+ },
+ saveComment(
+ key: string,
+ target: PreparedAnnotationTarget,
+ comment: string | null,
+ ) {
+ const field = find(key);
+ if (!field?.id) return;
+ const next = { ...field, comment };
+ const tracked = beginSave(
+ comment ? "update_comment" : "delete_comment",
+ next,
+ );
+ replaceField(indexOf(key), next);
+ const { targetKey: _targetKey, ...score } = field;
+ trackSave(
+ updateScore({
+ ...score,
+ ...target.scoreMetadata,
+ scoreTarget: target.scoreTarget,
+ comment,
+ } as UpdateAnnotationScoreData),
+ tracked,
+ next,
+ () => {
+ const index = indexOf(key);
+ if (index < 0) return;
+ replaceField(index, field);
+ form.setError(`scoreData.${index}.comment`, {
+ type: "server",
+ message: "Failed to update comment",
+ });
+ },
+ );
+ },
+ };
+}
+
+export type AnnotationFormActions = ReturnType<
+ typeof createAnnotationFormActions
+>;
diff --git a/web/src/features/scores/components/AddScoreCategoryDialog.tsx b/web/src/features/scores/components/AddScoreCategoryDialog.tsx
index 172873f27a..0ef80def52 100644
--- a/web/src/features/scores/components/AddScoreCategoryDialog.tsx
+++ b/web/src/features/scores/components/AddScoreCategoryDialog.tsx
@@ -16,20 +16,20 @@ import { Label } from "@/src/components/ui/label";
import { api } from "@/src/utils/api";
import { usePostHogClientCapture } from "@/src/features/posthog-analytics";
import { validateNewCategoryLabel } from "@/src/features/scores/lib/annotationFormHelpers";
-import { type AnalyticsData } from "@/src/features/scores/types";
+import { type AnnotationAnalyticsContext } from "@/src/features/scores/lib/annotationAnalytics";
export function AddScoreCategoryDialog({
projectId,
config,
initialLabel,
- source,
+ analyticsData,
onClose,
onCategoryAdded,
}: {
projectId: string;
config: ScoreConfigDomain;
initialLabel: string;
- source: AnalyticsData["source"] | undefined;
+ analyticsData: AnnotationAnalyticsContext;
onClose: () => void;
onCategoryAdded: (label: string, numericValue: number) => void;
}) {
@@ -41,10 +41,7 @@ export function AddScoreCategoryDialog({
const appendCategory = api.scoreConfigs.appendCategory.useMutation({
onSuccess: async (data, variables) => {
- capture(
- "score_configs:add_category_inline",
- source ? { source } : undefined,
- );
+ capture("score_configs:add_category_inline", analyticsData);
await Promise.all([
utils.scoreConfigs.invalidate(),
utils.annotationQueues.invalidate(),
diff --git a/web/src/features/scores/components/AnnotateDrawerContent.tsx b/web/src/features/scores/components/AnnotateDrawerContent.tsx
index c28b10677f..bb72b99f12 100644
--- a/web/src/features/scores/components/AnnotateDrawerContent.tsx
+++ b/web/src/features/scores/components/AnnotateDrawerContent.tsx
@@ -29,7 +29,7 @@ export function AnnotateDrawerContent<Target extends ScoreTarget>({
);
return (
- <DrawerContent className="p-3">
+ <DrawerContent className="p-3 [--annotation-surface:var(--modal)]">
<AnnotationForm
serverScores={scores}
scoreTarget={scoreTarget}
diff --git a/web/src/features/scores/components/AnnotateDrawerController.tsx b/web/src/features/scores/components/AnnotateDrawerController.tsx
index b566f963af..85e1a95fc3 100644
--- a/web/src/features/scores/components/AnnotateDrawerController.tsx
+++ b/web/src/features/scores/components/AnnotateDrawerController.tsx
@@ -5,7 +5,8 @@ import {
type AnalyticsData,
type ScoreTarget,
} from "@/src/features/scores/types";
-import { type ReactNode, useEffect, useMemo, useRef } from "react";
+import { type ReactNode, useMemo } from "react";
+import { getAnnotationTargetType } from "@/src/features/scores/lib/annotationAnalytics";
import { AnnotateDrawerContent } from "@/src/features/scores/components/AnnotateDrawerContent";
import {
filterAndValidateDbScoreList,
@@ -52,10 +53,8 @@ type AnnotateDrawerPayload<Target extends ScoreTarget> =
function ConnectedAnnotateDrawerContent({
state,
- capture,
}: {
state: AnnotateDrawerState;
- capture: ReturnType<typeof usePostHogClientCapture>;
}) {
const shouldFetchScores =
state.scores === undefined && state.scoreTarget.type === "trace";
@@ -89,18 +88,6 @@ function ConnectedAnnotateDrawerContent({
return toDomainArrayWithStringifiedMetadata(scores);
}, [scoresQuery.data, state.scoreTarget]);
const scores = state.scores ?? fetchedScores;
- const capturedState = useRef<typeof state>(null);
-
- useEffect(() => {
- if (state.scores !== undefined || scores === undefined) return;
- if (capturedState.current === state) return;
-
- capturedState.current = state;
- capture(
- scores.length ? "score:update_form_open" : "score:create_form_open",
- state.analyticsData,
- );
- }, [capture, scores, state]);
if (scoresQuery.isError && scores === undefined) {
return (
@@ -150,7 +137,7 @@ export function AnnotateDrawerController<Target extends ScoreTarget>({
return (
<DrawerController<AnnotateDrawerState>
renderContent={({ state }) => (
- <ConnectedAnnotateDrawerContent state={state} capture={capture} />
+ <ConnectedAnnotateDrawerContent state={state} />
)}
>
{({ openDrawer }) =>
@@ -159,14 +146,11 @@ export function AnnotateDrawerController<Target extends ScoreTarget>({
openDrawer: (payload) => {
if (disabled) return;
- if (payload.scores !== undefined) {
- capture(
- payload.scores.length
- ? "score:update_form_open"
- : "score:create_form_open",
- payload.analyticsData,
- );
- }
+ capture("annotation:entry_click", {
+ ...payload.analyticsData,
+ targetType: getAnnotationTargetType(payload.scoreTarget),
+ entryPoint: "annotate_button",
+ });
openDrawer(payload);
},
})
diff --git a/web/src/features/scores/components/AnnotationForm.tsx b/web/src/features/scores/components/AnnotationForm.tsx
index affa6eed09..7f62157e60 100644
--- a/web/src/features/scores/components/AnnotationForm.tsx
+++ b/web/src/features/scores/components/AnnotationForm.tsx
@@ -1,171 +1,56 @@
-/* eslint-disable no-nested-ternary */
-import React, { useEffect, useMemo, useRef, useState } from "react";
-import { Button } from "@/src/components/ui/button";
-import {
- MessageCircleMore,
- MessageCircle,
- X,
- Archive,
- Check,
- Trash,
-} from "lucide-react";
+import React, { useEffect, useRef, useState } from "react";
+import Link from "next/link";
+import { Plus, Settings2 } from "lucide-react";
import { useFieldArray, useForm } from "react-hook-form";
import { zodResolver } from "@hookform/resolvers/zod";
+import { Button } from "@/src/components/ui/button";
+import { Skeleton } from "@/src/components/ui/skeleton";
+import Header from "@/src/components/layouts/header";
+import { PopoverController } from "@/src/components/ui/popover";
import {
- Form,
- FormControl,
- FormField,
- FormItem,
- FormLabel,
- FormMessage,
-} from "@/src/components/ui/form";
-import {
- isPresent,
- type ScoreConfigDomain,
- type UpdateAnnotationScoreData,
- type CreateAnnotationScoreData,
- TEXT_SCORE_MAX_LENGTH,
-} from "@langfuse/shared";
-import { Input } from "@/src/components/ui/input";
-import {
- Popover,
- PopoverClose,
- PopoverContent,
- PopoverTrigger,
-} from "@/src/components/ui/popover";
-import { Textarea } from "@/src/components/ui/textarea";
-import { HoverCardContent } from "@radix-ui/react-hover-card";
-import { HoverCard, HoverCardTrigger } from "@/src/components/ui/hover-card";
+ InputCommand,
+ InputCommandEmpty,
+ InputCommandGroup,
+ InputCommandInput,
+ InputCommandItem,
+ InputCommandList,
+} from "@/src/components/ui/input-command";
+import { KeyboardShortcut } from "@/src/components/design-system/KeyboardShortcut/KeyboardShortcut";
+import { usePostHogClientCapture } from "@/src/features/posthog-analytics";
import {
formatAnnotateDescription,
isTextDataType,
isNumericDataType,
- isScoreUnsaved,
} from "@/src/features/scores/lib/helpers";
-import Header from "@/src/components/layouts/header";
-import { usePostHogClientCapture } from "@/src/features/posthog-analytics";
-import { cn } from "@/src/utils/tailwind";
-import {
- type AnnotationScoreFormData,
- type InnerAnnotationFormProps,
- type ScoreTarget,
- type AnnotationForm as AnnotationFormType,
-} from "@/src/features/scores/types";
import { AnnotateFormSchema } from "@/src/features/scores/schema";
-import { ScoreConfigDetails } from "@/src/features/score-configs/components/ScoreConfigDetails";
+import { getAnnotationTargetType } from "@/src/features/scores/lib/annotationAnalytics";
import {
- enrichCategoryOptionsWithStaleScoreValue,
- resolveCategoricalNumericValue,
- resolveConfigValue,
- validateNumericScore,
-} from "@/src/features/scores/lib/annotationFormHelpers";
+ annotationFieldKey,
+ getScoreConfigSelection,
+} from "@/src/features/scores/lib/annotationConfigSelection";
import { useMergedAnnotationScores } from "@/src/features/scores/lib/useMergedAnnotationScores";
+import { prepareAnnotationFormData } from "@/src/features/scores/lib/prepareAnnotationFormData";
import { transformToAnnotationScores } from "@/src/features/scores/lib/transformScores";
-import { v4 as uuid } from "uuid";
import { useScoreMutations } from "@/src/features/scores/hooks/useScoreMutations";
-import { MultiSelectKeyValues } from "@/src/features/scores/components/multi-select-key-values";
-import { CategoricalScoreInput } from "@/src/features/scores/components/CategoricalScoreInput";
-import { DropdownMenuItemWithSecondaryAction } from "@/src/components/ui/dropdown-menu";
-import { useScoreConfigSelection } from "@/src/features/scores/hooks/useScoreConfigSelection";
-import { KeyboardShortcut } from "@/src/components/design-system/KeyboardShortcut/KeyboardShortcut";
-import {
- hasBlockingOverlay,
- hasModifier,
-} from "@/src/features/scores/lib/keyboardShortcuts";
import { useAnnotationScoreConfigs } from "@/src/features/scores/hooks/useScoreConfigs";
-import { Skeleton } from "@/src/components/ui/skeleton";
-import { Spinner } from "@/src/components/design-system/Spinner/Spinner";
-
-function CommentField({
- savedComment,
- disabled,
- loading,
- onSave,
-}: {
- savedComment: string | null;
- disabled: boolean;
- loading: boolean;
- onSave: (comment: string | null) => void;
-}) {
- const [localValue, setLocalValue] = useState(savedComment || "");
-
- // Reset local value when saved comment changes (after mutation completes)
- useEffect(() => {
- setLocalValue(savedComment || "");
- }, [savedComment]);
-
- const hasChanges = localValue.trim() !== (savedComment || "");
-
- return (
- <div className="relative">
- <div className="mb-1 flex items-center justify-between">
- <FormLabel className="text-sm">Score Comment</FormLabel>
- <div className="relative">
- {savedComment && (
- <PopoverClose asChild>
- <Button
- variant="ghost"
- type="button"
- size="icon-xs"
- loading={loading}
- onClick={() => onSave(null)}
- >
- <Trash className="h-3 w-3" />
- </Button>
- </PopoverClose>
- )}
- </div>
- </div>
- <Textarea
- className="text-xs"
- value={localValue}
- onChange={(e) => setLocalValue(e.target.value)}
- disabled={disabled}
- />
-
- {hasChanges && (
- <div className="mt-2 flex justify-end gap-1">
- <PopoverClose asChild>
- <Button
- type="button"
- variant="secondary"
- size="sm"
- className="text-xs"
- disabled={disabled}
- loading={loading}
- onClick={() => {
- setLocalValue(savedComment || "");
- }}
- >
- Discard Changes
- </Button>
- </PopoverClose>
- <PopoverClose asChild>
- <Button
- type="button"
- size="sm"
- className="text-xs"
- disabled={disabled}
- loading={loading}
- onClick={() => {
- onSave(localValue);
- }}
- >
- Save Changes
- </Button>
- </PopoverClose>
- </div>
- )}
- </div>
- );
-}
+import { useAnnotationKeyboard } from "@/src/features/scores/hooks/useAnnotationKeyboard";
+import { createAnnotationFormActions } from "@/src/features/scores/actions/annotationFormActions";
+import { AnnotationScoreRow } from "@/src/features/scores/components/AnnotationScoreRow";
+import { AnnotationSaveStatus } from "@/src/features/scores/components/AnnotationSaveStatus";
+import type {
+ AnnotateFormSchemaType,
+ AnnotationScoreSchemaType,
+ PreparedAnnotationTarget,
+ ScoreTarget,
+ AnnotationForm as AnnotationFormType,
+} from "@/src/features/scores/types";
function AnnotateHeader({
- showSaving,
+ saveStatus,
actionButtons,
description,
}: {
- showSaving: boolean;
+ saveStatus: React.ReactNode;
actionButtons: React.ReactNode;
description: string;
}) {
@@ -178,27 +63,21 @@ function AnnotateHeader({
className: "leading-relaxed",
}}
actionButtons={[
- <div className="flex items-center justify-end" key="saving-spinner">
- <div className="mr-1 items-center justify-center">
- {showSaving ? (
- <Spinner size="xxs" />
- ) : (
- <Check className="h-3 w-3" />
- )}
- </div>
- <span className="text-muted-foreground text-xs">
- {showSaving ? "Saving score data" : "Score data saved"}
- </span>
- </div>,
- actionButtons,
+ <React.Fragment key="annotation-save-status">
+ {saveStatus}
+ </React.Fragment>,
+ <React.Fragment key="annotation-actions">
+ {actionButtons}
+ </React.Fragment>,
]}
/>
);
}
-const isInputDisabled = (config: ScoreConfigDomain) => {
- return config.isArchived;
-};
+function annotationTargetLabel(scoreTarget: ScoreTarget) {
+ if (scoreTarget.type === "session") return "Session";
+ return scoreTarget.observationId ? "Observation" : "Trace";
+}
const getEmptySelectedConfigIdsStorageKey = (scoreTarget: ScoreTarget) => {
if (scoreTarget.type === "session") {
@@ -210,1069 +89,377 @@ const getEmptySelectedConfigIdsStorageKey = (scoreTarget: ScoreTarget) => {
: "emptySelectedConfigIds:trace";
};
-function InnerAnnotationForm<Target extends ScoreTarget>({
- scoreTarget,
- initialFormData,
- scoreMetadata,
- analyticsData,
+export function AnnotationFormContent({
+ targets,
actionButtons,
- configControl,
-}: InnerAnnotationFormProps<Target>) {
+}: {
+ targets: PreparedAnnotationTarget[];
+ actionButtons?: React.ReactNode;
+}) {
const capture = usePostHogClientCapture();
- const { configs, allowManualSelection } = configControl;
-
- // Initialize form with initial data (never updates)
- const form = useForm({
+ const primaryTarget = targets[0]!;
+ const { scoreMetadata, analyticsData } = primaryTarget;
+ const allowManualSelection = targets.some(
+ (target) => target.configControl.allowManualSelection,
+ );
+ const initialFormData = targets
+ .flatMap((target) =>
+ target.initialFormData.map((field) => ({
+ ...field,
+ targetKey: target.key,
+ })),
+ )
+ .sort((a, b) => a.name.localeCompare(b.name));
+ const form = useForm<AnnotateFormSchemaType>({
resolver: zodResolver(AnnotateFormSchema),
defaultValues: { scoreData: initialFormData },
});
-
const { fields, update, remove, insert } = useFieldArray({
control: form.control,
name: "scoreData",
});
+ const { createMutation, updateMutation, deleteMutation } =
+ useScoreMutations();
+ const [actions] = useState(() =>
+ createAnnotationFormActions({
+ form,
+ replaceField: update,
+ insertField: insert,
+ initialTargets: targets,
+ capture,
+ createScore: createMutation.mutateAsync,
+ updateScore: updateMutation.mutateAsync,
+ deleteScore: deleteMutation.mutateAsync,
+ }),
+ );
+ // The analytics session follows the mounted form, not query refetches.
+ useEffect(() => {
+ actions.open();
+ return () => actions.close();
+ }, [actions]);
+ const formRootRef = useRef<HTMLDivElement | null>(null);
+ useAnnotationKeyboard({ formRootRef, form, actions, targets });
- // Watch form values to keep fields in sync
- const watchedScoreData = form.watch("scoreData");
- const controlledFields = fields.map((field, index) => {
- return {
- ...field,
- ...watchedScoreData[index],
- };
- });
-
- const description = formatAnnotateDescription(scoreTarget);
-
- // Mutations - write to cache but form doesn't consume cache updates
- const { createMutation, updateMutation, deleteMutation } = useScoreMutations({
- scoreTarget,
- scoreMetadata,
- });
-
- // Config selection
- const { selectionOptions, handleSelectionChange } = useScoreConfigSelection({
- configs,
- controlledFields,
- isInputDisabled,
+ const targetFor = (field: AnnotationScoreSchemaType) =>
+ targets.find((target) => target.key === field.targetKey)!;
+ const configFor = (field: AnnotationScoreSchemaType) =>
+ targetFor(field).configControl.configs.find(
+ (config) => config.id === field.configId,
+ );
+ const showSelectedTargets =
+ new Set(
+ fields.map((field) =>
+ getAnnotationTargetType(targetFor(field).scoreTarget),
+ ),
+ ).size > 1;
+ const description =
+ targets.length > 1
+ ? "Annotate the trace and observation with scores to capture human evaluation across different dimensions."
+ : formatAnnotateDescription(primaryTarget.scoreTarget);
+ const { selectionOptions } = getScoreConfigSelection({
+ targets,
+ controlledFields: [],
insert,
remove,
- emptySelectedConfigIdsStorageKey:
- configControl.emptySelectedConfigIdsStorageKey,
});
-
- const [showSaving, setShowSaving] = useState(false);
-
- // LFE-7628 — root of this form, used to scope the global keydown listener so
- // it doesn't double-fire across multiple mounted forms (DualAnnotationContent
- // mounts an observation form and a trace form side-by-side).
- const formRootRef = useRef<HTMLDivElement | null>(null);
-
- // LFE-7628 — keyboard-first scoring. Real DOM focus is the single source of
- // truth: `↑`/`↓` (and `Tab`) move focus between fields, `1`-`9` pick an option
- // on the *focused* row, and the focused row is highlighted via `:focus-within`.
- // There is deliberately no parallel "active row" state — it previously fought
- // the browser's own focus (a focused True/False toggle showed two outlines and
- // arrows moved both the field and the toggle at once).
- const isKeyboardSelectable = (
- field: (typeof controlledFields)[number] | undefined,
- ) => {
- if (!field) return false;
- if (isTextDataType(field.dataType) || isNumericDataType(field.dataType))
- return false;
- const config = configs.find((c) => c.id === field.configId);
+ const addableOptions = selectionOptions.filter(
+ (option) =>
+ !option.disabled &&
+ !fields.some((field) => field.configId === option.value),
+ );
+ const visibleFields = fields.filter((field) => configFor(field));
+ const rowCount = visibleFields.length;
+ const optionRowCount = visibleFields.filter((field) => {
+ const config = configFor(field)!;
return (
- !!config && !config.isArchived && (config.categories?.length ?? 0) > 0
+ !isTextDataType(field.dataType) &&
+ !isNumericDataType(field.dataType) &&
+ !config.isArchived &&
+ (config.categories?.length ?? 0) > 0
);
- };
-
- // Counts for the keyboard legend.
- const optionRowCount = controlledFields.filter((field) =>
- isKeyboardSelectable(field),
- ).length;
- const rowCount = controlledFields.filter((field) =>
- configs.some((c) => c.id === field.configId),
- ).length;
- const hasEditableRow = controlledFields.some(
+ }).length;
+ const hasEditableRow = visibleFields.some(
(field) =>
- configs.some((c) => c.id === field.configId) &&
- (isTextDataType(field.dataType) || isNumericDataType(field.dataType)),
+ isTextDataType(field.dataType) || isNumericDataType(field.dataType),
);
- useEffect(() => {
- const isPending =
- createMutation.isPending ||
- updateMutation.isPending ||
- deleteMutation.isPending;
- setShowSaving(isPending);
- }, [
- createMutation.isPending,
- updateMutation.isPending,
- deleteMutation.isPending,
- ]);
-
- const rollbackDeleteError = (
- index: number,
- field: (typeof controlledFields)[number],
- previousScore: {
- id: string | null;
- value?: number | null;
- stringValue?: string | null;
- comment?: string | null;
- timestamp?: Date | null;
- },
- ) => {
- // Rollback field array
- update(index, {
- name: field.name,
- dataType: field.dataType,
- configId: field.configId,
- ...previousScore,
- });
- // Rollback form values directly to ensure sync
- form.setValue(`scoreData.${index}.id`, previousScore.id);
- form.setValue(`scoreData.${index}.value`, previousScore.value);
- form.setValue(`scoreData.${index}.stringValue`, previousScore.stringValue);
- form.setValue(`scoreData.${index}.comment`, previousScore.comment);
- form.setValue(`scoreData.${index}.timestamp`, previousScore.timestamp);
- if (isTextDataType(field.dataType)) {
- form.setError(`scoreData.${index}.stringValue`, {
- type: "server",
- message: "Failed to delete score",
- });
- } else {
- form.setError(`scoreData.${index}.value`, {
- type: "server",
- message: "Failed to delete score",
- });
- }
- };
-
- const handleDeleteScore = (index: number) => {
- const field = controlledFields[index];
-
- // Capture previous state for rollback
- const previousScore = {
- id: field.id,
- value: field.value,
- stringValue: field.stringValue,
- comment: field.comment,
- timestamp: field.timestamp,
- };
-
- // Optimistically clear form
- if (isTextDataType(field.dataType)) {
- form.clearErrors(`scoreData.${index}.stringValue`);
- } else {
- form.clearErrors(`scoreData.${index}.value`);
- }
- update(index, {
- name: field.name,
- dataType: field.dataType,
- configId: field.configId,
- id: null,
- value: null,
- stringValue: null,
- comment: null,
- });
-
- // Fire mutation with rollback
- if (previousScore.id) {
- deleteMutation.mutate(
- {
- id: previousScore.id,
- projectId: scoreMetadata.projectId,
- },
- {
- onError: () => rollbackDeleteError(index, field, previousScore),
- },
- );
- }
-
- // Capture delete event
- capture("score:delete", analyticsData);
- };
-
- const rollbackUpdateError = (
- index: number,
- previousValue?: number | null,
- previousStringValue?: string | null,
- ) => {
- form.setValue(`scoreData.${index}.value`, previousValue);
- form.setValue(`scoreData.${index}.stringValue`, previousStringValue);
- if (isTextDataType(controlledFields[index]?.dataType)) {
- form.setError(`scoreData.${index}.stringValue`, {
- type: "server",
- message: "Failed to update score",
- });
- } else {
- form.setError(`scoreData.${index}.value`, {
- type: "server",
- message: "Failed to update score",
- });
- }
- };
-
- const rollbackCreateError = (
- index: number,
- previousValue?: number | null,
- previousStringValue?: string | null,
- previousId?: string | null,
- previousTimestamp?: Date | null,
- ) => {
- form.setValue(`scoreData.${index}.id`, previousId);
- form.setValue(`scoreData.${index}.timestamp`, previousTimestamp);
- form.setValue(`scoreData.${index}.value`, previousValue);
- form.setValue(`scoreData.${index}.stringValue`, previousStringValue);
- if (isTextDataType(controlledFields[index]?.dataType)) {
- form.setError(`scoreData.${index}.stringValue`, {
- type: "server",
- message: "Failed to create score",
- });
- } else {
- form.setError(`scoreData.${index}.value`, {
- type: "server",
- message: "Failed to create score",
- });
- }
- };
-
- const handleUpsert = (
- index: number,
- value: number | null,
- stringValue: string | null,
- ) => {
- const field = controlledFields[index];
- if (!field) return;
-
- // Capture previous form state for rollback
- const previousValue = field.value;
- const previousStringValue = field.stringValue;
- const previousId = field.id;
- const previousTimestamp = field.timestamp;
-
- // Clear errors and update form optimistically
- form.clearErrors(`scoreData.${index}.value`);
- form.setValue(`scoreData.${index}.value`, value);
- form.setValue(`scoreData.${index}.stringValue`, stringValue);
-
- // Fire mutation
- const {
- id: scoreId,
- timestamp: scoreTimestamp,
- ...fieldWithoutIdAndTimestamp
- } = field;
-
- const baseScoreData = {
- ...fieldWithoutIdAndTimestamp,
- ...scoreMetadata,
- value,
- stringValue,
- scoreTarget,
- };
-
- if (scoreId) {
- updateMutation.mutate(
- {
- ...baseScoreData,
- id: scoreId,
- timestamp: scoreTimestamp ?? undefined,
- } as UpdateAnnotationScoreData,
- {
- onError: () =>
- rollbackUpdateError(index, previousValue, previousStringValue),
- },
- );
- } else {
- const id = uuid();
- const timestamp = new Date();
- form.setValue(`scoreData.${index}.id`, id);
- form.setValue(`scoreData.${index}.timestamp`, timestamp);
- createMutation.mutate(
- {
- ...baseScoreData,
- id,
- timestamp,
- } as CreateAnnotationScoreData,
- {
- onError: () =>
- rollbackCreateError(
- index,
- previousValue,
- previousStringValue,
- previousId,
- previousTimestamp,
- ),
- },
- );
- }
- };
-
- const handleNumericUpsert = (index: number) => {
- const field = controlledFields[index];
- const config = configs.find((c) => c.id === field.configId);
-
- if (!config || !field) return;
-
- if (field.value === null || field.value === undefined) {
- // Cleared to empty: remove an existing score (mirrors the text field),
- // otherwise nothing to do.
- if (field.id) handleDeleteScore(index);
- return;
- }
-
- // Client-side validation - don't fire mutation if invalid
- const errorMessage = validateNumericScore({
- value: field.value,
- maxValue: config.maxValue,
- minValue: config.minValue,
- });
-
- if (!!errorMessage) {
- form.setError(`scoreData.${index}.value`, {
- type: "custom",
- message: errorMessage,
- });
- return;
- }
-
- form.clearErrors(`scoreData.${index}.value`);
- handleUpsert(index, field.value as number, null);
- };
-
- const handleCategoricalUpsert = (
- index: number,
- stringValue: string,
- numericValue?: number,
- ) => {
- const field = controlledFields[index];
- const config = configs.find((c) => c.id === field.configId);
-
- if (!config || !field) return;
-
- const numericCategoryValue = resolveCategoricalNumericValue({
- categories: config.categories,
- stringValue,
- numericValue,
- });
-
- if (!isPresent(numericCategoryValue)) return;
-
- handleUpsert(index, numericCategoryValue, stringValue);
- };
-
- const handleTextUpsert = (index: number) => {
- const field = controlledFields[index];
- const config = configs.find((c) => c.id === field.configId);
-
- if (!config || !field) return;
- if (!field.stringValue) {
- if (field.id) {
- handleDeleteScore(index);
- }
- return;
- }
-
- handleUpsert(index, 0, field.stringValue);
- };
-
- const rollbackCommentError = (
- index: number,
- field: (typeof controlledFields)[number],
- previousComment?: string | null,
- ) => {
- update(index, {
- ...field,
- comment: previousComment,
- });
- form.setError(`scoreData.${index}.comment`, {
- type: "server",
- message: "Failed to update comment",
- });
- };
-
- const handleCommentUpdate = (index: number, newComment: string | null) => {
- const field = controlledFields[index];
- if (!field || !field.id) return;
-
- const previousComment = field.comment;
-
- // Optimistically update form
- update(index, {
- ...field,
- comment: newComment,
- });
-
- // Fire mutation
- updateMutation.mutate(
- {
- ...field,
- ...scoreMetadata,
- scoreTarget,
- comment: newComment,
- } as UpdateAnnotationScoreData,
- {
- onError: () => rollbackCommentError(index, field, previousComment),
- },
- );
- };
-
- // LFE-7628 — keyboard-first scoring, driven by real DOM focus with a
- // spreadsheet-style navigate-vs-edit split (single source of truth, one
- // outline, never trapped):
- // - `↑` / `↓` move focus between *rows* (the row container, never into a text
- // field) so navigation keeps working even when the active row is text.
- // - `Enter` drills into the focused row's control (a text/number field to
- // type, the combobox to open, a toggle to use ←/→).
- // - `Esc` pops back out of an editing text field to its row.
- // - `1`-`9` pick the Nth option of the focused row (option rows only).
- // A focused text field owns its keys; an open popover/drawer suspends these.
- useEffect(() => {
- const handleKeyDown = (event: KeyboardEvent) => {
- if (hasModifier(event)) return;
-
- const root = formRootRef.current;
- if (!root) return;
- // Suspend for an overlapping popover/drawer (e.g. the comment editor), but
- // NOT for a drawer this form is mounted inside (the Annotate drawer) — that
- // is an ancestor of the form, so the scheme stays alive there.
- if (hasBlockingOverlay(root)) return;
- const target = event.target;
- const editing =
- target instanceof HTMLInputElement ||
- target instanceof HTMLTextAreaElement ||
- (target instanceof HTMLElement && target.isContentEditable);
-
- // (`Esc` to leave a field is handled by a separate capture-phase listener
- // below, so it can stop a wrapping drawer from also dismissing.)
-
- // `Enter` in a single-line field (e.g. a numeric score) commits the value
- // and returns to row navigation — there is no newline to insert, so this is
- // the spreadsheet "confirm cell" gesture. Multi-line text (textarea) keeps
- // Enter for newlines and is excluded here.
- if (
- event.key === "Enter" &&
- target instanceof HTMLInputElement &&
- root.contains(target)
- ) {
- // Out-of-range numeric: surface the constraint and stay so the value
- // isn't committed-and-dropped (mirrors the ⌘/Ctrl+Enter complete gate).
- if (
- target.type === "number" &&
- (target.validity.rangeOverflow || target.validity.rangeUnderflow)
- ) {
- event.preventDefault();
- target.reportValidity();
- return;
- }
- event.preventDefault();
- const row = target.closest<HTMLElement>("[data-score-row]");
- // Focusing the row blurs the input (its onBlur saves) and resumes ↑/↓.
- if (row) row.focus();
- else target.blur();
- return;
- }
-
- // While editing a text field, leave its keys (typing, caret, number step)
- // alone — `Esc` / `Tab` move out.
- if (editing) return;
- if (rowCount === 0) return;
-
- // Scope to a single form (DualAnnotationContent mounts two): the form that
- // contains focus acts; if focus is on the body (nothing focused) the first
- // form acts. A control focused *outside* any form (e.g. the Mark Completed
- // / Skip / Back / "?" page buttons) must NOT drive the form — otherwise
- // ↑/↓ would hijack focus off that button into the score rows.
- const active = document.activeElement;
- const focusedForm =
- active instanceof HTMLElement
- ? active.closest("[data-annotation-form]")
- : null;
- if (focusedForm) {
- if (focusedForm !== root) return;
- } else {
- if (active && active !== document.body) return;
- if (document.querySelector("[data-annotation-form]") !== root) return;
- }
-
- const rowEls = Array.from(
- root.querySelectorAll<HTMLElement>("[data-score-row]"),
- );
- if (rowEls.length === 0) return;
- const currentRow =
- active instanceof HTMLElement
- ? (active.closest("[data-score-row]") as HTMLElement | null)
- : null;
- const currentPos = currentRow ? rowEls.indexOf(currentRow) : -1;
-
- // `↑` / `↓` move focus between rows (to the row container itself, never
- // into a text field — so navigation is never trapped).
- if (event.key === "ArrowDown" || event.key === "ArrowUp") {
- if (rowEls.length < 2) return;
- // Navigate from a focused row, or enter from the body — but NOT from an
- // in-form non-row control (e.g. the config-picker trigger), which would
- // otherwise teleport focus to a row.
- if (currentPos < 0 && active !== document.body) return;
- event.preventDefault();
- const delta = event.key === "ArrowDown" ? 1 : -1;
- const nextPos =
- currentPos < 0
- ? delta > 0
- ? 0
- : rowEls.length - 1
- : (currentPos + delta + rowEls.length) % rowEls.length;
- rowEls[nextPos].focus();
- return;
- }
-
- // `Enter` drills into the focused row's control (only from the row
- // container itself — once a control is focused, Enter is left to it).
- if (event.key === "Enter") {
- if (currentRow && active === currentRow) {
- // First *enabled* control — skip the disabled stale-category chip that
- // enrichCategoryOptionsWithStaleScoreValue prepends (focusing a disabled
- // control is a no-op, so Enter would otherwise appear to do nothing).
- const control = Array.from(
- currentRow.querySelectorAll<HTMLElement>(
- "[data-score-control] :is(textarea, input, button)",
- ),
- ).find((el) => !el.matches(":disabled"));
- if (control) {
- event.preventDefault();
- // A dropdown trigger (combobox, `aria-haspopup`) opens directly so a
- // single Enter is enough (not focus-then-Enter). Radix's Popover
- // trigger opens on click. Other controls (number/text input, toggle)
- // just take focus.
- if (
- control.tagName === "BUTTON" &&
- control.getAttribute("aria-haspopup")
- ) {
- control.click();
- } else {
- control.focus();
- }
- }
- }
- return;
- }
-
- // `1`-`9` pick an option on the focused row (option rows only).
- if (/^[1-9]$/.test(event.key)) {
- if (currentPos < 0 || !currentRow) return;
- // Only from the row container itself or its value control — never the
- // in-row Comment / Delete buttons (else a stray digit writes a phantom
- // score). Mirrors the Enter branch's `active === currentRow` gate.
- if (
- active !== currentRow &&
- !(
- active instanceof HTMLElement &&
- active.closest("[data-score-control]")
- )
- )
- return;
- const rowIndex = Number(currentRow.getAttribute("data-score-row"));
- const field = controlledFields[rowIndex];
- if (!isKeyboardSelectable(field)) return;
- const config = configs.find((c) => c.id === field?.configId);
- const category = (config?.categories ?? [])[Number(event.key) - 1];
- if (!category) return;
- event.preventDefault();
- handleCategoricalUpsert(rowIndex, category.label);
- }
- };
- window.addEventListener("keydown", handleKeyDown);
- return () => window.removeEventListener("keydown", handleKeyDown);
- // eslint-disable-next-line react-hooks/exhaustive-deps
- }, [controlledFields, configs, rowCount]);
-
- // LFE-7628 — `Esc` leaves a focused score field (back to its row) WITHOUT
- // dismissing a wrapping drawer. Vaul/Radix DismissableLayer listens for Esc on
- // `document` with `{capture:true}`; a window capture-phase listener runs first
- // (window is the ancestor), so stopping propagation here prevents the drawer
- // from closing. Scoped to fields inside this form (a portaled comment popover
- // is not inside the form root, so its own Esc-to-close still works).
- useEffect(() => {
- const onEscapeCapture = (event: KeyboardEvent) => {
- if (event.key !== "Escape") return;
- const root = formRootRef.current;
- if (!root) return;
- const target = event.target;
- const inField =
- (target instanceof HTMLInputElement ||
- target instanceof HTMLTextAreaElement) &&
- root.contains(target);
- if (!inField) return;
- event.stopPropagation();
- const row = (target as HTMLElement).closest<HTMLElement>(
- "[data-score-row]",
- );
- if (row) row.focus();
- else (target as HTMLElement).blur();
- };
- window.addEventListener("keydown", onEscapeCapture, true);
- return () => window.removeEventListener("keydown", onEscapeCapture, true);
- }, []);
-
return (
<div
ref={formRootRef}
data-annotation-form
- className="mx-auto w-full space-y-2 overflow-y-auto md:max-h-full"
+ className="ph-no-capture mx-auto w-full space-y-4 overflow-y-auto p-1 md:max-h-full"
>
- <div className="bg-background sticky top-0 z-10 rounded-sm">
+ <div className="sticky top-0 z-10 flex flex-col gap-4 rounded-sm bg-[hsl(var(--annotation-surface,var(--background)))] pb-2">
<AnnotateHeader
- showSaving={showSaving}
- actionButtons={actionButtons}
+ saveStatus={<AnnotationSaveStatus form={form} actions={actions} />}
+ actionButtons={
+ <>
+ {allowManualSelection ? (
+ <Button
+ variant="outline"
+ size="sm"
+ className="bg-accent gap-1.5 text-xs"
+ asChild
+ >
+ <Link
+ href={`/project/${scoreMetadata.projectId}/settings/scores`}
+ target="_blank"
+ onClick={() => {
+ capture(
+ "score_configs:manage_configs_item_click",
+ analyticsData,
+ );
+ }}
+ onAuxClick={(event) => {
+ if (event.button === 1) {
+ capture(
+ "score_configs:manage_configs_item_click",
+ analyticsData,
+ );
+ }
+ }}
+ >
+ <Settings2 className="size-3" aria-hidden="true" />
+ Manage score configs
+ </Link>
+ </Button>
+ ) : null}
+ {actionButtons}
+ </>
+ }
description={description}
/>
- {allowManualSelection ? (
- <div className="grid grid-flow-col items-center">
- <MultiSelectKeyValues
- placeholder="Value"
- align="end"
- items="empty scores"
- className="grid grid-cols-[auto_1fr_auto_auto] gap-2"
- options={selectionOptions}
- onValueChange={handleSelectionChange}
- values={fields
- .filter((field) => !!field.configId)
- .map((field) => ({
- key: field.configId as string,
- value: resolveConfigValue({
- dataType: field.dataType,
- name: field.name,
- }),
- }))}
- controlButtons={
- <DropdownMenuItemWithSecondaryAction
- title="Manage score configs"
- href={`/project/${scoreMetadata.projectId}/settings/scores`}
- target="_blank"
- onBeforeAction={() => {
- capture(
- "score_configs:manage_configs_item_click",
- analyticsData,
- );
- }}
- />
- }
- />
- </div>
- ) : null}
</div>
- <Form {...form}>
- {/* No real submit: scores save per-field. Prevent the browser's implicit
+ {/* No real submit: scores save per-field. Prevent the browser's implicit
form submission (Enter in a single-line input would otherwise reload
the page). */}
- <form
- className="flex flex-col gap-4"
- onSubmit={(e) => e.preventDefault()}
- >
- <div className="grid grid-flow-row gap-2.5">
- <FormField
- control={form.control}
- name="scoreData"
- render={() => (
- <>
- {controlledFields.map((score, index) => {
- const config = configs.find(
- (config) => config.id === score.configId,
- );
- if (!config) return null;
- const categories = enrichCategoryOptionsWithStaleScoreValue(
- config.categories ?? [],
- score.stringValue,
- );
-
- return (
- <div
- key={fields[index]?.id}
- data-score-row={index}
- // `tabIndex={-1}` makes the row programmatically focusable
- // so ↑/↓ can land on the row itself (navigate) without
- // entering its text field. The focused row highlights via
- // `:focus-within` (single source of truth = real focus;
- // `ring-inset` so the scroll container's overflow can't
- // clip it), and `group` shows the option badges only on it.
- tabIndex={-1}
- role="group"
- aria-label={score.name}
- className={cn(
- "group grid w-full grid-cols-[1fr_2fr] items-center gap-3 rounded-md px-3 py-1 text-left transition-colors outline-none",
- "focus-within:ring-primary/30 focus-within:bg-accent/40 focus-within:ring-1 focus-within:ring-inset",
- )}
- >
- <div className="flex h-full min-w-0 items-center">
- {config.description ||
- isPresent(config.maxValue) ||
- isPresent(config.minValue) ? (
- <HoverCard>
- <HoverCardTrigger asChild>
- <span
- className={cn(
- "decoration-muted-gray line-clamp-2 min-w-0 text-xs font-bold wrap-break-word underline decoration-dashed underline-offset-2",
- config.isArchived
- ? "text-foreground/40"
- : "",
- )}
- >
- {score.name}
- </span>
- </HoverCardTrigger>
- <HoverCardContent className="z-20 max-h-[60vh] max-w-64 overflow-y-auto rounded border">
- <ScoreConfigDetails config={config} />
- </HoverCardContent>
- </HoverCard>
- ) : (
- <span
- className={cn(
- "line-clamp-2 min-w-0 text-xs font-bold wrap-break-word",
- config.isArchived ? "text-foreground/40" : "",
- )}
- title={score.name}
- >
- {score.name}
- </span>
- )}
- <Popover>
- <PopoverTrigger asChild>
- <Button
- variant="link"
- type="button"
- size="xs"
- title="Add or view score comment"
- // LFE-7628: center the comment icon vertically
- // against the score label instead of stretching
- // to the full (possibly multi-line) row height,
- // and keep it hugging the label (shrink-0) rather
- // than floating in the middle of the row.
- className="disabled:text-primary/50 flex h-auto shrink-0 items-center self-center px-0 pl-1 disabled:opacity-100"
- disabled={
- isScoreUnsaved(score.id) ||
- (config.isArchived && !score.comment)
- }
- >
- {score.comment ? (
- <MessageCircleMore className="h-4 w-4" />
- ) : (
- <MessageCircle className="h-4 w-4" />
- )}
- </Button>
- </PopoverTrigger>
- <PopoverContent>
- <FormField
- control={form.control}
- name={`scoreData.${index}.comment`}
- render={() => (
- <FormItem className="space-y-4">
- <FormControl>
- <CommentField
- savedComment={score.comment ?? null}
- disabled={isInputDisabled(config)}
- loading={updateMutation.isPending}
- onSave={(newComment) => {
- const trimmed = newComment?.trim();
- handleCommentUpdate(
- index,
- trimmed || null,
- );
- }}
- />
- </FormControl>
- <FormMessage className="text-xs" />
- </FormItem>
- )}
- />
- </PopoverContent>
- </Popover>
- </div>
- <div className="grid grid-cols-[11fr_1fr] items-center py-1">
- {/* data-score-control wraps only the value control so
- keyboard 1-9 scoring targets it, not the in-row
- Comment / Delete buttons. */}
- <div data-score-control>
- {isTextDataType(score.dataType) ? (
- <FormField
- control={form.control}
- name={`scoreData.${index}.stringValue`}
- render={({ field }) => (
- <FormItem>
- <FormControl>
- <Textarea
- {...field}
- value={field.value ?? ""}
- maxLength={TEXT_SCORE_MAX_LENGTH}
- className="text-xs"
- disabled={isInputDisabled(config)}
- placeholder="Enter free form text..."
- onBlur={() => handleTextUpsert(index)}
- />
- </FormControl>
- <FormMessage className="text-xs" />
- </FormItem>
- )}
- />
- ) : isNumericDataType(score.dataType) ? (
- <FormField
- control={form.control}
- name={`scoreData.${index}.value`}
- render={({ field }) => (
- <FormItem>
- <FormControl>
- <Input
- {...field}
- value={field.value ?? ""}
- onChange={(e) => {
- const value = e.target.value;
- // Empty → null so the field can be
- // cleared back to blank (returning here
- // instead trapped the last digit — the
- // form kept the old value and re-rendered
- // it). onBlur deletes the score when null.
- field.onChange(
- value === "" ? null : Number(value),
- );
- }}
- type="number"
- // Mirror the config range as native
- // constraints so out-of-range values are
- // catchable via the ⌘/Ctrl+Enter complete
- // gate's rangeOverflow/Underflow check, on
- // top of the existing onBlur JS validation.
- // `step="any"` keeps decimals valid (config
- // validation is range-only, not integer).
- min={config.minValue ?? undefined}
- max={config.maxValue ?? undefined}
- step="any"
- className="text-xs"
- disabled={isInputDisabled(config)}
- onBlur={() =>
- handleNumericUpsert(index)
- }
- />
- </FormControl>
- <FormMessage className="text-xs" />
- </FormItem>
- )}
- />
- ) : (
- <FormField
- control={form.control}
- name={`scoreData.${index}.stringValue`}
- render={({ field }) => (
- <FormItem>
- <FormControl>
- <CategoricalScoreInput
- projectId={scoreMetadata.projectId}
- config={config}
- categories={categories}
- name={field.name}
- value={field.value ?? ""}
- disabled={isInputDisabled(config)}
- source={analyticsData?.source}
- onValueChange={(
- value,
- numericValue,
- ) => {
- field.onChange(value);
- handleCategoricalUpsert(
- index,
- value,
- numericValue,
- );
- }}
- />
- </FormControl>
- <FormMessage className="text-xs" />
- </FormItem>
- )}
- />
- )}
- </div>
- {config.isArchived ? (
- <Popover>
- <PopoverTrigger asChild>
- <Button
- variant="link"
- type="button"
- className="px-0 pl-1"
- title="Delete archived score"
- disabled={isScoreUnsaved(score.id)}
- >
- <Archive className="h-4 w-4"></Archive>
- </Button>
- </PopoverTrigger>
- <PopoverContent>
- <h2 className="mb-3 font-bold">
- Your score is archived
- </h2>
- <p className="mb-3 text-sm">
- This action will delete your score
- irreversibly.
- </p>
- <div className="flex justify-end space-x-4">
- <Button
- type="button"
- variant="destructive"
- loading={deleteMutation.isPending}
- onClick={() => handleDeleteScore(index)}
- >
- Delete
- </Button>
- </div>
- </PopoverContent>
- </Popover>
- ) : (
- <Button
- variant="link"
- type="button"
- className="px-0 pl-1"
- title="Delete score from trace/observation"
- disabled={
- isScoreUnsaved(score.id) ||
- updateMutation.isPending
- }
- loading={
- deleteMutation.isPending &&
- !isScoreUnsaved(score.id)
- }
- onClick={() => handleDeleteScore(index)}
- >
- <X className="h-4 w-4" />
- </Button>
- )}
- </div>
- </div>
- );
- })}
- </>
- )}
- />
- </div>
- {rowCount > 0 && (
- // This legend only exists to advertise keyboard shortcuts, so hide
- // the whole strip on touch viewports rather than just the kbd
- // chips inside it (LFE-11067) — the shortcuts themselves still
- // work if a physical keyboard is attached.
- <div className="text-muted-foreground hidden flex-wrap items-center gap-x-2 gap-y-1 px-0.5 text-[11px] md:flex">
- {rowCount > 1 && (
- <span className="flex items-center gap-1">
- <KeyboardShortcut size="sm" keys={["ArrowUp"]} />
- <KeyboardShortcut size="sm" keys={["ArrowDown"]} />
- move between fields
- </span>
- )}
- {optionRowCount > 0 && (
- <span className="flex items-center gap-1">
- <KeyboardShortcut size="sm" keys={["1"]} />
- <span className="text-muted-foreground">…</span>
- <KeyboardShortcut size="sm" keys={["9"]} />
- select option
- </span>
+ <form
+ className="flex flex-col gap-4"
+ onSubmit={(e) => e.preventDefault()}
+ >
+ <div className="grid grid-flow-row gap-2.5">
+ {fields.map((field, index) => {
+ const target = targetFor(field);
+ const config = configFor(field);
+ return config ? (
+ <AnnotationScoreRow
+ key={field.id}
+ onRemove={() => {
+ if (actions.isSaving(annotationFieldKey(field))) return;
+ getScoreConfigSelection({
+ targets,
+ controlledFields: form.getValues("scoreData"),
+ insert,
+ remove,
+ }).removeEmptyField(annotationFieldKey(field));
+ }}
+ form={form}
+ actions={actions}
+ index={index}
+ fieldKey={annotationFieldKey(field)}
+ target={target}
+ config={config}
+ showTarget={showSelectedTargets}
+ formRootRef={formRootRef}
+ commentSaving={updateMutation.isPending}
+ targetOptions={
+ target.configControl.allowManualSelection
+ ? targets
+ .filter(
+ (candidate) =>
+ candidate.key !== target.key &&
+ candidate.configControl.allowManualSelection &&
+ candidate.configControl.configs.some(
+ (candidateConfig) =>
+ candidateConfig.id === config.id,
+ ),
+ )
+ .map((candidate) => ({
+ target: candidate,
+ hasField: fields.some(
+ (current) =>
+ current.configId === config.id &&
+ current.targetKey === candidate.key,
+ ),
+ }))
+ : []
+ }
+ />
+ ) : null;
+ })}
+ </div>
+ {allowManualSelection ? (
+ <div>
+ <PopoverController
+ align="start"
+ contentClassName="w-64 p-0"
+ disabled={addableOptions.length === 0}
+ modal={false}
+ renderContent={({ closePopover }) => (
+ <InputCommand>
+ <InputCommandInput
+ placeholder="Search scores..."
+ variant="bottom"
+ />
+ <InputCommandList className="max-h-72">
+ <InputCommandEmpty>No scores found.</InputCommandEmpty>
+ <InputCommandGroup>
+ {addableOptions.map((option) => (
+ <InputCommandItem
+ key={option.value}
+ value={option.value}
+ keywords={[option.label]}
+ className="cursor-pointer"
+ onSelect={() => {
+ closePopover();
+ const controlledFields =
+ form.getValues("scoreData");
+ getScoreConfigSelection({
+ targets,
+ controlledFields,
+ insert,
+ remove,
+ }).addScore(option.value);
+ }}
+ >
+ {option.label}
+ </InputCommandItem>
+ ))}
+ </InputCommandGroup>
+ </InputCommandList>
+ </InputCommand>
)}
- {hasEditableRow && (
- <span className="flex items-center gap-1">
- <KeyboardShortcut size="sm" keys={["Enter"]} />
- edit field
- </span>
+ >
+ {({ Trigger, disabled }) => (
+ <Trigger asChild>
+ <Button
+ type="button"
+ variant="outline"
+ size="sm"
+ className="gap-1.5 text-xs"
+ disabled={disabled}
+ data-add-score
+ >
+ <Plus className="size-3.5" aria-hidden="true" />
+ Add score
+ </Button>
+ </Trigger>
)}
- </div>
- )}
- </form>
- </Form>
+ </PopoverController>
+ </div>
+ ) : null}
+ {rowCount > 0 && (
+ // This legend only exists to advertise keyboard shortcuts, so hide
+ // the whole strip on touch viewports rather than just the kbd
+ // chips inside it — the shortcuts themselves still
+ // work if a physical keyboard is attached.
+ <div className="text-muted-foreground hidden flex-wrap items-center gap-x-2 gap-y-1 px-0.5 text-[11px] md:flex">
+ {rowCount > 1 && (
+ <span className="flex items-center gap-1">
+ <KeyboardShortcut size="sm" keys={["ArrowUp"]} />
+ <KeyboardShortcut size="sm" keys={["ArrowDown"]} />
+ move between fields
+ </span>
+ )}
+ {optionRowCount > 0 && (
+ <span className="flex items-center gap-1">
+ <KeyboardShortcut size="sm" keys={["1"]} />
+ <span className="text-muted-foreground">…</span>
+ <KeyboardShortcut size="sm" keys={["9"]} />
+ select option
+ </span>
+ )}
+ {hasEditableRow && (
+ <span className="flex items-center gap-1">
+ <KeyboardShortcut size="sm" keys={["Enter"]} />
+ edit field
+ </span>
+ )}
+ </div>
+ )}
+ </form>
</div>
);
}
-export function AnnotationForm<Target extends ScoreTarget>({
+export function usePreparedAnnotationFormTarget<Target extends ScoreTarget>({
scoreTarget,
serverScores,
scoreMetadata,
analyticsData,
- actionButtons,
configSelection = { mode: "selectable" },
}: AnnotationFormType<Target>) {
const { projectId } = scoreMetadata;
const emptySelectedConfigIdsStorageKey =
getEmptySelectedConfigIdsStorageKey(scoreTarget);
- const { isLoading, availableConfigs, selectedConfigIds } =
- useAnnotationScoreConfigs({
- projectId,
- configSelection,
- emptySelectedConfigIdsStorageKey,
- });
+ const {
+ isLoading,
+ availableConfigs,
+ selectedConfigIds,
+ setSelectedConfigIds,
+ } = useAnnotationScoreConfigs({
+ projectId,
+ configSelection,
+ emptySelectedConfigIdsStorageKey,
+ });
- // Step 1: Transform server scores to annotation scores
- const serverAnnotationScores = useMemo(() => {
- if (Array.isArray(serverScores)) {
- // Flat scores from trace/session detail
- return transformToAnnotationScores(serverScores, availableConfigs);
- }
- // Aggregates from compare view
- return transformToAnnotationScores(
- serverScores,
- availableConfigs,
- scoreTarget.type === "trace" ? scoreTarget.traceId : "",
- scoreTarget.type === "trace" ? scoreTarget.observationId : undefined,
- );
- }, [serverScores, availableConfigs, scoreTarget]);
+ const serverAnnotationScores = Array.isArray(serverScores)
+ ? transformToAnnotationScores(serverScores, availableConfigs)
+ : transformToAnnotationScores(
+ serverScores,
+ availableConfigs,
+ scoreTarget.type === "trace" ? scoreTarget.traceId : "",
+ scoreTarget.type === "trace" ? scoreTarget.observationId : undefined,
+ );
- // Step 2: Merge with cache
const annotationScores = useMergedAnnotationScores(
serverAnnotationScores,
scoreTarget,
);
- const initialFormData: AnnotationScoreFormData[] = [];
- const configIds = new Set<string>();
- annotationScores.forEach((score) => {
- configIds.add(score.configId);
- initialFormData.push({
- id: score.id,
- configId: score.configId,
- name: score.name,
- dataType: score.dataType,
- value: score.value,
- stringValue: score.stringValue,
- comment: score.comment,
- timestamp: score.timestamp,
- });
- });
-
- selectedConfigIds.forEach((configId) => {
- if (!configIds.has(configId)) {
- const config = availableConfigs.find((c) => c.id === configId);
- if (!config) return;
- initialFormData.push({
- id: null,
- configId,
- name: config.name,
- dataType: config.dataType,
- value: null,
- stringValue: null,
- comment: null,
- timestamp: null,
- });
- }
- });
-
- const sortedInitialFormData = initialFormData.sort((a, b) =>
- a.name.localeCompare(b.name),
+ const initialFormData = prepareAnnotationFormData(
+ annotationScores,
+ availableConfigs,
+ selectedConfigIds,
);
- return isLoading ? (
- <Skeleton className="h-full w-full" />
- ) : (
- <InnerAnnotationForm
- scoreTarget={scoreTarget}
- initialFormData={sortedInitialFormData}
- scoreMetadata={scoreMetadata}
- analyticsData={analyticsData}
- actionButtons={actionButtons}
- configControl={{
+ return {
+ isLoading,
+ target: {
+ key: JSON.stringify([
+ scoreMetadata.projectId,
+ scoreMetadata.queueId,
+ scoreTarget,
+ analyticsData.source,
+ analyticsData.isV4,
+ ]),
+ label: annotationTargetLabel(scoreTarget),
+ scoreTarget,
+ initialFormData,
+ scoreMetadata,
+ analyticsData,
+ configControl: {
configs: availableConfigs,
allowManualSelection: configSelection.mode === "selectable",
emptySelectedConfigIdsStorageKey,
- }}
+ selectedConfigIds,
+ setSelectedConfigIds,
+ },
+ } satisfies PreparedAnnotationTarget,
+ };
+}
+
+export function AnnotationForm<Target extends ScoreTarget>(
+ props: AnnotationFormType<Target>,
+) {
+ const { isLoading, target } = usePreparedAnnotationFormTarget(props);
+ return isLoading ? (
+ <Skeleton className="h-full w-full" />
+ ) : (
+ <AnnotationFormContent
+ key={target.key}
+ targets={[target]}
+ actionButtons={props.actionButtons}
/>
);
}
diff --git a/web/src/features/scores/components/AnnotationSaveStatus.tsx b/web/src/features/scores/components/AnnotationSaveStatus.tsx
new file mode 100644
index 0000000000..d71f16caa1
--- /dev/null
+++ b/web/src/features/scores/components/AnnotationSaveStatus.tsx
@@ -0,0 +1,54 @@
+import { useFormState, useWatch, type UseFormReturn } from "react-hook-form";
+import { useStore } from "zustand";
+import { Check } from "lucide-react";
+import { Spinner } from "@/src/components/design-system/Spinner/Spinner";
+import type { AnnotationFormActions } from "@/src/features/scores/actions/annotationFormActions";
+import type { AnnotateFormSchemaType } from "@/src/features/scores/types";
+import { annotationFieldKey } from "@/src/features/scores/lib/annotationConfigSelection";
+import { hasChangedAnnotationValue } from "@/src/features/scores/state/annotationSaveStore";
+
+export function AnnotationSaveStatus({
+ form,
+ actions,
+}: {
+ form: UseFormReturn<AnnotateFormSchemaType>;
+ actions: AnnotationFormActions;
+}) {
+ const fields = useWatch({ control: form.control, name: "scoreData" });
+ const { errors } = useFormState({ control: form.control, name: "scoreData" });
+ const state = useStore(actions.saveStore);
+ const dirty = fields.some((field) =>
+ hasChangedAnnotationValue(
+ { ...field, id: field.id ?? null },
+ state.confirmedFields.get(annotationFieldKey(field))?.field,
+ ),
+ );
+ let status: "idle" | "saving" | "saved" | "error" = "idle";
+ if (state.pending) status = "saving";
+ else if (state.failed) status = "error";
+ else if (state.saved && !dirty && !errors.scoreData) status = "saved";
+
+ return (
+ <>
+ {status !== "idle" ? (
+ <div
+ role="status"
+ aria-label="Score save status"
+ className="flex items-center justify-end"
+ >
+ <div className="mr-1 items-center justify-center">
+ {status === "saving" ? <Spinner size="xxs" /> : null}
+ {status === "saved" ? <Check className="h-3 w-3" /> : null}
+ </div>
+ <span className="text-muted-foreground text-xs">
+ {
+ { saving: "Saving…", saved: "Saved", error: "Could not save" }[
+ status
+ ]
+ }
+ </span>
+ </div>
+ ) : null}
+ </>
+ );
+}
diff --git a/web/src/features/scores/components/AnnotationScoreRow.tsx b/web/src/features/scores/components/AnnotationScoreRow.tsx
new file mode 100644
index 0000000000..94782399f9
--- /dev/null
+++ b/web/src/features/scores/components/AnnotationScoreRow.tsx
@@ -0,0 +1,572 @@
+import React, { useRef, useState } from "react";
+import { useFormState, useWatch, type UseFormReturn } from "react-hook-form";
+import { useStore } from "zustand";
+import {
+ isPresent,
+ TEXT_SCORE_MAX_LENGTH,
+ type ScoreConfigDomain,
+} from "@langfuse/shared";
+import {
+ MessageCircleMore,
+ MessageCircle,
+ MoreHorizontal,
+ Trash,
+} from "lucide-react";
+import { Button } from "@/src/components/ui/button";
+import {
+ Form,
+ FormControl,
+ FormField,
+ FormItem,
+ FormLabel,
+ FormMessage,
+} from "@/src/components/ui/form";
+import { Input } from "@/src/components/ui/input";
+import { Textarea } from "@/src/components/ui/textarea";
+import {
+ Popover,
+ PopoverClose,
+ PopoverContent,
+ PopoverTrigger,
+} from "@/src/components/ui/popover";
+import { HoverCardContent } from "@radix-ui/react-hover-card";
+import { HoverCard, HoverCardTrigger } from "@/src/components/ui/hover-card";
+import { Badge } from "@/src/components/ui/badge";
+import {
+ DropdownMenuController,
+ DropdownMenuItem,
+ DropdownMenuLabel,
+ DropdownMenuSeparator,
+} from "@/src/components/ui/dropdown-menu";
+import { ScoreConfigDetails } from "@/src/features/score-configs/components/ScoreConfigDetails";
+import { CategoricalScoreInput } from "@/src/features/scores/components/CategoricalScoreInput";
+import {
+ isTextDataType,
+ isNumericDataType,
+ isScoreUnsaved,
+} from "@/src/features/scores/lib/helpers";
+import { enrichCategoryOptionsWithStaleScoreValue } from "@/src/features/scores/lib/annotationFormHelpers";
+import { getAnnotationTargetType } from "@/src/features/scores/lib/annotationAnalytics";
+import type { AnnotationFormActions } from "@/src/features/scores/actions/annotationFormActions";
+import type {
+ AnnotateFormSchemaType,
+ PreparedAnnotationTarget,
+} from "@/src/features/scores/types";
+import { cn } from "@/src/utils/tailwind";
+
+function CommentField({
+ savedComment,
+ disabled,
+ loading,
+ onSave,
+}: {
+ savedComment: string | null;
+ disabled: boolean;
+ loading: boolean;
+ onSave: (comment: string | null) => void;
+}) {
+ const [localValue, setLocalValue] = useState(savedComment || "");
+
+ const hasChanges = localValue.trim() !== (savedComment || "");
+
+ return (
+ <div className="relative">
+ <div className="mb-1 flex items-center justify-between">
+ <FormLabel className="text-sm">Score Comment</FormLabel>
+ <div className="relative">
+ {savedComment && (
+ <PopoverClose asChild>
+ <Button
+ variant="ghost"
+ type="button"
+ size="icon-xs"
+ loading={loading}
+ onClick={() => onSave(null)}
+ >
+ <Trash className="h-3 w-3" />
+ </Button>
+ </PopoverClose>
+ )}
+ </div>
+ </div>
+ <Textarea
+ className="text-xs"
+ value={localValue}
+ onChange={(e) => setLocalValue(e.target.value)}
+ disabled={disabled}
+ />
+
+ {hasChanges && (
+ <div className="mt-2 flex justify-end gap-1">
+ <PopoverClose asChild>
+ <Button
+ type="button"
+ variant="secondary"
+ size="sm"
+ className="text-xs"
+ disabled={disabled}
+ loading={loading}
+ onClick={() => {
+ setLocalValue(savedComment || "");
+ }}
+ >
+ Discard Changes
+ </Button>
+ </PopoverClose>
+ <PopoverClose asChild>
+ <Button
+ type="button"
+ size="sm"
+ className="text-xs"
+ disabled={disabled}
+ loading={loading}
+ onClick={() => {
+ onSave(localValue);
+ }}
+ >
+ Save Changes
+ </Button>
+ </PopoverClose>
+ </div>
+ )}
+ </div>
+ );
+}
+
+export function AnnotationScoreRow({
+ form,
+ actions,
+ index,
+ fieldKey,
+ target,
+ config,
+ showTarget,
+ formRootRef,
+ commentSaving,
+ targetOptions,
+ onRemove,
+}: {
+ form: UseFormReturn<AnnotateFormSchemaType>;
+ actions: AnnotationFormActions;
+ index: number;
+ fieldKey: string;
+ target: PreparedAnnotationTarget;
+ config: ScoreConfigDomain;
+ showTarget: boolean;
+ formRootRef: React.RefObject<HTMLDivElement | null>;
+ commentSaving: boolean;
+ targetOptions: { target: PreparedAnnotationTarget; hasField: boolean }[];
+ onRemove: () => void;
+}) {
+ const score = useWatch({ control: form.control, name: `scoreData.${index}` });
+ const fieldState = useFormState({
+ control: form.control,
+ name: `scoreData.${index}`,
+ });
+ const saving = useStore(actions.saveStore, (state) => state.pending > 0);
+ const fieldSaving = useStore(
+ actions.saveStore,
+ (state) => state.pending > 0 && actions.isSaving(fieldKey),
+ );
+ const categories = enrichCategoryOptionsWithStaleScoreValue(
+ config.categories ?? [],
+ score.stringValue,
+ );
+ const rowRef = useRef<HTMLDivElement | null>(null);
+ const deferredInput = useRef<HTMLInputElement | HTMLTextAreaElement | null>(
+ null,
+ );
+ const focusFieldKey = useRef(fieldKey);
+ const hasScoreValue = Boolean(
+ score.id || isPresent(score.value) || score.stringValue || score.comment,
+ );
+ const invalid =
+ form.getFieldState(`scoreData.${index}.value`, fieldState).invalid ||
+ form.getFieldState(`scoreData.${index}.stringValue`, fieldState).invalid;
+ const canRemove =
+ target.configControl.allowManualSelection &&
+ !config.isArchived &&
+ !hasScoreValue &&
+ !invalid;
+ const isMovingToScoreActions = (
+ event: React.FocusEvent<HTMLInputElement | HTMLTextAreaElement>,
+ ) => {
+ const next = event.relatedTarget;
+ const row = rowRef.current;
+ const ownActions =
+ row &&
+ ((next instanceof HTMLElement &&
+ row.contains(next) &&
+ next.closest("[data-score-actions]")) ||
+ row.querySelector('[data-score-actions][data-state="open"]'));
+ if (!ownActions) return false;
+ deferredInput.current = event.currentTarget;
+ return true;
+ };
+ const commitDeferredScore = () => {
+ const input = deferredInput.current;
+ deferredInput.current = null;
+ if (input instanceof HTMLInputElement)
+ actions.saveNumeric(fieldKey, target, input);
+ else if (input) actions.saveText(fieldKey, target);
+ };
+ return (
+ <Form {...form} formState={fieldState}>
+ <div
+ ref={rowRef}
+ data-score-row={index}
+ // `tabIndex={-1}` makes the row programmatically focusable
+ // so ↑/↓ can land on the row itself (navigate) without
+ // entering its text field. The focused row highlights via
+ // `:focus-within` (single source of truth = real focus;
+ // `ring-inset` so the scroll container's overflow can't
+ // clip it), and `group` shows the option badges only on it.
+ tabIndex={-1}
+ role="group"
+ aria-label={showTarget ? `${score.name} (${target.label})` : score.name}
+ className={cn(
+ "group grid w-full grid-cols-[1fr_2fr] items-center gap-3 rounded-md px-3 py-1 text-left transition-colors outline-none",
+ "focus-within:ring-primary/30 focus-within:bg-accent/40 focus-within:ring-1 focus-within:ring-inset",
+ )}
+ >
+ <div className="flex h-full min-w-0 flex-col items-start justify-center gap-1">
+ <div className="flex max-w-full min-w-0 items-center gap-1">
+ {config.description ||
+ isPresent(config.maxValue) ||
+ isPresent(config.minValue) ? (
+ <HoverCard>
+ <HoverCardTrigger asChild>
+ <span
+ className={cn(
+ "decoration-muted-gray line-clamp-2 min-w-0 text-xs font-bold wrap-break-word underline decoration-dashed underline-offset-2",
+ config.isArchived ? "text-foreground/40" : "",
+ )}
+ >
+ {score.name}
+ </span>
+ </HoverCardTrigger>
+ <HoverCardContent className="z-20 max-h-[60vh] max-w-64 overflow-y-auto rounded border">
+ <ScoreConfigDetails config={config} />
+ </HoverCardContent>
+ </HoverCard>
+ ) : (
+ <span
+ className={cn(
+ "line-clamp-2 min-w-0 text-xs font-bold wrap-break-word",
+ config.isArchived ? "text-foreground/40" : "",
+ )}
+ title={score.name}
+ >
+ {score.name}
+ </span>
+ )}
+ <Popover>
+ <PopoverTrigger asChild>
+ <Button
+ variant="link"
+ type="button"
+ size="xs"
+ title="Add or view score comment"
+ // Center the comment icon vertically
+ // against the score label instead of stretching
+ // to the full (possibly multi-line) row height,
+ // and keep it hugging the label (shrink-0) rather
+ // than floating in the middle of the row.
+ className="disabled:text-primary/50 flex h-auto shrink-0 items-center self-center px-0 pl-1 disabled:opacity-100"
+ disabled={
+ isScoreUnsaved(score.id) ||
+ (config.isArchived && !score.comment)
+ }
+ >
+ {score.comment ? (
+ <MessageCircleMore className="h-4 w-4" />
+ ) : (
+ <MessageCircle className="h-4 w-4" />
+ )}
+ </Button>
+ </PopoverTrigger>
+ <PopoverContent>
+ <FormField
+ control={form.control}
+ name={`scoreData.${index}.comment`}
+ render={() => (
+ <FormItem className="space-y-4">
+ <FormControl>
+ <CommentField
+ savedComment={score.comment ?? null}
+ disabled={config.isArchived}
+ loading={commentSaving}
+ onSave={(newComment) => {
+ const trimmed = newComment?.trim();
+ actions.saveComment(
+ fieldKey,
+ target,
+ trimmed || null,
+ );
+ }}
+ />
+ </FormControl>
+ <FormMessage className="text-xs" />
+ </FormItem>
+ )}
+ />
+ </PopoverContent>
+ </Popover>
+ </div>
+ {showTarget ? (
+ <Badge variant="outline-solid" size="sm">
+ {target.label}
+ </Badge>
+ ) : null}
+ </div>
+ <div className="grid grid-cols-[minmax(0,1fr)_1.5rem] items-center gap-1 py-1">
+ {/* data-score-control wraps only the value control so
+ keyboard 1-9 scoring targets it, not the in-row
+ comment or score actions. */}
+ <div data-score-control>
+ {isTextDataType(score.dataType) ? (
+ <FormField
+ control={form.control}
+ name={`scoreData.${index}.stringValue`}
+ render={({ field }) => (
+ <FormItem>
+ <FormControl>
+ <Textarea
+ {...field}
+ value={field.value ?? ""}
+ maxLength={TEXT_SCORE_MAX_LENGTH}
+ className="text-xs"
+ disabled={config.isArchived}
+ placeholder="Enter free form text..."
+ onBlur={(event) => {
+ field.onBlur();
+ if (!isMovingToScoreActions(event))
+ actions.saveText(fieldKey, target);
+ }}
+ />
+ </FormControl>
+ <FormMessage className="text-xs" />
+ </FormItem>
+ )}
+ />
+ ) : null}
+ {isNumericDataType(score.dataType) ? (
+ <FormField
+ control={form.control}
+ name={`scoreData.${index}.value`}
+ render={({ field }) => (
+ <FormItem className="space-y-1.5">
+ <FormControl>
+ <Input
+ {...field}
+ value={field.value ?? ""}
+ onChange={(event) => {
+ const value = event.currentTarget.valueAsNumber;
+ field.onChange(Number.isFinite(value) ? value : null);
+ }}
+ onInput={(event) =>
+ actions.validateNumericInput(
+ fieldKey,
+ target,
+ event.currentTarget,
+ )
+ }
+ type="number"
+ // Mirror the config range as native
+ // constraints so out-of-range values are
+ // catchable via the ⌘/Ctrl+Enter complete
+ // gate's rangeOverflow/Underflow check, on
+ // top of the existing onBlur JS validation.
+ // `step="any"` keeps decimals valid (config
+ // validation is range-only, not integer).
+ min={config.minValue ?? undefined}
+ max={config.maxValue ?? undefined}
+ step="any"
+ className="aria-invalid:border-destructive text-xs"
+ disabled={config.isArchived}
+ onBlur={(event) => {
+ field.onBlur();
+ if (!isMovingToScoreActions(event))
+ actions.saveNumeric(
+ fieldKey,
+ target,
+ event.currentTarget,
+ );
+ }}
+ />
+ </FormControl>
+ <FormMessage className="text-xs leading-snug font-normal" />
+ </FormItem>
+ )}
+ />
+ ) : null}
+ {!isTextDataType(score.dataType) &&
+ !isNumericDataType(score.dataType) ? (
+ <FormField
+ control={form.control}
+ name={`scoreData.${index}.stringValue`}
+ render={({ field }) => (
+ <FormItem>
+ <FormControl>
+ <CategoricalScoreInput
+ projectId={target.scoreMetadata.projectId}
+ config={config}
+ categories={categories}
+ name={field.name}
+ value={field.value ?? ""}
+ disabled={config.isArchived}
+ analyticsData={{
+ ...target.analyticsData,
+ targetType: getAnnotationTargetType(
+ target.scoreTarget,
+ ),
+ }}
+ onValueChange={(value, numericValue) =>
+ actions.saveCategory(
+ fieldKey,
+ target,
+ value,
+ numericValue,
+ )
+ }
+ />
+ </FormControl>
+ <FormMessage className="text-xs" />
+ </FormItem>
+ )}
+ />
+ ) : null}
+ </div>
+ {hasScoreValue || invalid || canRemove || targetOptions.length > 0 ? (
+ <DropdownMenuController
+ align="end"
+ onCloseAutoFocus={(event) => {
+ event.preventDefault();
+ const nextFocus = focusFieldKey.current;
+ focusFieldKey.current = fieldKey;
+ commitDeferredScore();
+ const nextRow = formRootRef.current?.querySelector<HTMLElement>(
+ `[data-score-row="${actions.indexOf(nextFocus)}"]`,
+ );
+ const focusTarget =
+ nextRow ??
+ formRootRef.current?.querySelector<HTMLElement>(
+ "[data-add-score]",
+ );
+ focusTarget?.focus();
+ }}
+ renderMenu={() => (
+ <>
+ {targetOptions.length > 0 ? (
+ <>
+ <DropdownMenuLabel>
+ Applies to: {target.label}
+ </DropdownMenuLabel>
+ {score.id ? (
+ <div className="text-muted-foreground px-2 pb-2 text-xs">
+ This score is saved on the{" "}
+ {target.label.toLowerCase()} and cannot be moved.
+ </div>
+ ) : (
+ targetOptions.map(
+ ({ target: destination, hasField }) => (
+ <DropdownMenuItem
+ key={destination.key}
+ disabled={
+ saving ||
+ invalid ||
+ hasField ||
+ config.isArchived
+ }
+ onSelect={() => {
+ const input = deferredInput.current;
+ deferredInput.current = null;
+ const nextKey = actions.changeDraftTarget(
+ fieldKey,
+ destination,
+ );
+ if (nextKey) focusFieldKey.current = nextKey;
+ else deferredInput.current = input;
+ }}
+ >
+ Score {destination.label.toLowerCase()} instead
+ {hasField ? " (already selected)" : ""}
+ </DropdownMenuItem>
+ ),
+ )
+ )}
+ {targetOptions.map(
+ ({ target: destination, hasField }) => (
+ <DropdownMenuItem
+ key={`add-${destination.key}`}
+ disabled={hasField || config.isArchived}
+ onSelect={() => {
+ commitDeferredScore();
+ const nextKey = actions.addDraftTarget(
+ fieldKey,
+ destination,
+ );
+ if (nextKey) focusFieldKey.current = nextKey;
+ }}
+ >
+ Also score {destination.label.toLowerCase()}
+ {hasField ? " (already selected)" : ""}
+ </DropdownMenuItem>
+ ),
+ )}
+ </>
+ ) : null}
+ {(hasScoreValue || invalid || canRemove) &&
+ targetOptions.length > 0 ? (
+ <DropdownMenuSeparator />
+ ) : null}
+ {canRemove ? (
+ <DropdownMenuItem
+ disabled={fieldSaving}
+ onSelect={() => {
+ deferredInput.current = null;
+ onRemove();
+ }}
+ >
+ Remove score
+ </DropdownMenuItem>
+ ) : null}
+ {hasScoreValue || invalid ? (
+ <DropdownMenuItem
+ disabled={saving}
+ onSelect={() => {
+ deferredInput.current = null;
+ actions.clear(fieldKey, target);
+ }}
+ >
+ Clear score
+ </DropdownMenuItem>
+ ) : null}
+ </>
+ )}
+ >
+ {({ Trigger }) => (
+ <Trigger asChild>
+ <Button
+ variant="ghost"
+ type="button"
+ size="icon-xs"
+ data-score-actions
+ onBlur={(event) => {
+ if (event.currentTarget.dataset.state !== "open")
+ commitDeferredScore();
+ }}
+ aria-label={`Score actions for ${score.name}${showTarget ? ` (${target.label})` : ""}`}
+ title="Score actions"
+ >
+ <MoreHorizontal className="size-4" />
+ </Button>
+ </Trigger>
+ )}
+ </DropdownMenuController>
+ ) : null}
+ </div>
+ </div>
+ </Form>
+ );
+}
diff --git a/web/src/features/scores/components/CategoricalScoreInput.tsx b/web/src/features/scores/components/CategoricalScoreInput.tsx
index 53c9aa5b29..b42674847e 100644
--- a/web/src/features/scores/components/CategoricalScoreInput.tsx
+++ b/web/src/features/scores/components/CategoricalScoreInput.tsx
@@ -12,12 +12,12 @@ import { useHasProjectAccess } from "@/src/features/rbac";
import { isCategoricalDataType } from "@/src/features/scores/lib/helpers";
import { getAddCategoryActionLabel } from "@/src/features/scores/lib/annotationFormHelpers";
import { AddScoreCategoryDialog } from "@/src/features/scores/components/AddScoreCategoryDialog";
-import { type AnalyticsData } from "@/src/features/scores/types";
+import { type AnnotationAnalyticsContext } from "@/src/features/scores/lib/annotationAnalytics";
const CHAR_CUTOFF = 6;
const DIGIT_SHORTCUTS = ["1", "2", "3", "4", "5", "6", "7", "8", "9"] as const;
-function shouldUseCombobox(
+export function shouldUseCombobox(
categories: Pick<ScoreConfigCategoryDomain, "label">[],
) {
const hasMoreThanThreeCategories = categories.length > 3;
@@ -38,7 +38,7 @@ export function CategoricalScoreInput({
value,
disabled,
name,
- source,
+ analyticsData,
onValueChange,
}: {
projectId: string;
@@ -47,7 +47,7 @@ export function CategoricalScoreInput({
value: string;
disabled: boolean;
name: string;
- source: AnalyticsData["source"] | undefined;
+ analyticsData: AnnotationAnalyticsContext;
onValueChange: (value: string, numericValue?: number) => void;
}) {
const hasConfigCudAccess = useHasProjectAccess({
@@ -170,7 +170,7 @@ export function CategoricalScoreInput({
projectId={projectId}
config={config}
initialLabel={pendingLabel}
- source={source}
+ analyticsData={analyticsData}
onClose={() => setPendingLabel(null)}
onCategoryAdded={onValueChange}
/>
diff --git a/web/src/features/scores/components/DualAnnotationContent.clienttest.tsx b/web/src/features/scores/components/DualAnnotationContent.clienttest.tsx
new file mode 100644
index 0000000000..8c53872f2e
--- /dev/null
+++ b/web/src/features/scores/components/DualAnnotationContent.clienttest.tsx
@@ -0,0 +1,923 @@
+import {
+ act,
+ fireEvent,
+ render,
+ screen,
+ waitFor,
+ within,
+} from "@testing-library/react";
+import {
+ QueryClient,
+ QueryClientProvider,
+ type UseMutationOptions,
+} from "@tanstack/react-query";
+import type {
+ CreateAnnotationScoreData,
+ ScoreConfigDomain,
+ ScoreDomain,
+} from "@langfuse/shared";
+import type HeaderComponent from "@/src/components/layouts/header";
+import { LayerProvider } from "@/src/context/LayerContext/LayerContext";
+import {
+ ScoreCacheProvider,
+ useScoreCache,
+} from "@/src/features/scores/contexts/ScoreCacheContext";
+import { DualAnnotationContent } from "./DualAnnotationContent";
+import { AnnotationForm } from "./AnnotationForm";
+
+const mocks = vi.hoisted(() => ({
+ create: vi.fn(),
+ update: vi.fn(),
+ remove: vi.fn(),
+ capture: vi.fn(),
+ headerRender: vi.fn(),
+}));
+const defaultConfig = {
+ id: "quality",
+ name: "Quality",
+ projectId: "project",
+ dataType: "BOOLEAN",
+ categories: [
+ { label: "False", value: 0 },
+ { label: "True", value: 1 },
+ ],
+ isArchived: false,
+ createdAt: new Date(),
+ updatedAt: new Date(),
+} satisfies ScoreConfigDomain;
+const configs: ScoreConfigDomain[] = [defaultConfig];
+
+vi.mock("@/src/components/layouts/header", async (importOriginal) => {
+ const { default: Header } = await importOriginal<{
+ default: typeof HeaderComponent;
+ }>();
+ return {
+ default: (props: React.ComponentProps<typeof Header>) => {
+ mocks.headerRender();
+ return <Header {...props} />;
+ },
+ };
+});
+
+vi.mock("@/src/features/rbac", () => ({ useHasProjectAccess: () => false }));
+vi.mock("@/src/features/notifications", () => ({ showErrorToast: vi.fn() }));
+vi.mock("@/src/features/posthog-analytics", () => ({
+ usePostHogClientCapture: () => mocks.capture,
+}));
+vi.mock("@/src/features/posthog-analytics/usePostHogClientCapture", () => ({
+ usePostHogClientCapture: () => mocks.capture,
+}));
+vi.mock("@/src/utils/api", async () => {
+ const { useMutation } = await import("@tanstack/react-query");
+ return {
+ api: {
+ scoreConfigs: {
+ all: { useQuery: () => ({ isLoading: false, data: { configs } }) },
+ },
+ scores: {
+ createAnnotationScore: {
+ useMutation: (
+ options: UseMutationOptions<
+ unknown,
+ Error,
+ CreateAnnotationScoreData
+ >,
+ ) => useMutation({ ...options, mutationFn: mocks.create }),
+ },
+ updateAnnotationScore: {
+ useMutation: (options: UseMutationOptions) =>
+ useMutation({ ...options, mutationFn: mocks.update }),
+ },
+ deleteAnnotationScore: {
+ useMutation: (options: UseMutationOptions) =>
+ useMutation({ ...options, mutationFn: mocks.remove }),
+ },
+ },
+ },
+ };
+});
+
+function CacheProbe() {
+ const cache = useScoreCache();
+ return (
+ <output aria-label="Cached scores">{JSON.stringify(cache.getAll())}</output>
+ );
+}
+
+const content = (
+ <DualAnnotationContent
+ projectId="project"
+ isV4
+ traceId="trace"
+ observationId="observation"
+ traceEnvironment="trace-env"
+ observationEnvironment="observation-env"
+ traceScores={[]}
+ observationScores={[]}
+ />
+);
+
+function renderContent(children = content) {
+ const queryClient = new QueryClient({
+ defaultOptions: { mutations: { retry: false } },
+ });
+ const wrap = (children: React.ReactNode) => (
+ <QueryClientProvider client={queryClient}>
+ <LayerProvider>
+ <ScoreCacheProvider>
+ {children}
+ <CacheProbe />
+ </ScoreCacheProvider>
+ </LayerProvider>
+ </QueryClientProvider>
+ );
+ const rendered = render(wrap(children));
+ return {
+ ...rendered,
+ rerenderContent: (next: React.ReactNode) => rendered.rerender(wrap(next)),
+ };
+}
+
+describe("unified annotation targets", () => {
+ beforeEach(() => {
+ vi.clearAllMocks();
+ Element.prototype.scrollIntoView = vi.fn();
+ localStorage.clear();
+ localStorage.setItem(
+ "emptySelectedConfigIds:observation",
+ JSON.stringify(["quality"]),
+ );
+ localStorage.setItem(
+ "emptySelectedConfigIds:trace",
+ JSON.stringify(["quality"]),
+ );
+ vi.stubGlobal(
+ "ResizeObserver",
+ class {
+ observe() {}
+ unobserve() {}
+ disconnect() {}
+ },
+ );
+ });
+
+ afterEach(() => {
+ configs.splice(1);
+ vi.unstubAllGlobals();
+ });
+
+ it("adds scores once and removes only the chosen empty row", async () => {
+ configs.push({ ...defaultConfig, id: "accuracy", name: "Accuracy" });
+ const rendered = renderContent();
+ fireEvent.click(screen.getByRole("button", { name: "Add score" }));
+ expect(
+ screen.queryByRole("option", { name: /Quality/ }),
+ ).not.toBeInTheDocument();
+ fireEvent.click(await screen.findByRole("option", { name: /Accuracy/ }));
+ expect(screen.getAllByRole("group", { name: "Accuracy" })).toHaveLength(1);
+ expect(screen.getByRole("button", { name: "Add score" })).toBeDisabled();
+ fireEvent.keyDown(
+ screen.getByRole("button", { name: "Score actions for Accuracy" }),
+ { key: "ArrowDown" },
+ );
+ fireEvent.click(
+ await screen.findByRole("menuitem", { name: "Remove score" }),
+ );
+ expect(
+ screen.queryByRole("group", { name: "Accuracy" }),
+ ).not.toBeInTheDocument();
+ expect(screen.getByRole("group", { name: "Quality" })).toBeInTheDocument();
+ expect(mocks.remove).not.toHaveBeenCalled();
+ await waitFor(() =>
+ expect(screen.getByRole("button", { name: "Add score" })).toHaveFocus(),
+ );
+ rendered.unmount();
+ renderContent();
+ expect(
+ screen.queryByRole("group", { name: "Accuracy" }),
+ ).not.toBeInTheDocument();
+ });
+
+ it("chooses a target before the first save and cannot change it during or after saving", async () => {
+ const pending = deferred();
+ mocks.create.mockReturnValue(pending.promise);
+ renderContent();
+ expect(screen.getAllByRole("group", { name: "Quality" })).toHaveLength(1);
+ expect(screen.getByRole("button", { name: "Add score" })).toBeDisabled();
+ fireEvent.keyDown(
+ screen.getByRole("button", { name: "Score actions for Quality" }),
+ { key: "ArrowDown" },
+ );
+ fireEvent.click(
+ await screen.findByRole("menuitem", { name: "Score trace instead" }),
+ );
+ await waitFor(() =>
+ expect(screen.queryByRole("menu")).not.toBeInTheDocument(),
+ );
+ expect(mocks.create).not.toHaveBeenCalled();
+ expect(mocks.update).not.toHaveBeenCalled();
+ expect(mocks.remove).not.toHaveBeenCalled();
+ expect(mocks.capture).toHaveBeenCalledWith("score:level_changed", {
+ type: "trace",
+ source: "TraceDetail",
+ isV4: true,
+ previousTargetType: "observation",
+ targetType: "trace",
+ dataType: "BOOLEAN",
+ });
+ fireEvent.click(screen.getByRole("radio", { name: /True/ }));
+ await waitFor(() =>
+ expect(mocks.create).toHaveBeenCalledExactlyOnceWith(
+ expect.objectContaining({
+ scoreTarget: { type: "trace", traceId: "trace" },
+ value: 1,
+ }),
+ ),
+ );
+ fireEvent.keyDown(
+ screen.getByRole("button", { name: "Score actions for Quality" }),
+ { key: "ArrowDown" },
+ );
+ expect(
+ await screen.findByText(
+ "This score is saved on the trace and cannot be moved.",
+ ),
+ ).toBeVisible();
+ expect(
+ screen.queryByRole("menuitem", { name: "Score observation instead" }),
+ ).not.toBeInTheDocument();
+ await act(async () => pending.resolve({}));
+ expect(
+ screen.queryByRole("menuitem", { name: "Score observation instead" }),
+ ).not.toBeInTheDocument();
+ fireEvent.click(
+ screen.getByRole("menuitem", { name: "Also score observation" }),
+ );
+ await waitFor(() =>
+ expect(screen.queryByRole("menu")).not.toBeInTheDocument(),
+ );
+ const observationRow = screen.getByRole("group", {
+ name: "Quality (Observation)",
+ });
+ const traceRow = screen.getByRole("group", { name: "Quality (Trace)" });
+ expect(
+ within(observationRow).getByRole("radio", { name: /True/ }),
+ ).toHaveAttribute("aria-checked", "false");
+ expect(
+ within(traceRow).getByRole("radio", { name: /True/ }),
+ ).toHaveAttribute("aria-checked", "true");
+ fireEvent.keyDown(
+ within(traceRow).getByRole("button", { name: /Score actions/ }),
+ { key: "ArrowDown" },
+ );
+ expect(
+ await screen.findByRole("menuitem", {
+ name: "Also score observation (already selected)",
+ }),
+ ).toHaveAttribute("aria-disabled", "true");
+ fireEvent.keyDown(screen.getByRole("menu"), { key: "Escape" });
+ await waitFor(() => expect(traceRow).toHaveFocus());
+ expect(mocks.create).toHaveBeenCalledTimes(1);
+ expect(mocks.update).not.toHaveBeenCalled();
+ expect(mocks.remove).not.toHaveBeenCalled();
+ });
+
+ it("keeps a text draft while changing level without saving it on menu close", async () => {
+ configs.push({
+ ...defaultConfig,
+ id: "feedback",
+ name: "Feedback",
+ dataType: "TEXT",
+ categories: null,
+ });
+ localStorage.setItem(
+ "emptySelectedConfigIds:observation",
+ JSON.stringify(["feedback"]),
+ );
+ localStorage.setItem("emptySelectedConfigIds:trace", JSON.stringify([]));
+ mocks.create.mockResolvedValue({});
+ renderContent();
+ const input = screen.getByRole("textbox");
+ act(() => input.focus());
+ fireEvent.change(input, { target: { value: "My unsaved review" } });
+ const menu = screen.getByRole("button", {
+ name: "Score actions for Feedback",
+ });
+ act(() => menu.focus());
+ fireEvent.keyDown(menu, { key: "ArrowDown" });
+ fireEvent.click(
+ await screen.findByRole("menuitem", { name: "Score trace instead" }),
+ );
+ await waitFor(() =>
+ expect(screen.queryByRole("menu")).not.toBeInTheDocument(),
+ );
+ expect(screen.getByRole("textbox")).toHaveValue("My unsaved review");
+ expect(mocks.create).not.toHaveBeenCalled();
+ fireEvent.blur(screen.getByRole("textbox"));
+ await waitFor(() =>
+ expect(mocks.create).toHaveBeenCalledExactlyOnceWith(
+ expect.objectContaining({
+ scoreTarget: { type: "trace", traceId: "trace" },
+ stringValue: "My unsaved review",
+ }),
+ ),
+ );
+ });
+
+ it("keeps draft edits local, preserves them through refetch, and resets only for another target", () => {
+ const text = {
+ ...defaultConfig,
+ id: "feedback",
+ name: "Feedback",
+ dataType: "TEXT" as const,
+ categories: null,
+ };
+ const score = (sessionId: string, stringValue: string): ScoreDomain => ({
+ id: `score-${sessionId}`,
+ projectId: "project",
+ environment: "default",
+ name: "Feedback",
+ value: 0,
+ dataType: "TEXT",
+ stringValue,
+ longStringValue: "",
+ source: "ANNOTATION",
+ authorUserId: null,
+ comment: null,
+ metadata: {},
+ configId: "feedback",
+ queueId: null,
+ executionTraceId: null,
+ createdAt: new Date(),
+ updatedAt: new Date(),
+ timestamp: new Date(),
+ traceId: null,
+ sessionId,
+ datasetRunId: null,
+ observationId: null,
+ });
+ const contentFor = (sessionId: string, value: string) => (
+ <AnnotationForm
+ scoreTarget={{ type: "session", sessionId }}
+ serverScores={[{ ...score(sessionId, value), metadata: "{}" }]}
+ scoreMetadata={{ projectId: "project" }}
+ analyticsData={{ type: "session", source: "SessionDetail", isV4: true }}
+ configSelection={{ mode: "fixed", configs: [text] }}
+ />
+ );
+ const view = renderContent(contentFor("session", "Original"));
+ const input = screen.getByRole("textbox");
+ mocks.headerRender.mockClear();
+ fireEvent.change(input, { target: { value: "My unsaved draft" } });
+ expect(mocks.headerRender).not.toHaveBeenCalled();
+ expect(mocks.update).not.toHaveBeenCalled();
+ view.rerenderContent(contentFor("session", "Background refetch"));
+ expect(screen.getByRole("textbox")).toBe(input);
+ expect(input).toHaveValue("My unsaved draft");
+ view.rerenderContent(contentFor("next-session", "Other session score"));
+ expect(screen.getByRole("textbox")).toHaveValue("Other session score");
+ expect(mocks.update).not.toHaveBeenCalled();
+ });
+
+ it("navigates one form and saves the same config independently for observation and trace", async () => {
+ mocks.create.mockResolvedValue({});
+ await renderBothTargets();
+
+ expect(screen.getAllByRole("button", { name: "Add score" })).toHaveLength(
+ 1,
+ );
+ expect(
+ screen.queryByText(/Score data saved|^Saved$/),
+ ).not.toBeInTheDocument();
+ const observationRow = screen.getByRole("group", {
+ name: "Quality (Observation)",
+ });
+ const traceRow = screen.getByRole("group", { name: "Quality (Trace)" });
+ act(() => observationRow.focus());
+ fireEvent.keyDown(observationRow, { key: "1" });
+ fireEvent.keyDown(observationRow, { key: "ArrowDown" });
+ expect(traceRow).toHaveFocus();
+ fireEvent.keyDown(traceRow, { key: "2" });
+
+ await waitFor(() => expect(mocks.create).toHaveBeenCalledTimes(2));
+ const writes = mocks.create.mock.calls.map(([variables]) => variables);
+ expect(writes).toEqual([
+ expect.objectContaining({
+ configId: "quality",
+ value: 0,
+ environment: "observation-env",
+ scoreTarget: {
+ type: "trace",
+ traceId: "trace",
+ observationId: "observation",
+ },
+ }),
+ expect.objectContaining({
+ configId: "quality",
+ value: 1,
+ environment: "trace-env",
+ scoreTarget: { type: "trace", traceId: "trace" },
+ }),
+ ]);
+ expect(writes[0].id).not.toBe(writes[1].id);
+ await waitFor(() =>
+ expect(
+ screen.getByRole("status", { name: "Score save status" }),
+ ).toHaveTextContent("Saved"),
+ );
+ expect(screen.getByRole("button", { name: "Add score" })).toBeDisabled();
+ const cached = JSON.parse(
+ screen.getByLabelText("Cached scores").textContent!,
+ );
+ expect(cached).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({
+ observationId: "observation",
+ configId: "quality",
+ value: 0,
+ environment: "observation-env",
+ }),
+ expect.objectContaining({
+ observationId: null,
+ configId: "quality",
+ value: 1,
+ environment: "trace-env",
+ }),
+ ]),
+ );
+ expect(
+ within(observationRow).getByRole("radio", { name: /False/ }),
+ ).toHaveAttribute("aria-checked", "true");
+ expect(
+ within(traceRow).getByRole("radio", { name: /True/ }),
+ ).toHaveAttribute("aria-checked", "true");
+ });
+
+ it("restores a categorical score after a failed clear and keeps its field after retry", async () => {
+ mocks.create.mockResolvedValue({});
+ const firstClear = deferred();
+ mocks.remove.mockReturnValueOnce(firstClear.promise).mockResolvedValue({});
+ await renderBothTargets();
+ const observationRow = screen.getByRole("group", {
+ name: "Quality (Observation)",
+ });
+ const traceRow = screen.getByRole("group", { name: "Quality (Trace)" });
+ act(() => observationRow.focus());
+ fireEvent.keyDown(observationRow, { key: "1" });
+ act(() => traceRow.focus());
+ fireEvent.keyDown(traceRow, { key: "2" });
+ await waitFor(() =>
+ expect(
+ screen.getByRole("status", { name: "Score save status" }),
+ ).toHaveTextContent("Saved"),
+ );
+ fireEvent.keyDown(
+ screen.getByRole("button", {
+ name: "Score actions for Quality (Observation)",
+ }),
+ { key: "ArrowDown" },
+ );
+ expect(
+ screen.queryByRole("menuitem", { name: "Remove score" }),
+ ).not.toBeInTheDocument();
+ fireEvent.click(
+ await screen.findByRole("menuitem", { name: "Clear score" }),
+ );
+ fireEvent.keyDown(
+ screen.getByRole("button", {
+ name: "Score actions for Quality (Observation)",
+ }),
+ { key: "ArrowDown" },
+ );
+ const pendingRemoval = await screen.findByRole("menuitem", {
+ name: "Remove score",
+ });
+ expect(pendingRemoval).toHaveAttribute("aria-disabled", "true");
+ fireEvent.click(pendingRemoval);
+ fireEvent.keyDown(pendingRemoval, { key: "Escape" });
+ expect(
+ screen.getByRole("group", { name: "Quality (Observation)" }),
+ ).toBeInTheDocument();
+ await act(async () =>
+ firstClear.reject(new Error("Could not clear score")),
+ );
+ expect(await screen.findByText("Failed to clear score")).toBeVisible();
+ const restored = screen.getByRole("group", {
+ name: "Quality (Observation)",
+ });
+ expect(
+ within(restored).getByRole("radio", { name: /False/ }),
+ ).toHaveAttribute("aria-checked", "true");
+ fireEvent.keyDown(
+ screen.getByRole("button", {
+ name: "Score actions for Quality (Observation)",
+ }),
+ { key: "ArrowDown" },
+ );
+ fireEvent.click(
+ await screen.findByRole("menuitem", { name: "Clear score" }),
+ );
+ await waitFor(() => expect(mocks.remove).toHaveBeenCalledTimes(2));
+ expect(mocks.remove).toHaveBeenLastCalledWith(
+ expect.objectContaining({
+ id: mocks.create.mock.calls[0]![0].id,
+ projectId: "project",
+ }),
+ );
+ expect(screen.queryByText("Failed to clear score")).not.toBeInTheDocument();
+ const cleared = screen.getByRole("group", {
+ name: "Quality (Observation)",
+ });
+ expect(
+ within(cleared).getByRole("radio", { name: /False/ }),
+ ).toHaveAttribute("aria-checked", "false");
+ expect(
+ within(traceRow).getByRole("radio", { name: /True/ }),
+ ).toHaveAttribute("aria-checked", "true");
+ fireEvent.keyDown(
+ screen.getByRole("button", {
+ name: "Score actions for Quality (Observation)",
+ }),
+ { key: "ArrowDown" },
+ );
+ fireEvent.click(
+ await screen.findByRole("menuitem", { name: "Remove score" }),
+ );
+ expect(screen.getAllByRole("group", { name: "Quality" })).toHaveLength(1);
+ expect(mocks.remove).toHaveBeenCalledTimes(2);
+ });
+
+ it("keeps the latest saved status when an older edit of the same field fails", async () => {
+ const older = deferred();
+ const newer = deferred();
+ mocks.create.mockResolvedValue({});
+ mocks.update
+ .mockReturnValueOnce(older.promise)
+ .mockReturnValueOnce(newer.promise);
+ renderContent();
+ const row = screen.getByRole("group", { name: "Quality" });
+ fireEvent.click(within(row).getByRole("radio", { name: /False/ }));
+ await waitFor(() =>
+ expect(
+ screen.getByRole("status", { name: "Score save status" }),
+ ).toHaveTextContent("Saved"),
+ );
+ fireEvent.click(within(row).getByRole("radio", { name: /True/ }));
+ fireEvent.click(within(row).getByRole("radio", { name: /False/ }));
+ await waitFor(() => expect(mocks.update).toHaveBeenCalledTimes(2));
+ await act(async () => newer.resolve({}));
+ await act(async () => older.reject(new Error("Earlier edit failed")));
+ expect(
+ screen.getByRole("status", { name: "Score save status" }),
+ ).toHaveTextContent("Saved");
+ expect(within(row).getByRole("radio", { name: /False/ })).toHaveAttribute(
+ "aria-checked",
+ "true",
+ );
+ });
+
+ it.each([false, true])(
+ "rolls back a target failure and retries while another save is pending: %s",
+ async (retryWhilePending) => {
+ const first = deferred();
+ const second = deferred();
+ mocks.create
+ .mockReturnValueOnce(first.promise)
+ .mockReturnValueOnce(second.promise)
+ .mockResolvedValue({});
+ await renderBothTargets();
+ expect(
+ screen.queryByRole("status", { name: "Score save status" }),
+ ).not.toBeInTheDocument();
+ const observationRow = screen.getByRole("group", {
+ name: "Quality (Observation)",
+ });
+ const traceRow = screen.getByRole("group", { name: "Quality (Trace)" });
+ fireEvent.click(
+ within(observationRow).getByRole("radio", { name: /False/ }),
+ );
+ fireEvent.click(within(traceRow).getByRole("radio", { name: /True/ }));
+ await waitFor(() => expect(mocks.create).toHaveBeenCalledTimes(2));
+ expect(
+ screen.getByRole("status", { name: "Score save status" }),
+ ).toHaveTextContent("Saving");
+ await act(async () => {
+ first.reject(new Error("Could not write observation"));
+ if (!retryWhilePending) second.resolve({});
+ });
+ await waitFor(() =>
+ expect(
+ screen.getByRole("status", { name: "Score save status" }),
+ ).toHaveTextContent(retryWhilePending ? "Saving" : "Could not save"),
+ );
+ expect(
+ within(observationRow).getByRole("radio", { name: /False/ }),
+ ).toHaveAttribute("aria-checked", "false");
+ expect(
+ within(traceRow).getByRole("radio", { name: /True/ }),
+ ).toHaveAttribute("aria-checked", "true");
+ act(() => observationRow.focus());
+ fireEvent.keyDown(observationRow, { key: "1" });
+ await waitFor(() => expect(mocks.create).toHaveBeenCalledTimes(3));
+ expect(mocks.update).not.toHaveBeenCalled();
+ if (retryWhilePending) await act(async () => second.resolve({}));
+ await waitFor(() =>
+ expect(
+ screen.getByRole("status", { name: "Score save status" }),
+ ).toHaveTextContent("Saved"),
+ );
+ },
+ );
+
+ it("rolls back the same target after an earlier empty row is removed during its save", async () => {
+ const pending = deferred();
+ mocks.create.mockReturnValueOnce(pending.promise).mockResolvedValue({});
+ await renderBothTargets();
+ fireEvent.click(
+ within(screen.getByRole("group", { name: "Quality (Trace)" })).getByRole(
+ "radio",
+ { name: /True/ },
+ ),
+ );
+ await waitFor(() => expect(mocks.create).toHaveBeenCalledTimes(1));
+ fireEvent.keyDown(
+ screen.getByRole("button", {
+ name: "Score actions for Quality (Observation)",
+ }),
+ { key: "ArrowDown" },
+ );
+ fireEvent.click(
+ await screen.findByRole("menuitem", { name: "Remove score" }),
+ );
+ await act(async () => pending.reject(new Error("Trace save failed")));
+ const remaining = screen.getByRole("group", { name: "Quality" });
+ expect(
+ within(remaining).getByRole("radio", { name: /True/ }),
+ ).toHaveAttribute("aria-checked", "false");
+ expect(within(remaining).getByText("Failed to create score")).toBeVisible();
+ fireEvent.click(within(remaining).getByRole("radio", { name: /False/ }));
+ await waitFor(() => expect(mocks.create).toHaveBeenCalledTimes(2));
+ expect(mocks.create).toHaveBeenLastCalledWith(
+ expect.objectContaining({
+ scoreTarget: { type: "trace", traceId: "trace" },
+ value: 0,
+ }),
+ );
+ expect(mocks.update).not.toHaveBeenCalled();
+ });
+
+ it("keeps a single-target numeric form untagged and hides Saved while its next draft is invalid", async () => {
+ const numeric = {
+ ...defaultConfig,
+ id: "numeric",
+ name: "Rating",
+ dataType: "NUMERIC" as const,
+ minValue: 0,
+ maxValue: 10,
+ categories: null,
+ };
+ mocks.create.mockResolvedValue({});
+ renderContent(
+ <AnnotationForm
+ scoreTarget={{ type: "session", sessionId: "session" }}
+ serverScores={[]}
+ scoreMetadata={{ projectId: "project", environment: "session-env" }}
+ analyticsData={{ type: "session", source: "SessionDetail", isV4: true }}
+ configSelection={{ mode: "fixed", configs: [numeric] }}
+ />,
+ );
+ expect(
+ screen.queryByRole("status", { name: "Score save status" }),
+ ).not.toBeInTheDocument();
+ expect(screen.getByRole("group", { name: "Rating" })).toBeInTheDocument();
+ const input = screen.getByRole("spinbutton");
+ fireEvent.input(input, { target: { value: "5" } });
+ fireEvent.blur(input);
+ await waitFor(() =>
+ expect(
+ screen.getByRole("status", { name: "Score save status" }),
+ ).toHaveTextContent("Saved"),
+ );
+ expect(mocks.create).toHaveBeenCalledWith(
+ expect.objectContaining({
+ scoreTarget: { type: "session", sessionId: "session" },
+ environment: "session-env",
+ }),
+ );
+ fireEvent.input(input, { target: { value: "11" } });
+ expect(
+ screen.queryByRole("status", { name: "Score save status" }),
+ ).not.toBeInTheDocument();
+ fireEvent.blur(input);
+ expect(mocks.update).not.toHaveBeenCalled();
+ expect(screen.queryByText("Saved")).not.toBeInTheDocument();
+ });
+
+ it("does not delete a saved score for Firefox badInput and recovers from number and range errors", async () => {
+ const numeric = {
+ ...defaultConfig,
+ id: "numeric",
+ name: "Rating",
+ dataType: "NUMERIC" as const,
+ minValue: 1,
+ maxValue: 5,
+ categories: null,
+ };
+ mocks.create.mockResolvedValue({});
+ mocks.update.mockResolvedValue({});
+ mocks.remove.mockResolvedValue({});
+ renderContent(
+ <AnnotationForm
+ scoreTarget={{ type: "session", sessionId: "session" }}
+ serverScores={[]}
+ scoreMetadata={{ projectId: "project", environment: "session-env" }}
+ analyticsData={{ type: "session", source: "SessionDetail", isV4: true }}
+ configSelection={{ mode: "fixed", configs: [numeric] }}
+ />,
+ );
+ const input = screen.getByRole("spinbutton") as HTMLInputElement;
+ fireEvent.input(input, { target: { value: "5" } });
+ fireEvent.blur(input);
+ await waitFor(() =>
+ expect(
+ screen.getByRole("status", { name: "Score save status" }),
+ ).toHaveTextContent("Saved"),
+ );
+
+ fireEvent.input(input, { target: { value: "" } });
+ const badInput = vi
+ .spyOn(input.validity, "badInput", "get")
+ .mockReturnValue(true);
+ // Firefox emits another input event while its sanitized value stays empty.
+ fireEvent.input(input);
+ expect(screen.getByText("Enter a number")).toBeVisible();
+ expect(input).toHaveAttribute("aria-invalid", "true");
+ await act(async () => fireEvent.blur(input));
+ expect(mocks.remove).not.toHaveBeenCalled();
+ expect(screen.getByText("Enter a number")).toBeVisible();
+ expect(input).toHaveAttribute("aria-invalid", "true");
+ expect(input).toHaveAccessibleDescription("Enter a number");
+
+ badInput.mockReturnValue(false);
+ fireEvent.input(input, { target: { value: "6" } });
+ expect(screen.getByText("Enter a value between 1 and 5")).toBeVisible();
+ fireEvent.blur(input);
+ expect(mocks.update).not.toHaveBeenCalled();
+ fireEvent.input(input, { target: { value: "3" } });
+ expect(input).toHaveAttribute("aria-invalid", "false");
+ expect(
+ screen.queryByText("Enter a value between 1 and 5"),
+ ).not.toBeInTheDocument();
+ fireEvent.blur(input);
+ await waitFor(() =>
+ expect(mocks.update).toHaveBeenCalledWith(
+ expect.objectContaining({
+ value: 3,
+ scoreTarget: { type: "session", sessionId: "session" },
+ }),
+ ),
+ );
+ await act(async () => {
+ fireEvent.change(input, { target: { value: "" } });
+ fireEvent.blur(input);
+ });
+ expect(mocks.remove).toHaveBeenCalledTimes(1);
+ badInput.mockRestore();
+ });
+
+ it("saves on another row's actions and when dismissing its own actions without clearing", async () => {
+ const text = {
+ ...defaultConfig,
+ id: "feedback",
+ name: "Feedback",
+ dataType: "TEXT" as const,
+ categories: null,
+ };
+ mocks.create.mockResolvedValue({});
+ mocks.update.mockResolvedValue({});
+ renderContent(
+ <AnnotationForm
+ scoreTarget={{ type: "session", sessionId: "session" }}
+ serverScores={[]}
+ scoreMetadata={{ projectId: "project" }}
+ analyticsData={{ type: "session", source: "SessionDetail", isV4: true }}
+ configSelection={{
+ mode: "fixed",
+ configs: [text, { ...text, id: "notes", name: "Notes" }],
+ }}
+ />,
+ );
+ const input = within(
+ screen.getByRole("group", { name: "Feedback" }),
+ ).getByRole("textbox");
+ const otherInput = within(
+ screen.getByRole("group", { name: "Notes" }),
+ ).getByRole("textbox");
+ fireEvent.change(otherInput, { target: { value: "Other draft" } });
+ act(() => input.focus());
+ fireEvent.change(input, { target: { value: "Save feedback" } });
+ const otherActions = screen.getByRole("button", {
+ name: "Score actions for Notes",
+ });
+ act(() => otherActions.focus());
+ await waitFor(() => expect(mocks.create).toHaveBeenCalledTimes(1));
+ expect(mocks.create).toHaveBeenCalledWith(
+ expect.objectContaining({
+ configId: "feedback",
+ stringValue: "Save feedback",
+ }),
+ );
+
+ act(() => input.focus());
+ fireEvent.change(input, { target: { value: "Keep this edit" } });
+ const actions = screen.getByRole("button", {
+ name: "Score actions for Feedback",
+ });
+ act(() => actions.focus());
+ fireEvent.keyDown(actions, { key: "ArrowDown" });
+ const clearItem = await screen.findByRole("menuitem", {
+ name: "Clear score",
+ });
+ expect(mocks.update).not.toHaveBeenCalled();
+ fireEvent.keyDown(clearItem, { key: "Escape" });
+ await waitFor(() =>
+ expect(mocks.update).toHaveBeenCalledWith(
+ expect.objectContaining({
+ configId: "feedback",
+ stringValue: "Keep this edit",
+ }),
+ ),
+ );
+ expect(mocks.remove).not.toHaveBeenCalled();
+ expect(otherInput).toHaveValue("Other draft");
+ });
+
+ it("clears an unsaved draft from the labeled menu without posting it or removing a fixed field", async () => {
+ const text = {
+ ...defaultConfig,
+ id: "feedback",
+ name: "Feedback",
+ dataType: "TEXT" as const,
+ categories: null,
+ };
+ renderContent(
+ <AnnotationForm
+ scoreTarget={{ type: "session", sessionId: "session" }}
+ serverScores={[]}
+ scoreMetadata={{ projectId: "project" }}
+ analyticsData={{ type: "session", source: "SessionDetail", isV4: true }}
+ configSelection={{ mode: "fixed", configs: [text] }}
+ />,
+ );
+ const input = screen.getByRole("textbox");
+ expect(
+ screen.queryByRole("button", { name: "Score actions for Feedback" }),
+ ).not.toBeInTheDocument();
+ act(() => input.focus());
+ fireEvent.change(input, { target: { value: "Unsent feedback" } });
+ const actions = screen.getByRole("button", {
+ name: "Score actions for Feedback",
+ });
+ act(() => actions.focus());
+ fireEvent.keyDown(actions, { key: "ArrowDown" });
+ fireEvent.click(
+ await screen.findByRole("menuitem", { name: "Clear score" }),
+ );
+ await waitFor(() =>
+ expect(screen.queryByRole("menu")).not.toBeInTheDocument(),
+ );
+ expect(screen.getByRole("textbox")).toHaveValue("");
+ expect(screen.getByRole("group", { name: "Feedback" })).toBeInTheDocument();
+ expect(mocks.create).not.toHaveBeenCalled();
+ expect(mocks.remove).not.toHaveBeenCalled();
+ expect(
+ screen.queryByRole("status", { name: "Score save status" }),
+ ).not.toBeInTheDocument();
+ });
+});
+
+function deferred() {
+ let resolve!: (value: unknown) => void;
+ let reject!: (error: Error) => void;
+ const promise = new Promise<unknown>((onResolve, onReject) => {
+ resolve = onResolve;
+ reject = onReject;
+ });
+ return { promise, resolve, reject };
+}
+
+async function renderBothTargets() {
+ renderContent();
+ fireEvent.keyDown(
+ screen.getByRole("button", { name: "Score actions for Quality" }),
+ { key: "ArrowDown" },
+ );
+ fireEvent.click(
+ await screen.findByRole("menuitem", { name: "Also score trace" }),
+ );
+ await waitFor(() =>
+ expect(screen.queryByRole("menu")).not.toBeInTheDocument(),
+ );
+ expect(screen.getAllByRole("group", { name: /Quality/ })).toHaveLength(2);
+ expect(mocks.create).not.toHaveBeenCalled();
+}
diff --git a/web/src/features/scores/components/DualAnnotationContent.tsx b/web/src/features/scores/components/DualAnnotationContent.tsx
index 49342d9ca4..5136cf3071 100644
--- a/web/src/features/scores/components/DualAnnotationContent.tsx
+++ b/web/src/features/scores/components/DualAnnotationContent.tsx
@@ -1,10 +1,15 @@
import { type ScoreDomain } from "@langfuse/shared";
import { type WithStringifiedMetadata } from "@/src/utils/clientSideDomainTypes";
-import { AnnotationForm } from "@/src/features/scores/components/AnnotationForm";
-import { Separator } from "@/src/components/ui/separator";
+import {
+ AnnotationFormContent,
+ usePreparedAnnotationFormTarget,
+} from "@/src/features/scores/components/AnnotationForm";
+import { Skeleton } from "@/src/components/ui/skeleton";
+import { prepareCombinedAnnotationTargets } from "@/src/features/scores/lib/prepareAnnotationFormData";
interface DualAnnotationContentProps {
projectId: string;
+ isV4: boolean;
traceId: string;
observationId: string;
traceEnvironment: string;
@@ -15,6 +20,7 @@ interface DualAnnotationContentProps {
export function DualAnnotationContent({
projectId,
+ isV4,
traceId,
observationId,
traceEnvironment,
@@ -22,64 +28,30 @@ export function DualAnnotationContent({
observationScores,
traceScores,
}: DualAnnotationContentProps) {
- const hasNonAnnotationScores = [...observationScores, ...traceScores].some(
- (score) => score.source !== "ANNOTATION",
- );
-
+ const observation = usePreparedAnnotationFormTarget({
+ serverScores: observationScores,
+ scoreTarget: { type: "trace", traceId, observationId },
+ scoreMetadata: { projectId, environment: observationEnvironment },
+ analyticsData: { type: "trace", source: "TraceDetail", isV4 },
+ });
+ const trace = usePreparedAnnotationFormTarget({
+ serverScores: traceScores,
+ scoreTarget: { type: "trace", traceId },
+ scoreMetadata: { projectId, environment: traceEnvironment },
+ analyticsData: { type: "trace", source: "TraceDetail", isV4 },
+ });
return (
- <div className="flex max-h-[95vh] flex-col gap-4 overflow-y-auto">
- {/* Observation-level scores */}
- <div>
- <div className="text-muted-foreground mb-2 text-xs font-bold tracking-wide uppercase">
- Observation Scores
- </div>
- <AnnotationForm
- serverScores={observationScores}
- scoreTarget={{
- type: "trace",
- traceId,
- observationId,
- }}
- scoreMetadata={{
- projectId,
- environment: observationEnvironment,
- }}
- analyticsData={{
- type: "trace",
- source: "TraceDetail",
- }}
- />
- </div>
-
- <Separator />
-
- {/* Trace-level scores */}
- <div>
- <div className="text-muted-foreground mb-2 text-xs font-bold tracking-wide uppercase">
- Trace Scores
- </div>
- <AnnotationForm
- serverScores={traceScores}
- scoreTarget={{
- type: "trace",
- traceId,
- }}
- scoreMetadata={{
- projectId,
- environment: traceEnvironment,
- }}
- analyticsData={{
- type: "trace",
- source: "TraceDetail",
- }}
+ <div className="flex max-h-[95vh] flex-col overflow-y-auto">
+ {observation.isLoading || trace.isLoading ? (
+ <Skeleton className="h-full w-full" />
+ ) : (
+ <AnnotationFormContent
+ key={JSON.stringify([observation.target.key, trace.target.key])}
+ targets={prepareCombinedAnnotationTargets([
+ observation.target,
+ trace.target,
+ ])}
/>
- </div>
-
- {hasNonAnnotationScores && (
- <div className="text-muted-foreground text-xs">
- API and eval scores are hidden from this annotation drawer. Add manual
- annotations above.
- </div>
)}
</div>
);
diff --git a/web/src/features/scores/hooks/useAnnotationKeyboard.ts b/web/src/features/scores/hooks/useAnnotationKeyboard.ts
new file mode 100644
index 0000000000..6f1d3fa11f
--- /dev/null
+++ b/web/src/features/scores/hooks/useAnnotationKeyboard.ts
@@ -0,0 +1,266 @@
+import { useEffect, type RefObject } from "react";
+import type { UseFormReturn } from "react-hook-form";
+import type {
+ AnnotateFormSchemaType,
+ AnnotationScoreSchemaType,
+ PreparedAnnotationTarget,
+} from "@/src/features/scores/types";
+import type { AnnotationFormActions } from "@/src/features/scores/actions/annotationFormActions";
+import { annotationFieldKey } from "@/src/features/scores/lib/annotationConfigSelection";
+import {
+ hasBlockingOverlay,
+ hasModifier,
+} from "@/src/features/scores/lib/keyboardShortcuts";
+import {
+ isTextDataType,
+ isNumericDataType,
+} from "@/src/features/scores/lib/helpers";
+
+export function useAnnotationKeyboard({
+ formRootRef,
+ form,
+ actions,
+ targets,
+}: {
+ formRootRef: RefObject<HTMLDivElement | null>;
+ form: UseFormReturn<AnnotateFormSchemaType>;
+ actions: AnnotationFormActions;
+ targets: PreparedAnnotationTarget[];
+}) {
+ // Keyboard navigation uses real DOM focus with a
+ // spreadsheet-style navigate-vs-edit split (single source of truth, one
+ // outline, never trapped):
+ // - `↑` / `↓` move focus between *rows* (the row container, never into a text
+ // field) so navigation keeps working even when the active row is text.
+ // - `Enter` drills into the focused row's control (a text/number field to
+ // type, the combobox to open, a toggle to use ←/→).
+ // - `Esc` pops back out of an editing text field to its row.
+ // - `1`-`9` pick the Nth option of the focused row (option rows only).
+ // A focused text field owns its keys; an open popover/drawer suspends these.
+ useEffect(() => {
+ const targetFor = (field: AnnotationScoreSchemaType) =>
+ targets.find((target) => target.key === field.targetKey)!;
+ const configFor = (field: AnnotationScoreSchemaType | undefined) =>
+ field
+ ? targetFor(field).configControl.configs.find(
+ (config) => config.id === field.configId,
+ )
+ : undefined;
+ const isKeyboardSelectable = (
+ field: AnnotationScoreSchemaType | undefined,
+ ) => {
+ const config = configFor(field);
+ return (
+ !!field &&
+ !isTextDataType(field.dataType) &&
+ !isNumericDataType(field.dataType) &&
+ !!config &&
+ !config.isArchived &&
+ (config.categories?.length ?? 0) > 0
+ );
+ };
+ const handleKeyDown = (event: KeyboardEvent) => {
+ if (hasModifier(event)) return;
+ const controlledFields = form.getValues("scoreData");
+ const rowCount = controlledFields.filter((field) =>
+ configFor(field),
+ ).length;
+
+ const root = formRootRef.current;
+ if (!root) return;
+ // Suspend for an overlapping popover/drawer (e.g. the comment editor), but
+ // NOT for a drawer this form is mounted inside (the Annotate drawer) — that
+ // is an ancestor of the form, so the scheme stays alive there.
+ if (hasBlockingOverlay(root)) return;
+ const target = event.target;
+ const editing =
+ target instanceof HTMLInputElement ||
+ target instanceof HTMLTextAreaElement ||
+ (target instanceof HTMLElement && target.isContentEditable);
+
+ // (`Esc` to leave a field is handled by a separate capture-phase listener
+ // below, so it can stop a wrapping drawer from also dismissing.)
+
+ // `Enter` in a single-line field (e.g. a numeric score) commits the value
+ // and returns to row navigation — there is no newline to insert, so this is
+ // the spreadsheet "confirm cell" gesture. Multi-line text (textarea) keeps
+ // Enter for newlines and is excluded here.
+ if (
+ event.key === "Enter" &&
+ target instanceof HTMLInputElement &&
+ root.contains(target)
+ ) {
+ // Out-of-range numeric: surface the constraint and stay so the value
+ // isn't committed-and-dropped (mirrors the ⌘/Ctrl+Enter complete gate).
+ if (
+ target.type === "number" &&
+ (target.validity.badInput || !target.validity.valid)
+ ) {
+ event.preventDefault();
+ const row = target.closest<HTMLElement>("[data-score-row]");
+ if (row) {
+ const field = controlledFields[Number(row.dataset.scoreRow)];
+ if (field)
+ actions.saveNumeric(
+ annotationFieldKey(field),
+ targetFor(field),
+ target,
+ );
+ }
+ target.reportValidity();
+ return;
+ }
+ event.preventDefault();
+ const row = target.closest<HTMLElement>("[data-score-row]");
+ // Focusing the row blurs the input (its onBlur saves) and resumes ↑/↓.
+ if (row) row.focus();
+ else target.blur();
+ return;
+ }
+
+ // While editing a text field, leave its keys (typing, caret, number step)
+ // alone — `Esc` / `Tab` move out.
+ if (editing) return;
+ if (rowCount === 0) return;
+
+ // The form containing focus acts; if focus is on the body the first
+ // form acts. A control focused *outside* any form (e.g. the Mark Completed
+ // / Skip / Back / "?" page buttons) must NOT drive the form — otherwise
+ // ↑/↓ would hijack focus off that button into the score rows.
+ const active = document.activeElement;
+ const focusedForm =
+ active instanceof HTMLElement
+ ? active.closest("[data-annotation-form]")
+ : null;
+ if (focusedForm) {
+ if (focusedForm !== root) return;
+ } else {
+ if (active && active !== document.body) return;
+ if (document.querySelector("[data-annotation-form]") !== root) return;
+ }
+
+ const rowEls = Array.from(
+ root.querySelectorAll<HTMLElement>("[data-score-row]"),
+ );
+ if (rowEls.length === 0) return;
+ const currentRow =
+ active instanceof HTMLElement
+ ? (active.closest("[data-score-row]") as HTMLElement | null)
+ : null;
+ const currentPos = currentRow ? rowEls.indexOf(currentRow) : -1;
+
+ // Score action triggers own their menu keyboard navigation.
+ if (
+ active instanceof HTMLElement &&
+ active.closest("[data-score-actions]")
+ )
+ return;
+
+ // `↑` / `↓` move focus between rows (to the row container itself, never
+ // into a text field — so navigation is never trapped).
+ if (event.key === "ArrowDown" || event.key === "ArrowUp") {
+ if (rowEls.length < 2) return;
+ // Navigate from a focused row, or enter from the body — but NOT from an
+ // in-form non-row control (e.g. the config-picker trigger), which would
+ // otherwise teleport focus to a row.
+ if (currentPos < 0 && active !== document.body) return;
+ event.preventDefault();
+ const delta = event.key === "ArrowDown" ? 1 : -1;
+ let nextPos = (currentPos + delta + rowEls.length) % rowEls.length;
+ if (currentPos < 0) nextPos = delta > 0 ? 0 : rowEls.length - 1;
+ rowEls[nextPos].focus();
+ return;
+ }
+
+ // `Enter` drills into the focused row's control (only from the row
+ // container itself — once a control is focused, Enter is left to it).
+ if (event.key === "Enter") {
+ if (currentRow && active === currentRow) {
+ // First *enabled* control — skip the disabled stale-category chip that
+ // enrichCategoryOptionsWithStaleScoreValue prepends (focusing a disabled
+ // control is a no-op, so Enter would otherwise appear to do nothing).
+ const control = Array.from(
+ currentRow.querySelectorAll<HTMLElement>(
+ "[data-score-control] :is(textarea, input, button)",
+ ),
+ ).find((el) => !el.matches(":disabled"));
+ if (control) {
+ event.preventDefault();
+ // A dropdown trigger (combobox, `aria-haspopup`) opens directly so a
+ // single Enter is enough (not focus-then-Enter). Radix's Popover
+ // trigger opens on click. Other controls (number/text input, toggle)
+ // just take focus.
+ if (
+ control.tagName === "BUTTON" &&
+ control.getAttribute("aria-haspopup")
+ ) {
+ control.click();
+ } else {
+ control.focus();
+ }
+ }
+ }
+ return;
+ }
+
+ // `1`-`9` pick an option on the focused row (option rows only).
+ if (/^[1-9]$/.test(event.key)) {
+ if (currentPos < 0 || !currentRow) return;
+ // Only from the row container itself or its value control — never the
+ // in-row comment or score actions (else a stray digit writes a phantom
+ // score). Mirrors the Enter branch's `active === currentRow` gate.
+ if (
+ active !== currentRow &&
+ !(
+ active instanceof HTMLElement &&
+ active.closest("[data-score-control]")
+ )
+ )
+ return;
+ const rowIndex = Number(currentRow.getAttribute("data-score-row"));
+ const field = controlledFields[rowIndex];
+ if (!isKeyboardSelectable(field)) return;
+ const config = field ? configFor(field) : undefined;
+ const category = (config?.categories ?? [])[Number(event.key) - 1];
+ if (!category) return;
+ event.preventDefault();
+ if (field)
+ actions.saveCategory(
+ annotationFieldKey(field),
+ targetFor(field),
+ category.label,
+ );
+ }
+ };
+ window.addEventListener("keydown", handleKeyDown);
+ return () => window.removeEventListener("keydown", handleKeyDown);
+ }, [actions, form, formRootRef, targets]);
+
+ // `Esc` leaves a focused score field (back to its row) without
+ // dismissing a wrapping drawer. Vaul/Radix DismissableLayer listens for Esc on
+ // `document` with `{capture:true}`; a window capture-phase listener runs first
+ // (window is the ancestor), so stopping propagation here prevents the drawer
+ // from closing. Scoped to fields inside this form (a portaled comment popover
+ // is not inside the form root, so its own Esc-to-close still works).
+ useEffect(() => {
+ const onEscapeCapture = (event: KeyboardEvent) => {
+ if (event.key !== "Escape") return;
+ const root = formRootRef.current;
+ if (!root) return;
+ const target = event.target;
+ const inField =
+ (target instanceof HTMLInputElement ||
+ target instanceof HTMLTextAreaElement) &&
+ root.contains(target);
+ if (!inField) return;
+ event.stopPropagation();
+ const row = (target as HTMLElement).closest<HTMLElement>(
+ "[data-score-row]",
+ );
+ if (row) row.focus();
+ else (target as HTMLElement).blur();
+ };
+ window.addEventListener("keydown", onEscapeCapture, true);
+ return () => window.removeEventListener("keydown", onEscapeCapture, true);
+ }, [formRootRef]);
+}
diff --git a/web/src/features/scores/hooks/useScoreConfigSelection.clienttest.ts b/web/src/features/scores/hooks/useScoreConfigSelection.clienttest.ts
new file mode 100644
index 0000000000..1186b69c6d
--- /dev/null
+++ b/web/src/features/scores/hooks/useScoreConfigSelection.clienttest.ts
@@ -0,0 +1,153 @@
+import { describe, expect, it, vi } from "vitest";
+import { type ScoreConfigDomain } from "@langfuse/shared";
+import {
+ annotationFieldKey,
+ getScoreConfigSelection,
+} from "../lib/annotationConfigSelection";
+import type { PreparedAnnotationTarget } from "../types";
+
+const config = {
+ id: "quality",
+ name: "Quality",
+ projectId: "project",
+ dataType: "BOOLEAN",
+ categories: [
+ { label: "False", value: 0 },
+ { label: "True", value: 1 },
+ ],
+ isArchived: false,
+ createdAt: new Date(),
+ updatedAt: new Date(),
+} satisfies ScoreConfigDomain;
+
+function makeTarget(key: string): PreparedAnnotationTarget {
+ return {
+ key,
+ label: key,
+ scoreTarget: {
+ type: "trace",
+ traceId: "trace",
+ observationId: key === "observation" ? "observation" : undefined,
+ },
+ scoreMetadata: { projectId: "project" },
+ analyticsData: { type: "trace", source: "TraceDetail", isV4: true },
+ initialFormData: [],
+ configControl: {
+ configs: [config],
+ allowManualSelection: true,
+ selectedConfigIds: [config.id],
+ setSelectedConfigIds: vi.fn(),
+ },
+ };
+}
+
+const empty = {
+ targetKey: "observation",
+ configId: config.id,
+ name: config.name,
+ dataType: config.dataType,
+ id: null,
+};
+
+describe("score field selection", () => {
+ it("lists each config once, defaults additions to observation, and prevents duplicate or archived additions", () => {
+ const observation = makeTarget("observation");
+ const trace = makeTarget("trace");
+ const insert = vi.fn();
+ const selection = getScoreConfigSelection({
+ targets: [trace, observation],
+ controlledFields: [],
+ insert,
+ remove: vi.fn(),
+ });
+ expect(selection.selectionOptions).toHaveLength(1);
+ selection.addScore(config.id);
+ expect(insert).toHaveBeenCalledExactlyOnceWith(
+ 0,
+ expect.objectContaining({
+ configId: config.id,
+ targetKey: observation.key,
+ }),
+ );
+ insert.mockClear();
+ getScoreConfigSelection({
+ targets: [trace, observation],
+ controlledFields: [empty],
+ insert,
+ remove: vi.fn(),
+ }).addScore(config.id);
+ observation.configControl.configs = [{ ...config, isArchived: true }];
+ getScoreConfigSelection({
+ targets: [observation],
+ controlledFields: [],
+ insert,
+ remove: vi.fn(),
+ }).addScore(config.id);
+ expect(insert).not.toHaveBeenCalled();
+ });
+
+ it("removes only an empty row and clears remembered preferences without hiding another target's score", () => {
+ const observation = makeTarget("observation");
+ const trace = makeTarget("trace");
+ const remove = vi.fn();
+ const saved = { ...empty, targetKey: "trace", id: "saved-score", value: 0 };
+ const selection = getScoreConfigSelection({
+ targets: [trace, observation],
+ controlledFields: [empty, saved],
+ insert: vi.fn(),
+ remove,
+ });
+ selection.removeEmptyField(annotationFieldKey(saved));
+ expect(remove).not.toHaveBeenCalled();
+ selection.removeEmptyField(annotationFieldKey(empty));
+ expect(remove).toHaveBeenCalledExactlyOnceWith(0);
+ expect(
+ observation.configControl.setSelectedConfigIds,
+ ).toHaveBeenCalledExactlyOnceWith([]);
+ expect(trace.configControl.setSelectedConfigIds).not.toHaveBeenCalled();
+ remove.mockClear();
+ for (const draft of [
+ { ...empty, value: 0 },
+ { ...empty, stringValue: "Draft" },
+ { ...empty, comment: "Note" },
+ ]) {
+ getScoreConfigSelection({
+ targets: [observation],
+ controlledFields: [draft],
+ insert: vi.fn(),
+ remove,
+ }).removeEmptyField(annotationFieldKey(draft));
+ }
+ observation.configControl.configs = [{ ...config, isArchived: true }];
+ getScoreConfigSelection({
+ targets: [observation],
+ controlledFields: [empty],
+ insert: vi.fn(),
+ remove,
+ }).removeEmptyField(annotationFieldKey(empty));
+ expect(remove).not.toHaveBeenCalled();
+ observation.configControl.configs = [config];
+ observation.configControl.allowManualSelection = false;
+ getScoreConfigSelection({
+ targets: [observation],
+ controlledFields: [empty],
+ insert: vi.fn(),
+ remove,
+ }).removeEmptyField(annotationFieldKey(empty));
+ expect(remove).not.toHaveBeenCalled();
+ observation.configControl.allowManualSelection = true;
+ vi.mocked(observation.configControl.setSelectedConfigIds).mockClear();
+ getScoreConfigSelection({
+ targets: [trace, observation],
+ controlledFields: [empty],
+ insert: vi.fn(),
+ remove,
+ }).removeEmptyField(annotationFieldKey(empty));
+ expect(
+ observation.configControl.setSelectedConfigIds,
+ ).toHaveBeenCalledExactlyOnceWith([]);
+ expect(
+ trace.configControl.setSelectedConfigIds,
+ ).toHaveBeenCalledExactlyOnceWith([]);
+ });
+});
diff --git a/web/src/features/scores/hooks/useScoreConfigSelection.ts b/web/src/features/scores/hooks/useScoreConfigSelection.ts
deleted file mode 100644
index 1420681882..0000000000
--- a/web/src/features/scores/hooks/useScoreConfigSelection.ts
+++ /dev/null
@@ -1,132 +0,0 @@
-import { useEmptyScoreConfigs } from "@/src/features/scores/hooks/useEmptyConfigs";
-import { resolveConfigValue } from "@/src/features/scores/lib/annotationFormHelpers";
-import {
- type AnnotateFormSchemaType,
- type AnnotationScoreSchemaType,
-} from "@/src/features/scores/types";
-import { type ScoreConfigDomain } from "@langfuse/shared";
-import { useCallback, useMemo } from "react";
-import {
- type UseFieldArrayRemove,
- type UseFieldArrayInsert,
-} from "react-hook-form";
-import { toast } from "sonner";
-
-const resolveAlphabeticalPosition = (
- name: string,
- controlledFields: AnnotationScoreSchemaType[],
-) => {
- const alphabeticalIndex = controlledFields.findIndex(
- (f) => f.name.localeCompare(name) > 0,
- );
- // If no field comes after this one alphabetically (-1), insert at the end
- if (alphabeticalIndex === -1) return controlledFields.length;
-
- return alphabeticalIndex;
-};
-
-export function useScoreConfigSelection({
- configs,
- controlledFields,
- isInputDisabled,
- insert,
- remove,
- emptySelectedConfigIdsStorageKey,
-}: {
- configs: ScoreConfigDomain[];
- controlledFields: AnnotationScoreSchemaType[];
- isInputDisabled: (config: ScoreConfigDomain) => boolean;
- insert: UseFieldArrayInsert<AnnotateFormSchemaType, "scoreData">;
- remove: UseFieldArrayRemove;
- emptySelectedConfigIdsStorageKey?: string;
-}): {
- selectionOptions: {
- key: string;
- value: string;
- disabled: boolean;
- }[];
- handleSelectionChange: (
- values: Record<string, string>[],
- changedValueId?: string,
- ) => void;
-} {
- const { emptySelectedConfigIds, setEmptySelectedConfigIds } =
- useEmptyScoreConfigs(emptySelectedConfigIdsStorageKey);
-
- const selectionOptions = useMemo(() => {
- return configs
- .sort((a, b) => a.name.localeCompare(b.name))
- .map((config) => {
- return {
- key: config.id,
- value: resolveConfigValue({
- dataType: config.dataType,
- name: config.name,
- }),
- disabled: isInputDisabled(config),
- };
- });
- }, [configs, isInputDisabled]);
-
- const handleSelectionChange = useCallback(
- (values: Record<string, string>[], changedValueId?: string) => {
- if (!changedValueId) return;
-
- const fieldIndex = controlledFields.findIndex(
- (f) => f.configId === changedValueId,
- );
- const isCurrentlyInForm = fieldIndex !== -1;
-
- if (!isCurrentlyInForm) {
- // Config was just selected -> add empty row to form
- const config = configs.find((c) => c.id === changedValueId);
- if (config) {
- // find correct alphabetical position to insert new field at
- const position = resolveAlphabeticalPosition(
- config.name,
- controlledFields,
- );
-
- insert(position, {
- id: null,
- configId: config.id,
- name: config.name,
- dataType: config.dataType,
- value: null,
- stringValue: null,
- comment: null,
- });
- }
- setEmptySelectedConfigIds?.([
- ...emptySelectedConfigIds,
- changedValueId,
- ]);
- } else {
- // Config was deselected
- const field = controlledFields[fieldIndex];
- if (field?.id) {
- toast.error("Cannot deselect a populated score");
- return;
- }
- // No score -> remove row from form and empty selected config ids
- remove(fieldIndex);
- setEmptySelectedConfigIds?.(
- emptySelectedConfigIds.filter((id) => id !== changedValueId),
- );
- }
- },
- [
- controlledFields,
- configs,
- insert,
- remove,
- emptySelectedConfigIds,
- setEmptySelectedConfigIds,
- ],
- );
-
- return {
- selectionOptions,
- handleSelectionChange,
- };
-}
diff --git a/web/src/features/scores/hooks/useScoreConfigs.ts b/web/src/features/scores/hooks/useScoreConfigs.ts
index 664786be62..c254f95c93 100644
--- a/web/src/features/scores/hooks/useScoreConfigs.ts
+++ b/web/src/features/scores/hooks/useScoreConfigs.ts
@@ -28,10 +28,10 @@ export function useAnnotationScoreConfigs({
isLoading: boolean;
availableConfigs: ScoreConfigDomain[];
selectedConfigIds: string[];
+ setSelectedConfigIds: (ids: string[]) => void;
} {
- const { emptySelectedConfigIds } = useEmptyScoreConfigs(
- emptySelectedConfigIdsStorageKey,
- );
+ const { emptySelectedConfigIds, setEmptySelectedConfigIds } =
+ useEmptyScoreConfigs(emptySelectedConfigIdsStorageKey);
const configs = api.scoreConfigs.all.useQuery(
{
@@ -47,11 +47,13 @@ export function useAnnotationScoreConfigs({
isLoading: false,
selectedConfigIds: configSelection.configs.map((c) => c.id),
availableConfigs: configSelection.configs,
+ setSelectedConfigIds: setEmptySelectedConfigIds,
};
}
return {
isLoading: configs.isLoading,
selectedConfigIds: emptySelectedConfigIds,
availableConfigs: configs.data?.configs ?? [],
+ setSelectedConfigIds: setEmptySelectedConfigIds,
};
}
diff --git a/web/src/features/scores/hooks/useScoreMutations.ts b/web/src/features/scores/hooks/useScoreMutations.ts
index dbcec26529..a5f18f9ece 100644
--- a/web/src/features/scores/hooks/useScoreMutations.ts
+++ b/web/src/features/scores/hooks/useScoreMutations.ts
@@ -4,20 +4,9 @@ import {
isTraceScore,
} from "@/src/features/scores/lib/helpers";
import { useScoreCache } from "@/src/features/scores/contexts/ScoreCacheContext";
-import { type ScoreTarget } from "@langfuse/shared";
import { showErrorToast } from "@/src/features/notifications";
-export function useScoreMutations({
- scoreTarget,
- scoreMetadata,
-}: {
- scoreTarget: ScoreTarget;
- scoreMetadata: {
- projectId: string;
- queueId?: string;
- environment?: string;
- };
-}) {
+export function useScoreMutations() {
const {
set: cacheSet,
get: cacheGet,
@@ -31,6 +20,7 @@ export function useScoreMutations({
const createMutation = api.scores.createAnnotationScore.useMutation({
onMutate: (variables) => {
if (!variables.id) return;
+ const scoreTarget = variables.scoreTarget;
// Write to columns cache
cacheSetColumn({
@@ -42,8 +32,8 @@ export function useScoreMutations({
// Write to cache for optimistic update
cacheSet(variables.id, {
id: variables.id,
- projectId: scoreMetadata.projectId,
- environment: scoreMetadata.environment ?? "default",
+ projectId: variables.projectId,
+ environment: variables.environment ?? "default",
traceId: isTraceScore(scoreTarget) ? scoreTarget.traceId : null,
observationId: isTraceScore(scoreTarget)
? (scoreTarget.observationId ?? null)
@@ -71,14 +61,15 @@ export function useScoreMutations({
const updateMutation = api.scores.updateAnnotationScore.useMutation({
onMutate: (variables) => {
+ const scoreTarget = variables.scoreTarget;
const previousCacheValue = cacheGet(variables.id);
if (!previousCacheValue) {
// Write to cache for optimistic update
cacheSet(variables.id, {
id: variables.id,
- projectId: scoreMetadata.projectId,
- environment: scoreMetadata.environment ?? "default",
+ projectId: variables.projectId,
+ environment: variables.environment ?? "default",
traceId: isTraceScore(scoreTarget) ? scoreTarget.traceId : null,
observationId: isTraceScore(scoreTarget)
? (scoreTarget.observationId ?? null)
diff --git a/web/src/features/scores/lib/annotationAnalytics.clienttest.ts b/web/src/features/scores/lib/annotationAnalytics.clienttest.ts
new file mode 100644
index 0000000000..4f1d7a9f3e
--- /dev/null
+++ b/web/src/features/scores/lib/annotationAnalytics.clienttest.ts
@@ -0,0 +1,136 @@
+// @vitest-environment node
+import { createAnnotationAnalytics } from "./annotationAnalytics";
+
+describe("annotation analytics", () => {
+ beforeEach(() => vi.useFakeTimers());
+ afterEach(() => vi.useRealTimers());
+
+ const context = {
+ type: "trace" as const,
+ source: "AnnotationQueue" as const,
+ targetType: "observation" as const,
+ isV4: true,
+ };
+ const field = {
+ id: null,
+ configId: "private-config-id",
+ name: "private-config-name",
+ dataType: "TEXT" as const,
+ stringValue: null,
+ };
+
+ it("counts one visible form across effect replay and abandons only on close", () => {
+ const capture = vi.fn();
+ const analytics = createAnnotationAnalytics(capture, context, [field]);
+ analytics.open();
+ analytics.close();
+ analytics.open();
+ vi.runAllTimers();
+ expect(capture.mock.calls.map(([event]) => event)).toEqual([
+ "score:create_form_open",
+ ]);
+ analytics.close();
+ vi.runAllTimers();
+ expect(capture.mock.calls.at(-1)).toEqual([
+ "score:form_abandoned",
+ expect.objectContaining({
+ ...context,
+ fieldCount: 1,
+ filledFieldCount: 0,
+ }),
+ ]);
+ });
+
+ it("deduplicates unchanged values, waits for saves after close, and emits metadata only", () => {
+ const capture = vi.fn();
+ const analytics = createAnnotationAnalytics(capture, context, [field]);
+ analytics.open();
+ const saved = {
+ ...field,
+ id: "private-score-id",
+ stringValue: "private-text",
+ comment: "private-comment",
+ };
+ const save = analytics.beginSave("create", saved, [saved], {
+ control: "text",
+ optionCount: 0,
+ });
+ const duplicate = analytics.beginSave("update", saved, [saved], {
+ control: "text",
+ optionCount: 0,
+ });
+ analytics.close();
+ vi.runAllTimers();
+ expect(capture.mock.calls.map(([event]) => event)).toEqual([
+ "score:create_form_open",
+ "score:value_set",
+ ]);
+ save?.success();
+ duplicate?.success();
+ expect(capture.mock.calls.map(([event]) => event)).toEqual([
+ "score:create_form_open",
+ "score:value_set",
+ "score:create",
+ ]);
+ expect(capture.mock.calls.at(-1)?.[1]).toEqual({
+ ...context,
+ dataType: "TEXT",
+ fieldCount: 1,
+ filledFieldCount: 1,
+ hasComment: true,
+ });
+ expect(JSON.stringify(capture.mock.calls)).not.toContain("private-");
+ });
+
+ it("does not report failed saves, permits retry, and marks corrections", () => {
+ const capture = vi.fn();
+ const existing = { ...field, id: "private-score-id", stringValue: "old" };
+ const analytics = createAnnotationAnalytics(capture, context, [existing]);
+ analytics.open();
+ const changed = { ...existing, stringValue: "new" };
+ const failed = analytics.beginSave("update", changed, [changed], {
+ control: "text",
+ optionCount: 0,
+ });
+ failed?.failure();
+ expect(capture).not.toHaveBeenCalledWith("score:update", expect.anything());
+ expect(capture).toHaveBeenCalledWith(
+ "score:value_set",
+ expect.objectContaining({ isCorrection: true }),
+ );
+ const retry = analytics.beginSave("update", changed, [changed], {
+ control: "text",
+ optionCount: 0,
+ });
+ expect(retry).toBeDefined();
+ retry?.success();
+ analytics.close();
+ vi.runAllTimers();
+ expect(
+ capture.mock.calls.filter(([event]) => event === "score:update"),
+ ).toHaveLength(1);
+ expect(
+ capture.mock.calls.filter(([event]) => event === "score:form_abandoned"),
+ ).toHaveLength(0);
+ });
+
+ it("permits retry after overlapping updates both fail", () => {
+ const capture = vi.fn();
+ const existing = { ...field, id: "private-score-id", stringValue: "old" };
+ const analytics = createAnnotationAnalytics(capture, context, [existing]);
+ analytics.open();
+ const firstValue = { ...existing, stringValue: "first" };
+ const secondValue = { ...existing, stringValue: "second" };
+ const first = analytics.beginSave("update", firstValue, [firstValue]);
+ const second = analytics.beginSave("update", secondValue, [secondValue]);
+ first?.failure();
+ second?.failure();
+
+ const retry = analytics.beginSave("update", firstValue, [firstValue]);
+ expect(retry).toBeDefined();
+ retry?.success();
+ expect(
+ capture.mock.calls.filter(([event]) => event === "score:update"),
+ ).toHaveLength(1);
+ });
+});
diff --git a/web/src/features/scores/lib/annotationAnalytics.ts b/web/src/features/scores/lib/annotationAnalytics.ts
new file mode 100644
index 0000000000..b4bfc3697f
--- /dev/null
+++ b/web/src/features/scores/lib/annotationAnalytics.ts
@@ -0,0 +1,242 @@
+import type {
+ AnalyticsData,
+ AnnotationScoreFormData,
+ ScoreTarget,
+} from "@/src/features/scores/types";
+import type { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
+
+export type AnnotationAnalyticsContext = AnalyticsData & {
+ targetType: "trace" | "observation" | "session" | "dataset_item";
+};
+
+type FieldCounts = { fieldCount: number; filledFieldCount: number };
+type ScoreProperties = AnnotationAnalyticsContext &
+ FieldCounts & {
+ dataType: AnnotationScoreFormData["dataType"];
+ hasComment: boolean;
+ };
+type ControlProperties = {
+ control: "segmented" | "select" | "slider" | "text" | "number";
+ optionCount: number;
+};
+type QueueProperties = AnnotationAnalyticsContext & {
+ objectType: "TRACE" | "OBSERVATION" | "SESSION";
+ queueCount: number;
+ itemCount?: number;
+};
+
+export type AnnotationEventMap = {
+ "score:create_form_open": AnnotationAnalyticsContext & FieldCounts;
+ "score:update_form_open": AnnotationAnalyticsContext & FieldCounts;
+ "score:form_abandoned": AnnotationAnalyticsContext &
+ FieldCounts & { durationMs: number };
+ "score:create": ScoreProperties;
+ "score:update": ScoreProperties;
+ "score:delete": ScoreProperties;
+ "score:update_comment": ScoreProperties;
+ "score:delete_comment": ScoreProperties;
+ "score:level_changed": AnnotationAnalyticsContext & {
+ previousTargetType: AnnotationAnalyticsContext["targetType"];
+ dataType: AnnotationScoreFormData["dataType"];
+ };
+ "score:level_added": AnnotationAnalyticsContext & {
+ dataType: AnnotationScoreFormData["dataType"];
+ };
+ "score:value_set": AnnotationAnalyticsContext &
+ ControlProperties & {
+ dataType: AnnotationScoreFormData["dataType"];
+ isCorrection: boolean;
+ };
+ "annotation:entry_click": AnnotationAnalyticsContext & {
+ entryPoint: "annotate_button" | "queue_button" | "tab";
+ };
+ "annotation_queues:item_added": QueueProperties;
+ "annotation_queues:item_removed": QueueProperties;
+ "annotation_queues:manage_click": QueueProperties;
+};
+
+export function getAnnotationTargetType(
+ target: ScoreTarget,
+): AnnotationAnalyticsContext["targetType"] {
+ if (target.type === "session") return "session";
+ return target.observationId ? "observation" : "trace";
+}
+
+function counts(
+ fields: Pick<AnnotationScoreFormData, "dataType" | "value" | "stringValue">[],
+): FieldCounts {
+ return {
+ fieldCount: fields.length,
+ filledFieldCount: fields.filter((field) =>
+ field.dataType === "TEXT"
+ ? Boolean(field.stringValue)
+ : field.value !== null && field.value !== undefined,
+ ).length,
+ };
+}
+
+type Capture = ReturnType<typeof usePostHogClientCapture>;
+type SaveKind =
+ | "create"
+ | "update"
+ | "delete"
+ | "update_comment"
+ | "delete_comment";
+
+export function createAnnotationAnalytics(
+ capture: Capture,
+ context: AnnotationAnalyticsContext,
+ initialFields: AnnotationScoreFormData[],
+) {
+ const metadata = {
+ type: context.type,
+ source: context.source,
+ targetType: context.targetType,
+ isV4: context.isV4,
+ };
+ const values = new Map(
+ initialFields.map((field) => [field.configId, { ...field }]),
+ );
+ const confirmed = new Map(values);
+ const pendingValues = new Map<
+ string,
+ { field: AnnotationScoreFormData; sequence: number }[]
+ >();
+ const confirmedSequence = new Map<string, number>();
+ let sequence = 0;
+ let fieldCounts = counts(initialFields);
+ let openedAt: number | undefined;
+ let closedAt: number | undefined;
+ let closeTimer: ReturnType<typeof setTimeout> | undefined;
+ let closed = false;
+ let pending = 0;
+ let saved = false;
+ let abandoned = false;
+
+ const abandon = () => {
+ if (!closed || pending || saved || abandoned || openedAt === undefined)
+ return;
+ abandoned = true;
+ capture("score:form_abandoned", {
+ ...metadata,
+ ...fieldCounts,
+ durationMs: Math.max(0, (closedAt ?? Date.now()) - openedAt),
+ });
+ };
+
+ return {
+ open() {
+ clearTimeout(closeTimer);
+ closed = false;
+ closedAt = undefined;
+ if (openedAt !== undefined) return;
+ openedAt = Date.now();
+ capture(
+ initialFields.some((field) => field.id)
+ ? "score:update_form_open"
+ : "score:create_form_open",
+ { ...metadata, ...fieldCounts },
+ );
+ },
+ close(
+ fields?: Pick<
+ AnnotationScoreFormData,
+ "dataType" | "value" | "stringValue"
+ >[],
+ ) {
+ if (fields) fieldCounts = counts(fields);
+ closedAt = Date.now();
+ // React effect replay must not look like a reviewer closing the form.
+ closeTimer = setTimeout(() => {
+ closed = true;
+ abandon();
+ }, 0);
+ },
+ beginSave(
+ kind: SaveKind,
+ field: AnnotationScoreFormData,
+ fields: AnnotationScoreFormData[],
+ control?: ControlProperties,
+ ) {
+ const previous = values.get(field.configId);
+ const isValueSave = kind === "create" || kind === "update";
+ const changed = (baseline: AnnotationScoreFormData | undefined) => {
+ if (kind === "delete") return Boolean(baseline?.id);
+ if (!isValueSave)
+ return (baseline?.comment ?? null) !== (field.comment ?? null);
+ if (!baseline) return true;
+ return field.dataType === "TEXT"
+ ? baseline.stringValue !== field.stringValue
+ : baseline.value !== field.value ||
+ baseline.stringValue !== field.stringValue;
+ };
+ const valueChanged = changed(previous);
+ if (!valueChanged && !pendingValues.get(field.configId)?.length) return;
+
+ const next = { ...field };
+ const operation = { field: next, sequence: ++sequence };
+ pendingValues.set(field.configId, [
+ ...(pendingValues.get(field.configId) ?? []),
+ operation,
+ ]);
+ values.set(field.configId, next);
+ fieldCounts = counts(fields);
+ const properties: ScoreProperties = {
+ ...metadata,
+ ...fieldCounts,
+ dataType: field.dataType,
+ hasComment: Boolean(field.comment),
+ };
+ if (isValueSave && valueChanged && control) {
+ capture("score:value_set", {
+ ...metadata,
+ ...control,
+ dataType: field.dataType,
+ isCorrection: Boolean(previous?.id),
+ });
+ }
+ pending++;
+ let settled = false;
+ const settle = (success: boolean) => {
+ pending--;
+ if (
+ success &&
+ operation.sequence > (confirmedSequence.get(field.configId) ?? 0)
+ ) {
+ confirmed.set(field.configId, next);
+ confirmedSequence.set(field.configId, operation.sequence);
+ }
+ const remaining = (pendingValues.get(field.configId) ?? []).filter(
+ (item) => item !== operation,
+ );
+ pendingValues.set(field.configId, remaining);
+ const latest = remaining.at(-1);
+ const current =
+ latest &&
+ latest.sequence > (confirmedSequence.get(field.configId) ?? 0)
+ ? latest.field
+ : confirmed.get(field.configId);
+ if (current) values.set(field.configId, current);
+ else values.delete(field.configId);
+ };
+ return {
+ success() {
+ if (settled) return;
+ settled = true;
+ const savedChange = changed(confirmed.get(field.configId));
+ settle(true);
+ if (savedChange) {
+ saved = true;
+ capture(`score:${kind}`, properties);
+ }
+ },
+ failure() {
+ if (settled) return;
+ settled = true;
+ settle(false);
+ abandon();
+ },
+ };
+ },
+ };
+}
diff --git a/web/src/features/scores/lib/annotationConfigSelection.ts b/web/src/features/scores/lib/annotationConfigSelection.ts
new file mode 100644
index 0000000000..c3a42a5ddc
--- /dev/null
+++ b/web/src/features/scores/lib/annotationConfigSelection.ts
@@ -0,0 +1,138 @@
+import { resolveConfigValue } from "@/src/features/scores/lib/annotationFormHelpers";
+import type {
+ AnnotateFormSchemaType,
+ AnnotationScoreSchemaType,
+ PreparedAnnotationTarget,
+} from "@/src/features/scores/types";
+import type { UseFieldArrayInsert, UseFieldArrayRemove } from "react-hook-form";
+import { isPresent } from "@langfuse/shared";
+
+export const annotationFieldKey = (field: {
+ targetKey?: string;
+ configId: string;
+}) => JSON.stringify([field.targetKey, field.configId]);
+
+export function preferredAnnotationTargets(
+ targets: PreparedAnnotationTarget[],
+) {
+ return [...targets].sort(
+ (left, right) =>
+ Number(
+ right.scoreTarget.type === "trace" && !!right.scoreTarget.observationId,
+ ) -
+ Number(
+ left.scoreTarget.type === "trace" && !!left.scoreTarget.observationId,
+ ),
+ );
+}
+
+export function getScoreConfigSelection({
+ targets,
+ controlledFields,
+ insert,
+ remove,
+}: {
+ targets: PreparedAnnotationTarget[];
+ controlledFields: AnnotationScoreSchemaType[];
+ insert: UseFieldArrayInsert<AnnotateFormSchemaType, "scoreData">;
+ remove: UseFieldArrayRemove;
+}) {
+ const seenConfigs = new Set<string>();
+ const selectionOptions = preferredAnnotationTargets(targets)
+ .flatMap((target) =>
+ target.configControl.allowManualSelection
+ ? target.configControl.configs.map((config) => ({
+ value: config.id,
+ label: resolveConfigValue(config),
+ targetKey: target.key,
+ targetLabel: target.label,
+ disabled: config.isArchived,
+ config,
+ }))
+ : [],
+ )
+ .filter((option) => {
+ if (seenConfigs.has(option.value)) return false;
+ seenConfigs.add(option.value);
+ return true;
+ })
+ .sort((a, b) => a.config.name.localeCompare(b.config.name));
+
+ const addScore = (configId: string) => {
+ const option = selectionOptions.find((option) => option.value === configId);
+ if (
+ !option ||
+ option.disabled ||
+ controlledFields.some((field) => field.configId === configId)
+ )
+ return;
+ const field = {
+ targetKey: option.targetKey,
+ id: null,
+ configId: option.config.id,
+ name: option.config.name,
+ dataType: option.config.dataType,
+ value: null,
+ stringValue: null,
+ comment: null,
+ };
+ const nextIndex = controlledFields.findIndex(
+ (current) => current.name.localeCompare(field.name) > 0,
+ );
+ insert(nextIndex < 0 ? controlledFields.length : nextIndex, field);
+ const target = targets.find((target) => target.key === option.targetKey)!;
+ if (!target.configControl.selectedConfigIds.includes(configId))
+ target.configControl.setSelectedConfigIds([
+ ...target.configControl.selectedConfigIds,
+ configId,
+ ]);
+ };
+
+ const removeEmptyField = (key: string) => {
+ const index = controlledFields.findIndex(
+ (field) => annotationFieldKey(field) === key,
+ );
+ const field = controlledFields[index];
+ if (
+ !field ||
+ field.id ||
+ isPresent(field.value) ||
+ field.stringValue ||
+ field.comment
+ )
+ return;
+ const owner = targets.find((target) => target.key === field.targetKey);
+ const config = owner?.configControl.configs.find(
+ (config) => config.id === field.configId,
+ );
+ if (
+ !owner?.configControl.allowManualSelection ||
+ !config ||
+ config.isArchived
+ )
+ return;
+ remove(index);
+ const remainingFields = controlledFields.filter(
+ (_, fieldIndex) => fieldIndex !== index,
+ );
+ for (const target of targets) {
+ if (!target.configControl.allowManualSelection) continue;
+ if (
+ remainingFields.some(
+ (remaining) =>
+ remaining.configId === field.configId &&
+ remaining.targetKey === target.key,
+ )
+ )
+ continue;
+ if (target.configControl.selectedConfigIds.includes(field.configId))
+ target.configControl.setSelectedConfigIds(
+ target.configControl.selectedConfigIds.filter(
+ (id) => id !== field.configId,
+ ),
+ );
+ }
+ };
+
+ return { selectionOptions, addScore, removeEmptyField };
+}
diff --git a/web/src/features/scores/lib/annotationFormHelpers.clienttest.ts b/web/src/features/scores/lib/annotationFormHelpers.clienttest.ts
index f5ac26ddc0..7a467b179d 100644
--- a/web/src/features/scores/lib/annotationFormHelpers.clienttest.ts
+++ b/web/src/features/scores/lib/annotationFormHelpers.clienttest.ts
@@ -6,8 +6,37 @@ import {
nextCategoryValue,
resolveCategoricalNumericValue,
validateNewCategoryLabel,
+ validateNumericScore,
} from "@/src/features/scores/lib/annotationFormHelpers";
+describe("numeric score validation", () => {
+ it.each([Number.NaN, Number.POSITIVE_INFINITY, Number.NEGATIVE_INFINITY])(
+ "rejects non-finite values (%s) as invalid numbers",
+ (value) => {
+ expect(validateNumericScore({ value, minValue: 1, maxValue: 5 })).toBe(
+ "Enter a number",
+ );
+ },
+ );
+ it("keeps empty values distinct from zero and explains bounded and one-sided ranges", () => {
+ expect(
+ validateNumericScore({ value: null, minValue: 1, maxValue: 5 }),
+ ).toBeNull();
+ expect(validateNumericScore({ value: 0, minValue: 1, maxValue: 5 })).toBe(
+ "Enter a value between 1 and 5",
+ );
+ expect(validateNumericScore({ value: 0, minValue: 1 })).toBe(
+ "Enter a value of at least 1",
+ );
+ expect(validateNumericScore({ value: 6, maxValue: 5 })).toBe(
+ "Enter a value of at most 5",
+ );
+ expect(
+ validateNumericScore({ value: 2.5, minValue: 1, maxValue: 5 }),
+ ).toBeNull();
+ });
+});
+
describe("nextCategoryValue", () => {
it("starts at 0 when there are no categories", () => {
expect(nextCategoryValue([])).toBe(0);
diff --git a/web/src/features/scores/lib/annotationFormHelpers.ts b/web/src/features/scores/lib/annotationFormHelpers.ts
index cb989de863..d931b29aaa 100644
--- a/web/src/features/scores/lib/annotationFormHelpers.ts
+++ b/web/src/features/scores/lib/annotationFormHelpers.ts
@@ -19,16 +19,27 @@ export const validateNumericScore = ({
value,
minValue,
maxValue,
+ badInput = false,
}: {
value?: number | null;
minValue?: number | null;
maxValue?: number | null;
+ badInput?: boolean;
}): string | null => {
+ if (badInput || (isPresent(value) && !Number.isFinite(value))) {
+ return "Enter a number";
+ }
+ if (!isPresent(value)) return null;
if (
- (isPresent(maxValue) && Number(value) > maxValue) ||
- (isPresent(minValue) && Number(value) < minValue)
+ (isPresent(maxValue) && value > maxValue) ||
+ (isPresent(minValue) && value < minValue)
) {
- return `Not in range: [${minValue ?? "-∞"},${maxValue ?? "∞"}]`;
+ if (isPresent(minValue) && isPresent(maxValue)) {
+ return `Enter a value between ${minValue} and ${maxValue}`;
+ }
+ return isPresent(minValue)
+ ? `Enter a value of at least ${minValue}`
+ : `Enter a value of at most ${maxValue}`;
}
return null;
};
diff --git a/web/src/features/scores/lib/prepareAnnotationFormData.ts b/web/src/features/scores/lib/prepareAnnotationFormData.ts
new file mode 100644
index 0000000000..094142d22e
--- /dev/null
+++ b/web/src/features/scores/lib/prepareAnnotationFormData.ts
@@ -0,0 +1,84 @@
+import type { ScoreConfigDomain } from "@langfuse/shared";
+import type {
+ AnnotationScore,
+ AnnotationScoreFormData,
+ PreparedAnnotationTarget,
+} from "@/src/features/scores/types";
+import { preferredAnnotationTargets } from "@/src/features/scores/lib/annotationConfigSelection";
+
+export function prepareCombinedAnnotationTargets(
+ targets: PreparedAnnotationTarget[],
+): PreparedAnnotationTarget[] {
+ const ordered = preferredAnnotationTargets(targets);
+ const savedConfigIds = new Set(
+ ordered.flatMap((target) =>
+ target.initialFormData
+ .filter((field) => field.id)
+ .map((field) => field.configId),
+ ),
+ );
+ const emptyFields = new Map<string, AnnotationScoreFormData>();
+ for (const target of ordered) {
+ for (const field of target.initialFormData) {
+ if (!field.id && !savedConfigIds.has(field.configId))
+ emptyFields.set(field.configId, field);
+ }
+ }
+ return ordered.map((target) => {
+ const fields = target.initialFormData.filter((field) => field.id);
+ for (const [configId, field] of emptyFields) {
+ if (
+ !target.configControl.configs.some((config) => config.id === configId)
+ )
+ continue;
+ fields.push(field);
+ emptyFields.delete(configId);
+ }
+ return { ...target, initialFormData: fields };
+ });
+}
+
+export function prepareAnnotationFormData(
+ scores: AnnotationScore[],
+ configs: ScoreConfigDomain[],
+ selectedConfigIds: string[],
+): AnnotationScoreFormData[] {
+ const fields = scores.map(
+ ({
+ id,
+ configId,
+ name,
+ dataType,
+ value,
+ stringValue,
+ comment,
+ timestamp,
+ }) => ({
+ id,
+ configId,
+ name,
+ dataType,
+ value,
+ stringValue,
+ comment,
+ timestamp,
+ }),
+ );
+ const scoredConfigs = new Set(fields.map((field) => field.configId));
+ for (const configId of selectedConfigIds) {
+ if (scoredConfigs.has(configId)) continue;
+ const config = configs.find((candidate) => candidate.id === configId);
+ if (!config) continue;
+ fields.push({
+ id: null,
+ configId,
+ name: config.name,
+ dataType: config.dataType,
+ value: null,
+ stringValue: null,
+ comment: null,
+ timestamp: null,
+ });
+ }
+ return fields.sort((left, right) => left.name.localeCompare(right.name));
+}
diff --git a/web/src/features/scores/schema.ts b/web/src/features/scores/schema.ts
index 51ad61558c..5b955a0343 100644
--- a/web/src/features/scores/schema.ts
+++ b/web/src/features/scores/schema.ts
@@ -2,6 +2,7 @@ import { ANNOTATION_SCORE_DATA_TYPES_ARRAY } from "@/src/features/scores/types";
import { z } from "zod";
export const AnnotationScoreDataSchema = z.object({
+ targetKey: z.string().optional(),
// Required for ClickHouse deduplication (not shown in UI)
id: z.string().nullish(),
timestamp: z.date().nullish(),
diff --git a/web/src/features/scores/state/annotationSaveStore.ts b/web/src/features/scores/state/annotationSaveStore.ts
new file mode 100644
index 0000000000..724ded0a8a
--- /dev/null
+++ b/web/src/features/scores/state/annotationSaveStore.ts
@@ -0,0 +1,30 @@
+import { createStore } from "zustand/vanilla";
+import type { AnnotationScoreFormData } from "@/src/features/scores/types";
+import { annotationFieldKey } from "@/src/features/scores/lib/annotationConfigSelection";
+
+export function createAnnotationSaveStore(
+ initialFields: AnnotationScoreFormData[],
+) {
+ return createStore(() => ({
+ pending: 0,
+ failed: false,
+ saved: false,
+ confirmedFields: new Map(
+ initialFields.map((field) => [
+ annotationFieldKey(field),
+ { field, sequence: 0 },
+ ]),
+ ),
+ }));
+}
+
+export function hasChangedAnnotationValue(
+ field: AnnotationScoreFormData,
+ confirmed: AnnotationScoreFormData | undefined,
+) {
+ return (
+ (field.value ?? null) !== (confirmed?.value ?? null) ||
+ (field.stringValue ?? "") !== (confirmed?.stringValue ?? "") ||
+ (field.comment ?? "") !== (confirmed?.comment ?? "")
+ );
+}
diff --git a/web/src/features/scores/types.ts b/web/src/features/scores/types.ts
index a639938df5..5508437ad5 100644
--- a/web/src/features/scores/types.ts
+++ b/web/src/features/scores/types.ts
@@ -63,11 +63,15 @@ export type AnnotationScore = {
export type AnalyticsData = {
type: "trace" | "session";
+ isV4: boolean;
source:
| "TraceDetail"
| "SessionDetail"
| "AnnotationQueue"
- | "DatasetCompare";
+ | "DatasetCompare"
+ | "TraceTable"
+ | "ObservationTable"
+ | "SessionTable";
};
export type AnnotateFormSchemaType = z.infer<typeof AnnotateFormSchema>;
@@ -106,11 +110,12 @@ export type AnnotationForm<Target extends ScoreTarget> = {
environment?: string;
};
configSelection?: ScoreConfigSelection;
- analyticsData?: AnalyticsData;
+ analyticsData: AnalyticsData;
actionButtons?: React.ReactNode;
};
export type AnnotationScoreFormData = {
+ targetKey?: string;
id: string | null;
configId: string;
name: string;
@@ -121,19 +126,26 @@ export type AnnotationScoreFormData = {
timestamp?: Date | null;
};
-export type InnerAnnotationFormProps<Target extends ScoreTarget> = {
+type InnerAnnotationFormProps<Target extends ScoreTarget> = {
scoreTarget: Target;
initialFormData: AnnotationScoreFormData[];
configControl: {
configs: ScoreConfigDomain[];
allowManualSelection: boolean;
emptySelectedConfigIdsStorageKey?: string;
+ setSelectedConfigIds: (ids: string[]) => void;
+ selectedConfigIds: string[];
};
scoreMetadata: {
projectId: string;
queueId?: string;
environment?: string;
};
- analyticsData?: AnalyticsData;
+ analyticsData: AnalyticsData;
actionButtons?: React.ReactNode;
};
+
+export type PreparedAnnotationTarget = InnerAnnotationFormProps<ScoreTarget> & {
+ key: string;
+ label: string;
+};
diff --git a/web/src/features/sessions/SessionConversationTimeline/ConnectedSessionConversationTimeline.tsx b/web/src/features/sessions/SessionConversationTimeline/ConnectedSessionConversationTimeline.tsx
index b33ca43edb..e3c2bcf1f4 100644
--- a/web/src/features/sessions/SessionConversationTimeline/ConnectedSessionConversationTimeline.tsx
+++ b/web/src/features/sessions/SessionConversationTimeline/ConnectedSessionConversationTimeline.tsx
@@ -241,6 +241,7 @@ export function ConnectedSessionConversationTimeline({
analyticsData: {
type: "trace",
source: "SessionDetail",
+ isV4: true,
},
scoreMetadata: {
projectId,
diff --git a/web/src/features/sessions/SessionPages.tsx b/web/src/features/sessions/SessionPages.tsx
index 6f79619e3d..b6828fa35e 100644
--- a/web/src/features/sessions/SessionPages.tsx
+++ b/web/src/features/sessions/SessionPages.tsx
@@ -634,6 +634,7 @@ export const SessionPage: React.FC<{
analyticsData: {
type: "trace",
source: "TraceDetail",
+ isV4: false,
},
scoreMetadata: {
projectId,
@@ -655,6 +656,7 @@ export const SessionPage: React.FC<{
projectId={projectId}
objectId={sessionId}
objectType="SESSION"
+ analyticsData={{ source: "SessionDetail", isV4: false }}
>
{({ disabled, totalCount, Trigger }) => (
<Trigger asChild>
@@ -756,6 +758,7 @@ export const SessionPage: React.FC<{
analyticsData: {
type: "trace",
source: "TraceDetail",
+ isV4: false,
},
scoreMetadata: {
projectId,
@@ -777,6 +780,7 @@ export const SessionPage: React.FC<{
projectId={projectId}
objectId={sessionId}
objectType="SESSION"
+ analyticsData={{ source: "SessionDetail", isV4: false }}
>
{({ disabled, totalCount, Trigger }) => (
<Trigger asChild>
@@ -787,7 +791,9 @@ export const SessionPage: React.FC<{
className="w-full justify-start gap-2 font-normal"
>
<ListPlus className="h-4 w-4" />
- <span className="text-sm">Add to queue</span>
+ <span className="text-sm">
+ Add to human annotation queue
+ </span>
{totalCount > 0 && (
<AnnotationQueueItemCountBadge
totalCount={totalCount}
@@ -1679,6 +1685,7 @@ const LoadedSessionEventsPage: React.FC<{
analyticsData: {
type: "trace",
source: "TraceDetail",
+ isV4: true,
},
scoreMetadata: {
projectId,
@@ -1705,6 +1712,7 @@ const LoadedSessionEventsPage: React.FC<{
projectId={projectId}
objectId={sessionId}
objectType="SESSION"
+ analyticsData={{ source: "SessionDetail", isV4: true }}
>
{({ disabled, totalCount, Trigger }) => (
<Trigger asChild>
@@ -1833,6 +1841,7 @@ const LoadedSessionEventsPage: React.FC<{
analyticsData: {
type: "trace",
source: "TraceDetail",
+ isV4: true,
},
scoreMetadata: {
projectId,
@@ -1859,6 +1868,7 @@ const LoadedSessionEventsPage: React.FC<{
projectId={projectId}
objectId={sessionId}
objectType="SESSION"
+ analyticsData={{ source: "SessionDetail", isV4: true }}
>
{({ disabled, totalCount, Trigger }) => (
<Trigger asChild>
@@ -1869,7 +1879,9 @@ const LoadedSessionEventsPage: React.FC<{
className="w-full justify-start gap-2 font-normal"
>
<ListPlus className="h-4 w-4" />
- <span className="text-sm">Add to queue</span>
+ <span className="text-sm">
+ Add to human annotation queue
+ </span>
{totalCount > 0 && (
<AnnotationQueueItemCountBadge
totalCount={totalCount}
diff --git a/web/src/features/sessions/SessionTraceActionButtons.tsx b/web/src/features/sessions/SessionTraceActionButtons.tsx
index 8f9cb425ca..8470bb2ed1 100644
--- a/web/src/features/sessions/SessionTraceActionButtons.tsx
+++ b/web/src/features/sessions/SessionTraceActionButtons.tsx
@@ -11,13 +11,11 @@ import {
NewDatasetItemFromExistingObjectDialogController,
useDatasetItemFromTraceOrObservation,
} from "@/src/features/datasets";
-import {
- AnnotationQueueItemDropdownMenuController,
- AnnotationQueueItemCountBadge,
-} from "@/src/features/annotation-queues";
+import { AnnotationQueueItemDropdownMenuController } from "@/src/features/annotation-queues";
import { cn } from "@/src/utils/tailwind";
import {
ChevronDown,
+ ListPlus,
LockIcon,
MessageSquare,
MessageSquareOff,
@@ -35,6 +33,7 @@ export function SessionTraceActionButtons({
environment,
scores,
traceCommentCounts,
+ isV4,
density = "default",
className,
}: {
@@ -44,6 +43,7 @@ export function SessionTraceActionButtons({
environment?: string | null;
scores: TraceScores;
traceCommentCounts: Map<string, number> | undefined;
+ isV4: boolean;
density?: "default" | "compact";
className?: string;
}) {
@@ -138,14 +138,13 @@ export function SessionTraceActionButtons({
)}
</NewDatasetItemFromExistingObjectDialogController>
) : null}
- <div className="flex items-start">
+ <div className="flex flex-wrap items-start gap-2">
<AnnotateDrawerController projectId={projectId}>
{({ disabled, openDrawer }) => (
<Button
variant="outline"
size={size}
disabled={disabled}
- className="rounded-r-none"
onClick={() =>
openDrawer({
scoreTarget: { type: "trace", traceId },
@@ -153,6 +152,7 @@ export function SessionTraceActionButtons({
analyticsData: {
type: "trace",
source: "SessionDetail",
+ isV4,
},
scoreMetadata: {
projectId,
@@ -174,6 +174,7 @@ export function SessionTraceActionButtons({
projectId={projectId}
objectId={traceId}
objectType="TRACE"
+ analyticsData={{ source: "SessionDetail", isV4 }}
>
{({ disabled, totalCount, Trigger }) => (
<Trigger asChild>
@@ -181,17 +182,14 @@ export function SessionTraceActionButtons({
variant="outline"
size={size}
disabled={disabled !== undefined}
- className="rounded-l-none rounded-r-md border-l-2"
+ className="gap-1.5"
>
- <span className="relative mr-1 text-xs">
- <ChevronDown className="h-3 w-3" />
- {totalCount > 0 && (
- <AnnotationQueueItemCountBadge
- totalCount={totalCount}
- layout="toolbar"
- />
- )}
- </span>
+ <ListPlus className="h-4 w-4" />
+ <span>Add to human annotation queue</span>
+ {totalCount > 0 && (
+ <ActionButtonCountBadge count={totalCount} />
+ )}
+ <ChevronDown className="h-3 w-3" />
</Button>
</Trigger>
)}
diff --git a/web/src/features/sessions/SessionsTable.tsx b/web/src/features/sessions/SessionsTable.tsx
index e0da023df4..d6b1dcb7db 100644
--- a/web/src/features/sessions/SessionsTable.tsx
+++ b/web/src/features/sessions/SessionsTable.tsx
@@ -70,6 +70,7 @@ import {
import { type RowSelectionState } from "@tanstack/react-table";
import { showSuccessToast } from "@/src/features/notifications";
import { useScoreColumns, scoreFilters } from "@/src/features/scores";
+import { usePostHogClientCapture } from "@/src/features/posthog-analytics";
import { BatchExportTableButton } from "@/src/components/BatchExportTableButton";
import { toObservedOptions, TableSearchBar } from "@/src/features/search-bar";
@@ -114,6 +115,7 @@ export default function SessionsTable({
isV4 = false,
showControlsInPageHeader = false,
}: SessionTableProps) {
+ const capture = usePostHogClientCapture();
const sessionsFilterConfig = useMemo(
() => getSessionFilterConfig(omittedFilter, isV4),
[isV4, omittedFilter],
@@ -381,7 +383,19 @@ export default function SessionsTable({
const sessionCountQuery = isV4 ? sessionCountQueryV4 : sessionCountQueryV3;
const addToQueueMutation = api.annotationQueueItems.createMany.useMutation({
- onSuccess: (data) => {
+ onMutate: () => ({ isV4 }),
+ onSuccess: (data, variables, context) => {
+ if (context && (variables.isBatchAction || data.createdCount > 0)) {
+ capture("annotation_queues:item_added", {
+ type: "session",
+ source: "SessionTable",
+ targetType: "session",
+ objectType: "SESSION",
+ queueCount: 1,
+ isV4: context.isV4,
+ ...(variables.isBatchAction ? {} : { itemCount: data.createdCount }),
+ });
+ }
showSuccessToast({
title: "Sessions added to queue",
description: `Selected sessions will be added to queue "${data.queueName}". This may take a minute.`,
diff --git a/web/src/features/sessions/TraceEventsRow.tsx b/web/src/features/sessions/TraceEventsRow.tsx
index 674f776a7f..c44bcb1ea5 100644
--- a/web/src/features/sessions/TraceEventsRow.tsx
+++ b/web/src/features/sessions/TraceEventsRow.tsx
@@ -463,6 +463,7 @@ export const TraceEventsRow = React.memo(
</div>
</Link>
<SessionTraceActionButtons
+ isV4={true}
projectId={projectId}
traceId={trace.id}
timestamp={new Date(trace.timestamp)}
diff --git a/web/src/features/sessions/TraceRow.tsx b/web/src/features/sessions/TraceRow.tsx
index 40f28fd798..714a4a51c5 100644
--- a/web/src/features/sessions/TraceRow.tsx
+++ b/web/src/features/sessions/TraceRow.tsx
@@ -89,6 +89,7 @@ const TraceRow = React.memo(
</div>
</Link>
<SessionTraceActionButtons
+ isV4={false}
projectId={projectId}
traceId={trace.id}
timestamp={new Date(trace.timestamp)}
diff --git a/web/src/features/traces/TracesTable.tsx b/web/src/features/traces/TracesTable.tsx
index 31a502e551..275d052b27 100644
--- a/web/src/features/traces/TracesTable.tsx
+++ b/web/src/features/traces/TracesTable.tsx
@@ -189,6 +189,7 @@ function TracesTableInternal({
}: TracesTableProps & {
openDeleteTraceDialog: (traceId: string) => void;
}) {
+ const capture = usePostHogClientCapture();
const peekContext = usePeekTableState();
const hasTraceDeleteAccess = useHasProjectAccess({
projectId,
@@ -613,7 +614,18 @@ function TracesTableInternal({
});
const addToQueueMutation = api.annotationQueueItems.createMany.useMutation({
- onSuccess: (data) => {
+ onSuccess: (data, variables) => {
+ if (variables.isBatchAction || data.createdCount > 0) {
+ capture("annotation_queues:item_added", {
+ type: "trace",
+ source: "TraceTable",
+ targetType: "trace",
+ objectType: "TRACE",
+ queueCount: 1,
+ isV4: false,
+ ...(variables.isBatchAction ? {} : { itemCount: data.createdCount }),
+ });
+ }
showSuccessToast({
title: "Traces added to queue",
description: `Selected traces will be added to queue "${data.queueName}". This may take a minute.`,
diff --git a/web/src/features/traces/components/ConnectedTraceObservationAddToDropdownMenuController.tsx b/web/src/features/traces/components/ConnectedTraceObservationAddToDropdownMenuController.tsx
index 01b00410d1..9dc6c62d4d 100644
--- a/web/src/features/traces/components/ConnectedTraceObservationAddToDropdownMenuController.tsx
+++ b/web/src/features/traces/components/ConnectedTraceObservationAddToDropdownMenuController.tsx
@@ -16,6 +16,8 @@ import {
useJumpToPlayground,
} from "@/src/features/playground/page/components/JumpToPlaygroundDropdownMenuController";
import { useHasProjectAccess } from "@/src/features/rbac";
+import { usePostHogClientCapture } from "@/src/features/posthog-analytics";
+import { type AnalyticsData } from "@/src/features/scores/types";
import { api, reportNonTrpcError } from "@/src/utils/api";
import type { MetadataDomainClient } from "@/src/utils/clientSideDomainTypes";
@@ -34,6 +36,7 @@ type ConnectedTraceObservationAddToDropdownMenuControllerProps = {
input: Prisma.JsonValue | null;
output: Prisma.JsonValue | null;
metadata: MetadataDomainClient;
+ analyticsData: Pick<AnalyticsData, "source" | "isV4">;
children: ComponentProps<typeof DropdownMenu>["children"];
} & (
| {
@@ -53,6 +56,7 @@ type ConnectedTraceObservationAddToDropdownMenuControllerProps = {
export function ConnectedTraceObservationAddToDropdownMenuController(
props: ConnectedTraceObservationAddToDropdownMenuControllerProps,
) {
+ const capture = usePostHogClientCapture();
const objectId =
props.variant === "observation" ? props.observationId : props.traceId;
const objectType =
@@ -92,10 +96,19 @@ export function ConnectedTraceObservationAddToDropdownMenuController(
loading={removeFromQueueMutation.isPending}
error={removeFromQueueMutation.error?.message}
onConfirm={async ({ itemId }) => {
- await removeFromQueueMutation.mutateAsync({
+ const result = await removeFromQueueMutation.mutateAsync({
projectId: props.projectId,
itemIds: [itemId],
});
+ if (result.deletedCount > 0) {
+ capture("annotation_queues:item_removed", {
+ ...props.analyticsData,
+ type: "trace",
+ targetType: props.variant,
+ objectType,
+ queueCount: 1,
+ });
+ }
await utils.annotationQueues.byObjectId.invalidate({
projectId: props.projectId,
objectId,
@@ -132,6 +145,7 @@ function ConnectedTraceObservationAddToDropdownMenuControllerContent({
input,
output,
metadata,
+ analyticsData,
generation,
children,
createDatasetDisabled,
@@ -152,6 +166,7 @@ function ConnectedTraceObservationAddToDropdownMenuControllerContent({
openQueueDialog: () => void;
openRemoveQueueDialog: (queueRemoval: QueueRemoval) => void;
}) {
+ const capture = usePostHogClientCapture();
const session = useSession();
const objectId = variant === "observation" ? observationId : traceId;
const objectType =
@@ -195,12 +210,21 @@ function ConnectedTraceObservationAddToDropdownMenuControllerContent({
async (queueId: string, queueName: string, itemId?: string) => {
try {
if (!itemId) {
- await addToQueueMutation.mutateAsync({
+ const result = await addToQueueMutation.mutateAsync({
projectId,
objectIds: [objectId],
objectType,
queueId,
});
+ if (result.createdCount > 0) {
+ capture("annotation_queues:item_added", {
+ ...analyticsData,
+ type: "trace",
+ targetType: variant,
+ objectType,
+ queueCount: 1,
+ });
+ }
} else {
openRemoveQueueDialog({ itemId, queueName });
return;
@@ -217,11 +241,14 @@ function ConnectedTraceObservationAddToDropdownMenuControllerContent({
},
[
addToQueueMutation,
+ analyticsData,
+ capture,
objectId,
objectType,
openRemoveQueueDialog,
projectId,
utils.annotationQueues,
+ variant,
],
);
diff --git a/web/src/features/traces/components/ObservationDetailView/components/ObservationDetailViewHeader/ObservationDetailViewHeader.tsx b/web/src/features/traces/components/ObservationDetailView/components/ObservationDetailViewHeader/ObservationDetailViewHeader.tsx
index ada16bfdcd..9d2e36d418 100644
--- a/web/src/features/traces/components/ObservationDetailView/components/ObservationDetailViewHeader/ObservationDetailViewHeader.tsx
+++ b/web/src/features/traces/components/ObservationDetailView/components/ObservationDetailViewHeader/ObservationDetailViewHeader.tsx
@@ -3,13 +3,14 @@
*
* Contains:
* - Title row with ItemBadge, observation name, options menu
- * - Action buttons (Add to, Score, Comment)
+ * - Action buttons (Add to, Annotate, Comment)
* - Metadata badges (timestamp, latency, environment, cost, usage, model, etc.)
*
* Memoized to prevent unnecessary re-renders when tab state changes.
*/
import { memo, useMemo } from "react";
+import { usePostHogClientCapture } from "@/src/features/posthog-analytics";
import {
type ObservationType,
isGenerationLike,
@@ -115,6 +116,15 @@ export const ObservationDetailViewHeader = memo(
const { isAnnotationMode } = useViewPreferences();
const isMobile = useIsMobile();
const { isV4: isV4Enabled } = useReadPath();
+ const capture = usePostHogClientCapture();
+ const captureAnnotationEntry = () =>
+ capture("annotation:entry_click", {
+ type: "trace",
+ entryPoint: "annotate_button",
+ source: "TraceDetail",
+ targetType: "observation",
+ isV4: isV4Enabled,
+ });
const { trace, serverScores } = useTraceData();
// Get trace-level scores for V4 dual annotation
@@ -224,6 +234,10 @@ export const ObservationDetailViewHeader = memo(
>
{observationWithIO && (
<ConnectedTraceObservationAddToDropdownMenuController
+ analyticsData={{
+ source: "TraceDetail",
+ isV4: isV4Enabled,
+ }}
projectId={projectId}
traceId={traceId}
variant="observation"
@@ -262,6 +276,7 @@ export const ObservationDetailViewHeader = memo(
variant="ghost"
size="sm"
disabled={!hasAnnotationAccess}
+ onClick={captureAnnotationEntry}
className="w-full justify-start gap-2 font-normal"
>
{!hasAnnotationAccess ? (
@@ -269,11 +284,12 @@ export const ObservationDetailViewHeader = memo(
) : (
<SquarePen className="h-4 w-4" />
)}
- <span className="text-sm">Score</span>
+ <span className="text-sm">Annotate</span>
</Button>
</DrawerTrigger>
<DrawerContent className="p-3">
<DualAnnotationContent
+ isV4={isV4Enabled}
projectId={projectId}
traceId={traceId}
observationId={observation.id}
@@ -303,6 +319,7 @@ export const ObservationDetailViewHeader = memo(
analyticsData: {
type: "trace",
source: "TraceDetail",
+ isV4: isV4Enabled,
},
scoreMetadata: {
projectId,
@@ -316,7 +333,7 @@ export const ObservationDetailViewHeader = memo(
) : (
<SquarePen className="h-4 w-4" />
)}
- <span className="text-sm">Score</span>
+ <span className="text-sm">Annotate</span>
</Button>
)}
</AnnotateDrawerController>
@@ -388,6 +405,7 @@ export const ObservationDetailViewHeader = memo(
<div className="flex h-full flex-wrap content-start items-start justify-start gap-0.5 @2xl:mr-1 @2xl:justify-end">
{observationWithIO && (
<ConnectedTraceObservationAddToDropdownMenuController
+ analyticsData={{ source: "TraceDetail", isV4: isV4Enabled }}
projectId={projectId}
key={observation.id}
traceId={traceId}
@@ -431,11 +449,12 @@ export const ObservationDetailViewHeader = memo(
) : (
<SquarePen className="mr-1.5 h-3.5 w-3.5" />
)}
- <span>Score</span>
+ <span>Annotate</span>
</Button>
</DrawerTrigger>
<DrawerContent className="p-3">
<DualAnnotationContent
+ isV4={isV4Enabled}
projectId={projectId}
traceId={traceId}
observationId={observation.id}
@@ -464,6 +483,7 @@ export const ObservationDetailViewHeader = memo(
analyticsData: {
type: "trace",
source: "TraceDetail",
+ isV4: isV4Enabled,
},
scoreMetadata: {
projectId,
@@ -477,7 +497,7 @@ export const ObservationDetailViewHeader = memo(
) : (
<SquarePen className="mr-1.5 h-3.5 w-3.5" />
)}
- <span>Score</span>
+ <span>Annotate</span>
</Button>
)}
</AnnotateDrawerController>
diff --git a/web/src/features/traces/components/TraceDetailView/components/TraceDetailViewHeader.tsx b/web/src/features/traces/components/TraceDetailView/components/TraceDetailViewHeader.tsx
index f4ae881ae9..be9f21a6c5 100644
--- a/web/src/features/traces/components/TraceDetailView/components/TraceDetailViewHeader.tsx
+++ b/web/src/features/traces/components/TraceDetailView/components/TraceDetailViewHeader.tsx
@@ -3,7 +3,7 @@
*
* Contains:
* - Title row with ItemBadge, trace name, options menu
- * - Action buttons (Add to, Score, Comment)
+ * - Action buttons (Add to, Annotate, Comment)
* - Metadata badges (timestamp, environment, release, version, target trace)
* - Trace-level score chips
*
@@ -11,6 +11,7 @@
*/
import { memo } from "react";
+import { useReadPath } from "@/src/features/events";
import {
type TraceDomain,
type ScoreDomain,
@@ -76,6 +77,7 @@ export const TraceDetailViewHeader = memo(function TraceDetailViewHeader({
commentDrawerControl,
}: TraceDetailViewHeaderProps) {
const { isAnnotationMode } = useViewPreferences();
+ const { isV4 } = useReadPath();
const isMobile = useIsMobile();
const targetTraceId =
trace.environment === LangfuseInternalTraceEnvironment.LLMJudge
@@ -145,6 +147,7 @@ export const TraceDetailViewHeader = memo(function TraceDetailViewHeader({
className="flex w-auto min-w-44 flex-col gap-0.5 p-1 data-[state=closed]:hidden"
>
<ConnectedTraceObservationAddToDropdownMenuController
+ analyticsData={{ source: "TraceDetail", isV4 }}
projectId={projectId}
traceId={trace.id}
variant="trace"
@@ -184,6 +187,7 @@ export const TraceDetailViewHeader = memo(function TraceDetailViewHeader({
analyticsData: {
type: "trace",
source: "TraceDetail",
+ isV4,
},
scoreMetadata: {
projectId,
@@ -197,7 +201,7 @@ export const TraceDetailViewHeader = memo(function TraceDetailViewHeader({
) : (
<SquarePen className="h-4 w-4" />
)}
- <span className="text-sm">Score</span>
+ <span className="text-sm">Annotate</span>
</Button>
)}
</AnnotateDrawerController>
@@ -229,6 +233,7 @@ export const TraceDetailViewHeader = memo(function TraceDetailViewHeader({
{!isMobile && (
<div className="flex h-full flex-wrap content-start items-start justify-start gap-0.5 @2xl:mr-1 @2xl:justify-end">
<ConnectedTraceObservationAddToDropdownMenuController
+ analyticsData={{ source: "TraceDetail", isV4 }}
projectId={projectId}
traceId={trace.id}
variant="trace"
@@ -267,6 +272,7 @@ export const TraceDetailViewHeader = memo(function TraceDetailViewHeader({
analyticsData: {
type: "trace",
source: "TraceDetail",
+ isV4,
},
scoreMetadata: {
projectId,
@@ -280,7 +286,7 @@ export const TraceDetailViewHeader = memo(function TraceDetailViewHeader({
) : (
<SquarePen className="mr-1.5 h-3.5 w-3.5" />
)}
- <span>Score</span>
+ <span>Annotate</span>
</Button>
)}
</AnnotateDrawerController>
diff --git a/web/src/features/tracing-tables/observations/ObservationsTable.tsx b/web/src/features/tracing-tables/observations/ObservationsTable.tsx
index ed7547ffb2..611f8ad8a3 100644
--- a/web/src/features/tracing-tables/observations/ObservationsTable.tsx
+++ b/web/src/features/tracing-tables/observations/ObservationsTable.tsx
@@ -113,6 +113,7 @@ import {
type TableAction,
} from "@/src/features/table";
import { showSuccessToast } from "@/src/features/notifications";
+import { usePostHogClientCapture } from "@/src/features/posthog-analytics";
import { type DataTablePeekViewProps } from "@/src/components/table/peek";
import { useScoreColumns, scoreFilters } from "@/src/features/scores";
import { AddObservationsToDatasetDialog } from "@/src/features/batch-actions";
@@ -208,6 +209,7 @@ export default function ObservationsTable({
limitRows,
showControlsInPageHeader = false,
}: ObservationsTableProps) {
+ const capture = usePostHogClientCapture();
const peekContext = usePeekTableState();
const observationsFilterConfig = useMemo(
@@ -661,7 +663,18 @@ export default function ObservationsTable({
const totalCount = totalCountQuery.data?.totalCount ?? null;
const addToQueueMutation = api.annotationQueueItems.createMany.useMutation({
- onSuccess: (data) => {
+ onSuccess: (data, variables) => {
+ if (variables.isBatchAction || data.createdCount > 0) {
+ capture("annotation_queues:item_added", {
+ type: "trace",
+ source: "ObservationTable",
+ targetType: "observation",
+ objectType: "OBSERVATION",
+ queueCount: 1,
+ isV4: false,
+ ...(variables.isBatchAction ? {} : { itemCount: data.createdCount }),
+ });
+ }
showSuccessToast({
title: "Observations added to queue",
description: `Selected observations will be added to queue "${data.queueName}". This may take a minute.`,