Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(scores): Allow creating scores from annotation sidebar (#17815)

Ben Bachem818526e07c2c
diff --git a/web/src/components/design-system/DropdownMenu/DropdownMenu.stories.tsx b/web/src/components/design-system/DropdownMenu/DropdownMenu.stories.tsx
index 1ee01fb1fe..94e3054d6e 100644
--- a/web/src/components/design-system/DropdownMenu/DropdownMenu.stories.tsx
+++ b/web/src/components/design-system/DropdownMenu/DropdownMenu.stories.tsx
@@ -50,6 +50,86 @@ export const ManyItems = meta.story({
},
});
+export const TestSearchBehavior = meta.story({
+ name: "(Test) Search behavior",
+ args: {
+ search: { placeholder: "Search items" },
+ items: [
+ {
+ type: "item",
+ id: "default",
+ title: "Create report",
+ onClick: fn(),
+ },
+ {
+ type: "item",
+ id: "hidden",
+ title: "Create config",
+ searchBehavior: "hide",
+ onClick: fn(),
+ },
+ {
+ type: "item",
+ id: "always-show",
+ title: "Help",
+ searchBehavior: "always-show",
+ onClick: fn(),
+ },
+ {
+ type: "item",
+ id: "no-results",
+ title: "Add item",
+ searchBehavior: "show-when-no-results",
+ onClick: fn(),
+ },
+ ],
+ },
+ play: async ({ canvasElement }) => {
+ const canvas = within(canvasElement);
+ const body = within(canvasElement.ownerDocument.body);
+
+ await userEvent.click(canvas.getByRole("button", { name: "Open menu" }));
+ await userEvent.type(
+ body.getByRole("searchbox", { name: "Search items" }),
+ " ",
+ );
+ await expect(
+ body.getByRole("menuitem", { name: "Create config" }),
+ ).toBeVisible();
+ await expect(
+ body.getByRole("menuitem", { name: "Add item" }),
+ ).toBeVisible();
+
+ await userEvent.clear(
+ body.getByRole("searchbox", { name: "Search items" }),
+ );
+ await userEvent.type(
+ body.getByRole("searchbox", { name: "Search items" }),
+ "create",
+ );
+
+ await expect(
+ body.getByRole("menuitem", { name: "Create report" }),
+ ).toBeVisible();
+ await expect(
+ body.queryByRole("menuitem", { name: "Create config" }),
+ ).toBeNull();
+ await expect(body.getByRole("menuitem", { name: "Help" })).toBeVisible();
+ await expect(body.queryByRole("menuitem", { name: "Add item" })).toBeNull();
+
+ const search = body.getByRole("searchbox", { name: "Search items" });
+ await userEvent.clear(search);
+ await userEvent.type(search, "missing");
+
+ await expect(
+ body.queryByRole("menuitem", { name: "Create report" }),
+ ).toBeNull();
+ await expect(
+ body.getByRole("menuitem", { name: "Add item" }),
+ ).toBeVisible();
+ },
+});
+
const getMenuActions = async (canvasElement: HTMLElement) => {
const canvas = within(canvasElement);
const body = within(canvasElement.ownerDocument.body);
diff --git a/web/src/components/design-system/DropdownMenu/DropdownMenu.tsx b/web/src/components/design-system/DropdownMenu/DropdownMenu.tsx
index 020cdedd61..4356a9d2f9 100644
--- a/web/src/components/design-system/DropdownMenu/DropdownMenu.tsx
+++ b/web/src/components/design-system/DropdownMenu/DropdownMenu.tsx
@@ -96,19 +96,40 @@ type MenuAction =
| { href: string; onClick?: never }
| { href?: never; onClick: () => void };
+type DropdownMenuItemAction =
+ | {
+ href: string;
+ linkTarget?: "_blank";
+ onClick?: never;
+ onAfterNavigate?: () => void;
+ }
+ | {
+ href?: never;
+ linkTarget?: never;
+ onClick: () => void;
+ onAfterNavigate?: never;
+ };
+
+type SearchBehavior =
+ | "default"
+ | "hide"
+ | "always-show"
+ | "show-when-no-results";
+
type DropdownMenuItem = {
disabled?: { reason: string };
id: string;
title: string;
tooltip?: string;
icon?: LucideIcon;
+ searchBehavior?: SearchBehavior;
type: "item";
variant?: "default" | "destructive";
secondaryAction?: MenuAction & {
ariaLabel: string;
icon: LucideIcon;
};
-} & MenuAction;
+} & DropdownMenuItemAction;
type DropdownMenuCheckboxItem = {
checked: boolean;
@@ -117,6 +138,7 @@ type DropdownMenuCheckboxItem = {
id: string;
title: string;
icon?: LucideIcon;
+ searchBehavior?: SearchBehavior;
onCheckedChange: (checked: boolean) => void;
type: "checkbox";
};
@@ -127,6 +149,7 @@ type DropdownMenuSubmenu = {
title: string;
icon?: LucideIcon;
items: DropdownMenuItemDefinition[];
+ searchBehavior?: SearchBehavior;
search?: { placeholder: string };
type: "submenu";
};
@@ -238,10 +261,21 @@ function DropdownMenuNode({
const visibleItems = React.useMemo(() => {
const normalizedQuery = searchQuery.trim().toLocaleLowerCase();
if (!search || !normalizedQuery) return items;
+ const hasDefaultMatch = items.some((item) => {
+ if (item.type === "separator" || item.type === "loading") return false;
+ if (item.searchBehavior && item.searchBehavior !== "default")
+ return false;
+ return item.title.toLocaleLowerCase().includes(normalizedQuery);
+ });
return items.filter((item) => {
if (item.type === "separator") return false;
if (item.type === "loading") return true;
+ if (item.searchBehavior === "hide") return false;
+ if (item.searchBehavior === "always-show") return true;
+ if (item.searchBehavior === "show-when-no-results") {
+ return !hasDefaultMatch;
+ }
return item.title.toLocaleLowerCase().includes(normalizedQuery);
});
}, [items, search, searchQuery]);
@@ -587,6 +621,12 @@ function DropdownMenuNode({
<Link
data-primary-action=""
href={item.href}
+ target={item.linkTarget}
+ rel={
+ item.linkTarget === "_blank"
+ ? "noopener"
+ : undefined
+ }
aria-disabled={item.disabled ? "true" : undefined}
tabIndex={item.disabled ? -1 : undefined}
className={primaryActionVariants()}
@@ -596,6 +636,10 @@ function DropdownMenuNode({
return;
}
tree?.events.emit("click");
+ item.onAfterNavigate?.();
+ }}
+ onAuxClick={(event) => {
+ if (event.button === 1) item.onAfterNavigate?.();
}}
>
{ItemIcon ? (
diff --git a/web/src/features/score-configs/components/UpsertScoreConfigDialogController.tsx b/web/src/features/score-configs/components/UpsertScoreConfigDialogController.tsx
index cb411e2dcd..dac9463e35 100644
--- a/web/src/features/score-configs/components/UpsertScoreConfigDialogController.tsx
+++ b/web/src/features/score-configs/components/UpsertScoreConfigDialogController.tsx
@@ -10,7 +10,7 @@ import {
type CreateConfig,
type UpdateConfig,
} from "@/src/features/score-configs/lib/upsertFormTypes";
-import { api } from "@/src/utils/api";
+import { api, type RouterOutputs } from "@/src/utils/api";
const createDefaultValues: CreateConfig = {
dataType: ScoreDataTypeEnum.NUMERIC,
@@ -21,9 +21,11 @@ const createDefaultValues: CreateConfig = {
export function CreateScoreConfigDialogController({
projectId,
+ onAfterCreate,
children,
}: {
projectId: string;
+ onAfterCreate?: (config: RouterOutputs["scoreConfigs"]["create"]) => void;
children: (control: {
disabled: { reason: string } | undefined;
isSubmitting: boolean;
@@ -51,7 +53,7 @@ export function CreateScoreConfigDialogController({
mode="create"
defaultValues={createDefaultValues}
onSubmit={async (values) => {
- await mutation.mutateAsync({
+ const config = await mutation.mutateAsync({
projectId,
...values,
description: values.description ?? null,
@@ -62,6 +64,7 @@ export function CreateScoreConfigDialogController({
capture("score_configs:create_form_submit", {
dataType: values.dataType,
});
+ onAfterCreate?.(config);
}}
onFormSuccess={closeDialog}
isSubmitting={mutation.isPending}
diff --git a/web/src/features/scores/components/AnnotationForm.tsx b/web/src/features/scores/components/AnnotationForm.tsx
index 24a172a39e..fa683c8548 100644
--- a/web/src/features/scores/components/AnnotationForm.tsx
+++ b/web/src/features/scores/components/AnnotationForm.tsx
@@ -1,20 +1,12 @@
import React, { useEffect, useImperativeHandle, useRef, useState } from "react";
-import Link from "next/link";
-import { Plus, Settings2 } from "lucide-react";
+import { ExternalLink, Plus } from "lucide-react";
+import { ScoreDataTypeEnum, type ScoreConfigDataType } from "@langfuse/shared";
import { useFieldArray, useForm } from "react-hook-form";
import { zodResolver } from "@hookform/resolvers/zod";
import { Button } from "@/src/components/ui/button";
import { Skeleton } from "@/src/components/ui/skeleton";
import Header from "@/src/components/layouts/header";
-import { PopoverController } from "@/src/components/ui/popover";
-import {
- InputCommand,
- InputCommandEmpty,
- InputCommandGroup,
- InputCommandInput,
- InputCommandItem,
- InputCommandList,
-} from "@/src/components/ui/input-command";
+import { DropdownMenu } from "@/src/components/design-system/DropdownMenu/DropdownMenu";
import { KeyboardShortcut } from "@/src/components/design-system/KeyboardShortcut/KeyboardShortcut";
import { usePostHogClientCapture } from "@/src/features/posthog-analytics";
import {
@@ -27,6 +19,7 @@ import { getAnnotationTargetType } from "@/src/features/scores/lib/annotationAna
import {
annotationFieldKey,
getScoreConfigSelection,
+ preferredAnnotationTargets,
} from "@/src/features/scores/lib/annotationConfigSelection";
import { useMergedAnnotationScores } from "@/src/features/scores/lib/useMergedAnnotationScores";
import { prepareAnnotationFormData } from "@/src/features/scores/lib/prepareAnnotationFormData";
@@ -37,6 +30,7 @@ import { useAnnotationKeyboard } from "@/src/features/scores/hooks/useAnnotation
import { createAnnotationFormActions } from "@/src/features/scores/actions/annotationFormActions";
import { AnnotationScoreRow } from "@/src/features/scores/components/AnnotationScoreRow";
import { AnnotationSaveStatus } from "@/src/features/scores/components/AnnotationSaveStatus";
+import { CreateScoreConfigDialogController } from "@/src/features/score-configs/components/UpsertScoreConfigDialogController";
import type {
AnnotateFormSchemaType,
AnnotationScoreSchemaType,
@@ -46,6 +40,13 @@ import type {
AnnotationRefreshHandle,
} from "@/src/features/scores/types";
+const scoreDataTypeLabels = {
+ [ScoreDataTypeEnum.NUMERIC]: "Numeric",
+ [ScoreDataTypeEnum.CATEGORICAL]: "Categorical",
+ [ScoreDataTypeEnum.BOOLEAN]: "Boolean",
+ [ScoreDataTypeEnum.TEXT]: "Text",
+} satisfies Record<ScoreConfigDataType, string>;
+
function AnnotateHeader({
saveStatus,
actionButtons,
@@ -221,89 +222,6 @@ export function AnnotationFormContent({
(field) =>
isTextDataType(field.dataType) || isNumericDataType(field.dataType),
);
- const addScoreControl =
- allowManualSelection && isActive ? (
- <PopoverController
- align="start"
- contentClassName="w-64 p-0"
- disabled={addableOptions.length === 0}
- modal={false}
- renderContent={({ closePopover }) => (
- <InputCommand>
- <InputCommandInput
- placeholder="Search scores..."
- variant="bottom"
- />
- <InputCommandList className="max-h-72">
- <InputCommandEmpty>No scores found.</InputCommandEmpty>
- <InputCommandGroup>
- {addableOptions.map((option) => (
- <InputCommandItem
- key={option.value}
- value={option.value}
- keywords={[option.label]}
- className="cursor-pointer"
- onSelect={() => {
- closePopover();
- const controlledFields = form.getValues("scoreData");
- getScoreConfigSelection({
- targets,
- controlledFields,
- insert,
- remove,
- }).addScore(option.value);
- }}
- >
- {option.label}
- </InputCommandItem>
- ))}
- </InputCommandGroup>
- </InputCommandList>
- </InputCommand>
- )}
- >
- {({ Trigger, disabled }) => (
- <Trigger asChild>
- <Button
- data-add-score
- type="button"
- variant="outline"
- size="sm"
- className="gap-1.5 text-xs"
- disabled={disabled}
- >
- <Plus className="size-3.5" aria-hidden="true" />
- Add score
- </Button>
- </Trigger>
- )}
- </PopoverController>
- ) : null;
- const manageScoreConfigsControl = allowManualSelection ? (
- <Button
- variant="outline"
- size="sm"
- className="bg-accent gap-1.5 text-xs"
- asChild
- >
- <Link
- href={`/project/${scoreMetadata.projectId}/settings/scores`}
- target="_blank"
- onClick={() => {
- capture("score_configs:manage_configs_item_click", analyticsData);
- }}
- onAuxClick={(event) => {
- if (event.button === 1) {
- capture("score_configs:manage_configs_item_click", analyticsData);
- }
- }}
- >
- <Settings2 className="size-3" aria-hidden="true" />
- Manage score configs
- </Link>
- </Button>
- ) : null;
-
return (
<div
ref={formRootRef}
@@ -316,7 +234,102 @@ export function AnnotationFormContent({
saveStatus={<AnnotationSaveStatus form={form} actions={actions} />}
actionButtons={
<>
- {addScoreControl}
+ {allowManualSelection && isActive ? (
+ <CreateScoreConfigDialogController
+ projectId={scoreMetadata.projectId}
+ onAfterCreate={(config) => {
+ const target = preferredAnnotationTargets(targets).find(
+ (target) => target.configControl.allowManualSelection,
+ );
+ if (!target) return;
+
+ const controlledFields = form.getValues("scoreData");
+ const nextIndex = controlledFields.findIndex(
+ (field) => field.name.localeCompare(config.name) > 0,
+ );
+ insert(
+ nextIndex < 0 ? controlledFields.length : nextIndex,
+ {
+ targetKey: target.key,
+ id: null,
+ configId: config.id,
+ name: config.name,
+ dataType: config.dataType,
+ value: null,
+ stringValue: null,
+ comment: null,
+ },
+ );
+ target.configControl.setSelectedConfigIds([
+ ...target.configControl.selectedConfigIds,
+ config.id,
+ ]);
+ }}
+ >
+ {(createControl) => (
+ <DropdownMenu
+ maxHeight="18rem"
+ search={{ placeholder: "Search scores..." }}
+ items={[
+ ...addableOptions.map((option) => ({
+ type: "item" as const,
+ id: option.value,
+ title: `${option.config.name} (${scoreDataTypeLabels[option.config.dataType]})`,
+ onClick: () => {
+ const controlledFields =
+ form.getValues("scoreData");
+ getScoreConfigSelection({
+ targets,
+ controlledFields,
+ insert,
+ remove,
+ }).addScore(option.value);
+ },
+ })),
+ { type: "separator", id: "score-config-actions" },
+ {
+ type: "item",
+ id: "create-score-config",
+ title: "Add score config",
+ icon: Plus,
+ disabled: createControl.disabled,
+ searchBehavior: "show-when-no-results",
+ onClick: createControl.openDialog,
+ },
+ {
+ type: "item",
+ id: "view-score-config-settings",
+ title: "Score settings",
+ icon: ExternalLink,
+ searchBehavior: "hide",
+ href: `/project/${scoreMetadata.projectId}/settings/scores`,
+ linkTarget: "_blank",
+ onAfterNavigate: () => {
+ capture(
+ "score_configs:manage_configs_item_click",
+ analyticsData,
+ );
+ },
+ },
+ ]}
+ >
+ {({ getTriggerProps }) => (
+ <Button
+ data-add-score
+ type="button"
+ variant="outline"
+ size="sm"
+ className="gap-1.5 text-xs"
+ {...getTriggerProps()}
+ >
+ <Plus className="size-3.5" aria-hidden="true" />
+ Add score
+ </Button>
+ )}
+ </DropdownMenu>
+ )}
+ </CreateScoreConfigDialogController>
+ ) : null}
{actionButtons}
</>
}
@@ -387,9 +400,6 @@ export function AnnotationFormContent({
) : null;
})}
</div>
- {manageScoreConfigsControl ? (
- <div>{manageScoreConfigsControl}</div>
- ) : null}
{rowCount > 0 && (
// This legend only exists to advertise keyboard shortcuts, so hide
// the whole strip on touch viewports rather than just the kbd
diff --git a/web/src/features/scores/components/DualAnnotationContent.clienttest.tsx b/web/src/features/scores/components/DualAnnotationContent.clienttest.tsx
index 83d3509eed..e0663be254 100644
--- a/web/src/features/scores/components/DualAnnotationContent.clienttest.tsx
+++ b/web/src/features/scores/components/DualAnnotationContent.clienttest.tsx
@@ -78,6 +78,10 @@ vi.mock("@/src/utils/api", async () => {
return {
api: {
scoreConfigs: {
+ create: {
+ useMutation: (options: UseMutationOptions) =>
+ useMutation({ ...options, mutationFn: vi.fn() }),
+ },
all: {
useQuery: () => ({
isLoading: mocks.configsLoading,
@@ -218,11 +222,11 @@ describe("unified annotation targets", () => {
const rendered = renderContent();
fireEvent.click(screen.getByRole("button", { name: "Add score" }));
expect(
- screen.queryByRole("option", { name: /Quality/ }),
+ screen.queryByRole("menuitem", { name: /Quality/ }),
).not.toBeInTheDocument();
- fireEvent.click(await screen.findByRole("option", { name: /Accuracy/ }));
+ fireEvent.click(await screen.findByRole("menuitem", { name: /Accuracy/ }));
expect(screen.getAllByRole("group", { name: "Accuracy" })).toHaveLength(1);
- expect(screen.getByRole("button", { name: "Add score" })).toBeDisabled();
+ expect(screen.getByRole("button", { name: "Add score" })).toBeEnabled();
fireEvent.keyDown(
screen.getByRole("button", { name: "Score actions for Accuracy" }),
{ key: "ArrowDown" },
@@ -397,7 +401,9 @@ describe("unified annotation targets", () => {
"combobox",
);
fireEvent.click(trigger);
- expect(await screen.findByRole("listbox")).toBeVisible();
+ expect(
+ await screen.findByRole(control === "picker" ? "menu" : "listbox"),
+ ).toBeVisible();
}
rendered.rerenderContent(cloneElement(content, { isActive: false }));
await waitFor(() => {
@@ -660,7 +666,7 @@ describe("unified annotation targets", () => {
mocks.create.mockReturnValue(pending.promise);
renderContent();
expect(screen.getAllByRole("group", { name: "Quality" })).toHaveLength(1);
- expect(screen.getByRole("button", { name: "Add score" })).toBeDisabled();
+ expect(screen.getByRole("button", { name: "Add score" })).toBeEnabled();
fireEvent.keyDown(
screen.getByRole("button", { name: "Score actions for Quality" }),
{ key: "ArrowDown" },
@@ -882,7 +888,7 @@ describe("unified annotation targets", () => {
screen.getByRole("status", { name: "Score save status" }),
).toHaveTextContent("Saved"),
);
- expect(screen.getByRole("button", { name: "Add score" })).toBeDisabled();
+ expect(screen.getByRole("button", { name: "Add score" })).toBeEnabled();
const cached = JSON.parse(
screen.getByLabelText("Cached scores").textContent!,
);