🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(scores): Allow creating scores from annotation sidebar (#17815)
Ben Bachem818526e07c2c
diff --git a/web/src/components/design-system/DropdownMenu/DropdownMenu.stories.tsx b/web/src/components/design-system/DropdownMenu/DropdownMenu.stories.tsxindex 1ee01fb1fe..94e3054d6e 100644--- a/web/src/components/design-system/DropdownMenu/DropdownMenu.stories.tsx+++ b/web/src/components/design-system/DropdownMenu/DropdownMenu.stories.tsx@@ -50,6 +50,86 @@ export const ManyItems = meta.story({},});+export const TestSearchBehavior = meta.story({+ name: "(Test) Search behavior",+ args: {+ search: { placeholder: "Search items" },+ items: [+ {+ type: "item",+ id: "default",+ title: "Create report",+ onClick: fn(),+ },+ {+ type: "item",+ id: "hidden",+ title: "Create config",+ searchBehavior: "hide",+ onClick: fn(),+ },+ {+ type: "item",+ id: "always-show",+ title: "Help",+ searchBehavior: "always-show",+ onClick: fn(),+ },+ {+ type: "item",+ id: "no-results",+ title: "Add item",+ searchBehavior: "show-when-no-results",+ onClick: fn(),+ },+ ],+ },+ play: async ({ canvasElement }) => {+ const canvas = within(canvasElement);+ const body = within(canvasElement.ownerDocument.body);++ await userEvent.click(canvas.getByRole("button", { name: "Open menu" }));+ await userEvent.type(+ body.getByRole("searchbox", { name: "Search items" }),+ " ",+ );+ await expect(+ body.getByRole("menuitem", { name: "Create config" }),+ ).toBeVisible();+ await expect(+ body.getByRole("menuitem", { name: "Add item" }),+ ).toBeVisible();++ await userEvent.clear(+ body.getByRole("searchbox", { name: "Search items" }),+ );+ await userEvent.type(+ body.getByRole("searchbox", { name: "Search items" }),+ "create",+ );++ await expect(+ body.getByRole("menuitem", { name: "Create report" }),+ ).toBeVisible();+ await expect(+ body.queryByRole("menuitem", { name: "Create config" }),+ ).toBeNull();+ await expect(body.getByRole("menuitem", { name: "Help" })).toBeVisible();+ await expect(body.queryByRole("menuitem", { name: "Add item" })).toBeNull();++ const search = body.getByRole("searchbox", { name: "Search items" });+ await userEvent.clear(search);+ await userEvent.type(search, "missing");++ await expect(+ body.queryByRole("menuitem", { name: "Create report" }),+ ).toBeNull();+ await expect(+ body.getByRole("menuitem", { name: "Add item" }),+ ).toBeVisible();+ },+});+const getMenuActions = async (canvasElement: HTMLElement) => {const canvas = within(canvasElement);const body = within(canvasElement.ownerDocument.body);diff --git a/web/src/components/design-system/DropdownMenu/DropdownMenu.tsx b/web/src/components/design-system/DropdownMenu/DropdownMenu.tsxindex 020cdedd61..4356a9d2f9 100644--- a/web/src/components/design-system/DropdownMenu/DropdownMenu.tsx+++ b/web/src/components/design-system/DropdownMenu/DropdownMenu.tsx@@ -96,19 +96,40 @@ type MenuAction =| { href: string; onClick?: never }| { href?: never; onClick: () => void };+type DropdownMenuItemAction =+ | {+ href: string;+ linkTarget?: "_blank";+ onClick?: never;+ onAfterNavigate?: () => void;+ }+ | {+ href?: never;+ linkTarget?: never;+ onClick: () => void;+ onAfterNavigate?: never;+ };++type SearchBehavior =+ | "default"+ | "hide"+ | "always-show"+ | "show-when-no-results";+type DropdownMenuItem = {disabled?: { reason: string };id: string;title: string;tooltip?: string;icon?: LucideIcon;+ searchBehavior?: SearchBehavior;type: "item";variant?: "default" | "destructive";secondaryAction?: MenuAction & {ariaLabel: string;icon: LucideIcon;};-} & MenuAction;+} & DropdownMenuItemAction;type DropdownMenuCheckboxItem = {checked: boolean;@@ -117,6 +138,7 @@ type DropdownMenuCheckboxItem = {id: string;title: string;icon?: LucideIcon;+ searchBehavior?: SearchBehavior;onCheckedChange: (checked: boolean) => void;type: "checkbox";};@@ -127,6 +149,7 @@ type DropdownMenuSubmenu = {title: string;icon?: LucideIcon;items: DropdownMenuItemDefinition[];+ searchBehavior?: SearchBehavior;search?: { placeholder: string };type: "submenu";};@@ -238,10 +261,21 @@ function DropdownMenuNode({const visibleItems = React.useMemo(() => {const normalizedQuery = searchQuery.trim().toLocaleLowerCase();if (!search || !normalizedQuery) return items;+ const hasDefaultMatch = items.some((item) => {+ if (item.type === "separator" || item.type === "loading") return false;+ if (item.searchBehavior && item.searchBehavior !== "default")+ return false;+ return item.title.toLocaleLowerCase().includes(normalizedQuery);+ });return items.filter((item) => {if (item.type === "separator") return false;if (item.type === "loading") return true;+ if (item.searchBehavior === "hide") return false;+ if (item.searchBehavior === "always-show") return true;+ if (item.searchBehavior === "show-when-no-results") {+ return !hasDefaultMatch;+ }return item.title.toLocaleLowerCase().includes(normalizedQuery);});}, [items, search, searchQuery]);@@ -587,6 +621,12 @@ function DropdownMenuNode({<Linkdata-primary-action=""href={item.href}+ target={item.linkTarget}+ rel={+ item.linkTarget === "_blank"+ ? "noopener"+ : undefined+ }aria-disabled={item.disabled ? "true" : undefined}tabIndex={item.disabled ? -1 : undefined}className={primaryActionVariants()}@@ -596,6 +636,10 @@ function DropdownMenuNode({return;}tree?.events.emit("click");+ item.onAfterNavigate?.();+ }}+ onAuxClick={(event) => {+ if (event.button === 1) item.onAfterNavigate?.();}}>{ItemIcon ? (diff --git a/web/src/features/score-configs/components/UpsertScoreConfigDialogController.tsx b/web/src/features/score-configs/components/UpsertScoreConfigDialogController.tsxindex cb411e2dcd..dac9463e35 100644--- a/web/src/features/score-configs/components/UpsertScoreConfigDialogController.tsx+++ b/web/src/features/score-configs/components/UpsertScoreConfigDialogController.tsx@@ -10,7 +10,7 @@ import {type CreateConfig,type UpdateConfig,} from "@/src/features/score-configs/lib/upsertFormTypes";-import { api } from "@/src/utils/api";+import { api, type RouterOutputs } from "@/src/utils/api";const createDefaultValues: CreateConfig = {dataType: ScoreDataTypeEnum.NUMERIC,@@ -21,9 +21,11 @@ const createDefaultValues: CreateConfig = {export function CreateScoreConfigDialogController({projectId,+ onAfterCreate,children,}: {projectId: string;+ onAfterCreate?: (config: RouterOutputs["scoreConfigs"]["create"]) => void;children: (control: {disabled: { reason: string } | undefined;isSubmitting: boolean;@@ -51,7 +53,7 @@ export function CreateScoreConfigDialogController({mode="create"defaultValues={createDefaultValues}onSubmit={async (values) => {- await mutation.mutateAsync({+ const config = await mutation.mutateAsync({projectId,...values,description: values.description ?? null,@@ -62,6 +64,7 @@ export function CreateScoreConfigDialogController({capture("score_configs:create_form_submit", {dataType: values.dataType,});+ onAfterCreate?.(config);}}onFormSuccess={closeDialog}isSubmitting={mutation.isPending}diff --git a/web/src/features/scores/components/AnnotationForm.tsx b/web/src/features/scores/components/AnnotationForm.tsxindex 24a172a39e..fa683c8548 100644--- a/web/src/features/scores/components/AnnotationForm.tsx+++ b/web/src/features/scores/components/AnnotationForm.tsx@@ -1,20 +1,12 @@import React, { useEffect, useImperativeHandle, useRef, useState } from "react";-import Link from "next/link";-import { Plus, Settings2 } from "lucide-react";+import { ExternalLink, Plus } from "lucide-react";+import { ScoreDataTypeEnum, type ScoreConfigDataType } from "@langfuse/shared";import { useFieldArray, useForm } from "react-hook-form";import { zodResolver } from "@hookform/resolvers/zod";import { Button } from "@/src/components/ui/button";import { Skeleton } from "@/src/components/ui/skeleton";import Header from "@/src/components/layouts/header";-import { PopoverController } from "@/src/components/ui/popover";-import {- InputCommand,- InputCommandEmpty,- InputCommandGroup,- InputCommandInput,- InputCommandItem,- InputCommandList,-} from "@/src/components/ui/input-command";+import { DropdownMenu } from "@/src/components/design-system/DropdownMenu/DropdownMenu";import { KeyboardShortcut } from "@/src/components/design-system/KeyboardShortcut/KeyboardShortcut";import { usePostHogClientCapture } from "@/src/features/posthog-analytics";import {@@ -27,6 +19,7 @@ import { getAnnotationTargetType } from "@/src/features/scores/lib/annotationAnaimport {annotationFieldKey,getScoreConfigSelection,+ preferredAnnotationTargets,} from "@/src/features/scores/lib/annotationConfigSelection";import { useMergedAnnotationScores } from "@/src/features/scores/lib/useMergedAnnotationScores";import { prepareAnnotationFormData } from "@/src/features/scores/lib/prepareAnnotationFormData";@@ -37,6 +30,7 @@ import { useAnnotationKeyboard } from "@/src/features/scores/hooks/useAnnotationimport { createAnnotationFormActions } from "@/src/features/scores/actions/annotationFormActions";import { AnnotationScoreRow } from "@/src/features/scores/components/AnnotationScoreRow";import { AnnotationSaveStatus } from "@/src/features/scores/components/AnnotationSaveStatus";+import { CreateScoreConfigDialogController } from "@/src/features/score-configs/components/UpsertScoreConfigDialogController";import type {AnnotateFormSchemaType,AnnotationScoreSchemaType,@@ -46,6 +40,13 @@ import type {AnnotationRefreshHandle,} from "@/src/features/scores/types";+const scoreDataTypeLabels = {+ [ScoreDataTypeEnum.NUMERIC]: "Numeric",+ [ScoreDataTypeEnum.CATEGORICAL]: "Categorical",+ [ScoreDataTypeEnum.BOOLEAN]: "Boolean",+ [ScoreDataTypeEnum.TEXT]: "Text",+} satisfies Record<ScoreConfigDataType, string>;+function AnnotateHeader({saveStatus,actionButtons,@@ -221,89 +222,6 @@ export function AnnotationFormContent({(field) =>isTextDataType(field.dataType) || isNumericDataType(field.dataType),);- const addScoreControl =- allowManualSelection && isActive ? (- <PopoverController- align="start"- contentClassName="w-64 p-0"- disabled={addableOptions.length === 0}- modal={false}- renderContent={({ closePopover }) => (- <InputCommand>- <InputCommandInput- placeholder="Search scores..."- variant="bottom"- />- <InputCommandList className="max-h-72">- <InputCommandEmpty>No scores found.</InputCommandEmpty>- <InputCommandGroup>- {addableOptions.map((option) => (- <InputCommandItem- key={option.value}- value={option.value}- keywords={[option.label]}- className="cursor-pointer"- onSelect={() => {- closePopover();- const controlledFields = form.getValues("scoreData");- getScoreConfigSelection({- targets,- controlledFields,- insert,- remove,- }).addScore(option.value);- }}- >- {option.label}- </InputCommandItem>- ))}- </InputCommandGroup>- </InputCommandList>- </InputCommand>- )}- >- {({ Trigger, disabled }) => (- <Trigger asChild>- <Button- data-add-score- type="button"- variant="outline"- size="sm"- className="gap-1.5 text-xs"- disabled={disabled}- >- <Plus className="size-3.5" aria-hidden="true" />- Add score- </Button>- </Trigger>- )}- </PopoverController>- ) : null;- const manageScoreConfigsControl = allowManualSelection ? (- <Button- variant="outline"- size="sm"- className="bg-accent gap-1.5 text-xs"- asChild- >- <Link- href={`/project/${scoreMetadata.projectId}/settings/scores`}- target="_blank"- onClick={() => {- capture("score_configs:manage_configs_item_click", analyticsData);- }}- onAuxClick={(event) => {- if (event.button === 1) {- capture("score_configs:manage_configs_item_click", analyticsData);- }- }}- >- <Settings2 className="size-3" aria-hidden="true" />- Manage score configs- </Link>- </Button>- ) : null;-return (<divref={formRootRef}@@ -316,7 +234,102 @@ export function AnnotationFormContent({saveStatus={<AnnotationSaveStatus form={form} actions={actions} />}actionButtons={<>- {addScoreControl}+ {allowManualSelection && isActive ? (+ <CreateScoreConfigDialogController+ projectId={scoreMetadata.projectId}+ onAfterCreate={(config) => {+ const target = preferredAnnotationTargets(targets).find(+ (target) => target.configControl.allowManualSelection,+ );+ if (!target) return;++ const controlledFields = form.getValues("scoreData");+ const nextIndex = controlledFields.findIndex(+ (field) => field.name.localeCompare(config.name) > 0,+ );+ insert(+ nextIndex < 0 ? controlledFields.length : nextIndex,+ {+ targetKey: target.key,+ id: null,+ configId: config.id,+ name: config.name,+ dataType: config.dataType,+ value: null,+ stringValue: null,+ comment: null,+ },+ );+ target.configControl.setSelectedConfigIds([+ ...target.configControl.selectedConfigIds,+ config.id,+ ]);+ }}+ >+ {(createControl) => (+ <DropdownMenu+ maxHeight="18rem"+ search={{ placeholder: "Search scores..." }}+ items={[+ ...addableOptions.map((option) => ({+ type: "item" as const,+ id: option.value,+ title: `${option.config.name} (${scoreDataTypeLabels[option.config.dataType]})`,+ onClick: () => {+ const controlledFields =+ form.getValues("scoreData");+ getScoreConfigSelection({+ targets,+ controlledFields,+ insert,+ remove,+ }).addScore(option.value);+ },+ })),+ { type: "separator", id: "score-config-actions" },+ {+ type: "item",+ id: "create-score-config",+ title: "Add score config",+ icon: Plus,+ disabled: createControl.disabled,+ searchBehavior: "show-when-no-results",+ onClick: createControl.openDialog,+ },+ {+ type: "item",+ id: "view-score-config-settings",+ title: "Score settings",+ icon: ExternalLink,+ searchBehavior: "hide",+ href: `/project/${scoreMetadata.projectId}/settings/scores`,+ linkTarget: "_blank",+ onAfterNavigate: () => {+ capture(+ "score_configs:manage_configs_item_click",+ analyticsData,+ );+ },+ },+ ]}+ >+ {({ getTriggerProps }) => (+ <Button+ data-add-score+ type="button"+ variant="outline"+ size="sm"+ className="gap-1.5 text-xs"+ {...getTriggerProps()}+ >+ <Plus className="size-3.5" aria-hidden="true" />+ Add score+ </Button>+ )}+ </DropdownMenu>+ )}+ </CreateScoreConfigDialogController>+ ) : null}{actionButtons}</>}@@ -387,9 +400,6 @@ export function AnnotationFormContent({) : null;})}</div>- {manageScoreConfigsControl ? (- <div>{manageScoreConfigsControl}</div>- ) : null}{rowCount > 0 && (// This legend only exists to advertise keyboard shortcuts, so hide// the whole strip on touch viewports rather than just the kbddiff --git a/web/src/features/scores/components/DualAnnotationContent.clienttest.tsx b/web/src/features/scores/components/DualAnnotationContent.clienttest.tsxindex 83d3509eed..e0663be254 100644--- a/web/src/features/scores/components/DualAnnotationContent.clienttest.tsx+++ b/web/src/features/scores/components/DualAnnotationContent.clienttest.tsx@@ -78,6 +78,10 @@ vi.mock("@/src/utils/api", async () => {return {api: {scoreConfigs: {+ create: {+ useMutation: (options: UseMutationOptions) =>+ useMutation({ ...options, mutationFn: vi.fn() }),+ },all: {useQuery: () => ({isLoading: mocks.configsLoading,@@ -218,11 +222,11 @@ describe("unified annotation targets", () => {const rendered = renderContent();fireEvent.click(screen.getByRole("button", { name: "Add score" }));expect(- screen.queryByRole("option", { name: /Quality/ }),+ screen.queryByRole("menuitem", { name: /Quality/ }),).not.toBeInTheDocument();- fireEvent.click(await screen.findByRole("option", { name: /Accuracy/ }));+ fireEvent.click(await screen.findByRole("menuitem", { name: /Accuracy/ }));expect(screen.getAllByRole("group", { name: "Accuracy" })).toHaveLength(1);- expect(screen.getByRole("button", { name: "Add score" })).toBeDisabled();+ expect(screen.getByRole("button", { name: "Add score" })).toBeEnabled();fireEvent.keyDown(screen.getByRole("button", { name: "Score actions for Accuracy" }),{ key: "ArrowDown" },@@ -397,7 +401,9 @@ describe("unified annotation targets", () => {"combobox",);fireEvent.click(trigger);- expect(await screen.findByRole("listbox")).toBeVisible();+ expect(+ await screen.findByRole(control === "picker" ? "menu" : "listbox"),+ ).toBeVisible();}rendered.rerenderContent(cloneElement(content, { isActive: false }));await waitFor(() => {@@ -660,7 +666,7 @@ describe("unified annotation targets", () => {mocks.create.mockReturnValue(pending.promise);renderContent();expect(screen.getAllByRole("group", { name: "Quality" })).toHaveLength(1);- expect(screen.getByRole("button", { name: "Add score" })).toBeDisabled();+ expect(screen.getByRole("button", { name: "Add score" })).toBeEnabled();fireEvent.keyDown(screen.getByRole("button", { name: "Score actions for Quality" }),{ key: "ArrowDown" },@@ -882,7 +888,7 @@ describe("unified annotation targets", () => {screen.getByRole("status", { name: "Score save status" }),).toHaveTextContent("Saved"),);- expect(screen.getByRole("button", { name: "Add score" })).toBeDisabled();+ expect(screen.getByRole("button", { name: "Add score" })).toBeEnabled();const cached = JSON.parse(screen.getByLabelText("Cached scores").textContent!,);