🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(scores): add inline categories and open configs in a new tab (#16271)
Nikita Kabardin6465f7306914
diff --git a/web/src/__tests__/server/scores-trpc.servertest.ts b/web/src/__tests__/server/scores-trpc.servertest.tsindex c6238d4b77..d26bea4dcd 100644--- a/web/src/__tests__/server/scores-trpc.servertest.ts+++ b/web/src/__tests__/server/scores-trpc.servertest.ts@@ -730,4 +730,84 @@ describe("scores trpc", () => {expect(new Set(tiedIds).size).toBe(configIds.length);});});++ describe("scoreConfigs.appendCategory", () => {+ it("keeps both categories when two appends race", async () => {+ const config = await prisma.scoreConfig.create({+ data: {+ projectId,+ name: `append-race-${randomUUID().slice(0, 8)}`,+ dataType: ScoreConfigDataType.CATEGORICAL,+ categories: [{ label: "internal_user", value: 0 }],+ },+ });++ await Promise.all([+ caller.scoreConfigs.appendCategory({+ projectId,+ id: config.id,+ label: "pen_testing",+ }),+ caller.scoreConfigs.appendCategory({+ projectId,+ id: config.id,+ label: "just_testing",+ }),+ ]);++ const latest = await caller.scoreConfigs.byId({+ projectId,+ id: config.id,+ });++ expect(+ latest.categories?.map((category) => category.label).sort(),+ ).toEqual(["internal_user", "just_testing", "pen_testing"]);+ });++ it("rejects a duplicate label", async () => {+ const config = await prisma.scoreConfig.create({+ data: {+ projectId,+ name: `append-dup-${randomUUID().slice(0, 8)}`,+ dataType: ScoreConfigDataType.CATEGORICAL,+ categories: [{ label: "internal_user", value: 0 }],+ },+ });++ await expect(+ caller.scoreConfigs.appendCategory({+ projectId,+ id: config.id,+ label: "internal_user",+ }),+ ).rejects.toMatchObject({+ code: "BAD_REQUEST",+ message: "A category with this name already exists",+ });+ });++ it("rejects a non-categorical config", async () => {+ const config = await prisma.scoreConfig.create({+ data: {+ projectId,+ name: `append-numeric-${randomUUID().slice(0, 8)}`,+ dataType: ScoreConfigDataType.NUMERIC,+ minValue: 0,+ maxValue: 1,+ },+ });++ await expect(+ caller.scoreConfigs.appendCategory({+ projectId,+ id: config.id,+ label: "pen_testing",+ }),+ ).rejects.toMatchObject({+ code: "BAD_REQUEST",+ message: "Only categorical score configs can append categories.",+ });+ });+ });});diff --git a/web/src/components/ui/combobox.clienttest.tsx b/web/src/components/ui/combobox.clienttest.tsxnew file mode 100644index 0000000000..377b721623--- /dev/null+++ b/web/src/components/ui/combobox.clienttest.tsx@@ -0,0 +1,56 @@+import { fireEvent, render, screen } from "@testing-library/react";+import { Combobox } from "@/src/components/ui/combobox";+import { LAYER_ORDER } from "@/src/components/ui/layer";++const installOverlayLayers = () => {+ const overlayRoot = document.createElement("div");+ overlayRoot.setAttribute("data-overlay-root", "");+ for (const layer of LAYER_ORDER) {+ const layerNode = document.createElement("div");+ layerNode.setAttribute("data-layer", layer);+ overlayRoot.appendChild(layerNode);+ }+ document.body.appendChild(overlayRoot);+};++describe("Combobox footer", () => {+ beforeAll(() => {+ vi.stubGlobal(+ "ResizeObserver",+ class {+ observe() {}+ unobserve() {}+ disconnect() {}+ },+ );+ Element.prototype.scrollIntoView = vi.fn();+ });++ beforeEach(() => {+ installOverlayLayers();+ });++ afterEach(() => {+ document.querySelector("[data-overlay-root]")?.remove();+ });++ it("renders a footer action after the category list", async () => {+ render(+ <Combobox+ options={[{ value: "internal_user" }, { value: "just_testing" }]}+ placeholder="Select category"+ footer={({ search }) => (+ <button type="button">+ {search ? `Add "${search}"` : "Add new category"}+ </button>+ )}+ />,+ );++ fireEvent.click(screen.getByRole("combobox"));++ expect(+ await screen.findByRole("button", { name: "Add new category" }),+ ).toBeInTheDocument();+ });+});diff --git a/web/src/components/ui/combobox.tsx b/web/src/components/ui/combobox.tsxindex b60f8cef8d..b1fcceb21e 100644--- a/web/src/components/ui/combobox.tsx+++ b/web/src/components/ui/combobox.tsx@@ -56,6 +56,7 @@ export interface ComboboxProps<disabled?: boolean;className?: string;name?: string;+ footer?: (args: { search: string; close: () => void }) => React.ReactNode;}function isGroupedOptions<T extends string | number | boolean | { id: string }>(@@ -96,8 +97,15 @@ export function Combobox<T extends string | number | boolean | { id: string }>({disabled = false,className,name,+ footer,}: ComboboxProps<T>) {const [open, setOpen] = React.useState(false);+ const [search, setSearch] = React.useState("");+ const close = () => {+ setOpen(false);+ setSearch("");+ };+ const footerContent = footer?.({ search, close });const selectedOption = React.useMemo(() => {if (isGroupedOptions(options)) {@@ -115,7 +123,13 @@ export function Combobox<T extends string | number | boolean | { id: string }>({: placeholder;return (- <Popover open={open} onOpenChange={setOpen}>+ <Popover+ open={open}+ onOpenChange={(nextOpen) => {+ setOpen(nextOpen);+ if (!nextOpen) setSearch("");+ }}+ ><PopoverTrigger asChild><Buttonvariant="outline"@@ -147,7 +161,12 @@ export function Combobox<T extends string | number | boolean | { id: string }>({</PopoverTrigger><PopoverContent className="w-(--radix-popover-trigger-width) p-0"><Command>- <CommandInput placeholder={searchPlaceholder} className="text-xs" />+ <CommandInput+ placeholder={searchPlaceholder}+ className="text-xs"+ value={search}+ onValueChange={setSearch}+ /><CommandList><CommandEmpty>{emptyText}</CommandEmpty>{isGroupedOptions(options) ? (@@ -171,7 +190,7 @@ export function Combobox<T extends string | number | boolean | { id: string }>({onSelect={() => {if (!option.disabled && onValueChange) {onValueChange(option.value as T);- setOpen(false);+ close();}}}className={cn(@@ -222,7 +241,7 @@ export function Combobox<T extends string | number | boolean | { id: string }>({onSelect={() => {if (!option.disabled && onValueChange) {onValueChange(option.value as T);- setOpen(false);+ close();}}}className={cn(@@ -255,6 +274,9 @@ export function Combobox<T extends string | number | boolean | { id: string }>({)}</CommandList></Command>+ {footerContent ? (+ <div className="border-border border-t p-1">{footerContent}</div>+ ) : null}</PopoverContent></Popover>);diff --git a/web/src/components/ui/dropdown-menu.clienttest.tsx b/web/src/components/ui/dropdown-menu.clienttest.tsxnew file mode 100644index 0000000000..9c2143f5dc--- /dev/null+++ b/web/src/components/ui/dropdown-menu.clienttest.tsx@@ -0,0 +1,46 @@+import { render, screen } from "@testing-library/react";+import {+ DropdownMenu,+ DropdownMenuContent,+ DropdownMenuItemWithSecondaryAction,+} from "@/src/components/ui/dropdown-menu";+import { LAYER_ORDER } from "@/src/components/ui/layer";++const installOverlayLayers = () => {+ const overlayRoot = document.createElement("div");+ overlayRoot.setAttribute("data-overlay-root", "");+ for (const layer of LAYER_ORDER) {+ const layerNode = document.createElement("div");+ layerNode.setAttribute("data-layer", layer);+ overlayRoot.appendChild(layerNode);+ }+ document.body.appendChild(overlayRoot);+};++describe("DropdownMenuItemWithSecondaryAction", () => {+ beforeEach(() => {+ installOverlayLayers();+ });++ afterEach(() => {+ document.querySelector("[data-overlay-root]")?.remove();+ });++ it("opens href items in a new tab when target is _blank", () => {+ render(+ <DropdownMenu open>+ <DropdownMenuContent>+ <DropdownMenuItemWithSecondaryAction+ title="Manage score configs"+ href="/project/demo/settings/scores"+ target="_blank"+ />+ </DropdownMenuContent>+ </DropdownMenu>,+ );++ const link = screen.getByRole("link", { name: "Manage score configs" });+ expect(link).toHaveAttribute("target", "_blank");+ expect(link).toHaveAttribute("rel", "noopener noreferrer");+ });+});diff --git a/web/src/components/ui/dropdown-menu.tsx b/web/src/components/ui/dropdown-menu.tsxindex dbe1126df6..b2bc876636 100644--- a/web/src/components/ui/dropdown-menu.tsx+++ b/web/src/components/ui/dropdown-menu.tsx@@ -326,10 +326,12 @@ type DropdownMenuItemAction = {} & (| {href: React.ComponentProps<typeof Link>["href"];+ target?: React.ComponentProps<typeof Link>["target"];onClick?: never;}| {href?: never;+ target?: never;onClick: () => void;});@@ -344,6 +346,9 @@ type DropdownMenuItemWithSecondaryActionProps = {};} & DropdownMenuItemAction;+const relForTarget = (target: React.ComponentProps<typeof Link>["target"]) =>+ target === "_blank" ? "noopener noreferrer" : undefined;+const dropdownMenuItemPrimaryActionVariants = cva("flex min-w-0 flex-1 cursor-pointer items-center px-2 py-1.5",);@@ -384,6 +389,8 @@ const DropdownMenuItemWithSecondaryAction = (secondaryActionContent = (<Linkhref={secondaryAction.href}+ target={secondaryAction.target}+ rel={relForTarget(secondaryAction.target)}aria-label={secondaryAction.ariaLabel}aria-disabled={isDisabled}tabIndex={isDisabled ? -1 : undefined}@@ -449,6 +456,8 @@ const DropdownMenuItemWithSecondaryAction = (primaryActionRef.current = element;}}href={props.href}+ target={props.target}+ rel={relForTarget(props.target)}aria-disabled={isDisabled}tabIndex={isDisabled ? -1 : undefined}className={dropdownMenuItemPrimaryActionVariants()}diff --git a/web/src/features/annotation-queues/components/AnnotationQueueFormDialogContent.tsx b/web/src/features/annotation-queues/components/AnnotationQueueFormDialogContent.tsxindex a6eef82434..5e9f1f325f 100644--- a/web/src/features/annotation-queues/components/AnnotationQueueFormDialogContent.tsx+++ b/web/src/features/annotation-queues/components/AnnotationQueueFormDialogContent.tsx@@ -192,6 +192,7 @@ export function AnnotationQueueFormDialogContent({<DropdownMenuItemWithSecondaryActiononBeforeAction={onManageScoreConfigsClick}href={`/project/${projectId}/settings/scores`}+ target="_blank"title="Manage score configs"/>}diff --git a/web/src/features/posthog-analytics/usePostHogClientCapture.ts b/web/src/features/posthog-analytics/usePostHogClientCapture.tsindex 27480d1eea..b711c298c0 100644--- a/web/src/features/posthog-analytics/usePostHogClientCapture.ts+++ b/web/src/features/posthog-analytics/usePostHogClientCapture.ts@@ -119,6 +119,7 @@ export const events = {"create_form_submit","update_form_submit","manage_configs_item_click",+ "add_category_inline","archive_form_open","archive_form_submit",],diff --git a/web/src/features/scores/components/AddScoreCategoryDialog.tsx b/web/src/features/scores/components/AddScoreCategoryDialog.tsxnew file mode 100644index 0000000000..a580403462--- /dev/null+++ b/web/src/features/scores/components/AddScoreCategoryDialog.tsx@@ -0,0 +1,132 @@+import { useState } from "react";+import { toast } from "sonner";+import { isPresent, type ScoreConfigDomain } from "@langfuse/shared";+import { Button } from "@/src/components/ui/button";+import {+ Dialog,+ DialogBody,+ DialogContent,+ DialogFooter,+ DialogHeader,+ DialogTitle,+ DialogDescription,+} from "@/src/components/ui/dialog";+import { Input } from "@/src/components/ui/input";+import { Label } from "@/src/components/ui/label";+import { api } from "@/src/utils/api";+import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";+import { validateNewCategoryLabel } from "@/src/features/scores/lib/annotationFormHelpers";+import { type AnalyticsData } from "@/src/features/scores/types";++export function AddScoreCategoryDialog({+ projectId,+ config,+ initialLabel,+ source,+ onClose,+ onCategoryAdded,+}: {+ projectId: string;+ config: ScoreConfigDomain;+ initialLabel: string;+ source: AnalyticsData["source"] | undefined;+ onClose: () => void;+ onCategoryAdded: (label: string, numericValue: number) => void;+}) {+ const capture = usePostHogClientCapture();+ const utils = api.useUtils();+ const [label, setLabel] = useState(initialLabel);+ const existingCategories = config.categories ?? [];+ const validationError = validateNewCategoryLabel(label, existingCategories);++ const appendCategory = api.scoreConfigs.appendCategory.useMutation({+ onSuccess: async (data, variables) => {+ capture(+ "score_configs:add_category_inline",+ source ? { source } : undefined,+ );+ await Promise.all([+ utils.scoreConfigs.invalidate(),+ utils.annotationQueues.invalidate(),+ ]);+ const added = data.categories?.find(+ (category) => category.label === variables.label,+ );+ if (isPresent(added?.value)) {+ onCategoryAdded(variables.label, added.value);+ }+ onClose();+ },+ onError: (error) => {+ toast.error(error.message ?? "Failed to add category");+ },+ });++ const handleConfirm = () => {+ if (validationError) return;+ appendCategory.mutate({+ projectId,+ id: config.id,+ label: label.trim(),+ });+ };++ return (+ <Dialog+ open+ onOpenChange={(open) => {+ if (!open) onClose();+ }}+ >+ <DialogContent closeOnInteractionOutside>+ <DialogHeader variant="action">+ <DialogTitle>Add category</DialogTitle>+ <DialogDescription>+ This adds a category to{" "}+ <span className="text-foreground font-bold">{config.name}</span> for+ everyone in this project.+ </DialogDescription>+ </DialogHeader>+ <form+ onSubmit={(event) => {+ event.preventDefault();+ handleConfirm();+ }}+ >+ <DialogBody>+ <div className="grid gap-2">+ <Label htmlFor="new-score-category-label">Category name</Label>+ <Input+ id="new-score-category-label"+ value={label}+ autoFocus+ onChange={(event) => setLabel(event.target.value)}+ placeholder="Enter a category name"+ />+ {label.trim() && validationError ? (+ <p className="text-destructive text-sm">{validationError}</p>+ ) : null}+ </div>+ </DialogBody>+ <DialogFooter variant="action">+ <Button+ type="button"+ variant="outline"+ onClick={onClose}+ disabled={appendCategory.isPending}+ >+ Cancel+ </Button>+ <Button+ type="submit"+ disabled={!!validationError}+ loading={appendCategory.isPending}+ >+ Add category+ </Button>+ </DialogFooter>+ </form>+ </DialogContent>+ </Dialog>+ );+}diff --git a/web/src/features/scores/components/AnnotationForm.tsx b/web/src/features/scores/components/AnnotationForm.tsxindex 78a6380c47..7a9b6435da 100644--- a/web/src/features/scores/components/AnnotationForm.tsx+++ b/web/src/features/scores/components/AnnotationForm.tsx@@ -21,7 +21,6 @@ import {import {isPresent,type ScoreConfigDomain,- type ScoreConfigCategoryDomain,type UpdateAnnotationScoreData,type CreateAnnotationScoreData,TEXT_SCORE_MAX_LENGTH,@@ -33,7 +32,6 @@ import {PopoverContent,PopoverTrigger,} from "@/src/components/ui/popover";-import { Combobox } from "@/src/components/ui/combobox";import { Textarea } from "@/src/components/ui/textarea";import { HoverCardContent } from "@radix-ui/react-hover-card";import { HoverCard, HoverCardTrigger } from "@/src/components/ui/hover-card";@@ -43,7 +41,6 @@ import {isNumericDataType,isScoreUnsaved,} from "@/src/features/scores/lib/helpers";-import { ToggleGroup, ToggleGroupItem } from "@/src/components/ui/toggle-group";import Header from "@/src/components/layouts/header";import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";import { cn } from "@/src/utils/tailwind";@@ -57,6 +54,7 @@ import { AnnotateFormSchema } from "@/src/features/scores/schema";import { ScoreConfigDetails } from "@/src/features/score-configs/components/ScoreConfigDetails";import {enrichCategoryOptionsWithStaleScoreValue,+ resolveCategoricalNumericValue,resolveConfigValue,validateNumericScore,} from "@/src/features/scores/lib/annotationFormHelpers";@@ -65,6 +63,7 @@ import { transformToAnnotationScores } from "@/src/features/scores/lib/transformimport { v4 as uuid } from "uuid";import { useScoreMutations } from "@/src/features/scores/hooks/useScoreMutations";import { MultiSelectKeyValues } from "@/src/features/scores/components/multi-select-key-values";+import { CategoricalScoreInput } from "@/src/features/scores/components/CategoricalScoreInput";import { DropdownMenuItemWithSecondaryAction } from "@/src/components/ui/dropdown-menu";import { useScoreConfigSelection } from "@/src/features/scores/hooks/useScoreConfigSelection";import { KeyboardShortcut } from "@/src/components/ui/keyboard-shortcut";@@ -76,8 +75,6 @@ import { useAnnotationScoreConfigs } from "@/src/features/scores/hooks/useScoreCimport { Skeleton } from "@/src/components/ui/skeleton";import Spinner from "@/src/components/design-system/Spinner/Spinner";-const CHAR_CUTOFF = 6;-function CommentField({savedComment,disabled,@@ -162,18 +159,6 @@ function CommentField({);}-const renderSelect = (categories: ScoreConfigCategoryDomain[]) => {- const hasMoreThanThreeCategories = categories.length > 3;- const hasLongCategoryNames = categories.some(- ({ label }) => label.length > CHAR_CUTOFF,- );-- return (- hasMoreThanThreeCategories ||- (categories.length > 1 && hasLongCategoryNames)- );-};-function AnnotateHeader({showSaving,actionButtons,@@ -553,15 +538,21 @@ function InnerAnnotationForm<Target extends ScoreTarget>({handleUpsert(index, field.value as number, null);};- const handleCategoricalUpsert = (index: number, stringValue: string) => {+ const handleCategoricalUpsert = (+ index: number,+ stringValue: string,+ numericValue?: number,+ ) => {const field = controlledFields[index];const config = configs.find((c) => c.id === field.configId);if (!config || !field) return;- const numericCategoryValue = config.categories?.find(- ({ label }) => label === stringValue,- )?.value;+ const numericCategoryValue = resolveCategoricalNumericValue({+ categories: config.categories,+ stringValue,+ numericValue,+ });if (!isPresent(numericCategoryValue)) return;@@ -854,6 +845,7 @@ function InnerAnnotationForm<Target extends ScoreTarget>({<DropdownMenuItemWithSecondaryActiontitle="Manage score configs"href={`/project/${scoreMetadata.projectId}/settings/scores`}+ target="_blank"onBeforeAction={() => {capture("score_configs:manage_configs_item_click",@@ -1061,113 +1053,38 @@ function InnerAnnotationForm<Target extends ScoreTarget>({</FormItem>)}/>- ) : config.categories &&- renderSelect(categories) ? (+ ) : (<FormFieldcontrol={form.control}name={`scoreData.${index}.stringValue`}render={({ field }) => (<FormItem><FormControl>- <Combobox+ <CategoricalScoreInput+ projectId={scoreMetadata.projectId}+ config={config}+ categories={categories}name={field.name}value={field.value ?? ""}disabled={isInputDisabled(config)}- onValueChange={(value) => {+ source={analyticsData?.source}+ onValueChange={(+ value,+ numericValue,+ ) => {field.onChange(value);- handleCategoricalUpsert(index, value);+ handleCategoricalUpsert(+ index,+ value,+ numericValue,+ );}}- options={categories.map((category) => ({- value: category.label,- disabled: category.isOutdated,- }))}- placeholder="Select category"- searchPlaceholder="Search categories..."- emptyText="No category found."/></FormControl><FormMessage className="text-xs" /></FormItem>)}/>- ) : (- <FormField- control={form.control}- name={`scoreData.${index}.stringValue`}- render={({ field }) => (- <FormItem>- <FormControl>- <ToggleGroup- type="single"- // Horizontal roving so Radix only uses- // ←/→ between True/False, leaving ↑/↓ for- // our field navigation (no double-handling).- orientation="horizontal"- value={field.value ?? ""}- disabled={isInputDisabled(config)}- className={`grid grid-cols-${categories.length}`}- onValueChange={(value) => {- field.onChange(value);- handleCategoricalUpsert(index, value);- }}- >- {categories.map((category) =>- category.isOutdated ? (- <ToggleGroupItem- key={category.value}- value={category.label}- disabled- variant="outline"- className="grid grid-flow-col gap-1 px-1 text-xs font-normal text-nowrap opacity-50"- >- <span- className="truncate"- title={category.label}- >- {category.label}- </span>- <span>{`(${category.value})`}</span>- </ToggleGroupItem>- ) : (- <ToggleGroupItem- key={category.value}- value={category.label}- variant="outline"- className="grid grid-flow-col gap-1 px-1 text-xs font-normal text-nowrap"- >- <span- className="truncate"- title={category.label}- >- {category.label}- </span>- {(() => {- // LFE-7628: number-key hint for this- // option, shown only while the row is- // focused (CSS `group-focus-within`,- // so it tracks real focus directly).- const digit =- (config.categories?.findIndex(- (c) =>- c.label ===- category.label,- ) ?? -1) + 1;- return digit >= 1 &&- digit <= 9 ? (- <KeyboardShortcut className="ml-0.5 h-3.5 min-w-3.5 px-1 text-[9px] md:hidden md:group-focus-within:inline-flex">- {digit}- </KeyboardShortcut>- ) : null;- })()}- </ToggleGroupItem>- ),- )}- </ToggleGroup>- </FormControl>- <FormMessage className="text-xs" />- </FormItem>- )}- />)}</div>{config.isArchived ? (diff --git a/web/src/features/scores/components/CategoricalScoreInput.tsx b/web/src/features/scores/components/CategoricalScoreInput.tsxnew file mode 100644index 0000000000..f52a28d0ca--- /dev/null+++ b/web/src/features/scores/components/CategoricalScoreInput.tsx@@ -0,0 +1,178 @@+import { useState } from "react";+import { Plus } from "lucide-react";+import {+ type ScoreConfigCategoryDomain,+ type ScoreConfigDomain,+} from "@langfuse/shared";+import { Button } from "@/src/components/ui/button";+import { Combobox } from "@/src/components/ui/combobox";+import { KeyboardShortcut } from "@/src/components/ui/keyboard-shortcut";+import { ToggleGroup, ToggleGroupItem } from "@/src/components/ui/toggle-group";+import { useHasProjectAccess } from "@/src/features/rbac/utils/checkProjectAccess";+import { isCategoricalDataType } from "@/src/features/scores/lib/helpers";+import { getAddCategoryActionLabel } from "@/src/features/scores/lib/annotationFormHelpers";+import { AddScoreCategoryDialog } from "@/src/features/scores/components/AddScoreCategoryDialog";+import { type AnalyticsData } from "@/src/features/scores/types";++const CHAR_CUTOFF = 6;++function shouldUseCombobox(+ categories: Pick<ScoreConfigCategoryDomain, "label">[],+) {+ const hasMoreThanThreeCategories = categories.length > 3;+ const hasLongCategoryNames = categories.some(+ ({ label }) => label.length > CHAR_CUTOFF,+ );++ return (+ hasMoreThanThreeCategories ||+ (categories.length > 1 && hasLongCategoryNames)+ );+}++export function CategoricalScoreInput({+ projectId,+ config,+ categories,+ value,+ disabled,+ name,+ source,+ onValueChange,+}: {+ projectId: string;+ config: ScoreConfigDomain;+ categories: (ScoreConfigCategoryDomain & { isOutdated: boolean })[];+ value: string;+ disabled: boolean;+ name: string;+ source: AnalyticsData["source"] | undefined;+ onValueChange: (value: string, numericValue?: number) => void;+}) {+ const hasConfigCudAccess = useHasProjectAccess({+ projectId,+ scope: "scoreConfigs:CUD",+ });+ const [pendingLabel, setPendingLabel] = useState<string | null>(null);+ const canAddCategory =+ hasConfigCudAccess &&+ isCategoricalDataType(config.dataType) &&+ !config.isArchived &&+ !disabled;+ const existingLabels = categories+ .filter((category) => !category.isOutdated)+ .map((category) => category.label);++ const addCategoryButton = (search: string, close?: () => void) => (+ <button+ type="button"+ className="hover:bg-accent hover:text-accent-foreground flex w-full items-center rounded-sm px-2 py-1.5 text-left text-xs"+ onClick={() => {+ close?.();+ setPendingLabel(search.trim());+ }}+ >+ <Plus className="mr-2 h-3.5 w-3.5 shrink-0" />+ {getAddCategoryActionLabel(search, existingLabels)}+ </button>+ );++ return (+ <>+ {shouldUseCombobox(categories) ? (+ <Combobox+ name={name}+ value={value}+ disabled={disabled}+ onValueChange={onValueChange}+ options={categories.map((category) => ({+ value: category.label,+ disabled: category.isOutdated,+ }))}+ placeholder="Select category"+ searchPlaceholder="Search categories..."+ emptyText="No category found."+ footer={+ canAddCategory+ ? ({ search, close }) => addCategoryButton(search, close)+ : undefined+ }+ />+ ) : (+ <div className="flex items-center gap-1">+ <ToggleGroup+ type="single"+ // Horizontal roving so Radix only uses+ // ←/→ between True/False, leaving ↑/↓ for+ // our field navigation (no double-handling).+ orientation="horizontal"+ value={value}+ disabled={disabled}+ className={`grid flex-1 grid-cols-${categories.length}`}+ onValueChange={onValueChange}+ >+ {categories.map((category) =>+ category.isOutdated ? (+ <ToggleGroupItem+ key={category.value}+ value={category.label}+ disabled+ variant="outline"+ className="grid grid-flow-col gap-1 px-1 text-xs font-normal text-nowrap opacity-50"+ >+ <span className="truncate" title={category.label}>+ {category.label}+ </span>+ <span>{`(${category.value})`}</span>+ </ToggleGroupItem>+ ) : (+ <ToggleGroupItem+ key={category.value}+ value={category.label}+ variant="outline"+ className="grid grid-flow-col gap-1 px-1 text-xs font-normal text-nowrap"+ >+ <span className="truncate" title={category.label}>+ {category.label}+ </span>+ {(() => {+ const digit =+ (config.categories?.findIndex(+ (c) => c.label === category.label,+ ) ?? -1) + 1;+ return digit >= 1 && digit <= 9 ? (+ <KeyboardShortcut className="ml-0.5 h-3.5 min-w-3.5 px-1 text-[9px] md:hidden md:group-focus-within:inline-flex">+ {digit}+ </KeyboardShortcut>+ ) : null;+ })()}+ </ToggleGroupItem>+ ),+ )}+ </ToggleGroup>+ {canAddCategory ? (+ <Button+ type="button"+ variant="ghost"+ size="icon-xs"+ title="Add new category"+ onClick={() => setPendingLabel("")}+ >+ <Plus className="h-3.5 w-3.5" />+ </Button>+ ) : null}+ </div>+ )}+ {pendingLabel !== null ? (+ <AddScoreCategoryDialog+ projectId={projectId}+ config={config}+ initialLabel={pendingLabel}+ source={source}+ onClose={() => setPendingLabel(null)}+ onCategoryAdded={onValueChange}+ />+ ) : null}+ </>+ );+}diff --git a/web/src/features/scores/lib/annotationFormHelpers.clienttest.ts b/web/src/features/scores/lib/annotationFormHelpers.clienttest.tsnew file mode 100644index 0000000000..f5ac26ddc0--- /dev/null+++ b/web/src/features/scores/lib/annotationFormHelpers.clienttest.ts@@ -0,0 +1,128 @@+// @vitest-environment node++import {+ appendCategoryToExisting,+ getAddCategoryActionLabel,+ nextCategoryValue,+ resolveCategoricalNumericValue,+ validateNewCategoryLabel,+} from "@/src/features/scores/lib/annotationFormHelpers";++describe("nextCategoryValue", () => {+ it("starts at 0 when there are no categories", () => {+ expect(nextCategoryValue([])).toBe(0);+ });++ it("increments from the highest existing value", () => {+ expect(nextCategoryValue([{ value: 0 }, { value: 2 }, { value: 1 }])).toBe(+ 3,+ );+ });+});++describe("validateNewCategoryLabel", () => {+ it("rejects a blank label", () => {+ expect(validateNewCategoryLabel(" ", [{ label: "internal_user" }])).toBe(+ "Category name is required",+ );+ });++ it("rejects a duplicate label", () => {+ expect(+ validateNewCategoryLabel("internal_user", [{ label: "internal_user" }]),+ ).toBe("A category with this name already exists");+ });++ it("accepts a unique trimmed label", () => {+ expect(+ validateNewCategoryLabel(" pen_testing ", [{ label: "internal_user" }]),+ ).toBeNull();+ });+});++describe("getAddCategoryActionLabel", () => {+ it("uses the typed name when it is new", () => {+ expect(getAddCategoryActionLabel("pen_testing", ["internal_user"])).toBe(+ 'Add "pen_testing"',+ );+ });++ it("falls back when the search is empty or already exists", () => {+ expect(getAddCategoryActionLabel("", ["internal_user"])).toBe(+ "Add new category",+ );+ expect(getAddCategoryActionLabel("internal_user", ["internal_user"])).toBe(+ "Add new category",+ );+ });+});++describe("resolveCategoricalNumericValue", () => {+ const staleCategories = [{ label: "internal_user", value: 0 }];++ it("cannot resolve a label that is missing from a stale category list", () => {+ expect(+ resolveCategoricalNumericValue({+ categories: staleCategories,+ stringValue: "pen_testing",+ }),+ ).toBeUndefined();+ });++ it("uses the server-returned value when the stale list does not include the new label", () => {+ expect(+ resolveCategoricalNumericValue({+ categories: staleCategories,+ stringValue: "pen_testing",+ numericValue: 1,+ }),+ ).toBe(1);+ });++ it("looks up an existing category when no numeric value is provided", () => {+ expect(+ resolveCategoricalNumericValue({+ categories: staleCategories,+ stringValue: "internal_user",+ }),+ ).toBe(0);+ });++ it("accepts a null category list when a numeric value is provided", () => {+ expect(+ resolveCategoricalNumericValue({+ categories: null,+ stringValue: "pen_testing",+ numericValue: 1,+ }),+ ).toBe(1);+ });+});++describe("appendCategoryToExisting", () => {+ const existing = [{ label: "internal_user", value: 0 }];++ it("appends onto the latest list so a later add does not drop an earlier one", () => {+ const first = appendCategoryToExisting(existing, "pen_testing");+ expect(first.ok).toBe(true);+ if (!first.ok) return;++ const second = appendCategoryToExisting(first.categories, "just_testing");+ expect(second.ok).toBe(true);+ if (!second.ok) return;++ expect(second.categories.map((category) => category.label)).toEqual([+ "internal_user",+ "pen_testing",+ "just_testing",+ ]);+ });++ it("rejects a duplicate against the list being appended to", () => {+ const result = appendCategoryToExisting(existing, "internal_user");+ expect(result).toEqual({+ ok: false,+ error: "A category with this name already exists",+ });+ });+});diff --git a/web/src/features/scores/lib/annotationFormHelpers.ts b/web/src/features/scores/lib/annotationFormHelpers.tsindex 93713a34bc..cb989de863 100644--- a/web/src/features/scores/lib/annotationFormHelpers.ts+++ b/web/src/features/scores/lib/annotationFormHelpers.ts@@ -61,3 +61,67 @@ export const enrichCategoryOptionsWithStaleScoreValue = (...enrichedCategories,];};++export const resolveCategoricalNumericValue = ({+ categories,+ stringValue,+ numericValue,+}: {+ categories?: Pick<ScoreConfigCategoryDomain, "label" | "value">[] | null;+ stringValue: string;+ numericValue?: number;+}): number | undefined => {+ if (isPresent(numericValue)) return numericValue;+ return categories?.find(({ label }) => label === stringValue)?.value;+};++export const nextCategoryValue = (+ categories: Pick<ScoreConfigCategoryDomain, "value">[],+): number => {+ if (categories.length === 0) return 0;+ return (+ categories.reduce((max, category) => Math.max(max, category.value), 0) + 1+ );+};++export const validateNewCategoryLabel = (+ label: string,+ categories: Pick<ScoreConfigCategoryDomain, "label">[],+): string | null => {+ const trimmed = label.trim();+ if (!trimmed) return "Category name is required";+ if (categories.some((category) => category.label === trimmed)) {+ return "A category with this name already exists";+ }+ return null;+};++export const getAddCategoryActionLabel = (+ search: string,+ existingLabels: string[],+): string => {+ const trimmed = search.trim();+ if (trimmed && !existingLabels.includes(trimmed)) {+ return `Add "${trimmed}"`;+ }+ return "Add new category";+};++export const appendCategoryToExisting = (+ categories: ScoreConfigCategoryDomain[],+ label: string,+):+ | { ok: true; categories: ScoreConfigCategoryDomain[] }+ | { ok: false; error: string } => {+ const trimmed = label.trim();+ const error = validateNewCategoryLabel(trimmed, categories);+ if (error) return { ok: false, error };++ return {+ ok: true,+ categories: [+ ...categories,+ { label: trimmed, value: nextCategoryValue(categories) },+ ],+ };+};diff --git a/web/src/features/scores/types.ts b/web/src/features/scores/types.tsindex 96a0674a84..d598eed9b6 100644--- a/web/src/features/scores/types.ts+++ b/web/src/features/scores/types.ts@@ -72,7 +72,7 @@ export type AnnotationScore = {timestamp?: Date | null;};-type AnalyticsData = {+export type AnalyticsData = {type: "trace" | "session";source:| "TraceDetail"diff --git a/web/src/server/api/routers/scoreConfigs.ts b/web/src/server/api/routers/scoreConfigs.tsindex a24c4d502a..8f711bfa88 100644--- a/web/src/server/api/routers/scoreConfigs.ts+++ b/web/src/server/api/routers/scoreConfigs.ts@@ -18,6 +18,7 @@ import {} from "@langfuse/shared";import { traceException } from "@langfuse/shared/src/server";import { auditLog } from "@/src/features/audit-logs/auditLog";+import { appendCategoryToExisting } from "@/src/features/scores/lib/annotationFormHelpers";const ScoreConfigAllInput = z.object({projectId: z.string(), // Required for protectedProjectProcedure@@ -155,6 +156,94 @@ export const scoreConfigsRouter = createTRPCRouter({return validateDbScoreConfig(config);}),+ appendCategory: protectedProjectProcedure+ .input(+ z.object({+ projectId: z.string(),+ id: z.string(),+ label: z.string().trim().min(1),+ }),+ )+ .mutation(async ({ input, ctx }) => {+ throwIfNoProjectAccess({+ session: ctx.session,+ projectId: input.projectId,+ scope: "scoreConfigs:CUD",+ });++ const { before, after } = await ctx.prisma.$transaction(async (tx) => {+ // Serialize concurrent appends so each reads the latest categories.+ await tx.$queryRaw`+ SELECT 1+ FROM score_configs+ WHERE id = ${input.id}+ AND project_id = ${input.projectId}+ FOR UPDATE+ `;++ const existingConfig = await tx.scoreConfig.findFirst({+ where: {+ id: input.id,+ projectId: input.projectId,+ },+ });+ if (!existingConfig) {+ throw new LangfuseNotFoundError(+ "No score config with this id in this project.",+ );+ }+ if (existingConfig.dataType !== ScoreConfigDataType.CATEGORICAL) {+ throw new InvalidRequestError(+ "Only categorical score configs can append categories.",+ );+ }+ if (existingConfig.isArchived) {+ throw new InvalidRequestError(+ "Cannot add a category to an archived score config.",+ );+ }++ const parsedExisting = validateDbScoreConfig(existingConfig);+ const appended = appendCategoryToExisting(+ parsedExisting.categories ?? [],+ input.label,+ );+ if (!appended.ok) {+ throw new InvalidRequestError(appended.error);+ }++ const result = validateDbScoreConfigSafe({+ ...existingConfig,+ categories: appended.categories,+ });+ if (!result.success) {+ throw new InvalidRequestError(+ result.error.issues.map((issue) => issue.message).join(", "),+ );+ }++ const updated = await tx.scoreConfig.update({+ where: {+ id: input.id,+ projectId: input.projectId,+ },+ data: { categories: appended.categories },+ });++ return { before: existingConfig, after: updated };+ });++ await auditLog({+ session: ctx.session,+ resourceType: "scoreConfig",+ resourceId: after.id,+ action: "update",+ before,+ after,+ });++ return validateDbScoreConfig(after);+ }),byId: protectedProjectProcedure.input(z.object({