Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(scores): add inline categories and open configs in a new tab (#16271)

Nikita Kabardin6465f7306914
diff --git a/web/src/__tests__/server/scores-trpc.servertest.ts b/web/src/__tests__/server/scores-trpc.servertest.ts
index c6238d4b77..d26bea4dcd 100644
--- a/web/src/__tests__/server/scores-trpc.servertest.ts
+++ b/web/src/__tests__/server/scores-trpc.servertest.ts
@@ -730,4 +730,84 @@ describe("scores trpc", () => {
expect(new Set(tiedIds).size).toBe(configIds.length);
});
});
+
+ describe("scoreConfigs.appendCategory", () => {
+ it("keeps both categories when two appends race", async () => {
+ const config = await prisma.scoreConfig.create({
+ data: {
+ projectId,
+ name: `append-race-${randomUUID().slice(0, 8)}`,
+ dataType: ScoreConfigDataType.CATEGORICAL,
+ categories: [{ label: "internal_user", value: 0 }],
+ },
+ });
+
+ await Promise.all([
+ caller.scoreConfigs.appendCategory({
+ projectId,
+ id: config.id,
+ label: "pen_testing",
+ }),
+ caller.scoreConfigs.appendCategory({
+ projectId,
+ id: config.id,
+ label: "just_testing",
+ }),
+ ]);
+
+ const latest = await caller.scoreConfigs.byId({
+ projectId,
+ id: config.id,
+ });
+
+ expect(
+ latest.categories?.map((category) => category.label).sort(),
+ ).toEqual(["internal_user", "just_testing", "pen_testing"]);
+ });
+
+ it("rejects a duplicate label", async () => {
+ const config = await prisma.scoreConfig.create({
+ data: {
+ projectId,
+ name: `append-dup-${randomUUID().slice(0, 8)}`,
+ dataType: ScoreConfigDataType.CATEGORICAL,
+ categories: [{ label: "internal_user", value: 0 }],
+ },
+ });
+
+ await expect(
+ caller.scoreConfigs.appendCategory({
+ projectId,
+ id: config.id,
+ label: "internal_user",
+ }),
+ ).rejects.toMatchObject({
+ code: "BAD_REQUEST",
+ message: "A category with this name already exists",
+ });
+ });
+
+ it("rejects a non-categorical config", async () => {
+ const config = await prisma.scoreConfig.create({
+ data: {
+ projectId,
+ name: `append-numeric-${randomUUID().slice(0, 8)}`,
+ dataType: ScoreConfigDataType.NUMERIC,
+ minValue: 0,
+ maxValue: 1,
+ },
+ });
+
+ await expect(
+ caller.scoreConfigs.appendCategory({
+ projectId,
+ id: config.id,
+ label: "pen_testing",
+ }),
+ ).rejects.toMatchObject({
+ code: "BAD_REQUEST",
+ message: "Only categorical score configs can append categories.",
+ });
+ });
+ });
});
diff --git a/web/src/components/ui/combobox.clienttest.tsx b/web/src/components/ui/combobox.clienttest.tsx
new file mode 100644
index 0000000000..377b721623
--- /dev/null
+++ b/web/src/components/ui/combobox.clienttest.tsx
@@ -0,0 +1,56 @@
+import { fireEvent, render, screen } from "@testing-library/react";
+import { Combobox } from "@/src/components/ui/combobox";
+import { LAYER_ORDER } from "@/src/components/ui/layer";
+
+const installOverlayLayers = () => {
+ const overlayRoot = document.createElement("div");
+ overlayRoot.setAttribute("data-overlay-root", "");
+ for (const layer of LAYER_ORDER) {
+ const layerNode = document.createElement("div");
+ layerNode.setAttribute("data-layer", layer);
+ overlayRoot.appendChild(layerNode);
+ }
+ document.body.appendChild(overlayRoot);
+};
+
+describe("Combobox footer", () => {
+ beforeAll(() => {
+ vi.stubGlobal(
+ "ResizeObserver",
+ class {
+ observe() {}
+ unobserve() {}
+ disconnect() {}
+ },
+ );
+ Element.prototype.scrollIntoView = vi.fn();
+ });
+
+ beforeEach(() => {
+ installOverlayLayers();
+ });
+
+ afterEach(() => {
+ document.querySelector("[data-overlay-root]")?.remove();
+ });
+
+ it("renders a footer action after the category list", async () => {
+ render(
+ <Combobox
+ options={[{ value: "internal_user" }, { value: "just_testing" }]}
+ placeholder="Select category"
+ footer={({ search }) => (
+ <button type="button">
+ {search ? `Add "${search}"` : "Add new category"}
+ </button>
+ )}
+ />,
+ );
+
+ fireEvent.click(screen.getByRole("combobox"));
+
+ expect(
+ await screen.findByRole("button", { name: "Add new category" }),
+ ).toBeInTheDocument();
+ });
+});
diff --git a/web/src/components/ui/combobox.tsx b/web/src/components/ui/combobox.tsx
index b60f8cef8d..b1fcceb21e 100644
--- a/web/src/components/ui/combobox.tsx
+++ b/web/src/components/ui/combobox.tsx
@@ -56,6 +56,7 @@ export interface ComboboxProps<
disabled?: boolean;
className?: string;
name?: string;
+ footer?: (args: { search: string; close: () => void }) => React.ReactNode;
}
function isGroupedOptions<T extends string | number | boolean | { id: string }>(
@@ -96,8 +97,15 @@ export function Combobox<T extends string | number | boolean | { id: string }>({
disabled = false,
className,
name,
+ footer,
}: ComboboxProps<T>) {
const [open, setOpen] = React.useState(false);
+ const [search, setSearch] = React.useState("");
+ const close = () => {
+ setOpen(false);
+ setSearch("");
+ };
+ const footerContent = footer?.({ search, close });
const selectedOption = React.useMemo(() => {
if (isGroupedOptions(options)) {
@@ -115,7 +123,13 @@ export function Combobox<T extends string | number | boolean | { id: string }>({
: placeholder;
return (
- <Popover open={open} onOpenChange={setOpen}>
+ <Popover
+ open={open}
+ onOpenChange={(nextOpen) => {
+ setOpen(nextOpen);
+ if (!nextOpen) setSearch("");
+ }}
+ >
<PopoverTrigger asChild>
<Button
variant="outline"
@@ -147,7 +161,12 @@ export function Combobox<T extends string | number | boolean | { id: string }>({
</PopoverTrigger>
<PopoverContent className="w-(--radix-popover-trigger-width) p-0">
<Command>
- <CommandInput placeholder={searchPlaceholder} className="text-xs" />
+ <CommandInput
+ placeholder={searchPlaceholder}
+ className="text-xs"
+ value={search}
+ onValueChange={setSearch}
+ />
<CommandList>
<CommandEmpty>{emptyText}</CommandEmpty>
{isGroupedOptions(options) ? (
@@ -171,7 +190,7 @@ export function Combobox<T extends string | number | boolean | { id: string }>({
onSelect={() => {
if (!option.disabled && onValueChange) {
onValueChange(option.value as T);
- setOpen(false);
+ close();
}
}}
className={cn(
@@ -222,7 +241,7 @@ export function Combobox<T extends string | number | boolean | { id: string }>({
onSelect={() => {
if (!option.disabled && onValueChange) {
onValueChange(option.value as T);
- setOpen(false);
+ close();
}
}}
className={cn(
@@ -255,6 +274,9 @@ export function Combobox<T extends string | number | boolean | { id: string }>({
)}
</CommandList>
</Command>
+ {footerContent ? (
+ <div className="border-border border-t p-1">{footerContent}</div>
+ ) : null}
</PopoverContent>
</Popover>
);
diff --git a/web/src/components/ui/dropdown-menu.clienttest.tsx b/web/src/components/ui/dropdown-menu.clienttest.tsx
new file mode 100644
index 0000000000..9c2143f5dc
--- /dev/null
+++ b/web/src/components/ui/dropdown-menu.clienttest.tsx
@@ -0,0 +1,46 @@
+import { render, screen } from "@testing-library/react";
+import {
+ DropdownMenu,
+ DropdownMenuContent,
+ DropdownMenuItemWithSecondaryAction,
+} from "@/src/components/ui/dropdown-menu";
+import { LAYER_ORDER } from "@/src/components/ui/layer";
+
+const installOverlayLayers = () => {
+ const overlayRoot = document.createElement("div");
+ overlayRoot.setAttribute("data-overlay-root", "");
+ for (const layer of LAYER_ORDER) {
+ const layerNode = document.createElement("div");
+ layerNode.setAttribute("data-layer", layer);
+ overlayRoot.appendChild(layerNode);
+ }
+ document.body.appendChild(overlayRoot);
+};
+
+describe("DropdownMenuItemWithSecondaryAction", () => {
+ beforeEach(() => {
+ installOverlayLayers();
+ });
+
+ afterEach(() => {
+ document.querySelector("[data-overlay-root]")?.remove();
+ });
+
+ it("opens href items in a new tab when target is _blank", () => {
+ render(
+ <DropdownMenu open>
+ <DropdownMenuContent>
+ <DropdownMenuItemWithSecondaryAction
+ title="Manage score configs"
+ href="/project/demo/settings/scores"
+ target="_blank"
+ />
+ </DropdownMenuContent>
+ </DropdownMenu>,
+ );
+
+ const link = screen.getByRole("link", { name: "Manage score configs" });
+ expect(link).toHaveAttribute("target", "_blank");
+ expect(link).toHaveAttribute("rel", "noopener noreferrer");
+ });
+});
diff --git a/web/src/components/ui/dropdown-menu.tsx b/web/src/components/ui/dropdown-menu.tsx
index dbe1126df6..b2bc876636 100644
--- a/web/src/components/ui/dropdown-menu.tsx
+++ b/web/src/components/ui/dropdown-menu.tsx
@@ -326,10 +326,12 @@ type DropdownMenuItemAction = {
} & (
| {
href: React.ComponentProps<typeof Link>["href"];
+ target?: React.ComponentProps<typeof Link>["target"];
onClick?: never;
}
| {
href?: never;
+ target?: never;
onClick: () => void;
}
);
@@ -344,6 +346,9 @@ type DropdownMenuItemWithSecondaryActionProps = {
};
} & DropdownMenuItemAction;
+const relForTarget = (target: React.ComponentProps<typeof Link>["target"]) =>
+ target === "_blank" ? "noopener noreferrer" : undefined;
+
const dropdownMenuItemPrimaryActionVariants = cva(
"flex min-w-0 flex-1 cursor-pointer items-center px-2 py-1.5",
);
@@ -384,6 +389,8 @@ const DropdownMenuItemWithSecondaryAction = (
secondaryActionContent = (
<Link
href={secondaryAction.href}
+ target={secondaryAction.target}
+ rel={relForTarget(secondaryAction.target)}
aria-label={secondaryAction.ariaLabel}
aria-disabled={isDisabled}
tabIndex={isDisabled ? -1 : undefined}
@@ -449,6 +456,8 @@ const DropdownMenuItemWithSecondaryAction = (
primaryActionRef.current = element;
}}
href={props.href}
+ target={props.target}
+ rel={relForTarget(props.target)}
aria-disabled={isDisabled}
tabIndex={isDisabled ? -1 : undefined}
className={dropdownMenuItemPrimaryActionVariants()}
diff --git a/web/src/features/annotation-queues/components/AnnotationQueueFormDialogContent.tsx b/web/src/features/annotation-queues/components/AnnotationQueueFormDialogContent.tsx
index a6eef82434..5e9f1f325f 100644
--- a/web/src/features/annotation-queues/components/AnnotationQueueFormDialogContent.tsx
+++ b/web/src/features/annotation-queues/components/AnnotationQueueFormDialogContent.tsx
@@ -192,6 +192,7 @@ export function AnnotationQueueFormDialogContent({
<DropdownMenuItemWithSecondaryAction
onBeforeAction={onManageScoreConfigsClick}
href={`/project/${projectId}/settings/scores`}
+ target="_blank"
title="Manage score configs"
/>
}
diff --git a/web/src/features/posthog-analytics/usePostHogClientCapture.ts b/web/src/features/posthog-analytics/usePostHogClientCapture.ts
index 27480d1eea..b711c298c0 100644
--- a/web/src/features/posthog-analytics/usePostHogClientCapture.ts
+++ b/web/src/features/posthog-analytics/usePostHogClientCapture.ts
@@ -119,6 +119,7 @@ export const events = {
"create_form_submit",
"update_form_submit",
"manage_configs_item_click",
+ "add_category_inline",
"archive_form_open",
"archive_form_submit",
],
diff --git a/web/src/features/scores/components/AddScoreCategoryDialog.tsx b/web/src/features/scores/components/AddScoreCategoryDialog.tsx
new file mode 100644
index 0000000000..a580403462
--- /dev/null
+++ b/web/src/features/scores/components/AddScoreCategoryDialog.tsx
@@ -0,0 +1,132 @@
+import { useState } from "react";
+import { toast } from "sonner";
+import { isPresent, type ScoreConfigDomain } from "@langfuse/shared";
+import { Button } from "@/src/components/ui/button";
+import {
+ Dialog,
+ DialogBody,
+ DialogContent,
+ DialogFooter,
+ DialogHeader,
+ DialogTitle,
+ DialogDescription,
+} from "@/src/components/ui/dialog";
+import { Input } from "@/src/components/ui/input";
+import { Label } from "@/src/components/ui/label";
+import { api } from "@/src/utils/api";
+import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
+import { validateNewCategoryLabel } from "@/src/features/scores/lib/annotationFormHelpers";
+import { type AnalyticsData } from "@/src/features/scores/types";
+
+export function AddScoreCategoryDialog({
+ projectId,
+ config,
+ initialLabel,
+ source,
+ onClose,
+ onCategoryAdded,
+}: {
+ projectId: string;
+ config: ScoreConfigDomain;
+ initialLabel: string;
+ source: AnalyticsData["source"] | undefined;
+ onClose: () => void;
+ onCategoryAdded: (label: string, numericValue: number) => void;
+}) {
+ const capture = usePostHogClientCapture();
+ const utils = api.useUtils();
+ const [label, setLabel] = useState(initialLabel);
+ const existingCategories = config.categories ?? [];
+ const validationError = validateNewCategoryLabel(label, existingCategories);
+
+ const appendCategory = api.scoreConfigs.appendCategory.useMutation({
+ onSuccess: async (data, variables) => {
+ capture(
+ "score_configs:add_category_inline",
+ source ? { source } : undefined,
+ );
+ await Promise.all([
+ utils.scoreConfigs.invalidate(),
+ utils.annotationQueues.invalidate(),
+ ]);
+ const added = data.categories?.find(
+ (category) => category.label === variables.label,
+ );
+ if (isPresent(added?.value)) {
+ onCategoryAdded(variables.label, added.value);
+ }
+ onClose();
+ },
+ onError: (error) => {
+ toast.error(error.message ?? "Failed to add category");
+ },
+ });
+
+ const handleConfirm = () => {
+ if (validationError) return;
+ appendCategory.mutate({
+ projectId,
+ id: config.id,
+ label: label.trim(),
+ });
+ };
+
+ return (
+ <Dialog
+ open
+ onOpenChange={(open) => {
+ if (!open) onClose();
+ }}
+ >
+ <DialogContent closeOnInteractionOutside>
+ <DialogHeader variant="action">
+ <DialogTitle>Add category</DialogTitle>
+ <DialogDescription>
+ This adds a category to{" "}
+ <span className="text-foreground font-bold">{config.name}</span> for
+ everyone in this project.
+ </DialogDescription>
+ </DialogHeader>
+ <form
+ onSubmit={(event) => {
+ event.preventDefault();
+ handleConfirm();
+ }}
+ >
+ <DialogBody>
+ <div className="grid gap-2">
+ <Label htmlFor="new-score-category-label">Category name</Label>
+ <Input
+ id="new-score-category-label"
+ value={label}
+ autoFocus
+ onChange={(event) => setLabel(event.target.value)}
+ placeholder="Enter a category name"
+ />
+ {label.trim() && validationError ? (
+ <p className="text-destructive text-sm">{validationError}</p>
+ ) : null}
+ </div>
+ </DialogBody>
+ <DialogFooter variant="action">
+ <Button
+ type="button"
+ variant="outline"
+ onClick={onClose}
+ disabled={appendCategory.isPending}
+ >
+ Cancel
+ </Button>
+ <Button
+ type="submit"
+ disabled={!!validationError}
+ loading={appendCategory.isPending}
+ >
+ Add category
+ </Button>
+ </DialogFooter>
+ </form>
+ </DialogContent>
+ </Dialog>
+ );
+}
diff --git a/web/src/features/scores/components/AnnotationForm.tsx b/web/src/features/scores/components/AnnotationForm.tsx
index 78a6380c47..7a9b6435da 100644
--- a/web/src/features/scores/components/AnnotationForm.tsx
+++ b/web/src/features/scores/components/AnnotationForm.tsx
@@ -21,7 +21,6 @@ import {
import {
isPresent,
type ScoreConfigDomain,
- type ScoreConfigCategoryDomain,
type UpdateAnnotationScoreData,
type CreateAnnotationScoreData,
TEXT_SCORE_MAX_LENGTH,
@@ -33,7 +32,6 @@ import {
PopoverContent,
PopoverTrigger,
} from "@/src/components/ui/popover";
-import { Combobox } from "@/src/components/ui/combobox";
import { Textarea } from "@/src/components/ui/textarea";
import { HoverCardContent } from "@radix-ui/react-hover-card";
import { HoverCard, HoverCardTrigger } from "@/src/components/ui/hover-card";
@@ -43,7 +41,6 @@ import {
isNumericDataType,
isScoreUnsaved,
} from "@/src/features/scores/lib/helpers";
-import { ToggleGroup, ToggleGroupItem } from "@/src/components/ui/toggle-group";
import Header from "@/src/components/layouts/header";
import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
import { cn } from "@/src/utils/tailwind";
@@ -57,6 +54,7 @@ import { AnnotateFormSchema } from "@/src/features/scores/schema";
import { ScoreConfigDetails } from "@/src/features/score-configs/components/ScoreConfigDetails";
import {
enrichCategoryOptionsWithStaleScoreValue,
+ resolveCategoricalNumericValue,
resolveConfigValue,
validateNumericScore,
} from "@/src/features/scores/lib/annotationFormHelpers";
@@ -65,6 +63,7 @@ import { transformToAnnotationScores } from "@/src/features/scores/lib/transform
import { v4 as uuid } from "uuid";
import { useScoreMutations } from "@/src/features/scores/hooks/useScoreMutations";
import { MultiSelectKeyValues } from "@/src/features/scores/components/multi-select-key-values";
+import { CategoricalScoreInput } from "@/src/features/scores/components/CategoricalScoreInput";
import { DropdownMenuItemWithSecondaryAction } from "@/src/components/ui/dropdown-menu";
import { useScoreConfigSelection } from "@/src/features/scores/hooks/useScoreConfigSelection";
import { KeyboardShortcut } from "@/src/components/ui/keyboard-shortcut";
@@ -76,8 +75,6 @@ import { useAnnotationScoreConfigs } from "@/src/features/scores/hooks/useScoreC
import { Skeleton } from "@/src/components/ui/skeleton";
import Spinner from "@/src/components/design-system/Spinner/Spinner";
-const CHAR_CUTOFF = 6;
-
function CommentField({
savedComment,
disabled,
@@ -162,18 +159,6 @@ function CommentField({
);
}
-const renderSelect = (categories: ScoreConfigCategoryDomain[]) => {
- const hasMoreThanThreeCategories = categories.length > 3;
- const hasLongCategoryNames = categories.some(
- ({ label }) => label.length > CHAR_CUTOFF,
- );
-
- return (
- hasMoreThanThreeCategories ||
- (categories.length > 1 && hasLongCategoryNames)
- );
-};
-
function AnnotateHeader({
showSaving,
actionButtons,
@@ -553,15 +538,21 @@ function InnerAnnotationForm<Target extends ScoreTarget>({
handleUpsert(index, field.value as number, null);
};
- const handleCategoricalUpsert = (index: number, stringValue: string) => {
+ const handleCategoricalUpsert = (
+ index: number,
+ stringValue: string,
+ numericValue?: number,
+ ) => {
const field = controlledFields[index];
const config = configs.find((c) => c.id === field.configId);
if (!config || !field) return;
- const numericCategoryValue = config.categories?.find(
- ({ label }) => label === stringValue,
- )?.value;
+ const numericCategoryValue = resolveCategoricalNumericValue({
+ categories: config.categories,
+ stringValue,
+ numericValue,
+ });
if (!isPresent(numericCategoryValue)) return;
@@ -854,6 +845,7 @@ function InnerAnnotationForm<Target extends ScoreTarget>({
<DropdownMenuItemWithSecondaryAction
title="Manage score configs"
href={`/project/${scoreMetadata.projectId}/settings/scores`}
+ target="_blank"
onBeforeAction={() => {
capture(
"score_configs:manage_configs_item_click",
@@ -1061,113 +1053,38 @@ function InnerAnnotationForm<Target extends ScoreTarget>({
</FormItem>
)}
/>
- ) : config.categories &&
- renderSelect(categories) ? (
+ ) : (
<FormField
control={form.control}
name={`scoreData.${index}.stringValue`}
render={({ field }) => (
<FormItem>
<FormControl>
- <Combobox
+ <CategoricalScoreInput
+ projectId={scoreMetadata.projectId}
+ config={config}
+ categories={categories}
name={field.name}
value={field.value ?? ""}
disabled={isInputDisabled(config)}
- onValueChange={(value) => {
+ source={analyticsData?.source}
+ onValueChange={(
+ value,
+ numericValue,
+ ) => {
field.onChange(value);
- handleCategoricalUpsert(index, value);
+ handleCategoricalUpsert(
+ index,
+ value,
+ numericValue,
+ );
}}
- options={categories.map((category) => ({
- value: category.label,
- disabled: category.isOutdated,
- }))}
- placeholder="Select category"
- searchPlaceholder="Search categories..."
- emptyText="No category found."
/>
</FormControl>
<FormMessage className="text-xs" />
</FormItem>
)}
/>
- ) : (
- <FormField
- control={form.control}
- name={`scoreData.${index}.stringValue`}
- render={({ field }) => (
- <FormItem>
- <FormControl>
- <ToggleGroup
- type="single"
- // Horizontal roving so Radix only uses
- // ←/→ between True/False, leaving ↑/↓ for
- // our field navigation (no double-handling).
- orientation="horizontal"
- value={field.value ?? ""}
- disabled={isInputDisabled(config)}
- className={`grid grid-cols-${categories.length}`}
- onValueChange={(value) => {
- field.onChange(value);
- handleCategoricalUpsert(index, value);
- }}
- >
- {categories.map((category) =>
- category.isOutdated ? (
- <ToggleGroupItem
- key={category.value}
- value={category.label}
- disabled
- variant="outline"
- className="grid grid-flow-col gap-1 px-1 text-xs font-normal text-nowrap opacity-50"
- >
- <span
- className="truncate"
- title={category.label}
- >
- {category.label}
- </span>
- <span>{`(${category.value})`}</span>
- </ToggleGroupItem>
- ) : (
- <ToggleGroupItem
- key={category.value}
- value={category.label}
- variant="outline"
- className="grid grid-flow-col gap-1 px-1 text-xs font-normal text-nowrap"
- >
- <span
- className="truncate"
- title={category.label}
- >
- {category.label}
- </span>
- {(() => {
- // LFE-7628: number-key hint for this
- // option, shown only while the row is
- // focused (CSS `group-focus-within`,
- // so it tracks real focus directly).
- const digit =
- (config.categories?.findIndex(
- (c) =>
- c.label ===
- category.label,
- ) ?? -1) + 1;
- return digit >= 1 &&
- digit <= 9 ? (
- <KeyboardShortcut className="ml-0.5 h-3.5 min-w-3.5 px-1 text-[9px] md:hidden md:group-focus-within:inline-flex">
- {digit}
- </KeyboardShortcut>
- ) : null;
- })()}
- </ToggleGroupItem>
- ),
- )}
- </ToggleGroup>
- </FormControl>
- <FormMessage className="text-xs" />
- </FormItem>
- )}
- />
)}
</div>
{config.isArchived ? (
diff --git a/web/src/features/scores/components/CategoricalScoreInput.tsx b/web/src/features/scores/components/CategoricalScoreInput.tsx
new file mode 100644
index 0000000000..f52a28d0ca
--- /dev/null
+++ b/web/src/features/scores/components/CategoricalScoreInput.tsx
@@ -0,0 +1,178 @@
+import { useState } from "react";
+import { Plus } from "lucide-react";
+import {
+ type ScoreConfigCategoryDomain,
+ type ScoreConfigDomain,
+} from "@langfuse/shared";
+import { Button } from "@/src/components/ui/button";
+import { Combobox } from "@/src/components/ui/combobox";
+import { KeyboardShortcut } from "@/src/components/ui/keyboard-shortcut";
+import { ToggleGroup, ToggleGroupItem } from "@/src/components/ui/toggle-group";
+import { useHasProjectAccess } from "@/src/features/rbac/utils/checkProjectAccess";
+import { isCategoricalDataType } from "@/src/features/scores/lib/helpers";
+import { getAddCategoryActionLabel } from "@/src/features/scores/lib/annotationFormHelpers";
+import { AddScoreCategoryDialog } from "@/src/features/scores/components/AddScoreCategoryDialog";
+import { type AnalyticsData } from "@/src/features/scores/types";
+
+const CHAR_CUTOFF = 6;
+
+function shouldUseCombobox(
+ categories: Pick<ScoreConfigCategoryDomain, "label">[],
+) {
+ const hasMoreThanThreeCategories = categories.length > 3;
+ const hasLongCategoryNames = categories.some(
+ ({ label }) => label.length > CHAR_CUTOFF,
+ );
+
+ return (
+ hasMoreThanThreeCategories ||
+ (categories.length > 1 && hasLongCategoryNames)
+ );
+}
+
+export function CategoricalScoreInput({
+ projectId,
+ config,
+ categories,
+ value,
+ disabled,
+ name,
+ source,
+ onValueChange,
+}: {
+ projectId: string;
+ config: ScoreConfigDomain;
+ categories: (ScoreConfigCategoryDomain & { isOutdated: boolean })[];
+ value: string;
+ disabled: boolean;
+ name: string;
+ source: AnalyticsData["source"] | undefined;
+ onValueChange: (value: string, numericValue?: number) => void;
+}) {
+ const hasConfigCudAccess = useHasProjectAccess({
+ projectId,
+ scope: "scoreConfigs:CUD",
+ });
+ const [pendingLabel, setPendingLabel] = useState<string | null>(null);
+ const canAddCategory =
+ hasConfigCudAccess &&
+ isCategoricalDataType(config.dataType) &&
+ !config.isArchived &&
+ !disabled;
+ const existingLabels = categories
+ .filter((category) => !category.isOutdated)
+ .map((category) => category.label);
+
+ const addCategoryButton = (search: string, close?: () => void) => (
+ <button
+ type="button"
+ className="hover:bg-accent hover:text-accent-foreground flex w-full items-center rounded-sm px-2 py-1.5 text-left text-xs"
+ onClick={() => {
+ close?.();
+ setPendingLabel(search.trim());
+ }}
+ >
+ <Plus className="mr-2 h-3.5 w-3.5 shrink-0" />
+ {getAddCategoryActionLabel(search, existingLabels)}
+ </button>
+ );
+
+ return (
+ <>
+ {shouldUseCombobox(categories) ? (
+ <Combobox
+ name={name}
+ value={value}
+ disabled={disabled}
+ onValueChange={onValueChange}
+ options={categories.map((category) => ({
+ value: category.label,
+ disabled: category.isOutdated,
+ }))}
+ placeholder="Select category"
+ searchPlaceholder="Search categories..."
+ emptyText="No category found."
+ footer={
+ canAddCategory
+ ? ({ search, close }) => addCategoryButton(search, close)
+ : undefined
+ }
+ />
+ ) : (
+ <div className="flex items-center gap-1">
+ <ToggleGroup
+ type="single"
+ // Horizontal roving so Radix only uses
+ // ←/→ between True/False, leaving ↑/↓ for
+ // our field navigation (no double-handling).
+ orientation="horizontal"
+ value={value}
+ disabled={disabled}
+ className={`grid flex-1 grid-cols-${categories.length}`}
+ onValueChange={onValueChange}
+ >
+ {categories.map((category) =>
+ category.isOutdated ? (
+ <ToggleGroupItem
+ key={category.value}
+ value={category.label}
+ disabled
+ variant="outline"
+ className="grid grid-flow-col gap-1 px-1 text-xs font-normal text-nowrap opacity-50"
+ >
+ <span className="truncate" title={category.label}>
+ {category.label}
+ </span>
+ <span>{`(${category.value})`}</span>
+ </ToggleGroupItem>
+ ) : (
+ <ToggleGroupItem
+ key={category.value}
+ value={category.label}
+ variant="outline"
+ className="grid grid-flow-col gap-1 px-1 text-xs font-normal text-nowrap"
+ >
+ <span className="truncate" title={category.label}>
+ {category.label}
+ </span>
+ {(() => {
+ const digit =
+ (config.categories?.findIndex(
+ (c) => c.label === category.label,
+ ) ?? -1) + 1;
+ return digit >= 1 && digit <= 9 ? (
+ <KeyboardShortcut className="ml-0.5 h-3.5 min-w-3.5 px-1 text-[9px] md:hidden md:group-focus-within:inline-flex">
+ {digit}
+ </KeyboardShortcut>
+ ) : null;
+ })()}
+ </ToggleGroupItem>
+ ),
+ )}
+ </ToggleGroup>
+ {canAddCategory ? (
+ <Button
+ type="button"
+ variant="ghost"
+ size="icon-xs"
+ title="Add new category"
+ onClick={() => setPendingLabel("")}
+ >
+ <Plus className="h-3.5 w-3.5" />
+ </Button>
+ ) : null}
+ </div>
+ )}
+ {pendingLabel !== null ? (
+ <AddScoreCategoryDialog
+ projectId={projectId}
+ config={config}
+ initialLabel={pendingLabel}
+ source={source}
+ onClose={() => setPendingLabel(null)}
+ onCategoryAdded={onValueChange}
+ />
+ ) : null}
+ </>
+ );
+}
diff --git a/web/src/features/scores/lib/annotationFormHelpers.clienttest.ts b/web/src/features/scores/lib/annotationFormHelpers.clienttest.ts
new file mode 100644
index 0000000000..f5ac26ddc0
--- /dev/null
+++ b/web/src/features/scores/lib/annotationFormHelpers.clienttest.ts
@@ -0,0 +1,128 @@
+// @vitest-environment node
+
+import {
+ appendCategoryToExisting,
+ getAddCategoryActionLabel,
+ nextCategoryValue,
+ resolveCategoricalNumericValue,
+ validateNewCategoryLabel,
+} from "@/src/features/scores/lib/annotationFormHelpers";
+
+describe("nextCategoryValue", () => {
+ it("starts at 0 when there are no categories", () => {
+ expect(nextCategoryValue([])).toBe(0);
+ });
+
+ it("increments from the highest existing value", () => {
+ expect(nextCategoryValue([{ value: 0 }, { value: 2 }, { value: 1 }])).toBe(
+ 3,
+ );
+ });
+});
+
+describe("validateNewCategoryLabel", () => {
+ it("rejects a blank label", () => {
+ expect(validateNewCategoryLabel(" ", [{ label: "internal_user" }])).toBe(
+ "Category name is required",
+ );
+ });
+
+ it("rejects a duplicate label", () => {
+ expect(
+ validateNewCategoryLabel("internal_user", [{ label: "internal_user" }]),
+ ).toBe("A category with this name already exists");
+ });
+
+ it("accepts a unique trimmed label", () => {
+ expect(
+ validateNewCategoryLabel(" pen_testing ", [{ label: "internal_user" }]),
+ ).toBeNull();
+ });
+});
+
+describe("getAddCategoryActionLabel", () => {
+ it("uses the typed name when it is new", () => {
+ expect(getAddCategoryActionLabel("pen_testing", ["internal_user"])).toBe(
+ 'Add "pen_testing"',
+ );
+ });
+
+ it("falls back when the search is empty or already exists", () => {
+ expect(getAddCategoryActionLabel("", ["internal_user"])).toBe(
+ "Add new category",
+ );
+ expect(getAddCategoryActionLabel("internal_user", ["internal_user"])).toBe(
+ "Add new category",
+ );
+ });
+});
+
+describe("resolveCategoricalNumericValue", () => {
+ const staleCategories = [{ label: "internal_user", value: 0 }];
+
+ it("cannot resolve a label that is missing from a stale category list", () => {
+ expect(
+ resolveCategoricalNumericValue({
+ categories: staleCategories,
+ stringValue: "pen_testing",
+ }),
+ ).toBeUndefined();
+ });
+
+ it("uses the server-returned value when the stale list does not include the new label", () => {
+ expect(
+ resolveCategoricalNumericValue({
+ categories: staleCategories,
+ stringValue: "pen_testing",
+ numericValue: 1,
+ }),
+ ).toBe(1);
+ });
+
+ it("looks up an existing category when no numeric value is provided", () => {
+ expect(
+ resolveCategoricalNumericValue({
+ categories: staleCategories,
+ stringValue: "internal_user",
+ }),
+ ).toBe(0);
+ });
+
+ it("accepts a null category list when a numeric value is provided", () => {
+ expect(
+ resolveCategoricalNumericValue({
+ categories: null,
+ stringValue: "pen_testing",
+ numericValue: 1,
+ }),
+ ).toBe(1);
+ });
+});
+
+describe("appendCategoryToExisting", () => {
+ const existing = [{ label: "internal_user", value: 0 }];
+
+ it("appends onto the latest list so a later add does not drop an earlier one", () => {
+ const first = appendCategoryToExisting(existing, "pen_testing");
+ expect(first.ok).toBe(true);
+ if (!first.ok) return;
+
+ const second = appendCategoryToExisting(first.categories, "just_testing");
+ expect(second.ok).toBe(true);
+ if (!second.ok) return;
+
+ expect(second.categories.map((category) => category.label)).toEqual([
+ "internal_user",
+ "pen_testing",
+ "just_testing",
+ ]);
+ });
+
+ it("rejects a duplicate against the list being appended to", () => {
+ const result = appendCategoryToExisting(existing, "internal_user");
+ expect(result).toEqual({
+ ok: false,
+ error: "A category with this name already exists",
+ });
+ });
+});
diff --git a/web/src/features/scores/lib/annotationFormHelpers.ts b/web/src/features/scores/lib/annotationFormHelpers.ts
index 93713a34bc..cb989de863 100644
--- a/web/src/features/scores/lib/annotationFormHelpers.ts
+++ b/web/src/features/scores/lib/annotationFormHelpers.ts
@@ -61,3 +61,67 @@ export const enrichCategoryOptionsWithStaleScoreValue = (
...enrichedCategories,
];
};
+
+export const resolveCategoricalNumericValue = ({
+ categories,
+ stringValue,
+ numericValue,
+}: {
+ categories?: Pick<ScoreConfigCategoryDomain, "label" | "value">[] | null;
+ stringValue: string;
+ numericValue?: number;
+}): number | undefined => {
+ if (isPresent(numericValue)) return numericValue;
+ return categories?.find(({ label }) => label === stringValue)?.value;
+};
+
+export const nextCategoryValue = (
+ categories: Pick<ScoreConfigCategoryDomain, "value">[],
+): number => {
+ if (categories.length === 0) return 0;
+ return (
+ categories.reduce((max, category) => Math.max(max, category.value), 0) + 1
+ );
+};
+
+export const validateNewCategoryLabel = (
+ label: string,
+ categories: Pick<ScoreConfigCategoryDomain, "label">[],
+): string | null => {
+ const trimmed = label.trim();
+ if (!trimmed) return "Category name is required";
+ if (categories.some((category) => category.label === trimmed)) {
+ return "A category with this name already exists";
+ }
+ return null;
+};
+
+export const getAddCategoryActionLabel = (
+ search: string,
+ existingLabels: string[],
+): string => {
+ const trimmed = search.trim();
+ if (trimmed && !existingLabels.includes(trimmed)) {
+ return `Add "${trimmed}"`;
+ }
+ return "Add new category";
+};
+
+export const appendCategoryToExisting = (
+ categories: ScoreConfigCategoryDomain[],
+ label: string,
+):
+ | { ok: true; categories: ScoreConfigCategoryDomain[] }
+ | { ok: false; error: string } => {
+ const trimmed = label.trim();
+ const error = validateNewCategoryLabel(trimmed, categories);
+ if (error) return { ok: false, error };
+
+ return {
+ ok: true,
+ categories: [
+ ...categories,
+ { label: trimmed, value: nextCategoryValue(categories) },
+ ],
+ };
+};
diff --git a/web/src/features/scores/types.ts b/web/src/features/scores/types.ts
index 96a0674a84..d598eed9b6 100644
--- a/web/src/features/scores/types.ts
+++ b/web/src/features/scores/types.ts
@@ -72,7 +72,7 @@ export type AnnotationScore = {
timestamp?: Date | null;
};
-type AnalyticsData = {
+export type AnalyticsData = {
type: "trace" | "session";
source:
| "TraceDetail"
diff --git a/web/src/server/api/routers/scoreConfigs.ts b/web/src/server/api/routers/scoreConfigs.ts
index a24c4d502a..8f711bfa88 100644
--- a/web/src/server/api/routers/scoreConfigs.ts
+++ b/web/src/server/api/routers/scoreConfigs.ts
@@ -18,6 +18,7 @@ import {
} from "@langfuse/shared";
import { traceException } from "@langfuse/shared/src/server";
import { auditLog } from "@/src/features/audit-logs/auditLog";
+import { appendCategoryToExisting } from "@/src/features/scores/lib/annotationFormHelpers";
const ScoreConfigAllInput = z.object({
projectId: z.string(), // Required for protectedProjectProcedure
@@ -155,6 +156,94 @@ export const scoreConfigsRouter = createTRPCRouter({
return validateDbScoreConfig(config);
}),
+ appendCategory: protectedProjectProcedure
+ .input(
+ z.object({
+ projectId: z.string(),
+ id: z.string(),
+ label: z.string().trim().min(1),
+ }),
+ )
+ .mutation(async ({ input, ctx }) => {
+ throwIfNoProjectAccess({
+ session: ctx.session,
+ projectId: input.projectId,
+ scope: "scoreConfigs:CUD",
+ });
+
+ const { before, after } = await ctx.prisma.$transaction(async (tx) => {
+ // Serialize concurrent appends so each reads the latest categories.
+ await tx.$queryRaw`
+ SELECT 1
+ FROM score_configs
+ WHERE id = ${input.id}
+ AND project_id = ${input.projectId}
+ FOR UPDATE
+ `;
+
+ const existingConfig = await tx.scoreConfig.findFirst({
+ where: {
+ id: input.id,
+ projectId: input.projectId,
+ },
+ });
+ if (!existingConfig) {
+ throw new LangfuseNotFoundError(
+ "No score config with this id in this project.",
+ );
+ }
+ if (existingConfig.dataType !== ScoreConfigDataType.CATEGORICAL) {
+ throw new InvalidRequestError(
+ "Only categorical score configs can append categories.",
+ );
+ }
+ if (existingConfig.isArchived) {
+ throw new InvalidRequestError(
+ "Cannot add a category to an archived score config.",
+ );
+ }
+
+ const parsedExisting = validateDbScoreConfig(existingConfig);
+ const appended = appendCategoryToExisting(
+ parsedExisting.categories ?? [],
+ input.label,
+ );
+ if (!appended.ok) {
+ throw new InvalidRequestError(appended.error);
+ }
+
+ const result = validateDbScoreConfigSafe({
+ ...existingConfig,
+ categories: appended.categories,
+ });
+ if (!result.success) {
+ throw new InvalidRequestError(
+ result.error.issues.map((issue) => issue.message).join(", "),
+ );
+ }
+
+ const updated = await tx.scoreConfig.update({
+ where: {
+ id: input.id,
+ projectId: input.projectId,
+ },
+ data: { categories: appended.categories },
+ });
+
+ return { before: existingConfig, after: updated };
+ });
+
+ await auditLog({
+ session: ctx.session,
+ resourceType: "scoreConfig",
+ resourceId: after.id,
+ action: "update",
+ before,
+ after,
+ });
+
+ return validateDbScoreConfig(after);
+ }),
byId: protectedProjectProcedure
.input(
z.object({