Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

15.5 KB
Raw Blame
/* eslint-disable no-nested-ternary, @typescript-eslint/no-deprecated, react/no-unstable-nested-components -- Legacy UI usage retained during migration to design-system components. */
import { DataTable } from "@/src/components/table/data-table";
import { type LangfuseColumnDef } from "@/src/components/table/types";
import { api } from "@/src/utils/api";
import { safeExtract } from "@/src/utils/map-utils";
import { useQueryParams, withDefault, NumberParam } from "use-query-params";
import { DataTableToolbar } from "@/src/components/table/data-table-toolbar";
import { default as useColumnOrder } from "@/src/features/column-visibility/hooks/useColumnOrder";
import { default as useColumnVisibility } from "@/src/features/column-visibility/hooks/useColumnVisibility";
import { type AnnotationQueueStatus } from "@langfuse/shared";
import {
  customRowHeightMenu,
  useRowHeightLocalStorage,
} from "@/src/components/table/data-table-row-height-switch";
import { ListTree, Trash } from "@langfuse/icons";
import { DropdownIndicator } from "@/src/components/design-system/DropdownIndicator/DropdownIndicator";
import { type RouterOutput } from "@/src/utils/types";
import { type RowSelectionState } from "@tanstack/react-table";
import { useState } from "react";
import { DropdownMenu } from "@/src/components/design-system/DropdownMenu/DropdownMenu";
import { Button } from "@/src/components/ui/button";
import {
  Dialog,
  DialogContent,
  DialogDescription,
  DialogFooter,
  DialogHeader,
  DialogTitle,
} from "@/src/components/ui/dialog";
import { Checkbox } from "@/src/components/design-system/Checkbox/Checkbox";
import { useHasProjectAccess } from "@/src/features/rbac/utils/checkProjectAccess";
import { useReadPath } from "@/src/features/events/hooks/useReadPath";
import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
import { createStatusTableColumn } from "@/src/components/design-system/table/columns/createStatusTableColumn";
import { type Status } from "@/src/components/ui/StatusBadge/StatusBadge";
import { createIdTableColumn } from "@/src/components/design-system/table/columns/createIdTableColumn";
import { createLinkTableColumn } from "@/src/components/design-system/table/columns/createLinkTableColumn";
import { createUserTableColumn } from "@/src/components/design-system/table/columns/createUserTableColumn";

const QueueItemTableMultiSelectAction = ({
  selectedItems,
  projectId,
  isV4,
  onDeleteSuccess,
}: {
  selectedItems: Pick<QueueItemRowData, "id" | "objectType">[];
  projectId: string;
  isV4: boolean;
  onDeleteSuccess: () => void;
}) => {
  const utils = api.useUtils();
  const capture = usePostHogClientCapture();
  const [open, setOpen] = useState(false);
  const selectedItemIds = selectedItems.map((item) => item.id);

  const hasDeleteAccess = useHasProjectAccess({
    projectId,
    scope: "annotationQueues:CUD",
  });
  const mutDeleteItems = api.annotationQueueItems.deleteMany.useMutation({
    onMutate: (variables) => {
      const itemIds = new Set(variables.itemIds);
      return {
        isV4,
        objectTypes: new Set(
          selectedItems
            .filter((item) => itemIds.has(item.id))
            .map((item) => item.objectType),
        ),
      };
    },
    onSuccess: (data, _variables, context) => {
      if (context && data.deletedCount > 0) {
        for (const objectType of context.objectTypes) {
          capture("annotation_queues:item_removed", {
            type: objectType === "SESSION" ? "session" : "trace",
            source: "AnnotationQueue",
            targetType: (
              {
                TRACE: "trace",
                OBSERVATION: "observation",
                SESSION: "session",
              } as const
            )[objectType],
            objectType,
            queueCount: 1,
            isV4: context.isV4,
          });
        }
      }
      onDeleteSuccess();
      utils.annotationQueueItems.itemsByQueueId.invalidate();
    },
  });

  return (
    <>
      <DropdownMenu
        disabled={selectedItemIds.length < 1}
        items={[
          {
            type: "item",
            id: "delete",
            title: "Delete",
            icon: Trash,
            disabled: hasDeleteAccess
              ? undefined
              : { reason: "Missing permission to delete queue items" },
            onClick: () => {
              setOpen(true);
            },
          },
        ]}
      >
        {({ getTriggerProps }) => (
          <Button disabled={selectedItemIds.length < 1} {...getTriggerProps()}>
            Actions ({selectedItemIds.length} selected)
            <DropdownIndicator nudge />
          </Button>
        )}
      </DropdownMenu>
      <Dialog
        open={open}
        onOpenChange={(isOpen) => {
          if (!isOpen) {
            setOpen(false);
          }
        }}
      >
        <DialogContent className="sm:max-w-md">
          <DialogHeader>
            <DialogTitle>Delete queue items</DialogTitle>
            <DialogDescription>
              This action cannot be undone and removes the selected annotation
              queue item(s), but
              <strong> does not delete associated scores.</strong>
            </DialogDescription>
          </DialogHeader>
          <DialogFooter className="sm:justify-start">
            <Button
              type="button"
              variant="destructive"
              loading={mutDeleteItems.isPending}
              disabled={mutDeleteItems.isPending}
              onClick={() => {
                mutDeleteItems
                  .mutateAsync({
                    itemIds: selectedItemIds,
                    projectId,
                  })
                  .then(() => {
                    setOpen(false);
                  });
              }}
            >
              Delete {selectedItemIds.length} item(s)
            </Button>
          </DialogFooter>
        </DialogContent>
      </Dialog>
    </>
  );
};

export type QueueItemRowData = {
  id: string;
  sourceId: string;
  status: AnnotationQueueStatus;
  completedAt: string;
  annotatorUser: {
    userId?: string;
    userName?: string;
    image?: string;
  };
} & (
  | {
      objectType: "OBSERVATION";
      source: {
        traceId: string;
        observationId: string;
      };
    }
  | {
      objectType: "TRACE";
      source: {
        traceId: string;
      };
    }
  | {
      objectType: "SESSION";
      source: {
        sessionId: string;
      };
    }
);

export function AnnotationQueueItemsTable({
  projectId,
  queueId,
}: {
  projectId: string;
  queueId: string;
}) {
  const { isV4 } = useReadPath();
  const [paginationState, setPaginationState] = useQueryParams({
    pageIndex: withDefault(NumberParam, 0),
    pageSize: withDefault(NumberParam, 50),
  });
  const [selectedRows, setSelectedRows] = useState<RowSelectionState>({});

  const [rowHeight, setRowHeight, rowHeights] = useRowHeightLocalStorage(
    "queueItems",
    "s",
  );
  const items = api.annotationQueueItems.itemsByQueueId.useQuery({
    projectId,
    queueId,
    page: paginationState.pageIndex,
    limit: paginationState.pageSize,
  });
  const selectedItems =
    items.data?.queueItems.filter((item) => selectedRows[item.id]) ?? [];

  const columns: LangfuseColumnDef<QueueItemRowData>[] = [
    {
      id: "select",
      accessorKey: "select",
      size: 30,
      isPinnedLeft: true,
      isFixedPosition: true,
      header: ({ table }) => {
        return (
          <div className="flex h-full items-center">
            <Checkbox
              checked={
                table.getIsAllPageRowsSelected()
                  ? true
                  : table.getIsSomePageRowsSelected()
                    ? "indeterminate"
                    : false
              }
              onCheckedChange={(value) => {
                table.toggleAllPageRowsSelected(!!value);
                if (!value) {
                  setSelectedRows({});
                }
              }}
              aria-label="Select all"
              variant="muted"
            />
          </div>
        );
      },
      cell: ({ row }) => {
        return (
          <span className="mt-1 inline-block has-data-[state=checked]:mt-[5px]">
            <Checkbox
              checked={row.getIsSelected()}
              onCheckedChange={(value) => row.toggleSelected(!!value)}
              aria-label="Select row"
              variant="muted"
            />
          </span>
        );
      },
    },
    createLinkTableColumn<QueueItemRowData>({
      accessorKey: "id",
      header: "Id",
      size: 70,
      isFixedPosition: true,
      getCell: (id) => {
        if (id) {
          return {
            type: "link",
            props: {
              path: `/project/${projectId}/annotation-queues/${queueId}/items/${id}?singleItem=true`,
              value: id,
            },
          };
        }

        return undefined;
      },
    }),
    {
      accessorKey: "objectType",
      header: "Type",
      id: "objectType",
      size: 50,
      cell: ({ row }) => {
        const objectType: QueueItemRowData["objectType"] =
          row.getValue("objectType");
        return <span className="capitalize">{objectType.toLowerCase()}</span>;
      },
    },
    createLinkTableColumn<QueueItemRowData, QueueItemRowData["source"]>({
      accessorKey: "source",
      header: "Source",
      headerTooltip: {
        description:
          "Link to the source trace, observation or session based on which this item was added",
      },
      size: 50,
      getCell: (_, { row }) => {
        const rowData = row.original;
        if (!rowData.source) return undefined;

        if (rowData.objectType === "OBSERVATION") {
          return {
            type: "link",
            props: {
              path: `/project/${projectId}/traces/${rowData.source.traceId}?observation=${rowData.source.observationId}`,
              value: `Observation: ${rowData.source.observationId}`,
              icon: ListTree,
            },
          };
        }

        if (rowData.objectType === "TRACE") {
          return {
            type: "link",
            props: {
              path: `/project/${projectId}/traces/${rowData.source.traceId}`,
              value: `Trace: ${rowData.source.traceId}`,
              icon: ListTree,
            },
          };
        }

        return {
          type: "link",
          props: {
            path: `/project/${projectId}/sessions/${rowData.source.sessionId}`,
            value: `Session: ${rowData.source.sessionId}`,
            icon: ListTree,
          },
        };
      },
    }),
    createIdTableColumn<QueueItemRowData>({
      accessorKey: "sourceId",
      header: "Source ID",
      size: 50,
      enableHiding: true,
      defaultHidden: true,
    }),
    createStatusTableColumn<QueueItemRowData, AnnotationQueueStatus>({
      accessorKey: "status",
      header: "Status",
      getStatus: (status) =>
        status
          ? (
              {
                PENDING: "pending",
                COMPLETED: "completed",
              } satisfies Record<AnnotationQueueStatus, Status>
            )[status]
          : undefined,
      size: 60,
      isLive: false,
    }),
    {
      accessorKey: "completedAt",
      header: "Completed At",
      id: "completedAt",
      defaultHidden: true,
      enableHiding: true,
      size: 60,
    },
    createUserTableColumn<QueueItemRowData, QueueItemRowData["annotatorUser"]>({
      accessorKey: "annotatorUser",
      header: "Completed by",
      enableHiding: true,
      size: 80,
      variant: "avatar",
      emptyValue: "",
      getUser: (annotatorUser) => {
        if (!annotatorUser || !annotatorUser.userId) return undefined;

        const { userId, userName, image } = annotatorUser;
        return {
          type: "user",
          user: { id: userId, name: userName, image },
        };
      },
    }),
  ];

  const convertToTableRow = (
    item: RouterOutput["annotationQueueItems"]["itemsByQueueId"]["queueItems"][number],
  ): QueueItemRowData => {
    const baseData = {
      id: item.id,
      completedAt: item.completedAt?.toLocaleString() ?? "",
      status: item.status,
      annotatorUser: {
        userId: item.annotatorUserId ?? undefined,
        userName: item.annotatorUserName ?? undefined,
        image: item.annotatorUserImage ?? undefined,
      },
      sourceId: item.objectId,
    };

    switch (item.objectType) {
      case "OBSERVATION":
        return {
          ...baseData,
          objectType: "OBSERVATION" as const,
          source: {
            traceId: item.parentTraceId ?? "",
            observationId: item.objectId,
          },
        };
      case "TRACE":
        return {
          ...baseData,
          objectType: "TRACE" as const,
          source: {
            traceId: item.objectId,
          },
        };
      case "SESSION":
        return {
          ...baseData,
          objectType: "SESSION" as const,
          source: {
            sessionId: item.objectId,
          },
        };
      default:
        throw new Error(`Unknown object type: ${item.objectType}`);
    }
  };

  const [columnVisibility, setColumnVisibility] =
    useColumnVisibility<QueueItemRowData>(
      `queueItemsColumnVisibility-${projectId}`,
      columns,
    );

  const [columnOrder, setColumnOrder] = useColumnOrder<QueueItemRowData>(
    "queueItemsColumnOrder",
    columns,
  );

  return (
    <>
      <DataTableToolbar
        tableName="annotation-queue-items"
        columns={columns}
        columnVisibility={columnVisibility}
        setColumnVisibility={setColumnVisibility}
        columnOrder={columnOrder}
        setColumnOrder={setColumnOrder}
        rowHeight={rowHeight}
        setRowHeight={setRowHeight}
        customRowHeight={customRowHeightMenu(rowHeights)}
        actionButtons={[
          selectedItems.length > 0 ? (
            // eslint-disable-next-line react/jsx-key
            <QueueItemTableMultiSelectAction
              selectedItems={selectedItems}
              projectId={projectId}
              isV4={isV4}
              onDeleteSuccess={() => {
                setSelectedRows({});
              }}
            />
          ) : null,
        ]}
      />
      <DataTable
        tableName="annotationQueueItems"
        columns={columns}
        data={
          items.isLoading
            ? { isLoading: true, isError: false }
            : items.isError
              ? {
                  isLoading: false,
                  isError: true,
                  error: items.error.message,
                }
              : {
                  isLoading: false,
                  isError: false,
                  data: safeExtract(items.data, "queueItems", []).map((item) =>
                    convertToTableRow(item),
                  ),
                }
        }
        help={{
          description:
            "Add traces and/or observations to your annotation queue to have them annotated by your team across predefined dimensions.",
          href: "https://langfuse.com/docs/evaluation/evaluation-methods/annotation-queues",
        }}
        pagination={{
          totalCount: items.data?.totalItems ?? null,
          onChange: setPaginationState,
          state: paginationState,
        }}
        rowSelection={selectedRows}
        setRowSelection={setSelectedRows}
        columnVisibility={columnVisibility}
        onColumnVisibilityChange={setColumnVisibility}
        columnOrder={columnOrder}
        onColumnOrderChange={setColumnOrder}
        rowHeight={rowHeight}
        customRowHeightPx={rowHeights.activeHeightPx}
        onCustomRowHeightChange={rowHeights.setCustomPx}
        onSelectRowHeight={setRowHeight}
      />
    </>
  );
}