Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

11.7 KB
Raw Blame
import { useEffect, useState, type ReactNode } from "react";
import { type FilterState } from "@langfuse/shared";
import { DialogController } from "@/src/components/design-system/DialogController/DialogController";
import { env } from "@/src/env.mjs";
import { type JudgeModel } from "@/src/features/evals/v2/judgeModel";
import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
import { useHasProjectAccess } from "@/src/features/rbac/utils/checkProjectAccess";
import { api } from "@/src/utils/api";
import { type ConfigureTopicsSettingsInput } from "./TopicModelSettings";
import {
  TopicSetupFlowDialog,
  type TopicSetupPastCount,
} from "./TopicSetupFlowDialog";
import {
  TOPIC_SETUP_BACKFILL_DAYS,
  TOPIC_SETUP_BACKFILL_LIMIT,
  followsTopicRecommendations,
  initialTopicSetupDraft,
  resolveTopicModelSlot,
  topicConnectionCaption,
  topicProviderGroups,
  type TopicSetupConnection,
  type TopicSetupDraft,
  type TopicSetupQuestion,
} from "./topic-setup-flow";

export function TopicSetupFlowController({
  canWrite,
  children,
  onSaveSampling,
  projectId,
  settings,
}: {
  projectId: string;
  canWrite: boolean;
  settings: ConfigureTopicsSettingsInput | null;
  onSaveSampling: (input: {
    enabled: boolean;
    sampling: number;
  }) => Promise<unknown>;
  children: (control: { openSetup: () => void }) => ReactNode;
}) {
  // Opens only from Start analyzing, never on its own, so the landing shows first.
  const [open, setOpen] = useState(false);
  function openSetup() {
    setOpen(true);
  }
  function closeSetup() {
    setOpen(false);
  }
  return (
    <>
      {children({ openSetup })}
      {open && settings ? (
        <DialogController<null>
          initialState={() => null}
          onDismiss={closeSetup}
          renderDialog={({ closeDialog }) => (
            <TopicSetupSession
              // Rebuilds the draft with recommended models once the first connection arrives.
              key={settings.connections.length > 0 ? "ready" : "none"}
              projectId={projectId}
              canWrite={canWrite}
              settings={settings}
              onSaveSampling={onSaveSampling}
              closeDialog={closeDialog}
            />
          )}
        >
          {() => null}
        </DialogController>
      ) : null}
    </>
  );
}

function TopicSetupSession({
  canWrite,
  closeDialog,
  onSaveSampling,
  projectId,
  settings,
}: {
  projectId: string;
  canWrite: boolean;
  settings: ConfigureTopicsSettingsInput;
  onSaveSampling: (input: {
    enabled: boolean;
    sampling: number;
  }) => Promise<unknown>;
  closeDialog: () => void;
}) {
  const capture = usePostHogClientCapture();
  const utils = api.useUtils();
  const canAddConnection = useHasProjectAccess({
    projectId,
    scope: "llmApiKeys:create",
  });
  // A connection added in another tab shows up when this tab regains focus.
  useEffect(() => {
    const refetch = () => {
      if (document.visibilityState === "visible")
        utils.llmApiKey.effective.invalidate({ projectId });
    };
    window.addEventListener("focus", refetch);
    document.addEventListener("visibilitychange", refetch);
    return () => {
      window.removeEventListener("focus", refetch);
      document.removeEventListener("visibilitychange", refetch);
    };
  }, [utils, projectId]);
  const save = api.topics.saveSetup.useMutation({
    onSuccess: () => {
      utils.topics.modelSettings.invalidate({ projectId });
      utils.topics.facets.invalidate({ projectId });
      utils.topics.rules.invalidate({ projectId });
      utils.topics.tracingReadinessSettings.invalidate({ projectId });
    },
  });
  const startSetup = api.topics.startSetup.useMutation();
  const connectionSettingsUrl = `${window.location.origin}${env.NEXT_PUBLIC_BASE_PATH ?? ""}/project/${projectId}/settings/llm-connections`;
  const connections = settings.connections.map(
    (connection): TopicSetupConnection => ({
      id: connection.id,
      provider: connection.provider,
      adapter: connection.adapter,
      customModels: connection.customModels,
      withDefaultModels: connection.withDefaultModels,
    }),
  );
  const [draft, setDraft] = useState(() =>
    initialTopicSetupDraft({
      stored: settings.stored,
      connections,
      facets: settings.facets,
      enabledFacetIds: settings.rule ? new Set(settings.rule.facetIds) : null,
      traceIdleMs: settings.traceIdleMs,
    }),
  );
  const [error, setError] = useState<string | null>(null);
  // One request per session, so a repeated Start resolves to the same backfill.
  const [requestId] = useState(() => crypto.randomUUID());
  const [pending, setPending] = useState(false);
  useEffect(() => {
    capture("topics:setup_opened", { source: "start_analyzing" });
  }, [capture]);
  // Counted once per step 2 visit; the slider only rescales the result.
  const counts = api.topics.setupTraceCounts.useQuery(
    { projectId },
    { enabled: draft.step !== "questions", retry: false },
  );
  const pastCount = pastTraceCount(
    counts.data,
    draft.samplePercent,
    counts.error?.message ?? null,
    counts.isFetching,
  );

  async function saveQuestions() {
    const summary = resolveTopicModelSlot(draft.summary, connections, "chat");
    const embedding = resolveTopicModelSlot(
      draft.embedding,
      connections,
      "embedding",
      settings.stored.embedding,
    );
    const clustering = resolveTopicModelSlot(
      draft.clustering,
      connections,
      "chat",
    );
    if (!summary || !embedding || !clustering)
      throw new Error("Choose a model from a connection in this project.");
    const filter: FilterState = [];
    await save.mutateAsync({
      projectId,
      summary,
      embedding,
      embeddingDimensions: settings.stored.embeddingDimensions,
      clustering,
      facets: draft.questions.map((question) => ({
        name: question.name,
        enabled: question.enabled,
      })),
      customFacetIds: settings.facets.flatMap((facet) =>
        !facet.isBuiltIn && settings.rule?.facetIds.includes(facet.id)
          ? [facet.id]
          : [],
      ),
      filter,
      traceIdleMs: draft.waitMinutes * 60_000,
      ...(settings.rule ? { ruleId: settings.rule.id } : {}),
    });
    capture("topics:setup_step_saved", {
      step: "questions",
      questionCount: draft.questions.filter((question) => question.enabled)
        .length,
      followsRecommendation: followsTopicRecommendations(draft),
      waitMinutes: draft.waitMinutes,
    });
  }

  async function persistSampling(enabled: boolean) {
    await onSaveSampling({
      enabled,
      sampling: draft.samplePercent / 100,
    });
  }

  async function startAnalyzing() {
    await persistSampling(true);
    const facets = await utils.topics.facets.fetch({ projectId });
    const enabled = new Set<string>(
      draft.questions
        .filter((question) => question.enabled)
        .map((question) => question.name),
    );
    const refs = facets.flatMap((facet) => {
      const version = facet.versions[0]?.version;
      if (!facet.isBuiltIn || !enabled.has(facet.name) || !version) return [];
      return [{ facetId: facet.id, version }];
    });
    if (refs.length === 0)
      throw new Error("Save at least one question before starting.");
    const { traceCount: backfillCount } = await startSetup.mutateAsync({
      projectId,
      requestId,
      sampling: draft.samplePercent / 100,
      facets: refs,
    });
    await utils.topics.executions.invalidate({ projectId });
    capture("topics:setup_started", {
      backfillCount,
      samplePercent: draft.samplePercent,
    });
    closeDialog();
  }

  async function handleNext() {
    if (pending) return;
    setPending(true);
    setError(null);
    try {
      if (draft.step === "questions") {
        await saveQuestions();
        setDraft((current) => ({ ...current, step: "traces" }));
        return;
      }
      if (draft.step === "traces") {
        await persistSampling(false);
        capture("topics:setup_step_saved", {
          step: "traces",
          samplePercent: draft.samplePercent,
        });
        setDraft((current) => ({ ...current, step: "review" }));
        return;
      }
      await startAnalyzing();
    } catch (cause) {
      setError(
        cause instanceof Error ? cause.message : "Could not save Topics.",
      );
    } finally {
      setPending(false);
    }
  }

  return (
    <TopicSetupFlowDialog
      step={draft.step}
      busy={pending}
      canWrite={canWrite}
      error={error}
      questions={draft.questions}
      summary={draft.summary}
      embedding={draft.embedding}
      clustering={draft.clustering}
      providerGroups={topicProviderGroups(
        connections,
        "chat",
        settings.stored.summary?.model,
      )}
      embeddingProviderGroups={topicProviderGroups(
        connections,
        "embedding",
        settings.stored.embedding?.model,
      )}
      connectionCaption={topicConnectionCaption(connections)}
      hasConnection={connections.length > 0}
      canAddConnection={canAddConnection}
      advancedOpen={draft.advancedOpen}
      waitMinutes={draft.waitMinutes}
      samplePercent={draft.samplePercent}
      pastCount={pastCount}
      projectId={projectId}
      onQuestionEnabled={(name, enabled) =>
        setDraft((current) => ({
          ...current,
          questions: setQuestionEnabled(current.questions, name, enabled),
        }))
      }
      onSummaryChange={(model) => updateModel(setDraft, "summary", model)}
      onEmbeddingChange={(model) => updateModel(setDraft, "embedding", model)}
      onClusteringChange={(model) => updateModel(setDraft, "clustering", model)}
      onConfigureProviders={() => {
        window.open(connectionSettingsUrl, "_blank", "noopener,noreferrer");
      }}
      connectionSettingsUrl={connectionSettingsUrl}
      onAdvancedOpenChange={(advancedOpen) =>
        setDraft((current) => ({ ...current, advancedOpen }))
      }
      onWaitMinutesChange={(waitMinutes) =>
        setDraft((current) => ({ ...current, waitMinutes }))
      }
      onSamplePercentChange={(samplePercent) =>
        setDraft((current) => ({ ...current, samplePercent }))
      }
      onEdit={(step) => setDraft((current) => ({ ...current, step }))}
      onBack={() =>
        setDraft((current) => ({
          ...current,
          step: current.step === "review" ? "traces" : "questions",
        }))
      }
      onNext={() => {
        handleNext().catch(() => undefined);
      }}
    />
  );
}

function pastTraceCount(
  counts: { finished1d: number; finished7d: number } | undefined,
  samplePercent: number,
  error: string | null,
  fetching: boolean,
): TopicSetupPastCount {
  if (counts) {
    // Mirrors the server: 1 day when it fills the batch, else 7 days.
    const rate = samplePercent / 100;
    const window =
      counts.finished1d * rate >= TOPIC_SETUP_BACKFILL_LIMIT
        ? { days: 1, count: counts.finished1d }
        : { days: TOPIC_SETUP_BACKFILL_DAYS, count: counts.finished7d };
    return {
      status: "ready",
      days: window.days,
      matched: Math.min(
        TOPIC_SETUP_BACKFILL_LIMIT,
        Math.round(window.count * rate),
      ),
      dailyTraces: counts.finished7d / TOPIC_SETUP_BACKFILL_DAYS,
    };
  }
  if (error && !fetching) return { status: "error", message: error };
  return { status: "loading" };
}

function setQuestionEnabled(
  questions: TopicSetupQuestion[],
  name: TopicSetupQuestion["name"],
  enabled: boolean,
) {
  return questions.map((question) =>
    question.name === name ? { ...question, enabled } : question,
  );
}

function updateModel(
  setDraft: (update: (current: TopicSetupDraft) => TopicSetupDraft) => void,
  slot: "summary" | "embedding" | "clustering",
  model: JudgeModel,
) {
  setDraft((current) => ({ ...current, [slot]: model }));
}