Skip to content

Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.

25.4 KB
Raw Blame
import { memo, useCallback, useEffect, useRef, useState, type CSSProperties } from "react";
import { useTranslation } from "react-i18next";
import { Check, Copy, Plus, X } from "../icons";
import { BrandMarkIcon } from "./BrandMarkIcon";
import { HelpEvidence } from "../chat/GroundedHelpEvidence";
import { extractHelpEvidence } from "../chat/helpEvidence";
import { helpAnswerPlainText } from "../chat/helpEvidenceText";
import { MarkdownRenderer } from "../ui/MarkdownRenderer";
import { TechnicalErrorDetails } from "../ui/TechnicalErrorDetails";
import { openProviderSettings } from "../../utils/describeProviderError";
import { Button } from "../ui/button";
import { useChatPersistence } from "../chat/useChatPersistence";
import { useChatStreaming } from "../chat/useChatStreaming";
import { useChatMessageSender } from "../chat/useChatMessageSender";
import { ChatInput } from "../chat/ChatInput";
import { useWindowDrag } from "../../hooks/useWindowDrag";
import { useSettingsStore } from "../../stores/settingsStore";
import { formatHotkeyListLabel } from "../../utils/hotkeys";
import {
  ASSISTANT_FOOTER_TRANSITION_TIMING,
  resolveAssistantFooterPresentation,
  resolveAssistantResponseReady,
} from "../../helpers/voicePillPresentation";
import {
  AGENT_TOOL_ACTIVITY_ROTATION_MS,
  AGENT_TOOL_ACTIVITY_VERB_KEYS,
  AGENT_TOOL_NAME_FALLBACK_KEY,
} from "../../helpers/agentToolPresentation";
import { useCopyFeedback } from "../../hooks/useCopyFeedback";
import type { AgentState, ChatImageAttachment, Message } from "../chat/types";
import {
  normalizeAgentSelectionContext,
  type AgentSelectionContext,
} from "../../utils/agentSelectionContext";
import {
  getSelectionForCopyShortcut,
  getSelectionInside,
  isEditableTarget,
} from "../../utils/assistantSelection";
import { AssistantEmptyState } from "./AssistantEmptyState";
import { useToast } from "../ui/useToast";
import {
  resolveAssistantPanelBusy,
  restoreAssistantConversation,
} from "../../helpers/assistantSessionState";
import {
  deliverAssistantResponse,
  type AssistantResponseDelivery,
} from "../../helpers/assistantResponseDelivery";
import { buildAssistantCommandSendOptions } from "./assistantCommandOptions";
import { ApprovalCard } from "../chat/ApprovalCard";
import { approvalKey, useConnectorApprovalStore } from "../../stores/connectorApprovalStore";

export interface AssistantCommand {
  id: number;
  text: string;
  attachment: ChatImageAttachment | null;
  selectedContext: AgentSelectionContext | null;
  delivery: AssistantResponseDelivery | null;
}

type AssistantFooterPhase =
  "pill" | "pill-entering" | "pill-exiting" | "actions-entering" | "actions" | "actions-exiting";

const MANUAL_COPY_FEEDBACK_MS = 1800;
const AUTO_COPY_FEEDBACK_MS = 6000;

interface AssistantPanelProps {
  /** Voice command waiting to be sent into the conversation (consumed on mount and on change). */
  pendingCommand: AssistantCommand | null;
  onCommandConsumed: (id: number) => void;
  onCommandDiscarded: (id: number) => void;
  onCommandSettled: (id: number, options?: { showPanel?: boolean }) => void;
  /** Conversation to resume when reopening the panel; null starts fresh on first message. */
  initialConversationId: number | null;
  onConversationIdChange: (id: number | null) => void;
  /** Live voice state while the user records a follow-up with the panel open. */
  voiceState: Extract<AgentState, "idle" | "listening" | "transcribing">;
  /** Keeps follow-up thinking feedback inside an already-open panel. */
  thinking: boolean;
  open: boolean;
  footerPhase: AssistantFooterPhase;
  horizontalDirection: "left" | "right";
  onClose: () => void;
  onBusyChange: (busy: boolean) => void;
  onResponseReadyChange: (ready: boolean) => void;
  onResponseContent: () => void;
  onConversationReset: () => void;
  onSelectionContextChange: (context: AgentSelectionContext | null) => void;
}

// Avoid reparsing Markdown when only the selection indicator changes.
const StableAssistantMarkdown = memo(MarkdownRenderer);

export function AssistantPanel({
  pendingCommand,
  onCommandConsumed,
  onCommandDiscarded,
  onCommandSettled,
  initialConversationId,
  onConversationIdChange,
  voiceState,
  thinking,
  open,
  footerPhase,
  horizontalDirection,
  onClose,
  onBusyChange,
  onResponseReadyChange,
  onResponseContent,
  onConversationReset,
  onSelectionContextChange,
}: AssistantPanelProps) {
  const { t, i18n } = useTranslation();
  const { toast } = useToast();
  const { handleMouseDown, handleMouseUp } = useWindowDrag();
  const voiceAgentKey = useSettingsStore((state) => state.voiceAgentKey);
  const readableVoiceHotkey = formatHotkeyListLabel(voiceAgentKey);

  const persistence = useChatPersistence({ conversationId: initialConversationId });
  const { messages, setMessages } = persistence;

  const streaming = useChatStreaming({
    messages,
    setMessages,
    // Spoken commands answer on the Voice Assistant scope, not the Chat one.
    inferenceScope: "dictationAgent",
    allowConnectors: true,
    onStreamComplete: (_assistantId, content, toolCalls) => {
      void persistence.saveAssistantMessage(content, toolCalls);
    },
    onResponseContent,
  });

  const createConversation = useCallback(
    (text: string) => {
      const title = text.slice(0, 50) + (text.length > 50 ? "..." : "");
      return persistence.createConversation(title);
    },
    [persistence]
  );

  const [submissionInFlight, setSubmissionInFlight] = useState(false);
  const sendMessage = useChatMessageSender({
    conversationId: persistence.conversationId,
    persistence,
    streaming,
    createConversation,
    onSendingChange: setSubmissionInFlight,
  });
  const latestAssistantMessage = [...messages]
    .reverse()
    .find((message) => message.role === "assistant");
  const responseContent = latestAssistantMessage?.content ?? "";
  // A follow-up keeps the previous answer, and its help sources, on screen
  // until the new answer has content.
  const displayedMessageRef = useRef<Message | undefined>(undefined);
  if (responseContent) displayedMessageRef.current = latestAssistantMessage;
  const displayedMessage = displayedMessageRef.current;
  const displayedResponse = displayedMessage?.content ?? "";
  const helpEvidence = extractHelpEvidence(displayedMessage?.toolCalls);
  // Keyed by the answer itself, which is what auto-copy confirms; explicit
  // Copy adds the help sources to what it writes.
  const { copied, copyText, confirmCopied } = useCopyFeedback(responseContent, {
    resetMs: MANUAL_COPY_FEEDBACK_MS,
  });
  const handleCopy = useCallback(async () => {
    const responseCopyText = helpAnswerPlainText(
      displayedResponse,
      helpEvidence,
      t,
      i18n?.resolvedLanguage
    );
    if (await copyText(responseCopyText.trim())) confirmCopied(responseContent);
  }, [
    copyText,
    confirmCopied,
    displayedResponse,
    helpEvidence,
    t,
    i18n?.resolvedLanguage,
    responseContent,
  ]);

  useEffect(() => {
    onConversationIdChange(persistence.conversationId);
  }, [persistence.conversationId, onConversationIdChange]);

  // Reopening the panel resumes the previous conversation. Sends must wait
  // for the load: a command sent against the still-empty message list would
  // reach the model without the conversation's history, and the load's
  // setMessages would then wipe the just-sent turn from the UI.
  const [historyReady, setHistoryReady] = useState(initialConversationId == null);
  useEffect(() => {
    if (initialConversationId == null) return undefined;

    let active = true;
    void restoreAssistantConversation({
      conversationId: initialConversationId,
      loadConversation: persistence.loadConversation,
      onReady: () => setHistoryReady(true),
      onReset: persistence.handleNewChat,
      onError: (error: unknown) => {
        console.error("Failed to restore Assistant conversation", error);
        toast({ title: t("common.error"), variant: "destructive" });
      },
      isActive: () => active,
    });
    return () => {
      active = false;
    };
    // eslint-disable-next-line react-hooks/exhaustive-deps
  }, []);

  // Voice commands arrive as pending commands: send each exactly once.
  const consumedCommandIdRef = useRef<number | null>(null);
  useEffect(() => {
    if (
      !historyReady ||
      submissionInFlight ||
      !pendingCommand ||
      consumedCommandIdRef.current === pendingCommand.id
    ) {
      return;
    }
    consumedCommandIdRef.current = pendingCommand.id;
    const commandId = pendingCommand.id;
    const { options: sendOptions, wasDelivered } = buildAssistantCommandSendOptions(
      pendingCommand,
      {
        onResponseContent,
        deliver: deliverAssistantResponse,
        confirmCopied: (content) => confirmCopied(content, AUTO_COPY_FEEDBACK_MS),
      }
    );
    if (pendingCommand.selectedContext) {
      setSelectedContext(null);
      onSelectionContextChange(null);
    }
    void sendMessage(pendingCommand.text, sendOptions)
      .then((sent) => {
        if (sent) {
          onCommandConsumed(commandId);
          return;
        }
        // Another submission held the lock; let the effect retry once
        // submissionInFlight flips back to false.
        consumedCommandIdRef.current = null;
      })
      .catch((error: unknown) => {
        const errorMessage =
          error instanceof Error && error.message
            ? error.message
            : error == null
              ? t("common.unknownError")
              : String(error);
        onCommandDiscarded(commandId);
        setMessages((prev) => [
          ...prev,
          {
            id: crypto.randomUUID(),
            role: "assistant",
            content: `${t("agentMode.chat.errorPrefix")}: ${errorMessage}`,
            isStreaming: false,
          },
        ]);
      })
      .finally(() => onCommandSettled(commandId, { showPanel: !wasDelivered() }));
  }, [
    historyReady,
    submissionInFlight,
    pendingCommand,
    onCommandConsumed,
    onCommandDiscarded,
    onCommandSettled,
    onSelectionContextChange,
    onResponseContent,
    confirmCopied,
    sendMessage,
    setMessages,
    t,
  ]);

  const approvalEntries = useConnectorApprovalStore((state) => state.entries);
  const panelApprovals = latestAssistantMessage
    ? (latestAssistantMessage.toolCalls ?? [])
        .map((toolCall) => approvalEntries[approvalKey(latestAssistantMessage.id, toolCall.id)])
        .filter((entry): entry is NonNullable<typeof entry> => Boolean(entry))
    : [];
  const approvalAwaitingUser = panelApprovals.some(
    (entry) => entry.state === "pending" || entry.state === "committing"
  );
  const isToolExecuting = Boolean(streaming.activeToolName) && !approvalAwaitingUser;
  const isBusy = resolveAssistantPanelBusy({
    agentState: streaming.agentState,
    activeToolName: streaming.activeToolName,
    submissionInFlight,
  });
  const showContentFlourish = thinking || isToolExecuting;
  const [toolVerbIndex, setToolVerbIndex] = useState(0);
  const footerPresentation = resolveAssistantFooterPresentation(footerPhase);

  useEffect(() => {
    setToolVerbIndex(0);
    if (!isToolExecuting) return;

    const timer = window.setInterval(() => {
      setToolVerbIndex((current) => (current + 1) % AGENT_TOOL_ACTIVITY_VERB_KEYS.length);
    }, AGENT_TOOL_ACTIVITY_ROTATION_MS);
    return () => window.clearInterval(timer);
  }, [isToolExecuting, streaming.activeToolName]);

  useEffect(() => {
    onBusyChange(isBusy);
    return () => onBusyChange(false);
  }, [isBusy, onBusyChange]);

  // Keep the previous response ineligible throughout a follow-up request. Audio
  // processing can return voiceState to idle one render before the chat stream
  // reports busy; without this latch, the old response briefly restores the
  // Close/Copy actions and reverses the footer animation.
  const isResponseReady = resolveAssistantResponseReady({
    responseContent,
    isBusy,
    isStreaming: Boolean(latestAssistantMessage?.isStreaming),
    voiceState,
    requestPending: thinking || pendingCommand != null,
  });
  const responseSelectionRootRef = useRef<HTMLDivElement | null>(null);
  const [selectedContext, setSelectedContext] = useState<AgentSelectionContext | null>(null);

  const clearSelectedContext = useCallback(() => {
    setSelectedContext(null);
    onSelectionContextChange(null);
    window.getSelection()?.removeAllRanges();
  }, [onSelectionContextChange]);

  const handleNewConversation = useCallback(() => {
    if (isBusy || submissionInFlight) return;
    persistence.handleNewChat();
    onConversationIdChange(null);
    clearSelectedContext();
    displayedMessageRef.current = undefined;
    onConversationReset();
  }, [
    clearSelectedContext,
    isBusy,
    onConversationIdChange,
    onConversationReset,
    persistence,
    submissionInFlight,
  ]);

  const handleTextSubmit = useCallback(
    (text: string): void => {
      if (!historyReady) return;
      const context = selectedContext;
      if (context) clearSelectedContext();
      void sendMessage(text, context ? { selectedContext: context } : undefined);
    },
    [clearSelectedContext, historyReady, selectedContext, sendMessage]
  );

  const showEmptyState =
    historyReady &&
    messages.length === 0 &&
    streaming.agentState === "idle" &&
    !submissionInFlight &&
    !thinking &&
    pendingCommand === null;

  useEffect(() => {
    if (!open && selectedContext) clearSelectedContext();
  }, [clearSelectedContext, open, selectedContext]);

  useEffect(() => {
    if (!open || !isResponseReady || !latestAssistantMessage) return undefined;

    const captureSelection = () => {
      const selectedText = getSelectionInside(responseSelectionRootRef.current);
      if (!selectedText) return;

      const context = normalizeAgentSelectionContext({
        text: selectedText,
        sourceMessageId: latestAssistantMessage.id,
      });
      if (!context) return;

      setSelectedContext(context);
      onSelectionContextChange(context);
    };

    document.addEventListener("selectionchange", captureSelection);
    return () => document.removeEventListener("selectionchange", captureSelection);
  }, [isResponseReady, latestAssistantMessage, onSelectionContextChange, open]);

  useEffect(
    () => () => {
      onSelectionContextChange(null);
    },
    [onSelectionContextChange]
  );

  useEffect(() => {
    onResponseReadyChange(isResponseReady);
  }, [isResponseReady, onResponseReadyChange]);

  // Esc cancels a running response or closes the panel. The screenshot's
  // compact `C` hint is also a keyboard shortcut once a response is final.
  useEffect(() => {
    const handleKeyDown = (e: KeyboardEvent) => {
      if (e.key === "Escape") {
        if (voiceState === "listening") return;
        if (isBusy) {
          streaming.cancelStream();
          // A hidden panel means the compact Beam circle owns the thinking
          // state; cancelling it must also dismiss that otherwise-idle shell.
          if (!open) onClose();
        } else onClose();
        return;
      }

      if (isResponseReady) {
        const selectedText = getSelectionForCopyShortcut(e, responseSelectionRootRef.current);
        if (selectedText) {
          e.preventDefault();
          void copyText(selectedText);
          return;
        }
      }
      const isEditable = isEditableTarget(e.target);
      if (
        isResponseReady &&
        footerPhase === "actions" &&
        e.key.toLowerCase() === "c" &&
        !e.metaKey &&
        !e.ctrlKey &&
        !e.altKey &&
        !isEditable
      ) {
        e.preventDefault();
        void handleCopy();
      }
    };
    document.addEventListener("keydown", handleKeyDown);
    return () => document.removeEventListener("keydown", handleKeyDown);
  }, [
    voiceState,
    isBusy,
    streaming,
    open,
    onClose,
    isResponseReady,
    footerPhase,
    handleCopy,
    copyText,
  ]);

  return (
    <>
      <header
        className="flex h-16 shrink-0 cursor-grab items-center gap-3 px-5 active:cursor-grabbing"
        onMouseDown={handleMouseDown}
        onMouseUp={handleMouseUp}
      >
        <BrandMarkIcon size={20} className="shrink-0 text-muted-foreground" />
        {messages.length > 0 && (
          <button
            type="button"
            className="inline-flex size-7 shrink-0 items-center justify-center rounded-full text-muted-foreground transition-colors hover:bg-foreground/10 hover:text-foreground"
            aria-label={t("assistant.panel.newConversation")}
            onPointerDown={(event) => {
              event.preventDefault();
              event.stopPropagation();
            }}
            onClick={(event) => {
              event.stopPropagation();
              handleNewConversation();
            }}
          >
            <Plus size={15} strokeWidth={2} />
          </button>
        )}

        <div className="flex min-w-0 flex-1 items-center justify-end text-end text-xs text-muted-foreground/70">
          {selectedContext ? (
            <div className="inline-flex min-w-0 max-w-full items-center gap-1.5 rounded-full border border-border/55 bg-foreground/[0.06] py-1 ps-2.5 pe-1.5 text-foreground/75">
              <span className="truncate">{t("assistant.panel.selectedContext")}</span>
              <button
                type="button"
                className="inline-flex size-4 shrink-0 items-center justify-center rounded-full text-muted-foreground transition-colors hover:bg-foreground/10 hover:text-foreground"
                aria-label={t("assistant.panel.removeSelectedContext")}
                onPointerDown={(event) => {
                  event.preventDefault();
                  event.stopPropagation();
                }}
                onClick={(event) => {
                  event.stopPropagation();
                  clearSelectedContext();
                }}
              >
                <X size={11} strokeWidth={2} />
              </button>
            </div>
          ) : (
            <>
              <span className="truncate">{t("assistant.panel.selectionHint")}</span>
              {readableVoiceHotkey && (
                <kbd
                  dir="ltr"
                  className="ms-2 inline-flex rounded-md border border-border/40 bg-foreground/5 px-2 py-1 font-mono text-[11px] tracking-normal text-foreground/65 shadow-sm"
                >
                  {readableVoiceHotkey}
                </kbd>
              )}
            </>
          )}
        </div>
      </header>

      <div className="relative mx-4 flex min-h-0 flex-1 flex-col overflow-hidden rounded-2xl border border-border/40 bg-surface-1 shadow-inner">
        <main
          data-panel-scroll-region
          className="agent-chat-scroll min-h-0 flex-1 overflow-y-auto px-5 py-4"
          aria-busy={!historyReady || thinking || isBusy}
        >
          <div
            data-panel-size-source
            className={`assistant-response-content min-h-full ${
              showContentFlourish ? "assistant-response-content-updating" : ""
            }`}
          >
            {displayedResponse ? (
              <>
                <div
                  ref={responseSelectionRootRef}
                  style={{ animation: "agent-message-in 160ms ease-out both" }}
                >
                  <StableAssistantMarkdown
                    content={displayedResponse}
                    className="text-[15px] leading-relaxed text-foreground selection:bg-agent-brand/35 selection:text-foreground [&_p]:text-[15px] [&_li]:text-[15px]"
                  />
                  {latestAssistantMessage?.isStreaming && (
                    <span
                      className="ms-0.5 inline-block h-4 w-0.5 align-middle bg-foreground/70"
                      style={{ animation: "agent-cursor-blink 1s ease-in-out infinite" }}
                    />
                  )}
                </div>
                {helpEvidence && <HelpEvidence evidence={helpEvidence} />}
                {/* Outside responseSelectionRootRef: a drag-select + copy over the
                    response must never pick up these affordances. */}
                {latestAssistantMessage?.error?.settingsTarget && (
                  <button
                    type="button"
                    onClick={() =>
                      openProviderSettings(latestAssistantMessage.error!.settingsTarget!)
                    }
                    className="mt-2 text-[13px] font-medium text-primary hover:underline focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring/40 rounded-sm"
                  >
                    {t("providerErrors.openSettings")}
                  </button>
                )}
                {latestAssistantMessage?.error?.technicalDetails && (
                  <TechnicalErrorDetails details={latestAssistantMessage.error.technicalDetails} />
                )}
              </>
            ) : null}
            {panelApprovals.map((entry) => (
              <ApprovalCard key={entry.key} entry={entry} />
            ))}
            {thinking && (
              <div role="status">
                <span className="sr-only">{t("agentMode.input.thinking")}</span>
              </div>
            )}
            {showEmptyState && (
              <AssistantEmptyState
                disabled={isBusy || submissionInFlight}
                onSelectSuggestion={handleTextSubmit}
              />
            )}
          </div>
        </main>

        <ChatInput
          agentState={historyReady && !submissionInFlight ? streaming.agentState : "thinking"}
          partialTranscript=""
          onTextSubmit={handleTextSubmit}
          onCancel={
            historyReady && (!submissionInFlight || streaming.agentState !== "idle")
              ? streaming.cancelStream
              : undefined
          }
          autoFocus={historyReady && open && messages.length === 0}
          className="px-3 pb-3"
        />

        {showContentFlourish && (
          <div
            className="assistant-content-area-shimmer pointer-events-none absolute inset-0 z-10 overflow-hidden rounded-2xl"
            aria-hidden="true"
          />
        )}
        {isToolExecuting && (
          <div
            className="assistant-tool-invocation pointer-events-none absolute inset-0 z-20 flex items-center justify-center px-6"
            role="status"
            aria-live="polite"
            aria-atomic="true"
            data-tool-invocation={streaming.activeToolName}
          >
            <span className="sr-only">{streaming.toolStatus}</span>
            <div
              className="assistant-tool-invocation-status inline-flex max-w-full items-center gap-2.5 rounded-xl border border-border/60 bg-surface-raised/60 px-4 py-2.5 shadow-[var(--shadow-card)] backdrop-blur-md"
              aria-hidden="true"
            >
              <span className="assistant-tool-invocation-pulse relative flex size-2 shrink-0 rounded-full bg-agent-brand" />
              <span
                key={`${streaming.activeToolName}-${toolVerbIndex}`}
                className="assistant-tool-invocation-verb w-16 text-end text-sm text-muted-foreground"
              >
                {t(AGENT_TOOL_ACTIVITY_VERB_KEYS[toolVerbIndex])}
              </span>
              <span className="h-4 w-px shrink-0 bg-border/70" />
              <span className="truncate text-sm font-medium text-foreground">
                {t(`agentMode.tools.${streaming.activeToolName}Name`, {
                  defaultValue: t(AGENT_TOOL_NAME_FALLBACK_KEY),
                })}
              </span>
            </div>
          </div>
        )}
      </div>

      <footer
        className={`flex h-16 shrink-0 items-center px-4 ${
          horizontalDirection === "left" ? "justify-start" : "justify-end"
        }`}
        data-assistant-footer-direction={horizontalDirection}
      >
        {footerPresentation.actionsMounted && (
          <div
            className={`assistant-response-actions flex items-center gap-2 ${
              horizontalDirection === "left" ? "flex-row-reverse" : ""
            }`}
            data-assistant-footer-phase={footerPhase}
            data-horizontal-direction={horizontalDirection}
            aria-hidden={footerPhase !== "actions"}
            style={
              {
                "--assistant-actions-retreat-duration": `${ASSISTANT_FOOTER_TRANSITION_TIMING.actionsRetreatMs}ms`,
                "--assistant-actions-entrance-duration": `${ASSISTANT_FOOTER_TRANSITION_TIMING.actionsEntranceMs}ms`,
              } as CSSProperties
            }
          >
            <Button
              type="button"
              variant="secondary"
              size="sm"
              className="px-4"
              onClick={onClose}
              tabIndex={footerPhase === "actions" ? 0 : -1}
            >
              <X aria-hidden="true" />
              {t("common.close")}
            </Button>
            <Button
              type="button"
              size="sm"
              className="bg-surface-raised bg-none px-4 font-medium text-foreground shadow-sm hover:bg-surface-3 dark:bg-white dark:text-neutral-950 dark:hover:bg-white/90"
              onClick={() => void handleCopy()}
              aria-live="polite"
              tabIndex={footerPhase === "actions" ? 0 : -1}
            >
              {copied ? <Check aria-hidden="true" /> : <Copy aria-hidden="true" />}
              {copied ? t("common.copied") : t("assistant.panel.copyToClipboard")}
              {!copied && (
                <kbd
                  dir="ltr"
                  className="ms-1 rounded bg-foreground/10 px-1.5 py-0.5 font-mono text-[10px] font-medium dark:bg-black/10"
                >
                  C
                </kbd>
              )}
            </Button>
          </div>
        )}
      </footer>
    </>
  );
}