Skip to content

Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.

36.7 KB
Raw Blame
import { useState, useRef, useEffect, useCallback } from "react";
import { useTranslation } from "react-i18next";
import ReasoningService, { type AgentStreamChunk } from "../../services/ReasoningService";
import { isEnterpriseProvider } from "../../models/ModelRegistry";
import { providerSupportsImages } from "../../services/ai/inferenceProviders";
import { getSettings, useSettingsStore } from "../../stores/settingsStore";
import { resolveChatStreamingInference } from "../../helpers/dictationAgentInference.js";
import { describeProviderError } from "../../utils/describeProviderError";
import { productHelpMetadata } from "../../services/help/productHelp";
import logger from "../../utils/logger";
import {
  isAgentAllowed,
  isConnectorsAllowed,
  isConnectorsBlockedByOrg,
  isLlmSelectionAllowed,
  isWebSearchAllowed,
  isWebSearchBlockedByOrg,
} from "../../stores/policyRules";
import { usePolicyStore } from "../../stores/policyStore";
import { useAuth } from "../../hooks/useAuth";
import { getUsageState } from "../../lib/usageStore";
import { readIsSubscribed } from "../../lib/subscriptionFlag";
import { hasConnectorPlan } from "../../utils/connectorEligibility";
import { gmailSendStatus, resolveEmailDraftTarget } from "../../utils/emailDraftTarget";
import {
  ensureConnectorStatus,
  readyConnectorIds,
  useConnectorStatusStore,
} from "../../stores/connectorStatusStore";
import {
  appendDictionarySuffix,
  appendPlainTextResponseSuffix,
  appendScreenContextSuffix,
  getAgentSystemPrompt,
} from "../../config/prompts";
import { resolveUnavailableCapabilities } from "../../config/agentCapabilities";
import type { ToneAppCategory } from "../../config/toneStyle";
import { getDictionaryHintWords } from "../../utils/snippets";
import { noteAttendeesContext, withoutAttendeesFence } from "../../utils/noteAttendees";
import { FULL_TRANSCRIPT_MAX_TOKENS } from "../../utils/transcriptEvidence";
import { createToolRegistry } from "../../services/tools";
import {
  executeTool,
  type HoldDeliveryOptions,
  type ToolRegistry,
} from "../../services/tools/ToolRegistry";
import { createToolExecutionScope, type ToolExecutionScope } from "./toolExecutionScope";
import { isQueryResultData } from "../../services/tools/connectors/runQueryAction";
import { getAgentToolActivityRemainingMs } from "../../helpers/agentToolPresentation";
import type { Message, AgentState, ChatImageAttachment, ToolCallInfo } from "./types";
import { toHistoryMessages, withoutEchoedToolTrace, type HistoryMessage } from "./historyMessages";
import type { ContainerScope } from "../../types/chat";
import type { NoteAttendeesRequest } from "../../types/connectors";
import {
  buildAgentRequestText,
  type AgentSelectionContext,
} from "../../utils/agentSelectionContext";

const RAG_NOTE_LIMIT = 5;
const RAG_NOTE_SNIPPET_LENGTH = 500;
const STREAM_FLUSH_INTERVAL_MS = 32;

const LOCAL_TOOL_MIN_PARAMS_B = 4;

function estimateModelSizeB(modelId: string): number {
  const match = modelId.match(/-([\d.]+)[bB]/);
  return match ? parseFloat(match[1]) : 0;
}

// Main adds the note's identified speakers and its calendar event's
// organizer (calendars often leave them out of the attendees), then drops
// the user (their OpenWhispr address included) and rooms with find_contact's
// rules; a failed lookup just leaves the block out.
async function buildNoteAttendeesContext(
  meeting: NoteAttendeesRequest | undefined
): Promise<string> {
  if (!meeting || !window.electronAPI?.connectorNoteAttendees) return "";
  try {
    const result = await window.electronAPI.connectorNoteAttendees(meeting);
    return noteAttendeesContext(result?.attendees ?? []);
  } catch {
    return "";
  }
}

async function buildRAGContext(userText: string, scope?: ContainerScope): Promise<string> {
  if (!window.electronAPI?.semanticSearchNotes) return "";
  try {
    const results = await window.electronAPI.semanticSearchNotes(
      userText,
      RAG_NOTE_LIMIT,
      scope?.spaceId ?? null,
      scope?.folderId ?? null
    );
    if (!results || results.length === 0) return "";

    const snippets = await Promise.all(
      results.map(async (r: { id: number; title: string; score?: number }) => {
        const note = await window.electronAPI.getNote(r.id);
        if (!note) return null;
        const content = (note.content || "").slice(0, RAG_NOTE_SNIPPET_LENGTH);
        return `<note id="${note.id}" title="${note.title}">\n${content}\n</note>`;
      })
    );

    return snippets.filter(Boolean).join("\n\n");
  } catch {
    return "";
  }
}

/**
 * Which settings scope answers a conversation. Typed chat surfaces stay on the
 * Chat scope; the voice assistant panel runs on the Voice Assistant scope so
 * the model picked under Settings > Voice Assistant is the one that answers
 * (see resolveChatStreamingInference for its Chat fallback).
 */
export type ChatStreamingScope = "chatIntelligence" | "dictationAgent";

export interface OpenNoteContext {
  text: string;
  /** Only the start of the transcript is included; get_note reads the rest. */
  transcriptPreview: boolean;
}

interface UseChatStreamingOptions {
  messages: Message[];
  setMessages: React.Dispatch<React.SetStateAction<Message[]>>;
  /** Settings scope the conversation resolves its provider and model from. */
  inferenceScope?: ChatStreamingScope;
  /** Notes listed in the system prompt as possibly relevant (container overview chat). */
  noteContext?: string;
  /**
   * The note a note's chat is about. The model answers from it first, and no
   * other notes are added up front: it reaches them through search_notes.
   */
  openNote?: string | ((maxTranscriptTokens: number) => OpenNoteContext);
  /** Optional container scope applied to RAG and the search_notes tool (container overview chat). */
  searchScope?: ContainerScope;
  /**
   * Offer connector tools (email drafts, contact lookup) when the plan and
   * policy allow them. Off unless a surface opts in: they act outside the app.
   */
  allowConnectors?: boolean;
  /**
   * Name capabilities that exist but aren't usable here, and how to turn them
   * on. Off only where the request already says what to do without them (the
   * onboarding demo answers with suggested times instead).
   */
  nameUnavailableCapabilities?: boolean;
  /** Disable product-help tools on guided surfaces such as the onboarding demo. */
  productHelpEnabled?: boolean;
  /**
   * The note's meeting (note chat). Its attendees are listed for the model
   * only in a send that offers connector tools, so recipients come from them.
   */
  noteMeeting?: NoteAttendeesRequest;
  onStreamComplete?: (assistantId: string, content: string, toolCalls?: ToolCallInfo[]) => void;
  /** Fires exactly once when displayable assistant content or tool activity becomes available. */
  onResponseContent?: () => void;
}

export interface SendToAIOptions {
  /** Screenshot for this message; attached only when the resolved model can see it. */
  attachment?: ChatImageAttachment;
  /** Agent-response selection attached to this request without changing chat history. */
  selectedContext?: AgentSelectionContext;
  /**
   * Sent to the model in place of this turn's visible message, which is what the
   * chat shows and saves (a note action shows its name while its prompt is sent).
   */
  requestText?: string;
  /** This turn isn't offered update_note, so it can't change a note (a note's chat action). */
  keepNotesUnchanged?: boolean;
  /** Keeps a caret-destined voice response in the compact pill while it streams. */
  suppressResponseContent?: boolean;
  /** Asks the model for plain prose because the answer will be pasted into a plain-text app. */
  plainTextResponse?: boolean;
  /** Tone & Style: the kind of app a caret-destined answer will be typed into. */
  destinationAppCategory?: ToneAppCategory;
  /** Per-request completion hook used to deliver a finished voice response. */
  onComplete?: (result: {
    assistantId: string;
    content: string;
    toolCalls?: ToolCallInfo[];
  }) => void | Promise<void>;
  /** Fires when a tool shows an approval card, so a hidden panel can open. */
  onApprovalRequested?: () => void;
  /** Fires when this turn's answer must not be pasted at the caret (see ToolExecutionContext). */
  onHoldDelivery?: (options?: HoldDeliveryOptions) => void;
}

export interface ChatStreaming {
  agentState: AgentState;
  toolStatus: string;
  activeToolName: string;
  sendToAI: (userText: string, allMessages: Message[], options?: SendToAIOptions) => Promise<void>;
  cancelStream: () => void;
}

// Walks backward to the newest user message; a null transform keeps walking.
function transformLastUserMessage(
  history: HistoryMessage[],
  transform: (message: HistoryMessage) => HistoryMessage | null
): void {
  for (let i = history.length - 1; i >= 0; i--) {
    if (history[i].role !== "user") continue;
    const replacement = transform(history[i]);
    if (replacement === null) continue;
    history[i] = replacement;
    break;
  }
}

export function useChatStreaming({
  messages,
  setMessages,
  inferenceScope = "chatIntelligence",
  noteContext: externalNoteContext,
  openNote,
  searchScope,
  allowConnectors = false,
  nameUnavailableCapabilities = true,
  productHelpEnabled = true,
  noteMeeting,
  onStreamComplete,
  onResponseContent,
}: UseChatStreamingOptions): ChatStreaming {
  const { t } = useTranslation();
  const [agentState, setAgentState] = useState<AgentState>("idle");
  const [toolStatus, setToolStatus] = useState("");
  const [activeToolName, setActiveToolName] = useState("");
  const mountedRef = useRef(true);
  const messagesRef = useRef<Message[]>([]);
  const noteContextRef = useRef(externalNoteContext);
  noteContextRef.current = externalNoteContext;
  const openNoteRef = useRef(openNote);
  openNoteRef.current = openNote;
  const searchScopeRef = useRef(searchScope);
  searchScopeRef.current = searchScope;
  const noteMeetingRef = useRef(noteMeeting);
  noteMeetingRef.current = noteMeeting;
  // Only an account bound to this session: after a switch, drafts must not
  // be signed with the previous account's name.
  const { user } = useAuth();
  const userNameRef = useRef<string | null>(null);
  userNameRef.current = user?.name ?? null;
  const toolRegistryRef = useRef<{ key: string; registry: ToolRegistry } | null>(null);
  const toolActivityStartedAtRef = useRef<number | null>(null);
  const toolActivityTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null);

  const clearToolActivityTimer = useCallback(() => {
    if (toolActivityTimerRef.current) {
      clearTimeout(toolActivityTimerRef.current);
      toolActivityTimerRef.current = null;
    }
  }, []);

  const clearToolActivity = useCallback(() => {
    clearToolActivityTimer();
    toolActivityStartedAtRef.current = null;
    if (!mountedRef.current) return;
    setToolStatus("");
    setActiveToolName("");
  }, [clearToolActivityTimer]);

  const beginToolActivity = useCallback(
    (name: string, status: string) => {
      clearToolActivityTimer();
      toolActivityStartedAtRef.current = Date.now();
      setActiveToolName(name);
      setToolStatus(status);
    },
    [clearToolActivityTimer]
  );

  const completeToolActivity = useCallback(() => {
    const startedAt = toolActivityStartedAtRef.current;
    if (startedAt == null) {
      clearToolActivity();
      return;
    }

    const remainingMs = getAgentToolActivityRemainingMs(startedAt);
    clearToolActivityTimer();
    if (remainingMs === 0) {
      clearToolActivity();
      return;
    }

    toolActivityTimerRef.current = setTimeout(clearToolActivity, remainingMs);
  }, [clearToolActivity, clearToolActivityTimer]);

  useEffect(() => {
    messagesRef.current = messages;
  }, [messages]);

  const sendGenerationRef = useRef(0);
  // Generation stamped by the most recent cancel issued while still mounted
  // (Esc, a Stop button). The unmount cleanup below flips mountedRef off
  // before routing through cancelStream, so it never stamps this — which is
  // how sendToAI tells a user's cancel from an unmount mid-stream.
  const explicitCancelGenerationRef = useRef(0);
  const toolScopeRef = useRef<ToolExecutionScope | null>(null);
  const cancelStream = useCallback(() => {
    sendGenerationRef.current += 1;
    if (mountedRef.current) {
      explicitCancelGenerationRef.current = sendGenerationRef.current;
    }
    toolScopeRef.current?.abort();
    toolScopeRef.current = null;
    ReasoningService.cancelActiveStream();
    setAgentState("idle");
    clearToolActivity();
  }, [clearToolActivity]);

  useEffect(() => {
    mountedRef.current = true;
    return () => {
      mountedRef.current = false;
      // Route through cancelStream so every cancellation path — Esc/Stop and
      // an unmount mid-stream alike — bumps the same generation counter.
      // Calling ReasoningService.cancelActiveStream() directly here (as this
      // used to) left sendToAI's cancelled() check unaware of an unmount
      // cancel, so its empty-response fallback still fired and persisted a
      // fabricated reply for a send the user never saw complete.
      cancelStream();
    };
  }, [cancelStream]);

  const sendToAI = useCallback(
    async (userText: string, allMessages: Message[], options?: SendToAIOptions) => {
      const sendGeneration = ++sendGenerationRef.current;
      const cancelled = () => sendGeneration !== sendGenerationRef.current;
      // This send supersedes any still running, so that turn's tools are
      // released now rather than when (or if) they settle.
      toolScopeRef.current?.abort();
      const toolScope = createToolExecutionScope({
        onApprovalRequested: options?.onApprovalRequested,
        onHoldDelivery: options?.onHoldDelivery,
      });
      toolScopeRef.current = toolScope;
      clearToolActivity();

      // Every exit, thrown errors included, releases tools still waiting on
      // this turn (an approval card would otherwise sit out its TTL). A newer
      // send may already own toolScopeRef.
      try {
        await runSend();
      } finally {
        toolScope.abort();
        if (toolScopeRef.current === toolScope) toolScopeRef.current = null;
      }

      async function runSend(): Promise<void> {
        let responseAnnounced = false;
        const announceResponse = () => {
          if (responseAnnounced) return;
          responseAnnounced = true;
          if (!options?.suppressResponseContent) onResponseContent?.();
        };
        const settings = getSettings();
        const { config: llmConfig, attachScreenContext } = resolveChatStreamingInference(settings, {
          inferenceScope,
          hasScreenContext: !!options?.attachment,
          isProviderImageWired: providerSupportsImages,
        });
        const requestedAttachment = attachScreenContext ? (options?.attachment ?? null) : null;
        const llmMode = llmConfig.mode || "openwhispr";
        const policyState = usePolicyStore.getState();
        const policyProvider =
          llmMode === "openwhispr"
            ? "openwhispr"
            : llmMode === "local"
              ? "local"
              : llmConfig.provider;
        if (
          !isAgentAllowed(policyState) ||
          !isLlmSelectionAllowed(policyState, { mode: llmMode, provider: policyProvider })
        ) {
          // The user message is already appended; answer it instead of dead-ending silently.
          const restriction = !isAgentAllowed(policyState)
            ? t("common.policyAgentRestricted")
            : t("common.policyAiProcessingRestricted");
          announceResponse();
          setMessages((prev) => [
            ...prev,
            {
              id: crypto.randomUUID(),
              role: "assistant",
              content: restriction,
              isStreaming: false,
            },
          ]);
          return;
        }

        setAgentState("thinking");
        const isCloudAgent = llmMode === "openwhispr" && settings.isSignedIn;
        const isLanAgent = llmMode === "self-hosted" && !!llmConfig.remoteUrl;
        const isCustomAgent = llmMode === "providers" && llmConfig.provider === "custom";
        const isLocalProvider =
          !isEnterpriseProvider(llmConfig.provider) &&
          ![
            "openai",
            "groq",
            "custom",
            "anthropic",
            "gemini",
            "tinfoil",
            "openrouter",
            "corti",
          ].includes(llmConfig.provider);
        const localModelCanUseTool =
          isLocalProvider && estimateModelSizeB(llmConfig.model) >= LOCAL_TOOL_MIN_PARAMS_B;
        const supportsTools = isCloudAgent || !isLocalProvider || localModelCanUseTool;

        const scope = searchScopeRef.current;
        const policy = usePolicyStore.getState();
        // The calendar tool reads the shared provider-deduped events table,
        // so any connected provider enables it.
        const calendarConnected =
          settings.gcalConnected || settings.mcalConnected || settings.appleCalendarConnected;
        const webSearchEnabled = isWebSearchAllowed(policy);
        const connectorPlan = hasConnectorPlan(getUsageState(), readIsSubscribed());
        let registry: ToolRegistry | null = null;
        let connectorsOffered = false;
        if (supportsTools) {
          const scopeKey = scope ? `${scope.spaceId}:${scope.folderId ?? ""}` : "";
          const connectorsAvailable =
            allowConnectors && settings.isSignedIn && connectorPlan && isConnectorsAllowed(policy);
          // The first send in a window must not miss a connector that is already connected.
          if (connectorsAvailable) await ensureConnectorStatus();
          const connectors = connectorsAvailable
            ? {
                emailDraftTarget: resolveEmailDraftTarget({
                  ...settings,
                  gmailStatus: gmailSendStatus(useConnectorStatusStore.getState().statuses.gmail),
                }),
                readyConnectorIds: readyConnectorIds(),
              }
            : undefined;
          connectorsOffered = connectors !== undefined;
          // Triggers ride in the tool description, so a snippet edit rebuilds the registry.
          const snippetKey = settings.snippets.map((s) => s.trigger).join("|");
          const keepNotesUnchanged = !!options?.keepNotesUnchanged;
          const cacheKey = `${settings.isSignedIn}-${calendarConnected}-${settings.cloudBackupEnabled}-${scopeKey}-${webSearchEnabled}-${productHelpEnabled}-${snippetKey}-${connectors?.emailDraftTarget ?? "no-connectors"}-${connectors?.readyConnectorIds.join(",") ?? ""}-${keepNotesUnchanged}`;
          if (toolRegistryRef.current?.key === cacheKey) {
            registry = toolRegistryRef.current.registry;
          } else {
            registry = createToolRegistry({
              isSignedIn: settings.isSignedIn,
              calendarConnected,
              cloudBackupEnabled: settings.cloudBackupEnabled,
              searchScope: scope,
              webSearchEnabled,
              productHelpEnabled,
              vocabulary: {
                getDictionary: () => getSettings().customDictionary,
                updateDictionary: (changes) =>
                  useSettingsStore.getState().updateCustomDictionary(changes),
                getSnippets: () => getSettings().snippets,
                setSnippets: (snippets) => useSettingsStore.getState().setSnippets(snippets),
              },
              connectors,
              keepNotesUnchanged,
            });
            toolRegistryRef.current = { key: cacheKey, registry };
          }
        }

        // Named in the prompt so the model says how to turn a capability on
        // instead of claiming it can't; built per send, outside the registry cache.
        const unavailable = resolveUnavailableCapabilities({
          supportsTools,
          isSignedIn: settings.isSignedIn,
          webSearch: { allowed: webSearchEnabled, blockedByOrg: isWebSearchBlockedByOrg(policy) },
          calendarConnected,
          connectors: allowConnectors
            ? {
                hasPlan: connectorPlan,
                allowed: isConnectorsAllowed(policy),
                blockedByOrg: isConnectorsBlockedByOrg(policy),
                statuses: useConnectorStatusStore.getState().statuses,
              }
            : undefined,
          locations: {
            account: `${t("sidebar.settings")} → ${t("settingsModal.sections.account.label")}`,
            plans: `${t("sidebar.settings")} → ${t("settingsModal.sections.plansBilling.label")}`,
            calendars: `${t("sidebar.integrations")} → ${t("integrations.nav.sections.calendars")}`,
            connectors: `${t("sidebar.integrations")} → ${t("integrations.nav.sections.connectors")}`,
            models: `${t("sidebar.settings")} → ${t("settingsModal.sections.llms.label")}`,
          },
        });

        // A note's chat is about its note: other notes up front would compete with
        // it, and search_notes still reaches them when it doesn't answer.
        const [ragContext, attendeesContext] = await Promise.all([
          openNoteRef.current ? "" : buildRAGContext(userText, scope),
          connectorsOffered ? buildNoteAttendeesContext(noteMeetingRef.current) : "",
        ]);
        if (cancelled() || !mountedRef.current) return;
        // A transcript too long for the model is read through get_note, except
        // by whole-note actions, which work from the complete source.
        const maxTranscriptTokens =
          registry?.get("get_note") && !options?.requestText
            ? FULL_TRANSCRIPT_MAX_TOKENS[
                !isCloudAgent && (isLocalProvider || isLanAgent) ? "selfHosted" : "hosted"
              ]
            : Infinity;
        const currentNote: OpenNoteContext =
          typeof openNoteRef.current === "function"
            ? openNoteRef.current(maxTranscriptTokens)
            : { text: openNoteRef.current ?? "", transcriptPreview: false };
        // Only main's attendee block may carry its fence: note text and search
        // results can't fake a second list.
        const openNoteContext = [withoutAttendeesFence(currentNote.text), attendeesContext]
          .filter(Boolean)
          .join("\n\n");
        const libraryContext = [
          withoutAttendeesFence(noteContextRef.current ?? ""),
          withoutAttendeesFence(ragContext),
        ]
          .filter(Boolean)
          .join("\n\n");
        // The user's dictionary rides on every conversation so replies use their
        // jargon — same suffix the dictation prompts carry.
        let systemPrompt = appendDictionarySuffix(
          getAgentSystemPrompt(registry?.getAll(), libraryContext || undefined, {
            unavailable: nameUnavailableCapabilities ? unavailable : [],
            toolTrace: registry !== null,
            openNote: openNoteContext || undefined,
            openNoteTranscriptPreview: currentNote.transcriptPreview,
            userName: userNameRef.current,
            writingStyle: settings.toneStyleEnabled
              ? { tones: settings.toneStyles, destination: options?.destinationAppCategory }
              : undefined,
          }),
          getDictionaryHintWords(settings),
          settings.uiLanguage
        );

        const history: HistoryMessage[] = toHistoryMessages(allMessages, {
          includeToolTrace: registry !== null,
        });

        const requestText = options?.requestText;
        if (requestText) {
          transformLastUserMessage(history, (message) => ({ ...message, content: requestText }));
        }

        const selectedContext = options?.selectedContext;
        if (selectedContext) {
          transformLastUserMessage(history, (message) =>
            typeof message.content === "string"
              ? { ...message, content: buildAgentRequestText(message.content, selectedContext) }
              : null
          );
        }

        // A screenshot the resolver kept rides with the command it came with:
        // BYOK models get it as an image part, the cloud agent as a dedicated
        // field the server vision-routes (older servers strip the unknown field,
        // which degrades to a plain command). A dropped one costs nothing but the
        // image — the command still runs.
        const attachment = requestedAttachment && !isCloudAgent ? requestedAttachment : null;
        const cloudScreenContext =
          requestedAttachment && isCloudAgent
            ? { data: requestedAttachment.image, mediaType: requestedAttachment.mediaType }
            : null;
        if (attachment) {
          // The screenshot needs its grounding instruction, exactly like the
          // dictation path pairs the suffix with an attached image. Restore it
          // for cloud context once openwhispr-api#157 vision-routes that field.
          systemPrompt = appendScreenContextSuffix(systemPrompt, settings.uiLanguage);
        }
        if (options?.plainTextResponse) {
          systemPrompt = appendPlainTextResponseSuffix(systemPrompt);
        }
        if (attachment) {
          transformLastUserMessage(history, (message) => ({
            role: "user",
            content: [
              { type: "text", text: message.content as string },
              { type: "image", image: attachment.image, mediaType: attachment.mediaType },
            ],
          }));
        }

        const llmMessages = [{ role: "system", content: systemPrompt }, ...history];

        const assistantId = crypto.randomUUID();
        setMessages((prev) => [
          ...prev,
          { id: assistantId, role: "assistant", content: "", isStreaming: true },
        ]);
        setAgentState("streaming");

        // Chat re-parses the whole answer through react-markdown on every
        // content write, so one write per streamed token made parse cost scale
        // with token count. Buffer and flush at most once per interval.
        let rawContent = "";
        let fullContent = "";
        let contentFlushTimer: ReturnType<typeof setTimeout> | null = null;
        const cancelContentFlush = () => {
          if (contentFlushTimer === null) return;
          clearTimeout(contentFlushTimer);
          contentFlushTimer = null;
        };
        const flushContentNow = () => {
          cancelContentFlush();
          setMessages((prev) =>
            prev.map((m) => (m.id === assistantId ? { ...m, content: fullContent } : m))
          );
        };
        const scheduleContentFlush = () => {
          if (contentFlushTimer !== null) return;
          contentFlushTimer = setTimeout(flushContentNow, STREAM_FLUSH_INTERVAL_MS);
        };

        try {
          let stream: AsyncGenerator<AgentStreamChunk>;
          // Each call's own step text on the AI SDK path, whose tool results
          // carry only the model-facing output (the cloud path yields it).
          const toolDisplayTexts = new Map<string, string>();

          if (isCloudAgent) {
            const executeToolCall = registry
              ? async (name: string, argsJson: string, toolCallId: string) => {
                  const tool = registry.get(name);
                  if (!tool)
                    return {
                      data: `Unknown tool: ${name}`,
                      displayText: t("agentMode.tools.unknownTool", { name }),
                    };
                  let args: Record<string, unknown>;
                  try {
                    args = JSON.parse(argsJson);
                  } catch {
                    return {
                      data: `Invalid tool arguments for ${name}`,
                      displayText: t("agentMode.tools.invalidArgs", { name }),
                    };
                  }
                  const result = await executeTool(
                    tool,
                    args,
                    toolScope.createContext({ messageId: assistantId, toolCallId })
                  );
                  const data = result.success
                    ? typeof result.data === "string"
                      ? result.data
                      : JSON.stringify(result.data)
                    : result.displayText;
                  const metadata =
                    result.success && result.data && typeof result.data === "object"
                      ? (result.data as Record<string, unknown> | Array<Record<string, unknown>>)
                      : undefined;
                  return { data, displayText: result.displayText, metadata };
                }
              : undefined;

            stream = ReasoningService.processTextStreamingCloud(llmMessages, {
              systemPrompt,
              tools: registry?.getAll().map((t) => ({
                name: t.name,
                description: t.description,
                parameters: t.parameters,
              })),
              executeToolCall,
              ...(cloudScreenContext ? { screenContext: cloudScreenContext } : {}),
            });
          } else {
            const aiTools = registry?.toAISDKFormat(
              (toolCallId, abortSignal) =>
                toolScope.createContext({
                  messageId: assistantId,
                  toolCallId,
                  signal: abortSignal,
                }),
              (id, text) => toolDisplayTexts.set(id, text)
            );
            stream = ReasoningService.processTextStreamingAI(
              llmMessages,
              llmConfig.model,
              llmConfig.provider,
              {
                systemPrompt,
                // Policy and managed enforcement judge the scope that actually
                // answers: the panel's Chat fallback as Chat, and the vision
                // override as the agent scope whose image lane it is.
                inferenceScope:
                  llmConfig.scope === "dictationAgentVision" ? "dictationAgent" : llmConfig.scope,
                lanUrl: isLanAgent ? llmConfig.remoteUrl : undefined,
                baseUrl: isCustomAgent ? llmConfig.cloudBaseUrl || undefined : undefined,
                customApiKey:
                  isCustomAgent || isLanAgent ? llmConfig.customApiKey || undefined : undefined,
                disableThinking: llmConfig.disableThinking,
                reasoningEffort: llmConfig.reasoningEffort,
              },
              aiTools
            );
          }

          for await (const chunk of stream) {
            if (!mountedRef.current) {
              ReasoningService.cancelActiveStream();
              break;
            }
            if (chunk.type === "content") {
              rawContent += chunk.text;
              fullContent = withoutEchoedToolTrace(rawContent);
              if (fullContent) announceResponse();
              scheduleContentFlush();
            } else if (chunk.type === "tool_calls") {
              // Text that arrived before a tool step must be on screen before the
              // step appears, not an interval after it.
              flushContentNow();
              if (chunk.calls.length > 0) announceResponse();
              for (const call of chunk.calls) {
                setAgentState("tool-executing");
                beginToolActivity(
                  call.name,
                  t(`agentMode.tools.${call.name}Status`, { defaultValue: `Using ${call.name}...` })
                );
                setMessages((prev) =>
                  prev.map((m) =>
                    m.id === assistantId
                      ? {
                          ...m,
                          toolCalls: [
                            ...(m.toolCalls || []),
                            {
                              id: call.id,
                              name: call.name,
                              arguments: call.arguments,
                              status: "executing" as const,
                            },
                          ],
                        }
                      : m
                  )
                );
              }
            } else if (chunk.type === "tool_result") {
              setMessages((prev) =>
                prev.map((m) =>
                  m.id === assistantId && m.toolCalls
                    ? {
                        ...m,
                        toolCalls: m.toolCalls.map((tc) =>
                          tc.id === chunk.callId
                            ? {
                                ...tc,
                                status: "completed" as const,
                                result: toolDisplayTexts.get(chunk.callId) ?? chunk.displayText,
                                ...(chunk.metadata && !isQueryResultData(chunk.metadata)
                                  ? {
                                      metadata:
                                        productHelpMetadata(tc.name, chunk.metadata) ??
                                        chunk.metadata,
                                    }
                                  : {}),
                              }
                            : tc
                        ),
                      }
                    : m
                )
              );
              setAgentState("streaming");
              completeToolActivity();
            }
          }

          if (cancelled() || !mountedRef.current) {
            flushContentNow();
            setMessages((prev) =>
              prev.map((message) =>
                message.id === assistantId ? { ...message, isStreaming: false } : message
              )
            );
            // An unmount mid-stream (page navigation) keeps the partial reply in
            // history so the saved conversation matches what the user last saw;
            // an explicit cancel drops it. The unmount cleanup cancels too, so
            // cancelled() alone cannot tell the two apart. Neither path may run
            // the per-request delivery hook.
            const explicitlyCancelled = explicitCancelGenerationRef.current > sendGeneration;
            if (!explicitlyCancelled && fullContent.trim().length > 0) {
              const finalMsg = messagesRef.current.find((m) => m.id === assistantId);
              onStreamComplete?.(assistantId, fullContent, finalMsg?.toolCalls);
            }
            return;
          }

          flushContentNow();
          const hasDeliverableContent = fullContent.trim().length > 0;
          if (!responseAnnounced && !cancelled()) {
            // The stream ended without a visible token or tool call (think-only
            // local model, empty completion). Show that as a reply so every
            // listener — the assistant panel's thinking state included — sees a
            // terminal outcome.
            fullContent = t("agentMode.chat.emptyResponse");
            announceResponse();
            setMessages((prev) =>
              prev.map((m) => (m.id === assistantId ? { ...m, content: fullContent } : m))
            );
          }

          setMessages((prev) =>
            prev.map((m) => (m.id === assistantId ? { ...m, isStreaming: false } : m))
          );

          const finalMsg = messagesRef.current.find((m) => m.id === assistantId);
          onStreamComplete?.(assistantId, fullContent, finalMsg?.toolCalls);
          if (hasDeliverableContent) {
            await options?.onComplete?.({
              assistantId,
              content: fullContent,
              toolCalls: finalMsg?.toolCalls,
            });
          }
        } catch (error) {
          if (cancelled()) {
            flushContentNow();
            setMessages((prev) =>
              prev.map((message) =>
                message.id === assistantId ? { ...message, isStreaming: false } : message
              )
            );
          } else {
            cancelContentFlush();
            logger.error(
              "Assistant request failed",
              {
                scope: llmConfig.scope,
                mode: llmMode,
                provider: llmConfig.provider,
                model: llmConfig.model,
                attachScreenContext,
                error: (error as Error).message,
              },
              "reasoning"
            );
            announceResponse();
            const failure = describeProviderError(error, t);
            const messageError =
              failure.technicalDetails || failure.settingsTarget
                ? {
                    technicalDetails: failure.technicalDetails,
                    settingsTarget: failure.settingsTarget,
                  }
                : undefined;
            setMessages((prev) =>
              prev.map((m) =>
                m.id === assistantId
                  ? {
                      ...m,
                      content: `${t("agentMode.chat.errorPrefix")}: ${failure.description}`,
                      isStreaming: false,
                      ...(messageError ? { error: messageError } : {}),
                    }
                  : m
              )
            );
          }
        }

        setAgentState("idle");
        completeToolActivity();
      }
    },
    [
      inferenceScope,
      allowConnectors,
      nameUnavailableCapabilities,
      productHelpEnabled,
      t,
      setMessages,
      onStreamComplete,
      onResponseContent,
      clearToolActivity,
      beginToolActivity,
      completeToolActivity,
    ]
  );

  return {
    agentState,
    toolStatus,
    activeToolName,
    sendToAI,
    cancelStream,
  };
}