Turn meetings into live agent loops. Record, transcribe, and analyze meetings with real-time AI intelligence — before, during, and after calls.
feat: live assist handling with skip flow and visual index
Om Gateaddc866e3abb
diff --git a/src/main/ipc/live-assist.ts b/src/main/ipc/live-assist.tsindex acec9cf..f3bf9fa 100644--- a/src/main/ipc/live-assist.ts+++ b/src/main/ipc/live-assist.ts@@ -7,6 +7,7 @@import { ipcMain, BrowserWindow } from 'electron';import { getLiveAssistService, resetLiveAssistService } from '../services/live-assist.service';+import type { MeetingContext } from '../services/live-assist.service';import { getMCPInferenceService, resetMCPInferenceService } from '../services/mcp-inference.service';import { createChildLogger } from '../lib/logger';import { updateWidgetLiveAssist } from './widget';@@ -29,8 +30,8 @@ export function setLiveAssistWindow(window: BrowserWindow): void {export function setupLiveAssistHandlers(): void {// Start live assist (also starts MCP inference)- ipcMain.handle('live-assist:start', async () => {- logger.info('Starting live assist and MCP inference');+ ipcMain.handle('live-assist:start', async (_event, context?: MeetingContext) => {+ logger.info({ hasContext: !!context }, 'Starting live assist and MCP inference');// Start Live Assist serviceconst liveAssistService = getLiveAssistService();@@ -44,7 +45,7 @@ export function setupLiveAssistHandlers(): void {askThis: event.insights.ask_this,});});- liveAssistService.start();+ liveAssistService.start(context);// Start MCP Inference serviceconst mcpInferenceService = getMCPInferenceService();@@ -83,6 +84,14 @@ export function setupLiveAssistHandlers(): void {return { success: true };});+ // Add visual index (called when screen analysis is received)+ ipcMain.handle('live-assist:add-visual-index', async (_event, text: string) => {+ const liveAssistService = getLiveAssistService();+ liveAssistService.addVisualIndex(text);++ return { success: true };+ });+// Clear live assist stateipcMain.handle('live-assist:clear', async () => {const liveAssistService = getLiveAssistService();diff --git a/src/main/services/live-assist.service.ts b/src/main/services/live-assist.service.tsindex a468291..efce35a 100644--- a/src/main/services/live-assist.service.ts+++ b/src/main/services/live-assist.service.ts@@ -10,6 +10,7 @@ import { EventEmitter } from 'events';import { logger } from '../lib/logger';import { getLLMService } from './llm.service';import type { LiveInsights } from '../../shared/types/live-assist.types';+import type { ProbingQuestion } from '../../shared/types/meeting-setup.types';const log = logger.child({ module: 'live-assist' });@@ -21,6 +22,15 @@ IMPORTANT: All assistance must be directed to the "User" (labeled as [User] in t---+## CONTEXT YOU MAY RECEIVE++You may receive additional context sections before the transcript:++- **MEETING CONTEXT**: Meeting name, purpose, prep notes, and goals. Use this to keep suggestions aligned with what the User wants to accomplish.+- **SCREEN CONTENT**: Description of what's on screen. Only use this if it's directly relevant to the current conversation — ignore generic UI descriptions or unrelated content.++---+## WHAT TO SURFACE**say_this** - Things the User could say:@@ -62,35 +72,51 @@ Return a JSON object with two arrays of strings:- Write as ready-to-use first-person lines.- Only output the JSON, nothing else.`;+export interface MeetingContext {+ name?: string;+ description?: string;+ questions?: ProbingQuestion[];+ checklist?: string[];+}+interface TranscriptChunk {text: string;source: 'mic' | 'system_audio';timestamp: number;}+interface VisualIndexChunk {+ text: string;+ timestamp: number;+}+class LiveAssistService extends EventEmitter {private intervalTimer: NodeJS.Timeout | null = null;private isRunning = false;private transcriptBuffer: TranscriptChunk[] = [];+ private visualIndexBuffer: VisualIndexChunk[] = [];private previousSayThis: Set<string> = new Set();private previousAskThis: Set<string> = new Set();private lastProcessedTimestamp = 0;+ private meetingContext: MeetingContext | null = null;/*** Start the live assist loop*/- start(): void {+ start(context?: MeetingContext): void {if (this.isRunning) {log.warn('Live assist already running');return;}- log.info('Starting live assist service');+ log.info({ context: context ? { name: context.name, hasDescription: !!context.description } : null }, 'Starting live assist service');this.isRunning = true;this.transcriptBuffer = [];+ this.visualIndexBuffer = [];this.previousSayThis.clear();this.previousAskThis.clear();this.lastProcessedTimestamp = Date.now();+ this.meetingContext = context || null;// Run immediately, then every 20 secondsthis.processTranscript();@@ -114,8 +140,10 @@ class LiveAssistService extends EventEmitter {}this.transcriptBuffer = [];+ this.visualIndexBuffer = [];this.previousSayThis.clear();this.previousAskThis.clear();+ this.meetingContext = null;}/**@@ -135,6 +163,68 @@ class LiveAssistService extends EventEmitter {this.transcriptBuffer = this.transcriptBuffer.filter(t => t.timestamp > cutoff);}+ /**+ * Add a visual index event to the buffer+ */+ addVisualIndex(text: string): void {+ if (!this.isRunning) return;++ this.visualIndexBuffer.push({+ text,+ timestamp: Date.now(),+ });++ // Keep only last 60 seconds of visual index for context+ const cutoff = Date.now() - 60000;+ this.visualIndexBuffer = this.visualIndexBuffer.filter(v => v.timestamp > cutoff);+ }++ /**+ * Build meeting context section for prompt (only if context exists)+ */+ private buildMeetingContextSection(): string {+ if (!this.meetingContext) return '';++ const parts: string[] = [];++ if (this.meetingContext.name) {+ parts.push(`Meeting: ${this.meetingContext.name}`);+ }++ if (this.meetingContext.description) {+ parts.push(`Purpose: ${this.meetingContext.description}`);+ }++ if (this.meetingContext.questions && this.meetingContext.questions.length > 0) {+ const answeredQuestions = this.meetingContext.questions+ .filter(q => q.answer)+ .map(q => `- ${q.question}: ${q.answer}`)+ .join('\n');+ if (answeredQuestions) {+ parts.push(`Key context from prep:\n${answeredQuestions}`);+ }+ }++ if (this.meetingContext.checklist && this.meetingContext.checklist.length > 0) {+ parts.push(`Goals to cover:\n${this.meetingContext.checklist.map(c => `- ${c}`).join('\n')}`);+ }++ if (parts.length === 0) return '';++ return `## MEETING CONTEXT\n${parts.join('\n\n')}\n\n---\n\n`;+ }++ /**+ * Build visual index section for prompt (only if recent visual data exists)+ */+ private buildVisualIndexSection(cutoff: number): string {+ const recentVisuals = this.visualIndexBuffer.filter(v => v.timestamp > cutoff);+ if (recentVisuals.length === 0) return '';++ const visualText = recentVisuals.map(v => v.text).join('\n');+ return `## SCREEN CONTENT (use only if relevant to the meeting)\n${visualText}\n\n---\n\n`;+ }+/*** Process transcript and generate assists*/@@ -158,13 +248,19 @@ class LiveAssistService extends EventEmitter {}).join('\n');- log.info({ chunkCount: recentChunks.length, textLength: transcriptText.length }, 'Processing transcript for live assist');+ // Build context sections (only included if they have content)+ const meetingContextSection = this.buildMeetingContextSection();+ const visualIndexSection = this.buildVisualIndexSection(cutoff);++ const userPrompt = `${meetingContextSection}${visualIndexSection}## TRANSCRIPT\n${transcriptText}`;++ log.info({ chunkCount: recentChunks.length, textLength: transcriptText.length, hasContext: !!meetingContextSection, hasVisual: !!visualIndexSection }, 'Processing transcript for live assist');try {const llm = getLLMService();const response = await llm.chatCompletionJSON<LiveInsights>([{ role: 'system', content: SYSTEM_PROMPT },- { role: 'user', content: `Analyze this transcript chunk:\n\n${transcriptText}` },+ { role: 'user', content: userPrompt },]);if (!response.success || !response.data) {@@ -215,8 +311,10 @@ class LiveAssistService extends EventEmitter {*/clear(): void {this.transcriptBuffer = [];+ this.visualIndexBuffer = [];this.previousSayThis.clear();this.previousAskThis.clear();+ this.meetingContext = null;}}diff --git a/src/preload/index.ts b/src/preload/index.tsindex 019bb69..1ac104e 100644--- a/src/preload/index.ts+++ b/src/preload/index.ts@@ -331,10 +331,13 @@ const api: IpcApi = {},liveAssist: {- start: () => ipcRenderer.invoke('live-assist:start'),+ start: (context?: { name?: string; description?: string; questions?: Array<{ question: string; answer: string }>; checklist?: string[] }) =>+ ipcRenderer.invoke('live-assist:start', context),stop: () => ipcRenderer.invoke('live-assist:stop'),addTranscript: (text: string, source: 'mic' | 'system_audio') =>ipcRenderer.invoke('live-assist:add-transcript', text, source),+ addVisualIndex: (text: string) =>+ ipcRenderer.invoke('live-assist:add-visual-index', text),clear: () => ipcRenderer.invoke('live-assist:clear'),},diff --git a/src/renderer/components/history/RecordingDetailPage.tsx b/src/renderer/components/history/RecordingDetailPage.tsxindex c13ff29..b7ad0a7 100644--- a/src/renderer/components/history/RecordingDetailPage.tsx+++ b/src/renderer/components/history/RecordingDetailPage.tsx@@ -168,13 +168,8 @@ function Header({ title, recordingId, createdAt, duration, playerUrl, onBack }:const [copyState, setCopyState] = useState<'idle' | 'copying' | 'copied'>('idle');const [exportOpen, setExportOpen] = useState(false);const [downloadingVideo, setDownloadingVideo] = useState(false);- const [downloadingTranscript, setDownloadingTranscript] = useState(false);const downloadVideoMutation = trpc.recordings.downloadVideo.useMutation();- const { data: transcript } = trpc.recordings.getTranscript.useQuery(- { recordingId },- { enabled: !!recordingId }- );const handleCopyLink = async () => {if (!playerUrl || copyState !== 'idle') return;@@ -198,29 +193,6 @@ function Header({ title, recordingId, createdAt, duration, playerUrl, onBack }:}};- const handleDownloadTranscript = () => {- if (!transcript || transcript.length === 0) return;- setDownloadingTranscript(true);- setExportOpen(false);-- const content = transcript.map(seg => {- const mins = Math.floor(seg.startTime / 60);- const secs = Math.floor(seg.startTime % 60);- const time = `${mins}:${secs.toString().padStart(2, '0')}`;- const speaker = seg.channel === 'me' ? 'You' : 'Them';- return `[${time}] ${speaker}: ${seg.text}`;- }).join('\n\n');-- const blob = new Blob([content], { type: 'text/plain' });- const url = URL.createObjectURL(blob);- const a = document.createElement('a');- a.href = url;- a.download = `${title.replace(/[^a-z0-9]/gi, '_')}_transcript.txt`;- a.click();- URL.revokeObjectURL(url);- setDownloadingTranscript(false);- };-return (<div className="flex gap-[12px] items-start p-[20px]">{/* Left: Back + Title */}@@ -267,7 +239,7 @@ function Header({ title, recordingId, createdAt, duration, playerUrl, onBack }:<div className="relative"><buttononClick={() => setExportOpen(!exportOpen)}- disabled={downloadingVideo || downloadingTranscript}+ disabled={downloadingVideo}className={cn("flex items-center gap-[6px] border rounded-[12px] px-[16px] py-[12px] shadow-[0px_1.27px_15.27px_0px_rgba(0,0,0,0.05)] transition-colors",exportOpen@@ -275,7 +247,7 @@ function Header({ title, recordingId, createdAt, duration, playerUrl, onBack }:: "bg-white border-[#efefef] hover:bg-[#efefef] hover:border-[#969696]")}>- {downloadingVideo || downloadingTranscript ? (+ {downloadingVideo ? (<Loader2 className="h-5 w-5 text-black animate-spin" />) : (<Upload className="h-5 w-5 text-black" />@@ -298,14 +270,6 @@ function Header({ title, recordingId, createdAt, duration, playerUrl, onBack }:<Video className="h-5 w-5 text-black" /><span className="text-[13px] font-medium text-black">Video</span></button>- <button- onClick={handleDownloadTranscript}- disabled={!transcript || transcript.length === 0}- className="w-full flex items-center gap-[6px] px-[10px] py-[8px] rounded-[10px] hover:bg-[#efefef] transition-colors disabled:opacity-50"- >- <FileText className="h-5 w-5 text-black" />- <span className="text-[13px] font-medium text-black">Transcript</span>- </button></div></>)}diff --git a/src/renderer/components/meeting-setup/MeetingSetupFlow.tsx b/src/renderer/components/meeting-setup/MeetingSetupFlow.tsxindex 168b1e6..de2c9e6 100644--- a/src/renderer/components/meeting-setup/MeetingSetupFlow.tsx+++ b/src/renderer/components/meeting-setup/MeetingSetupFlow.tsx@@ -53,22 +53,68 @@ export function MeetingSetupFlow({ onCancel }: MeetingSetupFlowProps) {setStep('info');}, [setStep]);- // Skip setup and start recording with whatever data is already filled- // Accepts optional overrides for name/description from local step state+ // Skip setup and start recording+ // If description or answered questions exist, generate checklist first+ // Otherwise skip directly to recordingconst handleSkipAndRecord = async (overrideName?: string, overrideDescription?: string) => {-const finalName = (overrideName ?? name).trim() || generateDefaultMeetingName();const finalDescription = (overrideDescription ?? description).trim();// Update store so RecordingHeader can display the correct namesetInfo(finalName, finalDescription);- await startRecording({- name: finalName,- description: finalDescription,- questions: questions.filter(q => q.answer), // Only include answered questions- checklist,- });+ const answeredQuestions = questions.filter(q => q.answer);+ const hasContext = finalDescription || answeredQuestions.length > 0;++ if (hasContext) {+ // Generate checklist first if we have description or answered questions+ setIsGenerating(true);+ setError(null);++ try {+ const result = await generateChecklistMutation.mutateAsync({+ name: finalName,+ description: finalDescription,+ questions: answeredQuestions,+ });++ if (result.success && result.checklist.length > 0) {+ setChecklist(result.checklist);+ await startRecording({+ name: finalName,+ description: finalDescription,+ questions: answeredQuestions,+ checklist: result.checklist,+ });+ } else {+ // Checklist generation returned empty, proceed to recording+ await startRecording({+ name: finalName,+ description: finalDescription,+ questions: answeredQuestions,+ checklist: [],+ });+ }+ } catch {+ // Checklist generation failed, proceed to recording+ await startRecording({+ name: finalName,+ description: finalDescription,+ questions: answeredQuestions,+ checklist: [],+ });+ } finally {+ setIsGenerating(false);+ }+ } else {+ // No context (only name or empty), skip directly to recording+ await startRecording({+ name: finalName,+ description: finalDescription,+ questions: [],+ checklist: [],+ });+ }};const handleInfoNext = async (newName: string, newDescription: string) => {diff --git a/src/renderer/hooks/useGlobalRecorderEvents.ts b/src/renderer/hooks/useGlobalRecorderEvents.tsindex 0c21be6..0814ab8 100644--- a/src/renderer/hooks/useGlobalRecorderEvents.ts+++ b/src/renderer/hooks/useGlobalRecorderEvents.ts@@ -101,9 +101,16 @@ export function useGlobalRecorderEvents() {rtstreamName: visualData.rtstreamName,});+ // Forward to live assist for real-time analysis+ const currentApi = getElectronAPI();+ if (currentApi) {+ currentApi.liveAssist.addVisualIndex(visualData.text).catch((err: Error) => {+ console.warn('[GlobalRecorderEvents] Error forwarding visual index to live assist:', err);+ });+ }+// Save to database for durable storageconst currentSession = useSessionStore.getState();- const currentApi = getElectronAPI();if (currentSession.recordingId && currentSession.sessionId && currentApi) {// Convert epoch ms timestamps to seconds from call startconst callStartSec = currentSession.startTime ? currentSession.startTime / 1000 : visualData.start;diff --git a/src/renderer/hooks/useLiveAssist.ts b/src/renderer/hooks/useLiveAssist.tsindex 699b1af..9a9fc78 100644--- a/src/renderer/hooks/useLiveAssist.ts+++ b/src/renderer/hooks/useLiveAssist.ts@@ -8,6 +8,7 @@import { useEffect, useRef } from 'react';import { useLiveAssistStore } from '../stores/live-assist.store';import { useSessionStore } from '../stores/session.store';+import { useMeetingSetupStore } from '../stores/meeting-setup.store';import { getElectronAPI } from '../api/ipc';export function useLiveAssist() {@@ -22,9 +23,20 @@ export function useLiveAssist() {if (!api) return;if (isRecording && !wasRecordingRef.current) {- // Recording just started- console.log('[LiveAssist] Starting live assist service');- api.liveAssist.start().catch(err => {+ // Recording just started - get meeting context+ const meetingSetup = useMeetingSetupStore.getState();+ const context = {+ name: meetingSetup.name || undefined,+ description: meetingSetup.description || undefined,+ questions: meetingSetup.questions.length > 0 ? meetingSetup.questions : undefined,+ checklist: meetingSetup.checklist.length > 0 ? meetingSetup.checklist : undefined,+ };++ // Only pass context if at least one field has content+ const hasContext = context.name || context.description || context.questions || context.checklist;++ console.log('[LiveAssist] Starting live assist service', hasContext ? 'with context' : 'without context');+ api.liveAssist.start(hasContext ? context : undefined).catch(err => {console.error('[LiveAssist] Failed to start:', err);});wasRecordingRef.current = true;diff --git a/src/shared/types/ipc.types.ts b/src/shared/types/ipc.types.tsindex 4b057bd..fa9503f 100644--- a/src/shared/types/ipc.types.ts+++ b/src/shared/types/ipc.types.ts@@ -168,10 +168,18 @@ export interface CopilotConfig {enableNudges: boolean;}+export interface LiveAssistMeetingContext {+ name?: string;+ description?: string;+ questions?: Array<{ question: string; answer: string }>;+ checklist?: string[];+}+export interface LiveAssistApi {- start: () => Promise<{ success: boolean }>;+ start: (context?: LiveAssistMeetingContext) => Promise<{ success: boolean }>;stop: () => Promise<{ success: boolean }>;addTranscript: (text: string, source: 'mic' | 'system_audio') => Promise<{ success: boolean }>;+ addVisualIndex: (text: string) => Promise<{ success: boolean }>;clear: () => Promise<{ success: boolean }>;}