Skip to content

Turn meetings into live agent loops. Record, transcribe, and analyze meetings with real-time AI intelligence — before, during, and after calls.

feat: live assist handling with skip flow and visual index

Om Gateaddc866e3abb
diff --git a/src/main/ipc/live-assist.ts b/src/main/ipc/live-assist.ts
index acec9cf..f3bf9fa 100644
--- a/src/main/ipc/live-assist.ts
+++ b/src/main/ipc/live-assist.ts
@@ -7,6 +7,7 @@
import { ipcMain, BrowserWindow } from 'electron';
import { getLiveAssistService, resetLiveAssistService } from '../services/live-assist.service';
+import type { MeetingContext } from '../services/live-assist.service';
import { getMCPInferenceService, resetMCPInferenceService } from '../services/mcp-inference.service';
import { createChildLogger } from '../lib/logger';
import { updateWidgetLiveAssist } from './widget';
@@ -29,8 +30,8 @@ export function setLiveAssistWindow(window: BrowserWindow): void {
export function setupLiveAssistHandlers(): void {
// Start live assist (also starts MCP inference)
- ipcMain.handle('live-assist:start', async () => {
- logger.info('Starting live assist and MCP inference');
+ ipcMain.handle('live-assist:start', async (_event, context?: MeetingContext) => {
+ logger.info({ hasContext: !!context }, 'Starting live assist and MCP inference');
// Start Live Assist service
const liveAssistService = getLiveAssistService();
@@ -44,7 +45,7 @@ export function setupLiveAssistHandlers(): void {
askThis: event.insights.ask_this,
});
});
- liveAssistService.start();
+ liveAssistService.start(context);
// Start MCP Inference service
const mcpInferenceService = getMCPInferenceService();
@@ -83,6 +84,14 @@ export function setupLiveAssistHandlers(): void {
return { success: true };
});
+ // Add visual index (called when screen analysis is received)
+ ipcMain.handle('live-assist:add-visual-index', async (_event, text: string) => {
+ const liveAssistService = getLiveAssistService();
+ liveAssistService.addVisualIndex(text);
+
+ return { success: true };
+ });
+
// Clear live assist state
ipcMain.handle('live-assist:clear', async () => {
const liveAssistService = getLiveAssistService();
diff --git a/src/main/services/live-assist.service.ts b/src/main/services/live-assist.service.ts
index a468291..efce35a 100644
--- a/src/main/services/live-assist.service.ts
+++ b/src/main/services/live-assist.service.ts
@@ -10,6 +10,7 @@ import { EventEmitter } from 'events';
import { logger } from '../lib/logger';
import { getLLMService } from './llm.service';
import type { LiveInsights } from '../../shared/types/live-assist.types';
+import type { ProbingQuestion } from '../../shared/types/meeting-setup.types';
const log = logger.child({ module: 'live-assist' });
@@ -21,6 +22,15 @@ IMPORTANT: All assistance must be directed to the "User" (labeled as [User] in t
---
+## CONTEXT YOU MAY RECEIVE
+
+You may receive additional context sections before the transcript:
+
+- **MEETING CONTEXT**: Meeting name, purpose, prep notes, and goals. Use this to keep suggestions aligned with what the User wants to accomplish.
+- **SCREEN CONTENT**: Description of what's on screen. Only use this if it's directly relevant to the current conversation — ignore generic UI descriptions or unrelated content.
+
+---
+
## WHAT TO SURFACE
**say_this** - Things the User could say:
@@ -62,35 +72,51 @@ Return a JSON object with two arrays of strings:
- Write as ready-to-use first-person lines.
- Only output the JSON, nothing else.`;
+export interface MeetingContext {
+ name?: string;
+ description?: string;
+ questions?: ProbingQuestion[];
+ checklist?: string[];
+}
+
interface TranscriptChunk {
text: string;
source: 'mic' | 'system_audio';
timestamp: number;
}
+interface VisualIndexChunk {
+ text: string;
+ timestamp: number;
+}
+
class LiveAssistService extends EventEmitter {
private intervalTimer: NodeJS.Timeout | null = null;
private isRunning = false;
private transcriptBuffer: TranscriptChunk[] = [];
+ private visualIndexBuffer: VisualIndexChunk[] = [];
private previousSayThis: Set<string> = new Set();
private previousAskThis: Set<string> = new Set();
private lastProcessedTimestamp = 0;
+ private meetingContext: MeetingContext | null = null;
/**
* Start the live assist loop
*/
- start(): void {
+ start(context?: MeetingContext): void {
if (this.isRunning) {
log.warn('Live assist already running');
return;
}
- log.info('Starting live assist service');
+ log.info({ context: context ? { name: context.name, hasDescription: !!context.description } : null }, 'Starting live assist service');
this.isRunning = true;
this.transcriptBuffer = [];
+ this.visualIndexBuffer = [];
this.previousSayThis.clear();
this.previousAskThis.clear();
this.lastProcessedTimestamp = Date.now();
+ this.meetingContext = context || null;
// Run immediately, then every 20 seconds
this.processTranscript();
@@ -114,8 +140,10 @@ class LiveAssistService extends EventEmitter {
}
this.transcriptBuffer = [];
+ this.visualIndexBuffer = [];
this.previousSayThis.clear();
this.previousAskThis.clear();
+ this.meetingContext = null;
}
/**
@@ -135,6 +163,68 @@ class LiveAssistService extends EventEmitter {
this.transcriptBuffer = this.transcriptBuffer.filter(t => t.timestamp > cutoff);
}
+ /**
+ * Add a visual index event to the buffer
+ */
+ addVisualIndex(text: string): void {
+ if (!this.isRunning) return;
+
+ this.visualIndexBuffer.push({
+ text,
+ timestamp: Date.now(),
+ });
+
+ // Keep only last 60 seconds of visual index for context
+ const cutoff = Date.now() - 60000;
+ this.visualIndexBuffer = this.visualIndexBuffer.filter(v => v.timestamp > cutoff);
+ }
+
+ /**
+ * Build meeting context section for prompt (only if context exists)
+ */
+ private buildMeetingContextSection(): string {
+ if (!this.meetingContext) return '';
+
+ const parts: string[] = [];
+
+ if (this.meetingContext.name) {
+ parts.push(`Meeting: ${this.meetingContext.name}`);
+ }
+
+ if (this.meetingContext.description) {
+ parts.push(`Purpose: ${this.meetingContext.description}`);
+ }
+
+ if (this.meetingContext.questions && this.meetingContext.questions.length > 0) {
+ const answeredQuestions = this.meetingContext.questions
+ .filter(q => q.answer)
+ .map(q => `- ${q.question}: ${q.answer}`)
+ .join('\n');
+ if (answeredQuestions) {
+ parts.push(`Key context from prep:\n${answeredQuestions}`);
+ }
+ }
+
+ if (this.meetingContext.checklist && this.meetingContext.checklist.length > 0) {
+ parts.push(`Goals to cover:\n${this.meetingContext.checklist.map(c => `- ${c}`).join('\n')}`);
+ }
+
+ if (parts.length === 0) return '';
+
+ return `## MEETING CONTEXT\n${parts.join('\n\n')}\n\n---\n\n`;
+ }
+
+ /**
+ * Build visual index section for prompt (only if recent visual data exists)
+ */
+ private buildVisualIndexSection(cutoff: number): string {
+ const recentVisuals = this.visualIndexBuffer.filter(v => v.timestamp > cutoff);
+ if (recentVisuals.length === 0) return '';
+
+ const visualText = recentVisuals.map(v => v.text).join('\n');
+ return `## SCREEN CONTENT (use only if relevant to the meeting)\n${visualText}\n\n---\n\n`;
+ }
+
/**
* Process transcript and generate assists
*/
@@ -158,13 +248,19 @@ class LiveAssistService extends EventEmitter {
})
.join('\n');
- log.info({ chunkCount: recentChunks.length, textLength: transcriptText.length }, 'Processing transcript for live assist');
+ // Build context sections (only included if they have content)
+ const meetingContextSection = this.buildMeetingContextSection();
+ const visualIndexSection = this.buildVisualIndexSection(cutoff);
+
+ const userPrompt = `${meetingContextSection}${visualIndexSection}## TRANSCRIPT\n${transcriptText}`;
+
+ log.info({ chunkCount: recentChunks.length, textLength: transcriptText.length, hasContext: !!meetingContextSection, hasVisual: !!visualIndexSection }, 'Processing transcript for live assist');
try {
const llm = getLLMService();
const response = await llm.chatCompletionJSON<LiveInsights>([
{ role: 'system', content: SYSTEM_PROMPT },
- { role: 'user', content: `Analyze this transcript chunk:\n\n${transcriptText}` },
+ { role: 'user', content: userPrompt },
]);
if (!response.success || !response.data) {
@@ -215,8 +311,10 @@ class LiveAssistService extends EventEmitter {
*/
clear(): void {
this.transcriptBuffer = [];
+ this.visualIndexBuffer = [];
this.previousSayThis.clear();
this.previousAskThis.clear();
+ this.meetingContext = null;
}
}
diff --git a/src/preload/index.ts b/src/preload/index.ts
index 019bb69..1ac104e 100644
--- a/src/preload/index.ts
+++ b/src/preload/index.ts
@@ -331,10 +331,13 @@ const api: IpcApi = {
},
liveAssist: {
- start: () => ipcRenderer.invoke('live-assist:start'),
+ start: (context?: { name?: string; description?: string; questions?: Array<{ question: string; answer: string }>; checklist?: string[] }) =>
+ ipcRenderer.invoke('live-assist:start', context),
stop: () => ipcRenderer.invoke('live-assist:stop'),
addTranscript: (text: string, source: 'mic' | 'system_audio') =>
ipcRenderer.invoke('live-assist:add-transcript', text, source),
+ addVisualIndex: (text: string) =>
+ ipcRenderer.invoke('live-assist:add-visual-index', text),
clear: () => ipcRenderer.invoke('live-assist:clear'),
},
diff --git a/src/renderer/components/history/RecordingDetailPage.tsx b/src/renderer/components/history/RecordingDetailPage.tsx
index c13ff29..b7ad0a7 100644
--- a/src/renderer/components/history/RecordingDetailPage.tsx
+++ b/src/renderer/components/history/RecordingDetailPage.tsx
@@ -168,13 +168,8 @@ function Header({ title, recordingId, createdAt, duration, playerUrl, onBack }:
const [copyState, setCopyState] = useState<'idle' | 'copying' | 'copied'>('idle');
const [exportOpen, setExportOpen] = useState(false);
const [downloadingVideo, setDownloadingVideo] = useState(false);
- const [downloadingTranscript, setDownloadingTranscript] = useState(false);
const downloadVideoMutation = trpc.recordings.downloadVideo.useMutation();
- const { data: transcript } = trpc.recordings.getTranscript.useQuery(
- { recordingId },
- { enabled: !!recordingId }
- );
const handleCopyLink = async () => {
if (!playerUrl || copyState !== 'idle') return;
@@ -198,29 +193,6 @@ function Header({ title, recordingId, createdAt, duration, playerUrl, onBack }:
}
};
- const handleDownloadTranscript = () => {
- if (!transcript || transcript.length === 0) return;
- setDownloadingTranscript(true);
- setExportOpen(false);
-
- const content = transcript.map(seg => {
- const mins = Math.floor(seg.startTime / 60);
- const secs = Math.floor(seg.startTime % 60);
- const time = `${mins}:${secs.toString().padStart(2, '0')}`;
- const speaker = seg.channel === 'me' ? 'You' : 'Them';
- return `[${time}] ${speaker}: ${seg.text}`;
- }).join('\n\n');
-
- const blob = new Blob([content], { type: 'text/plain' });
- const url = URL.createObjectURL(blob);
- const a = document.createElement('a');
- a.href = url;
- a.download = `${title.replace(/[^a-z0-9]/gi, '_')}_transcript.txt`;
- a.click();
- URL.revokeObjectURL(url);
- setDownloadingTranscript(false);
- };
-
return (
<div className="flex gap-[12px] items-start p-[20px]">
{/* Left: Back + Title */}
@@ -267,7 +239,7 @@ function Header({ title, recordingId, createdAt, duration, playerUrl, onBack }:
<div className="relative">
<button
onClick={() => setExportOpen(!exportOpen)}
- disabled={downloadingVideo || downloadingTranscript}
+ disabled={downloadingVideo}
className={cn(
"flex items-center gap-[6px] border rounded-[12px] px-[16px] py-[12px] shadow-[0px_1.27px_15.27px_0px_rgba(0,0,0,0.05)] transition-colors",
exportOpen
@@ -275,7 +247,7 @@ function Header({ title, recordingId, createdAt, duration, playerUrl, onBack }:
: "bg-white border-[#efefef] hover:bg-[#efefef] hover:border-[#969696]"
)}
>
- {downloadingVideo || downloadingTranscript ? (
+ {downloadingVideo ? (
<Loader2 className="h-5 w-5 text-black animate-spin" />
) : (
<Upload className="h-5 w-5 text-black" />
@@ -298,14 +270,6 @@ function Header({ title, recordingId, createdAt, duration, playerUrl, onBack }:
<Video className="h-5 w-5 text-black" />
<span className="text-[13px] font-medium text-black">Video</span>
</button>
- <button
- onClick={handleDownloadTranscript}
- disabled={!transcript || transcript.length === 0}
- className="w-full flex items-center gap-[6px] px-[10px] py-[8px] rounded-[10px] hover:bg-[#efefef] transition-colors disabled:opacity-50"
- >
- <FileText className="h-5 w-5 text-black" />
- <span className="text-[13px] font-medium text-black">Transcript</span>
- </button>
</div>
</>
)}
diff --git a/src/renderer/components/meeting-setup/MeetingSetupFlow.tsx b/src/renderer/components/meeting-setup/MeetingSetupFlow.tsx
index 168b1e6..de2c9e6 100644
--- a/src/renderer/components/meeting-setup/MeetingSetupFlow.tsx
+++ b/src/renderer/components/meeting-setup/MeetingSetupFlow.tsx
@@ -53,22 +53,68 @@ export function MeetingSetupFlow({ onCancel }: MeetingSetupFlowProps) {
setStep('info');
}, [setStep]);
- // Skip setup and start recording with whatever data is already filled
- // Accepts optional overrides for name/description from local step state
+ // Skip setup and start recording
+ // If description or answered questions exist, generate checklist first
+ // Otherwise skip directly to recording
const handleSkipAndRecord = async (overrideName?: string, overrideDescription?: string) => {
-
const finalName = (overrideName ?? name).trim() || generateDefaultMeetingName();
const finalDescription = (overrideDescription ?? description).trim();
// Update store so RecordingHeader can display the correct name
setInfo(finalName, finalDescription);
- await startRecording({
- name: finalName,
- description: finalDescription,
- questions: questions.filter(q => q.answer), // Only include answered questions
- checklist,
- });
+ const answeredQuestions = questions.filter(q => q.answer);
+ const hasContext = finalDescription || answeredQuestions.length > 0;
+
+ if (hasContext) {
+ // Generate checklist first if we have description or answered questions
+ setIsGenerating(true);
+ setError(null);
+
+ try {
+ const result = await generateChecklistMutation.mutateAsync({
+ name: finalName,
+ description: finalDescription,
+ questions: answeredQuestions,
+ });
+
+ if (result.success && result.checklist.length > 0) {
+ setChecklist(result.checklist);
+ await startRecording({
+ name: finalName,
+ description: finalDescription,
+ questions: answeredQuestions,
+ checklist: result.checklist,
+ });
+ } else {
+ // Checklist generation returned empty, proceed to recording
+ await startRecording({
+ name: finalName,
+ description: finalDescription,
+ questions: answeredQuestions,
+ checklist: [],
+ });
+ }
+ } catch {
+ // Checklist generation failed, proceed to recording
+ await startRecording({
+ name: finalName,
+ description: finalDescription,
+ questions: answeredQuestions,
+ checklist: [],
+ });
+ } finally {
+ setIsGenerating(false);
+ }
+ } else {
+ // No context (only name or empty), skip directly to recording
+ await startRecording({
+ name: finalName,
+ description: finalDescription,
+ questions: [],
+ checklist: [],
+ });
+ }
};
const handleInfoNext = async (newName: string, newDescription: string) => {
diff --git a/src/renderer/hooks/useGlobalRecorderEvents.ts b/src/renderer/hooks/useGlobalRecorderEvents.ts
index 0c21be6..0814ab8 100644
--- a/src/renderer/hooks/useGlobalRecorderEvents.ts
+++ b/src/renderer/hooks/useGlobalRecorderEvents.ts
@@ -101,9 +101,16 @@ export function useGlobalRecorderEvents() {
rtstreamName: visualData.rtstreamName,
});
+ // Forward to live assist for real-time analysis
+ const currentApi = getElectronAPI();
+ if (currentApi) {
+ currentApi.liveAssist.addVisualIndex(visualData.text).catch((err: Error) => {
+ console.warn('[GlobalRecorderEvents] Error forwarding visual index to live assist:', err);
+ });
+ }
+
// Save to database for durable storage
const currentSession = useSessionStore.getState();
- const currentApi = getElectronAPI();
if (currentSession.recordingId && currentSession.sessionId && currentApi) {
// Convert epoch ms timestamps to seconds from call start
const callStartSec = currentSession.startTime ? currentSession.startTime / 1000 : visualData.start;
diff --git a/src/renderer/hooks/useLiveAssist.ts b/src/renderer/hooks/useLiveAssist.ts
index 699b1af..9a9fc78 100644
--- a/src/renderer/hooks/useLiveAssist.ts
+++ b/src/renderer/hooks/useLiveAssist.ts
@@ -8,6 +8,7 @@
import { useEffect, useRef } from 'react';
import { useLiveAssistStore } from '../stores/live-assist.store';
import { useSessionStore } from '../stores/session.store';
+import { useMeetingSetupStore } from '../stores/meeting-setup.store';
import { getElectronAPI } from '../api/ipc';
export function useLiveAssist() {
@@ -22,9 +23,20 @@ export function useLiveAssist() {
if (!api) return;
if (isRecording && !wasRecordingRef.current) {
- // Recording just started
- console.log('[LiveAssist] Starting live assist service');
- api.liveAssist.start().catch(err => {
+ // Recording just started - get meeting context
+ const meetingSetup = useMeetingSetupStore.getState();
+ const context = {
+ name: meetingSetup.name || undefined,
+ description: meetingSetup.description || undefined,
+ questions: meetingSetup.questions.length > 0 ? meetingSetup.questions : undefined,
+ checklist: meetingSetup.checklist.length > 0 ? meetingSetup.checklist : undefined,
+ };
+
+ // Only pass context if at least one field has content
+ const hasContext = context.name || context.description || context.questions || context.checklist;
+
+ console.log('[LiveAssist] Starting live assist service', hasContext ? 'with context' : 'without context');
+ api.liveAssist.start(hasContext ? context : undefined).catch(err => {
console.error('[LiveAssist] Failed to start:', err);
});
wasRecordingRef.current = true;
diff --git a/src/shared/types/ipc.types.ts b/src/shared/types/ipc.types.ts
index 4b057bd..fa9503f 100644
--- a/src/shared/types/ipc.types.ts
+++ b/src/shared/types/ipc.types.ts
@@ -168,10 +168,18 @@ export interface CopilotConfig {
enableNudges: boolean;
}
+export interface LiveAssistMeetingContext {
+ name?: string;
+ description?: string;
+ questions?: Array<{ question: string; answer: string }>;
+ checklist?: string[];
+}
+
export interface LiveAssistApi {
- start: () => Promise<{ success: boolean }>;
+ start: (context?: LiveAssistMeetingContext) => Promise<{ success: boolean }>;
stop: () => Promise<{ success: boolean }>;
addTranscript: (text: string, source: 'mic' | 'system_audio') => Promise<{ success: boolean }>;
+ addVisualIndex: (text: string) => Promise<{ success: boolean }>;
clear: () => Promise<{ success: boolean }>;
}