Skip to content

Personal AI, On Personal Devices

Python

feat: Deep Research — personal deep research over Gmail with hybrid retrieval and agentic synthesis (#354)

Robby Manihani9af0dfe336f6
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 6f3ce052..dd1ccecb 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -72,6 +72,8 @@ jobs:
- name: Upload coverage report
if: always()
+ continue-on-error: true
+ timeout-minutes: 5
uses: actions/upload-artifact@v4
with:
name: coverage-xml
diff --git a/.gitignore b/.gitignore
index 92c34a67..4b7e2a7a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -99,6 +99,9 @@ src/openjarvis/channels/whatsapp_baileys_bridge/node_modules/
# SQLite in-memory artifacts
:memory:
+# Dogfood reports (regenerated locally; not for VCS)
+dogfood_report*.md
+
# Second Repos
Inline/
scratch/
diff --git a/frontend/src/components/Chat/ChatArea.tsx b/frontend/src/components/Chat/ChatArea.tsx
index 7bdd93a4..5df8b08c 100644
--- a/frontend/src/components/Chat/ChatArea.tsx
+++ b/frontend/src/components/Chat/ChatArea.tsx
@@ -146,14 +146,29 @@ export function ChatArea() {
</div>
) : (
<div className="max-w-[var(--chat-max-width)] mx-auto px-4 py-6">
- {messages.map((msg) => (
- <MessageBubble key={msg.id} message={msg} />
- ))}
- {streamState.isStreaming && streamState.content === '' && (
- <div className="flex justify-start mb-4">
- <StreamingDots phase={streamState.phase} />
- </div>
- )}
+ {messages.map((msg, i) => {
+ const isLastAssistant =
+ i === messages.length - 1 && msg.role === 'assistant';
+ return (
+ <MessageBubble
+ key={msg.id}
+ message={msg}
+ isLive={isLastAssistant && streamState.isStreaming}
+ />
+ );
+ })}
+ {(() => {
+ if (!streamState.isStreaming || streamState.content !== '') return null;
+ // For research messages the ResearchTimeline handles its own
+ // pre-content loading state — suppress the generic dots.
+ const last = messages[messages.length - 1];
+ if (last?.role === 'assistant' && last.isResearch) return null;
+ return (
+ <div className="flex justify-start mb-4">
+ <StreamingDots phase={streamState.phase} />
+ </div>
+ );
+ })()}
</div>
)}
</div>
diff --git a/frontend/src/components/Chat/InputArea.tsx b/frontend/src/components/Chat/InputArea.tsx
index 13cdce9d..dd841b25 100644
--- a/frontend/src/components/Chat/InputArea.tsx
+++ b/frontend/src/components/Chat/InputArea.tsx
@@ -1,11 +1,77 @@
import { useState, useRef, useCallback, useEffect } from 'react';
-import { Send, Square, Paperclip } from 'lucide-react';
+import { Send, Square, Paperclip, Search } from 'lucide-react';
import { useAppStore, generateId } from '../../lib/store';
-import { streamChat } from '../../lib/sse';
+import { streamChat, streamResearch } from '../../lib/sse';
import { fetchSavings, getBase } from '../../lib/api';
+import { listConnectors, getSyncStatus } from '../../lib/connectors-api';
import { MicButton } from './MicButton';
import { useSpeech } from '../../hooks/useSpeech';
-import type { ChatMessage, ToolCallInfo, TokenUsage, MessageTelemetry } from '../../types';
+import type {
+ ChatMessage,
+ MessageTelemetry,
+ ResearchSearchTrace,
+ ResearchSource,
+ TokenUsage,
+ ToolCallInfo,
+} from '../../types';
+
+// While Deep Research is toggled on, poll connected sources for sync
+// progress so we can surface "Searching over N items — sync in progress"
+// next to the toggle. Polling is gated on `enabled` so toggling DR off
+// stops the network chatter immediately.
+function useResearchCorpusSync(enabled: boolean): {
+ syncing: boolean;
+ itemsSynced: number;
+} {
+ const [state, setState] = useState({ syncing: false, itemsSynced: 0 });
+
+ useEffect(() => {
+ if (!enabled) {
+ setState({ syncing: false, itemsSynced: 0 });
+ return;
+ }
+ let cancelled = false;
+
+ const poll = async () => {
+ try {
+ const list = await listConnectors();
+ const connected = list.filter((c) => c.connected);
+ if (connected.length === 0) {
+ if (!cancelled) setState({ syncing: false, itemsSynced: 0 });
+ return;
+ }
+ const results = await Promise.all(
+ connected.map(async (c) => {
+ try {
+ return await getSyncStatus(c.connector_id);
+ } catch {
+ return null;
+ }
+ }),
+ );
+ let syncing = false;
+ let itemsSynced = 0;
+ for (const r of results) {
+ if (!r) continue;
+ if (r.state === 'syncing') syncing = true;
+ itemsSynced += r.items_synced ?? 0;
+ }
+ if (!cancelled) setState({ syncing, itemsSynced });
+ } catch {
+ // Network blip — leave previous state intact.
+ }
+ };
+
+ poll();
+ const interval = setInterval(poll, 5000);
+ return () => {
+ cancelled = true;
+ clearInterval(interval);
+ };
+ }, [enabled]);
+
+ return state;
+}
export function InputArea() {
const [input, setInput] = useState('');
@@ -26,6 +92,9 @@ export function InputArea() {
const setStreamState = useAppStore((s) => s.setStreamState);
const resetStream = useAppStore((s) => s.resetStream);
const modelLoading = useAppStore((s) => s.modelLoading);
+ const deepResearch = useAppStore((s) => s.deepResearch);
+ const setDeepResearch = useAppStore((s) => s.setDeepResearch);
+ const corpusSync = useResearchCorpusSync(deepResearch);
const { state: speechState, available: speechAvailable, startRecording, stopRecording } = useSpeech();
@@ -114,6 +183,7 @@ export function InputArea() {
role: 'assistant',
content: '',
timestamp: Date.now(),
+ isResearch: deepResearch || undefined,
};
addMessage(convId, assistantMsg);
@@ -131,12 +201,16 @@ export function InputArea() {
let usage: TokenUsage | undefined;
let complexity: { score: number; tier: string; suggested_max_tokens: number } | undefined;
const toolCalls: ToolCallInfo[] = [];
+ const researchTraces: ResearchSearchTrace[] = [];
+ const researchSourcesByRef = new Map<number, ResearchSource>();
+ const flushSources = () =>
+ Array.from(researchSourcesByRef.values()).sort((a, b) => a.ref - b.ref);
let lastFlush = 0;
let ttftMs: number | undefined;
setStreamState({
isStreaming: true,
- phase: 'Generating...',
+ phase: deepResearch ? 'Researching...' : 'Generating...',
elapsedMs: 0,
activeToolCalls: [],
content: '',
@@ -145,10 +219,116 @@ export function InputArea() {
timestamp: Date.now(),
level: 'info',
category: 'chat',
- message: `Request: "${content.slice(0, 80)}${content.length > 80 ? '...' : ''}" → ${selectedModel}`,
+ message: deepResearch
+ ? `Research: "${content.slice(0, 80)}${content.length > 80 ? '...' : ''}"`
+ : `Request: "${content.slice(0, 80)}${content.length > 80 ? '...' : ''}" → ${selectedModel}`,
});
try {
+ if (deepResearch) {
+ for await (const ev of streamResearch(content, controller.signal)) {
+ if (ev.type === 'search_call') {
+ const trace: ResearchSearchTrace = {
+ id: generateId(),
+ query: ev.arguments?.query ?? '',
+ person: ev.arguments?.person,
+ timeRange: ev.arguments?.time_range,
+ status: 'pending',
+ };
+ researchTraces.push(trace);
+ setStreamState({ phase: `Searching: ${trace.query}` });
+ updateLastAssistant(
+ convId,
+ accumulatedContent,
+ undefined,
+ undefined,
+ undefined,
+ undefined,
+ [...researchTraces],
+ flushSources(),
+ );
+ useAppStore.getState().addLogEntry({
+ timestamp: Date.now(),
+ level: 'info',
+ category: 'tool',
+ message: `Search: "${trace.query}"${trace.person ? ` (person: ${trace.person})` : ''}`,
+ });
+ } else if (ev.type === 'search_result') {
+ const pending = [...researchTraces].reverse().find((t) => t.status === 'pending');
+ if (pending) {
+ pending.status = 'complete';
+ pending.numHits = ev.num_hits;
+ pending.topTitles = ev.top_titles;
+ }
+ if (ev.sources) {
+ for (const src of ev.sources) {
+ if (src && typeof src.ref === 'number' && !researchSourcesByRef.has(src.ref)) {
+ researchSourcesByRef.set(src.ref, src);
+ }
+ }
+ }
+ updateLastAssistant(
+ convId,
+ accumulatedContent,
+ undefined,
+ undefined,
+ undefined,
+ undefined,
+ [...researchTraces],
+ flushSources(),
+ );
+ } else if (ev.type === 'synthesis') {
+ if (!ttftMs) ttftMs = Date.now() - startTime;
+ accumulatedContent += ev.text;
+ setStreamState({ content: accumulatedContent, phase: '' });
+ const now = Date.now();
+ if (now - lastFlush >= 80) {
+ updateLastAssistant(
+ convId,
+ accumulatedContent,
+ undefined,
+ undefined,
+ undefined,
+ undefined,
+ [...researchTraces],
+ flushSources(),
+ );
+ lastFlush = now;
+ }
+ } else if (ev.type === 'system_metrics') {
+ // Live GPU sample — feed straight to the System panel so Power
+ // (W) and Energy (kJ) tick up in real time as the agent runs.
+ useAppStore.getState().setLiveEnergy({
+ power_w: ev.power_w,
+ energy_j: ev.energy_j,
+ duration_s: ev.duration_s,
+ });
+ } else if (ev.type === 'done') {
+ if (ev.usage) {
+ usage = {
+ prompt_tokens: ev.usage.prompt_tokens ?? 0,
+ completion_tokens: ev.usage.completion_tokens ?? 0,
+ total_tokens:
+ ev.usage.total_tokens ??
+ (ev.usage.prompt_tokens ?? 0) +
+ (ev.usage.completion_tokens ?? 0),
+ };
+ // Optimistically roll this research turn into the session
+ // counters so the Session panel updates the moment the
+ // stream finishes, regardless of how /v1/savings aggregates
+ // research telemetry server-side.
+ useAppStore.getState().incrementSavings(usage);
+ }
+ // Hold the final live numbers visible for a beat so the panel
+ // doesn't flash to 0 between the SSE close and the next
+ // /v1/telemetry/energy poll picking up the persisted record.
+ window.setTimeout(() => {
+ useAppStore.getState().setLiveEnergy(null);
+ }, 1500);
+ break;
+ }
+ }
+ } else {
for await (const sseEvent of streamChat(
{ model: selectedModel, messages: apiMessages, stream: true, temperature, max_tokens: maxTokens },
controller.signal,
@@ -225,6 +405,7 @@ export function InputArea() {
} catch {}
}
}
+ }
} catch (err: any) {
if (err.name === 'AbortError') {
// User cancelled or model switch — keep whatever was accumulated
@@ -238,6 +419,9 @@ export function InputArea() {
message: `Stream error: ${errMsg}`,
});
}
+ // If we tore out mid-research, make sure the live System panel
+ // numbers don't get stuck on the last sample.
+ useAppStore.getState().setLiveEnergy(null);
} finally {
if (!accumulatedContent) {
accumulatedContent = 'No response was generated. Please try again.';
@@ -278,6 +462,8 @@ export function InputArea() {
usage,
telemetry,
audioMeta,
+ researchTraces.length > 0 ? researchTraces : undefined,
+ researchSourcesByRef.size > 0 ? flushSources() : undefined,
);
if (timerRef.current) {
clearInterval(timerRef.current);
@@ -290,9 +476,15 @@ export function InputArea() {
});
abortRef.current = null;
- fetchSavings()
- .then((data) => useAppStore.getState().setSavings(data))
- .catch(() => {});
+ // Research path updates session counters optimistically from the
+ // `done` event's usage payload — re-fetching here would overwrite
+ // it with a potentially stale snapshot if the server's research
+ // telemetry hasn't been merged into /v1/savings yet.
+ if (!deepResearch) {
+ fetchSavings()
+ .then((data) => useAppStore.getState().setSavings(data))
+ .catch(() => {});
+ }
}
}, [
input,
@@ -304,6 +496,9 @@ export function InputArea() {
updateLastAssistant,
setStreamState,
resetStream,
+ deepResearch,
+ temperature,
+ maxTokens,
]);
const handleKeyDown = (e: React.KeyboardEvent) => {
@@ -315,6 +510,38 @@ export function InputArea() {
return (
<div className="px-4 pb-4 pt-2" style={{ maxWidth: 'var(--chat-max-width)', margin: '0 auto', width: '100%' }}>
+ <div className="mb-2 flex flex-col gap-1">
+ <div className="flex items-center gap-2">
+ <button
+ type="button"
+ onClick={() => setDeepResearch(!deepResearch)}
+ disabled={streamState.isStreaming}
+ aria-pressed={deepResearch}
+ className="inline-flex items-center gap-1.5 px-2.5 py-1 rounded-full text-xs transition-colors cursor-pointer disabled:cursor-default disabled:opacity-50"
+ style={{
+ background: deepResearch ? 'var(--color-accent-subtle)' : 'transparent',
+ border: `1px solid ${deepResearch ? 'var(--color-accent)' : 'var(--color-border)'}`,
+ color: deepResearch ? 'var(--color-accent)' : 'var(--color-text-tertiary)',
+ }}
+ title={deepResearch ? 'Deep Research: on' : 'Deep Research: off'}
+ >
+ <Search size={12} />
+ Deep Research
+ </button>
+ </div>
+ {deepResearch && corpusSync.syncing && corpusSync.itemsSynced > 0 && (
+ <div
+ className="text-[11px] leading-snug"
+ style={{ color: 'var(--color-text-tertiary)' }}
+ >
+ Searching over{' '}
+ <span key={corpusSync.itemsSynced} className="sync-bump" style={{ color: 'var(--color-text-secondary)' }}>
+ {corpusSync.itemsSynced.toLocaleString()}
+ </span>{' '}
+ items — sync in progress, results will improve as more data is indexed.
+ </div>
+ )}
+ </div>
<div
className="flex items-center gap-2 rounded-2xl px-4 py-3 transition-shadow"
style={{
diff --git a/frontend/src/components/Chat/MessageBubble.tsx b/frontend/src/components/Chat/MessageBubble.tsx
index efe68e79..c221050b 100644
--- a/frontend/src/components/Chat/MessageBubble.tsx
+++ b/frontend/src/components/Chat/MessageBubble.tsx
@@ -8,6 +8,8 @@ import 'katex/dist/katex.min.css';
import { Copy, Check } from 'lucide-react';
import { AudioPlayer } from './AudioPlayer';
import { ToolCallCard } from './ToolCallCard';
+import { ResearchTimeline } from './ResearchTimeline';
+import { rehypeCitations } from '../../lib/rehype-citations';
import { XRayFooter } from './XRayFooter';
import type { ChatMessage } from '../../types';
@@ -19,6 +21,7 @@ function stripThinkTags(text: string): string {
interface Props {
message: ChatMessage;
+ isLive?: boolean;
}
function getTextContent(node: any): string {
@@ -97,7 +100,7 @@ function CopyMessageButton({ content }: { content: string }) {
);
}
-export function MessageBubble({ message }: Props) {
+export function MessageBubble({ message, isLive = false }: Props) {
const isUser = message.role === 'user';
if (isUser) {
@@ -121,8 +124,33 @@ export function MessageBubble({ message }: Props) {
const cleanContent = useMemo(() => stripThinkTags(message.content), [message.content]);
+ // Build a ref→source lookup once per render. Memoized so the rehype plugin
+ // identity stays stable until the source list actually changes.
+ const sourcesMap = useMemo(() => {
+ const m = new Map<number, NonNullable<ChatMessage['researchSources']>[number]>();
+ for (const s of message.researchSources ?? []) {
+ if (typeof s.ref === 'number') m.set(s.ref, s);
+ }
+ return m;
+ }, [message.researchSources]);
+
+ const rehypePlugins = useMemo(() => {
+ const base: any[] = [[rehypeHighlight, { detect: true }], rehypeKatex];
+ if (sourcesMap.size > 0) base.push([rehypeCitations, { sources: sourcesMap }]);
+ return base;
+ }, [sourcesMap]);
+
return (
<div className="group mb-6">
+ {/* Deep Research timeline (steps + status) */}
+ {(message.isResearch || (message.researchTraces && message.researchTraces.length > 0)) && (
+ <ResearchTimeline
+ traces={message.researchTraces ?? []}
+ isLive={isLive}
+ hasContent={cleanContent.length > 0}
+ />
+ )}
+
{/* Tool calls */}
{message.toolCalls && message.toolCalls.length > 0 && (
<div className="mb-3 flex flex-col gap-2">
@@ -140,7 +168,7 @@ export function MessageBubble({ message }: Props) {
<div className="prose max-w-none">
<ReactMarkdown
remarkPlugins={[remarkGfm, remarkMath]}
- rehypePlugins={[[rehypeHighlight, { detect: true }], rehypeKatex]}
+ rehypePlugins={rehypePlugins}
components={{
pre: CodeBlockPre,
}}
@@ -154,7 +182,11 @@ export function MessageBubble({ message }: Props) {
<div className="flex items-center gap-2 mt-1.5">
<CopyMessageButton content={cleanContent} />
</div>
- <XRayFooter usage={message.usage} telemetry={message.telemetry} />
+ <XRayFooter
+ usage={message.usage}
+ telemetry={message.telemetry}
+ isResearch={message.isResearch}
+ />
</div>
);
}
diff --git a/frontend/src/components/Chat/ResearchTimeline.tsx b/frontend/src/components/Chat/ResearchTimeline.tsx
new file mode 100644
index 00000000..be66a8af
--- /dev/null
+++ b/frontend/src/components/Chat/ResearchTimeline.tsx
@@ -0,0 +1,236 @@
+import { useEffect, useRef, useState } from 'react';
+import { ChevronDown, ChevronUp } from 'lucide-react';
+import type { ResearchSearchTrace, TimeRange } from '../../types';
+
+interface Props {
+ traces: ResearchSearchTrace[];
+ isLive: boolean;
+ hasContent: boolean;
+}
+
+const SHORT_DATE = new Intl.DateTimeFormat('en-US', {
+ month: 'short',
+ day: 'numeric',
+});
+
+function fmtDate(iso: string): string | null {
+ const d = new Date(iso);
+ if (Number.isNaN(d.getTime())) return null;
+ return SHORT_DATE.format(d);
+}
+
+function formatTimeRange(tr: TimeRange | string | undefined): string | null {
+ if (!tr) return null;
+ if (typeof tr === 'string') return tr.trim() || null;
+ const start = tr.start ? fmtDate(tr.start) : null;
+ const end = tr.end ? fmtDate(tr.end) : null;
+ if (start && end) return start === end ? start : `${start} – ${end}`;
+ if (start) return `after ${start}`;
+ if (end) return `before ${end}`;
+ return null;
+}
+
+function summarizeTraces(traces: ResearchSearchTrace[]): string {
+ const n = traces.length;
+ const hits = traces.reduce((s, t) => s + (t.numHits ?? 0), 0);
+ const searchLabel = `${n} ${n === 1 ? 'search' : 'searches'}`;
+ if (hits === 0) return searchLabel;
+ return `${searchLabel} · ${hits} ${hits === 1 ? 'result' : 'results'}`;
+}
+
+function StatusLine({ text }: { text: string }) {
+ return (
+ <div
+ className="text-xs leading-relaxed animate-pulse"
+ style={{ color: 'var(--color-text-tertiary)' }}
+ >
+ {text}…
+ </div>
+ );
+}
+
+function TimelineStep({
+ index,
+ trace,
+ isLive,
+}: {
+ index: number;
+ trace: ResearchSearchTrace;
+ isLive: boolean;
+}) {
+ const pending = trace.status === 'pending';
+ const active = pending && isLive;
+ const meta: string[] = [];
+ if (trace.person) meta.push(`person: ${trace.person}`);
+ const formattedTime = formatTimeRange(trace.timeRange);
+ if (formattedTime) meta.push(`time: ${formattedTime}`);
+
+ return (
+ <div className="relative pl-4">
+ {/* Step indicator dot, sitting on the vertical rail */}
+ <div
+ className="absolute left-0 top-[7px] w-1.5 h-1.5 rounded-full"
+ style={{
+ background: active
+ ? 'var(--color-accent)'
+ : 'var(--color-text-tertiary)',
+ opacity: active ? 1 : 0.6,
+ transform: 'translateX(-3px)',
+ boxShadow: active ? '0 0 0 3px var(--color-accent-subtle)' : 'none',
+ transition: 'background 200ms, box-shadow 200ms',
+ }}
+ />
+
+ <div
+ className="text-[10px] uppercase tracking-[0.08em] mb-0.5"
+ style={{
+ color: active ? 'var(--color-accent)' : 'var(--color-text-tertiary)',
+ }}
+ >
+ Search {index}
+ </div>
+
+ <div
+ className="text-sm"
+ style={{ color: 'var(--color-text)', fontWeight: 450 }}
+ >
+ “{trace.query}”
+ </div>
+
+ {meta.length > 0 && (
+ <div
+ className="text-[11px] mt-0.5"
+ style={{ color: 'var(--color-text-tertiary)' }}
+ >
+ {meta.join(' · ')}
+ </div>
+ )}
+
+ {active ? (
+ <div
+ className="mt-2 h-px overflow-hidden relative"
+ style={{ background: 'var(--color-border)' }}
+ >
+ <div
+ className="research-shimmer absolute inset-y-0 w-1/4"
+ style={{ background: 'var(--color-accent)' }}
+ />
+ </div>
+ ) : trace.numHits != null ? (
+ <div
+ className="text-[11px] mt-1"
+ style={{ color: 'var(--color-text-tertiary)' }}
+ >
+ {trace.numHits} {trace.numHits === 1 ? 'result' : 'results'}
+ </div>
+ ) : null}
+
+ {trace.topTitles && trace.topTitles.length > 0 && (
+ <div
+ className="text-[11px] mt-0.5 truncate"
+ style={{ color: 'var(--color-text-tertiary)', opacity: 0.75 }}
+ >
+ {trace.topTitles.slice(0, 2).join(' · ')}
+ </div>
+ )}
+ </div>
+ );
+}
+
+export function ResearchTimeline({ traces, isLive, hasContent }: Props) {
+ const showAnalyzing = isLive && traces.length === 0 && !hasContent;
+ const allComplete =
+ traces.length > 0 && traces.every((t) => t.status === 'complete');
+ const showSynthesizing = isLive && allComplete && !hasContent;
+
+ // Auto-collapse the timeline the moment synthesis text begins streaming.
+ // Subsequent user toggles win — we only fire the auto-collapse once per
+ // false→true transition of hasContent.
+ const [expanded, setExpanded] = useState(true);
+ const prevHasContent = useRef(false);
+ useEffect(() => {
+ if (hasContent && !prevHasContent.current) {
+ setExpanded(false);
+ }
+ prevHasContent.current = hasContent;
+ }, [hasContent]);
+
+ if (!showAnalyzing && traces.length === 0) return null;
+
+ // No collapse affordance before any traces have arrived — just the
+ // analyzing status sitting alone.
+ if (traces.length === 0) {
+ return (
+ <div className="mb-4">
+ <div className="relative pl-4">
+ <div
+ className="absolute left-0 top-[7px] w-1.5 h-1.5 rounded-full"
+ style={{
+ background: 'var(--color-accent)',
+ transform: 'translateX(-3px)',
+ boxShadow: '0 0 0 3px var(--color-accent-subtle)',
+ }}
+ />
+ <StatusLine text="Analyzing query" />
+ </div>
+ </div>
+ );
+ }
+
+ const summary = summarizeTraces(traces);
+ const Chevron = expanded ? ChevronUp : ChevronDown;
+
+ return (
+ <div className="mb-4">
+ <button
+ type="button"
+ onClick={() => setExpanded((e) => !e)}
+ className="flex items-center gap-2 text-[11px] mb-2 cursor-pointer transition-colors"
+ style={{
+ color: 'var(--color-text-tertiary)',
+ background: 'transparent',
+ border: 'none',
+ padding: 0,
+ }}
+ title={expanded ? 'Collapse search trace' : 'Expand search trace'}
+ >
+ <span>{summary}</span>
+ <Chevron size={12} />
+ </button>
+
+ {expanded && (
+ <div className="relative">
+ <div
+ aria-hidden
+ className="absolute top-1 bottom-1 left-0 w-px"
+ style={{ background: 'var(--color-border)' }}
+ />
+ <div className="flex flex-col gap-3">
+ {traces.map((t, i) => (
+ <TimelineStep
+ key={t.id}
+ index={i + 1}
+ trace={t}
+ isLive={isLive}
+ />
+ ))}
+
+ {showSynthesizing && (
+ <div className="relative pl-4">
+ <div
+ className="absolute left-0 top-[7px] w-1.5 h-1.5 rounded-full"
+ style={{
+ background: 'var(--color-accent)',
+ transform: 'translateX(-3px)',
+ boxShadow: '0 0 0 3px var(--color-accent-subtle)',
+ }}
+ />
+ <StatusLine text="Synthesizing findings" />
+ </div>
+ )}
+ </div>
+ </div>
+ )}
+ </div>
+ );
+}
diff --git a/frontend/src/components/Chat/SystemPanel.tsx b/frontend/src/components/Chat/SystemPanel.tsx
index e83b2e9b..0fc2a866 100644
--- a/frontend/src/components/Chat/SystemPanel.tsx
+++ b/frontend/src/components/Chat/SystemPanel.tsx
@@ -39,6 +39,7 @@ export function SystemPanel() {
const toggleSystemPanel = useAppStore((s) => s.toggleSystemPanel);
const optInEnabled = useAppStore((s) => s.optInEnabled);
const setOptInModalOpen = useAppStore((s) => s.setOptInModalOpen);
+ const liveEnergy = useAppStore((s) => s.liveEnergy);
const [energy, setEnergy] = useState<EnergyData | null>(null);
const [telemetry, setTelemetry] = useState<TelemetryStats | null>(null);
@@ -129,13 +130,15 @@ export function SystemPanel() {
<MiniStat
icon={Zap}
label="Power"
- value={(energy?.avg_power_w ?? 0).toFixed(1)}
+ value={(liveEnergy?.power_w ?? energy?.avg_power_w ?? 0).toFixed(1)}
unit="W"
/>
<MiniStat
icon={Activity}
label="Energy"
- value={((energy?.total_energy_j ?? 0) / 1000).toFixed(1)}
+ value={(
+ ((liveEnergy?.energy_j ?? energy?.total_energy_j ?? 0) / 1000)
+ ).toFixed(1)}
unit="kJ"
/>
</div>
diff --git a/frontend/src/components/Chat/XRayFooter.tsx b/frontend/src/components/Chat/XRayFooter.tsx
index b13126dc..08fb3468 100644
--- a/frontend/src/components/Chat/XRayFooter.tsx
+++ b/frontend/src/components/Chat/XRayFooter.tsx
@@ -5,19 +5,26 @@ import type { TokenUsage, MessageTelemetry } from '../../types';
interface Props {
usage?: TokenUsage;
telemetry?: MessageTelemetry;
+ isResearch?: boolean;
}
function formatMs(ms: number): string {
return ms < 1000 ? `${Math.round(ms)}ms` : `${(ms / 1000).toFixed(1)}s`;
}
-export function XRayFooter({ usage, telemetry }: Props) {
+export function XRayFooter({ usage, telemetry, isResearch = false }: Props) {
const [expanded, setExpanded] = useState(false);
- // Build collapsed summary parts
+ // Build collapsed summary parts. For Deep Research responses we hide the
+ // chat-side engine/model (which doesn't reflect the planner that actually
+ // produced the answer) and label the response with the mode instead.
const parts: string[] = [];
- if (telemetry?.engine) parts.push(telemetry.engine);
- if (telemetry?.model_id) parts.push(telemetry.model_id);
+ if (isResearch) {
+ parts.push('Deep Research');
+ } else {
+ if (telemetry?.engine) parts.push(telemetry.engine);
+ if (telemetry?.model_id) parts.push(telemetry.model_id);
+ }
if (telemetry?.complexity_tier) parts.push(telemetry.complexity_tier);
if (telemetry?.total_ms) parts.push(formatMs(telemetry.total_ms));
if (usage && (usage.prompt_tokens || usage.completion_tokens)) {
@@ -32,7 +39,9 @@ export function XRayFooter({ usage, telemetry }: Props) {
// Build expanded rows
const rows: Array<{ label: string; value: string; color?: string }> = [];
- if (telemetry?.engine) {
+ if (isResearch) {
+ rows.push({ label: 'Mode', value: 'Deep Research' });
+ } else if (telemetry?.engine) {
const modelDetail = telemetry.model_id || '';
rows.push({ label: 'Engine', value: `${telemetry.engine}${modelDetail ? ` (${modelDetail})` : ''}` });
}
diff --git a/frontend/src/components/Sidebar/Sidebar.tsx b/frontend/src/components/Sidebar/Sidebar.tsx
index de518405..a4a5b670 100644
--- a/frontend/src/components/Sidebar/Sidebar.tsx
+++ b/frontend/src/components/Sidebar/Sidebar.tsx
@@ -33,6 +33,7 @@ export function Sidebar() {
const serverInfo = useAppStore((s) => s.serverInfo);
const setCommandPaletteOpen = useAppStore((s) => s.setCommandPaletteOpen);
const modelLoading = useAppStore((s) => s.modelLoading);
+ const deepResearch = useAppStore((s) => s.deepResearch);
const settings = useAppStore((s) => s.settings);
const updateSettings = useAppStore((s) => s.updateSettings);
@@ -143,8 +144,13 @@ export function Sidebar() {
<Cpu size={14} />
)}
<div className="flex-1 min-w-0">
- <span className="truncate block text-left" style={{ color: 'var(--color-text)' }}>
- {selectedModel || serverInfo?.model || 'Select model'}
+ <span
+ className="truncate block text-left"
+ style={{ color: deepResearch ? 'var(--color-accent)' : 'var(--color-text)' }}
+ >
+ {deepResearch
+ ? 'Deep Research'
+ : selectedModel || serverInfo?.model || 'Select model'}
</span>
{modelLoading && (
<span className="text-[10px] block text-left" style={{ color: 'var(--color-accent)' }}>
diff --git a/frontend/src/index.css b/frontend/src/index.css
index b043b74a..a974c6e8 100644
--- a/frontend/src/index.css
+++ b/frontend/src/index.css
@@ -707,4 +707,50 @@
--color-foreground: var(--foreground);
--color-background: var(--background);
/* Keep project radius tokens — don't override with shadcn calc values */
+}
+
+/* Deep Research timeline — pending-step shimmer */
+@keyframes openjarvis-research-shimmer {
+ 0% { transform: translateX(-120%); }
+ 100% { transform: translateX(420%); }
+}
+.research-shimmer {
+ animation: openjarvis-research-shimmer 1.6s ease-in-out infinite;
+}
+
+/* Sync progress pulse — fires on each items_synced bump so the
+ "12,450 / 99,840" label visibly breathes between polls. */
+@keyframes openjarvis-sync-bump {
+ 0% { opacity: 0.55; }
+ 100% { opacity: 1; }
+}
+.sync-bump {
+ animation: openjarvis-sync-bump 400ms ease-out;
+}
+
+/* Deep Research inline citation pills — small rounded chips that flow
+ with the surrounding text and link out to the cited email. */
+.research-citation {
+ display: inline-block;
+ vertical-align: baseline;
+ margin: 0 1px;
+ padding: 0 6px;
+ min-width: 1.25em;
+ text-align: center;
+ font-size: 0.72em;
+ font-weight: 600;
+ line-height: 1.5;
+ color: var(--color-accent);
+ background: var(--color-accent-subtle);
+ border-radius: 9999px;
+ text-decoration: none;
+ cursor: pointer;
+ transition: background 150ms, color 150ms;
+ /* Keep the pill from inheriting prose link styles (underline, bold). */
+ border-bottom: none;
+}
+.research-citation:hover {
+ color: var(--color-on-accent, #ffffff);
+ background: var(--color-accent);
+ text-decoration: none;
}
\ No newline at end of file
diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts
index 32a4d173..b62d71b2 100644
--- a/frontend/src/lib/api.ts
+++ b/frontend/src/lib/api.ts
@@ -192,12 +192,26 @@ export async function checkHealth(): Promise<boolean> {
return false;
}
}
- try {
- const res = await fetch(`${getBase()}/health`);
- return res.ok;
- } catch {
- return false;
- }
+ // In the browser, hit /health relative to the page origin so the request
+ // flows through whatever path is already serving the SPA — the Vite
+ // proxy in dev, FastAPI's static mount in prod. This avoids the
+ // false-negative "Cannot reach backend" banner when getBase() points at
+ // an absolute URL the browser can't reach directly.
+ //
+ // If /health itself fails for any reason (proxy quirk, stale service
+ // worker, etc.) fall back to an arbitrary API endpoint we know the rest
+ // of the app polls successfully. If THAT also fails we genuinely can't
+ // reach the backend.
+ const probe = async (url: string): Promise<boolean> => {
+ try {
+ const res = await fetch(url, { cache: 'no-store' });
+ return res.ok;
+ } catch {
+ return false;
+ }
+ };
+ if (await probe('/health')) return true;
+ return probe('/v1/connectors');
}
export async function fetchEnergy(): Promise<unknown> {
diff --git a/frontend/src/lib/rehype-citations.ts b/frontend/src/lib/rehype-citations.ts
new file mode 100644
index 00000000..18332d1a
--- /dev/null
+++ b/frontend/src/lib/rehype-citations.ts
@@ -0,0 +1,203 @@
+import type { ResearchSource } from '../types';
+
+interface HastNode {
+ type: string;
+ tagName?: string;
+ value?: string;
+ properties?: Record<string, unknown>;
+ children?: HastNode[];
+}
+
+// Tags whose descendants should NOT be searched for citations.
+// `<a>` is handled specially below (markdown link to a single number is
+// promoted to a pill); other links + code/pre/script/style are skipped.
+const SKIP_TAGS = new Set(['code', 'pre', 'script', 'style']);
+
+// Matches `[N]` or `[N, M, ...]` — bracketed comma-separated digit lists.
+// Whitespace inside the brackets is tolerated. An optional trailing
+// whitespace run is consumed when followed by sentence punctuation, so we
+// can render `… San Francisco [1].` without a stray space before the period.
+const CITATION_RE =
+ /\[(\s*\d+(?:\s*,\s*\d+)*\s*)\](\s+(?=[.,;:!?]))?/g;
+
+const TOOLTIP_DATE = new Intl.DateTimeFormat('en-US', {
+ month: 'short',
+ day: 'numeric',
+ year: 'numeric',
+});
+
+function formatTooltipDate(iso: string | undefined): string | null {
+ if (!iso) return null;
+ const d = new Date(iso);
+ if (Number.isNaN(d.getTime())) return iso.trim() || null;
+ return TOOLTIP_DATE.format(d);
+}
+
+function buildTooltip(src: ResearchSource): string {
+ const parts: string[] = [];
+ if (src.title) parts.push(src.title);
+ const meta: string[] = [];
+ if (src.sender) meta.push(src.sender);
+ const date = formatTooltipDate(src.date);
+ if (date) meta.push(date);
+ if (meta.length > 0) parts.push(meta.join(' · '));
+ return parts.join('\n');
+}
+
+function buildPill(n: number, src: ResearchSource): HastNode {
+ return {
+ type: 'element',
+ tagName: 'a',
+ properties: {
+ href: src.url,
+ target: '_blank',
+ rel: 'noopener noreferrer',
+ className: ['research-citation'],
+ 'data-ref': String(n),
+ title: buildTooltip(src) || `Source ${n}`,
+ },
+ children: [{ type: 'text', value: String(n) }],
+ };
+}
+
+function parseRefList(inner: string): number[] {
+ return inner
+ .split(',')
+ .map((s) => parseInt(s.trim(), 10))
+ .filter((n) => !Number.isNaN(n));
+}
+
+/**
+ * Replace bracketed citations in the rendered markdown with small inline
+ * pill links to the underlying source.
+ *
+ * Patterns handled:
+ * • `[1]` — single ref
+ * • `[4, 7, 20]` — grouped refs → individual pills, no separators
+ * • `[1](https://…)` — markdown link whose text is just a number; the
+ * inline URL is discarded in favor of the source-map
+ * URL so all citations stay routed through Gmail.
+ *
+ * Citations whose ref is missing from the source map are preserved as
+ * literal text so we never silently lose information.
+ */
+export function rehypeCitations(options: {
+ sources: Map<number, ResearchSource>;
+}) {
+ const { sources } = options;
+ if (sources.size === 0) {
+ return () => undefined;
+ }
+
+ function transformText(text: string): HastNode[] | null {
+ if (!text.includes('[')) return null;
+
+ const pieces: HastNode[] = [];
+ let lastIndex = 0;
+ let m: RegExpExecArray | null;
+ CITATION_RE.lastIndex = 0;
+ let changed = false;
+
+ while ((m = CITATION_RE.exec(text)) !== null) {
+ const refs = parseRefList(m[1]);
+ const resolved = refs.map((n) => ({ n, src: sources.get(n) }));
+ const allMatched =
+ resolved.length > 0 && resolved.every((r) => r.src && r.src.url);
+
+ if (m.index > lastIndex) {
+ pieces.push({ type: 'text', value: text.slice(lastIndex, m.index) });
+ }
+
+ if (allMatched) {
+ for (const r of resolved) {
+ pieces.push(buildPill(r.n, r.src!));
+ }
+ changed = true;
+ } else {
+ // Conservative fallback: if any ref in this group is missing source
+ // data, keep the whole bracketed expression as the model wrote it.
+ pieces.push({ type: 'text', value: m[0] });
+ }
+
+ lastIndex = m.index + m[0].length;
+ }
+
+ if (!changed) return null;
+
+ if (lastIndex < text.length) {
+ pieces.push({ type: 'text', value: text.slice(lastIndex) });
+ }
+ return pieces;
+ }
+
+ function tryMarkdownCitationLink(node: HastNode): HastNode | null {
+ // `[1](url)` → <a href="url">1</a>. If the link text is just a number
+ // and we have a source for that ref, replace with our pill.
+ if (
+ node.type !== 'element' ||
+ node.tagName !== 'a' ||
+ !node.children ||
+ node.children.length !== 1
+ ) {
+ return null;
+ }
+ const child = node.children[0];
+ if (child.type !== 'text' || typeof child.value !== 'string') return null;
+
+ const trimmed = child.value.trim();
+ if (!/^\d+$/.test(trimmed)) return null;
+
+ const n = parseInt(trimmed, 10);
+ const src = sources.get(n);
+ if (!src || !src.url) return null;
+
+ return buildPill(n, src);
+ }
+
+ function walk(node: HastNode): void {
+ if (!node.children || node.children.length === 0) return;
+ if (node.tagName && SKIP_TAGS.has(node.tagName)) return;
+
+ const out: HastNode[] = [];
+ let changed = false;
+
+ for (const child of node.children) {
+ // 1. Markdown citation link: [1](url)
+ const promoted = tryMarkdownCitationLink(child);
+ if (promoted) {
+ out.push(promoted);
+ changed = true;
+ continue;
+ }
+
+ // 2. Other elements — recurse but don't transform inside <a>/code/etc.
+ if (child.type === 'element') {
+ if (child.tagName !== 'a') walk(child);
+ out.push(child);
+ continue;
+ }
+
+ // 3. Text node — look for [N] and [N, M, …] patterns.
+ if (child.type === 'text' && typeof child.value === 'string') {
+ const replaced = transformText(child.value);
+ if (replaced) {
+ out.push(...replaced);
+ changed = true;
+ } else {
+ out.push(child);
+ }
+ continue;
+ }
+
+ out.push(child);
+ }
+
+ if (changed) {
+ node.children = out;
+ }
+ }
+
+ return (tree: HastNode) => {
+ walk(tree);
+ };
+}
diff --git a/frontend/src/lib/sse.ts b/frontend/src/lib/sse.ts
index d00fae92..ffe7baff 100644
--- a/frontend/src/lib/sse.ts
+++ b/frontend/src/lib/sse.ts
@@ -1,4 +1,4 @@
-import type { SSEEvent } from '../types';
+import type { ResearchEvent, SSEEvent } from '../types';
import { getBase } from './api';
export interface ChatRequest {
@@ -57,3 +57,52 @@ export async function* streamChat(
reader.releaseLock();
}
}
+
+export async function* streamResearch(
+ query: string,
+ signal?: AbortSignal,
+): AsyncGenerator<ResearchEvent> {
+ // /api/research is mounted at the server root — strip any trailing /v1
+ // from the base so configurations like "http://host:8000/v1" still resolve.
+ const base = getBase().replace(/\/v1\/?$/, '');
+ const response = await fetch(`${base}/api/research`, {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json' },
+ body: JSON.stringify({ query }),
+ signal,
+ });
+
+ if (!response.ok) {
+ throw new Error(`Research request failed: ${response.status}`);
+ }
+
+ const reader = response.body!.getReader();
+ const decoder = new TextDecoder();
+ let buffer = '';
+
+ try {
+ while (true) {
+ const { done, value } = await reader.read();
+ if (done) break;
+
+ buffer += decoder.decode(value, { stream: true });
+ const lines = buffer.split('\n');
+ buffer = lines.pop() || '';
+
+ for (const line of lines) {
+ if (!line.startsWith('data: ')) continue;
+ const data = line.slice(6);
+ if (data === '[DONE]') return;
+ try {
+ const parsed = JSON.parse(data) as ResearchEvent;
+ yield parsed;
+ if (parsed.type === 'done') return;
+ } catch {
+ // skip malformed chunks
+ }
+ }
+ }
+ } finally {
+ reader.releaseLock();
+ }
+}
diff --git a/frontend/src/lib/store.ts b/frontend/src/lib/store.ts
index 2b52219a..37a7d84c 100644
--- a/frontend/src/lib/store.ts
+++ b/frontend/src/lib/store.ts
@@ -2,9 +2,12 @@ import { create } from 'zustand';
import type {
Conversation,
ChatMessage,
+ LiveEnergyMetrics,
LogEntry,
ModelInfo,
MessageTelemetry,
+ ResearchSearchTrace,
+ ResearchSource,
SavingsData,
ServerInfo,
StreamState,
@@ -158,16 +161,29 @@ interface AppState {
usage?: TokenUsage,
telemetry?: MessageTelemetry,
audio?: { url: string },
+ researchTraces?: ResearchSearchTrace[],
+ researchSources?: ResearchSource[],
) => void;
setStreamState: (state: Partial<StreamState>) => void;
resetStream: () => void;
+ // Deep Research toggle
+ deepResearch: boolean;
+ setDeepResearch: (on: boolean) => void;
+
// Actions: models & server
setModels: (models: ModelInfo[]) => void;
setModelsLoading: (loading: boolean) => void;
setSelectedModel: (model: string) => void;
setServerInfo: (info: ServerInfo | null) => void;
setSavings: (data: SavingsData | null) => void;
+ incrementSavings: (usage: TokenUsage) => void;
+
+ // Live GPU metrics — streamed from /api/research system_metrics events.
+ // When non-null, the System panel renders this instead of polled values
+ // so Power (W) and Energy (kJ) update in real time during a research run.
+ liveEnergy: LiveEnergyMetrics | null;
+ setLiveEnergy: (data: LiveEnergyMetrics | null) => void;
// Actions: settings
updateSettings: (partial: Partial<Settings>) => void;
@@ -387,6 +403,8 @@ export const useAppStore = create<AppState>((set, get) => {
usage?: TokenUsage,
telemetry?: MessageTelemetry,
audio?: { url: string },
+ researchTraces?: ResearchSearchTrace[],
+ researchSources?: ResearchSource[],
) => {
const store = loadConversations();
const conv = store.conversations[conversationId];
@@ -398,6 +416,8 @@ export const useAppStore = create<AppState>((set, get) => {
if (usage) lastMsg.usage = usage;
if (telemetry) lastMsg.telemetry = telemetry;
if (audio) lastMsg.audio = audio;
+ if (researchTraces) lastMsg.researchTraces = researchTraces;
+ if (researchSources) lastMsg.researchSources = researchSources;
conv.updatedAt = Date.now();
saveConversations(store);
set({ messages: [...conv.messages] });
@@ -412,6 +432,10 @@ export const useAppStore = create<AppState>((set, get) => {
set({ streamState: INITIAL_STREAM });
},
+ // ── Deep Research ─────────────────────────────────────────────
+ deepResearch: false,
+ setDeepResearch: (on: boolean) => set({ deepResearch: on }),
+
// ── Models & server ────────────────────────────────────────────
setModels: (models: ModelInfo[]) => set({ models }),
@@ -419,6 +443,26 @@ export const useAppStore = create<AppState>((set, get) => {
setSelectedModel: (model: string) => set({ selectedModel: model }),
setServerInfo: (info: ServerInfo | null) => set({ serverInfo: info }),
setSavings: (data: SavingsData | null) => set({ savings: data }),
+ incrementSavings: (usage: TokenUsage) => {
+ const cur = get().savings;
+ const prompt = usage.prompt_tokens ?? 0;
+ const completion = usage.completion_tokens ?? 0;
+ const total = usage.total_tokens ?? prompt + completion;
+ set({
+ savings: {
+ total_calls: (cur?.total_calls ?? 0) + 1,
+ total_prompt_tokens: (cur?.total_prompt_tokens ?? 0) + prompt,
+ total_completion_tokens: (cur?.total_completion_tokens ?? 0) + completion,
+ total_tokens: (cur?.total_tokens ?? 0) + total,
+ local_cost: cur?.local_cost ?? 0,
+ per_provider: cur?.per_provider ?? [],
+ token_counting_version: cur?.token_counting_version,
+ },
+ });
+ },
+
+ liveEnergy: null,
+ setLiveEnergy: (data: LiveEnergyMetrics | null) => set({ liveEnergy: data }),
cachedConnectors: null,
setCachedConnectors: (list) => set({ cachedConnectors: list }),
diff --git a/frontend/src/pages/DataSourcesPage.tsx b/frontend/src/pages/DataSourcesPage.tsx
index 5de4b908..a5fce03b 100644
--- a/frontend/src/pages/DataSourcesPage.tsx
+++ b/frontend/src/pages/DataSourcesPage.tsx
@@ -24,7 +24,7 @@ import {
import type { LucideIcon } from 'lucide-react';
import { SOURCE_CATALOG } from '../types/connectors';
import type { ConnectRequest } from '../types/connectors';
-import { listConnectors, connectSource, getSyncStatus, triggerSync } from '../lib/connectors-api';
+import { listConnectors, connectSource, disconnectSource, getSyncStatus, triggerSync } from '../lib/connectors-api';
import type { SyncStatus } from '../types/connectors';
// ---------------------------------------------------------------------------
@@ -312,11 +312,120 @@ const IconFor = ({ id, size = 18 }: { id: string; size?: number }) => {
return <Ico size={size} />;
};
+// The Gmail card unifies the OAuth (`gmail`) and IMAP (`gmail_imap`) backend
+// connectors — both should resolve to the gmail_imap catalog entry so the
+// connected card shows the same name, unit label, and troubleshooting tips
+// regardless of which underlying flow the user picked.
+function metaFor(connectorId: string) {
+ const id = connectorId === 'gmail' ? 'gmail_imap' : connectorId;
+ return SOURCE_CATALOG.find((s) => s.connector_id === id);
+}
+
+// Advanced OAuth disclosure for the unified Gmail card. Hidden by default;
+// expands to a Client ID + Client Secret form that POSTs to the OAuth
+// `gmail` backend connector. Lives here rather than in SOURCE_CATALOG
+// because the Gmail card is the only one with a dual-flow shape.
+function GmailOAuthAdvanced({
+ loading,
+ onConnect,
+}: {
+ loading: boolean;
+ onConnect: (req: ConnectRequest) => void;
+}) {
+ const [open, setOpen] = useState(false);
+ return (
+ <div style={{ marginTop: 12 }}>
+ <button
+ type="button"
+ onClick={() => setOpen((o) => !o)}
+ style={{
+ background: 'transparent',
+ border: 'none',
+ padding: 0,
+ fontSize: 11,
+ color: 'var(--color-text-tertiary)',
+ cursor: 'pointer',
+ textDecoration: 'underline',
+ }}
+ >
+ {open ? 'Hide advanced' : 'Advanced: Connect with Google OAuth'}
+ </button>
+ {open && (
+ <div
+ style={{
+ marginTop: 8,
+ padding: 10,
+ background: 'var(--color-bg)',
+ border: '1px solid var(--color-border)',
+ borderRadius: 6,
+ }}
+ >
+ <div style={{ fontSize: 11, color: 'var(--color-text-tertiary)', marginBottom: 8 }}>
+ For developers with an existing Google Cloud project. Enable the
+ Gmail API and create a Desktop OAuth client at{' '}
+ <a
+ href="https://console.cloud.google.com/apis/credentials"
+ target="_blank"
+ rel="noopener noreferrer"
+ style={{ color: 'var(--color-accent)', textDecoration: 'underline' }}
+ >
+ Google Cloud Credentials →
+ </a>{' '}
+ then paste the Client ID and Client Secret below.
+ </div>
+ <InlineConnectForm
+ fields={[
+ { name: 'email', placeholder: 'Client ID', type: 'text' },
+ { name: 'password', placeholder: 'Client Secret', type: 'password' },
+ ]}
+ loading={loading}
+ onSubmit={onConnect}
+ />
+ </div>
+ )}
+ </div>
+ );
+}
+
// ---------------------------------------------------------------------------
// Data Sources section
// ---------------------------------------------------------------------------
// Sync status display component with progress bar
+function formatTimeAgo(iso: string | null | undefined): string | null {
+ if (!iso) return null;
+ const t = new Date(iso).getTime();
+ if (Number.isNaN(t)) return null;
+ const diffSec = (Date.now() - t) / 1000;
+ if (diffSec < 30) return 'just now';
+ if (diffSec < 60) return 'less than a min ago';
+ if (diffSec < 3600) {
+ const m = Math.round(diffSec / 60);
+ return `${m} min${m === 1 ? '' : 's'} ago`;
+ }
+ if (diffSec < 86400) {
+ const h = Math.round(diffSec / 3600);
+ return `${h} hr${h === 1 ? '' : 's'} ago`;
+ }
+ const d = Math.round(diffSec / 86400);
+ return `${d} day${d === 1 ? '' : 's'} ago`;
+}
+
+/** Render how far back the corpus extends, given the oldest indexed
+ * item's timestamp. Returns null when there isn't enough data yet. */
+function formatBacklogRange(iso: string | null | undefined): string | null {
+ if (!iso) return null;
+ const t = new Date(iso).getTime();
+ if (Number.isNaN(t)) return null;
+ const days = (Date.now() - t) / 86400_000;
+ if (days < 7) return 'past few days';
+ if (days < 30) return 'past month';
+ if (days < 90) return 'past 3 months';
+ if (days < 365) return 'past year';
+ const years = Math.round(days / 365);
+ return `past ${years} year${years === 1 ? '' : 's'}`;
+}
+
function SyncStatusDisplay({
chunks,
sync,
@@ -368,13 +477,65 @@ function SyncStatusDisplay({
);
}
- // Done — has chunks
- if (chunks > 0) {
+ // Treat the SyncEngine's checkpointed items_synced as the source of
+ // truth for "total indexed" — `chunks` from listConnectors counts
+ // embedding chunks (often != source items) and the checkpoint is what
+ // both the syncing and idle branches need to display consistently.
+ const totalIndexed = sync?.items_synced ?? chunks;
+ const itemsTotal = sync?.items_total ?? 0;
+ const backlogRange = formatBacklogRange(sync?.oldest_item_date);
+ // "Complete inbox" — the user has indexed everything reachable. Only
+ // surface this label when idle (during a sync we always show how far
+ // back we've gotten so far).
+ const isComplete =
+ totalIndexed > 0 && itemsTotal > 0 && totalIndexed >= itemsTotal;
+
+ // Actively syncing — single status line + reassurance line.
+ if (sync?.state === 'syncing' || syncing) {
+ const rangeLabel = backlogRange ?? 'building corpus';
+ return (
+ <div>
+ <div style={{ fontSize: 11, color: 'var(--color-warning)', marginBottom: 4 }}>
+ Indexed{' '}
+ <span key={totalIndexed} className="sync-bump">
+ {totalIndexed.toLocaleString()} {unitLabel}
+ </span>{' '}
+ <span style={{ color: 'var(--color-text-tertiary)' }}>
+ ({rangeLabel})
+ </span>{' '}
+ <span style={{ color: 'var(--color-text-tertiary)' }}>
+ · Still indexing…
+ </span>
+ </div>
+ <div style={{ fontSize: 10.5, color: 'var(--color-text-tertiary)' }}>
+ Deep Research available now · results improve as more {unitLabel} are indexed
+ </div>
+ </div>
+ );
+ }
+
+ // Idle — already has indexed items: show the corpus size + range or
+ // "complete inbox" label, plus how long ago we last refreshed it.
+ if (totalIndexed > 0) {
+ const lastSyncLabel = formatTimeAgo(sync?.last_sync);
+ const rangeLabel = isComplete
+ ? 'complete inbox'
+ : backlogRange;
return (
<div>
<div style={{ display: 'flex', alignItems: 'center', gap: 8 }}>
<span style={{ fontSize: 12, color: 'var(--color-success)' }}>
- {chunks.toLocaleString()} {unitLabel}
+ Indexed {totalIndexed.toLocaleString()} {unitLabel}
+ {rangeLabel && (
+ <span style={{ color: 'var(--color-text-tertiary)' }}>
+ {' '}({rangeLabel})
+ </span>
+ )}
+ {lastSyncLabel && (
+ <span style={{ color: 'var(--color-text-tertiary)' }}>
+ {' · '}Last synced {lastSyncLabel}
+ </span>
+ )}
</span>
<button
onClick={handleSync}
@@ -397,70 +558,14 @@ function SyncStatusDisplay({
);
}
- // Actively syncing
- if (sync?.state === 'syncing' || syncing) {
- const pct = sync?.items_total && sync.items_total > 0
- ? Math.round((sync.items_synced / sync.items_total) * 100)
- : null;
- const label = sync?.items_total && sync.items_total > 0
- ? `${sync.items_synced.toLocaleString()} / ${sync.items_total.toLocaleString()}`
- : sync?.items_synced && sync.items_synced > 0
- ? `${sync.items_synced.toLocaleString()} items so far`
- : 'Starting...';
- return (
- <div>
- <div style={{ fontSize: 11, color: 'var(--color-warning)', marginBottom: 4 }}>
- Syncing — {label}
- </div>
- <div style={{
- height: 4, borderRadius: 2,
- background: 'var(--color-bg-tertiary)',
- overflow: 'hidden',
- }}>
- <div style={{
- height: '100%', borderRadius: 2,
- background: 'var(--color-warning)',
- width: pct != null ? `${pct}%` : '30%',
- transition: 'width 0.5s ease',
- animationName: pct == null ? 'pulse' : undefined,
- animationDuration: pct == null ? '1.5s' : undefined,
- animationIterationCount: pct == null ? 'infinite' : undefined,
- }} />
- </div>
- </div>
- );
- }
-
- // Idle with items synced but no chunks yet (indexing)
- if (sync?.state === 'idle' && sync.items_synced > 0) {
- return (
- <div>
- <div style={{ fontSize: 11, color: 'var(--color-warning)', marginBottom: 4 }}>
- Indexing {sync.items_synced.toLocaleString()} items...
- </div>
- <div style={{
- height: 4, borderRadius: 2,
- background: 'var(--color-bg-tertiary)',
- overflow: 'hidden',
- }}>
- <div style={{
- height: '100%', borderRadius: 2, background: 'var(--color-warning)',
- width: '60%',
- animationName: 'pulse', animationDuration: '1.5s', animationIterationCount: 'infinite',
- }} />
- </div>
- </div>
- );
- }
-
- // Connected but no chunks yet
+ // Connected but nothing ever ingested. Mirror the original copy.
const hasSynced = sync?.last_sync != null;
return (
<div>
<div style={{ display: 'flex', alignItems: 'center', gap: 8 }}>
<span style={{ fontSize: 12, color: 'var(--color-text-tertiary)' }}>
{hasSynced
- ? 'Synced — 0 items found'
+ ? `Synced — 0 ${unitLabel} found`
: 'Connected — not synced yet'}
</span>
<button
@@ -546,6 +651,21 @@ function DataSourcesSection() {
const [connectingId, setConnectingId] = useState<string | null>(null);
const [connectStage, setConnectStage] = useState<string>('');
const [connectError, setConnectError] = useState<string>('');
+ const [disconnectingId, setDisconnectingId] = useState<string | null>(null);
+
+ const handleDisconnect = async (id: string) => {
+ if (disconnectingId) return;
+ setDisconnectingId(id);
+ try {
+ await disconnectSource(id);
+ loadConnectors();
+ } catch {
+ // Surface failures silently — the connector list will refresh on the
+ // next poll and reflect the true state regardless.
+ } finally {
+ setDisconnectingId(null);
+ }
+ };
const handleConnect = async (id: string, req: ConnectRequest) => {
setLoading(true);
@@ -598,8 +718,32 @@ function DataSourcesSection() {
}
};
- const connected = connectors.filter((c) => c.connected);
- const notConnectedBase = connectors.filter((c) => !c.connected);
+ // Merge the OAuth Gmail (`gmail`) and IMAP Gmail (`gmail_imap`) backend
+ // connectors into a single user-facing Gmail card. IMAP is the default
+ // flow (no Google Cloud setup needed); OAuth lives behind an "Advanced"
+ // disclosure when the card is expanded. If both happen to be connected,
+ // keep whichever has more indexed chunks so the active source still
+ // surfaces its sync state.
+ const unifiedConnectors = (() => {
+ const gmail = connectors.find((c) => c.connector_id === 'gmail');
+ const gmailImap = connectors.find((c) => c.connector_id === 'gmail_imap');
+ if (!gmail || !gmailImap) return connectors;
+ if (gmail.connected && !gmailImap.connected) {
+ return connectors.filter((c) => c.connector_id !== 'gmail_imap');
+ }
+ if (gmailImap.connected && !gmail.connected) {
+ return connectors.filter((c) => c.connector_id !== 'gmail');
+ }
+ if (gmail.connected && gmailImap.connected) {
+ const dropId = gmail.chunks >= gmailImap.chunks ? 'gmail_imap' : 'gmail';
+ return connectors.filter((c) => c.connector_id !== dropId);
+ }
+ // Neither connected — show only the IMAP card as the default flow.
+ return connectors.filter((c) => c.connector_id !== 'gmail');
+ })();
+
+ const connected = unifiedConnectors.filter((c) => c.connected);
+ const notConnectedBase = unifiedConnectors.filter((c) => !c.connected);
// Always show the upload card in the not-connected list (it has no backend connector)
const uploadEntry = { connector_id: 'upload', display_name: 'Upload / Paste', connected: false, chunks: 0 };
const notConnected = notConnectedBase.some((c) => c.connector_id === 'upload')
@@ -642,10 +786,9 @@ function DataSourcesSection() {
</div>
<div className="flex flex-col gap-2">
{connected.map((c) => {
- const meta = SOURCE_CATALOG.find(s => s.connector_id === c.connector_id);
+ const meta = metaFor(c.connector_id);
const unit = meta?.unitLabel || 'items';
const sync = syncStatuses[c.connector_id];
- const isReconnecting = expandedId === c.connector_id;
const hasError = !!sync?.error;
return (
<div
@@ -663,7 +806,7 @@ function DataSourcesSection() {
}}>
<div style={{ flex: 1, minWidth: 0 }}>
<div className="font-semibold" style={{ fontSize: 14, fontWeight: 600, color: 'var(--color-text)' }}>
- {c.display_name}
+ {meta?.display_name ?? c.display_name}
</div>
<SyncStatusDisplay
chunks={c.chunks}
@@ -674,60 +817,23 @@ function DataSourcesSection() {
/>
</div>
<button
- onClick={() => setExpandedId(isReconnecting ? null : c.connector_id)}
+ onClick={() => handleDisconnect(c.connector_id)}
+ disabled={disconnectingId === c.connector_id}
className="hud-label"
style={{
padding: '6px 12px',
background: 'transparent',
color: 'var(--color-text-secondary)',
border: '1px solid var(--color-border)',
- borderRadius: 4, cursor: 'pointer',
+ borderRadius: 4,
+ cursor: disconnectingId === c.connector_id ? 'default' : 'pointer',
letterSpacing: '0.15em',
+ opacity: disconnectingId === c.connector_id ? 0.5 : 1,
}}
>
- {isReconnecting ? 'Cancel' : 'Reconnect'}
+ {disconnectingId === c.connector_id ? 'Disconnecting…' : 'Disconnect'}
</button>
</div>
- {isReconnecting && meta?.steps && (
- <div style={{ borderTop: '1px solid var(--color-border)', padding: 12 }}>
- <div style={{ fontSize: 12, color: 'var(--color-warning)', marginBottom: 8 }}>
- Re-enter credentials to reconnect this source.
- </div>
- {meta.steps.map((step, i) => (
- <div
- key={i}
- style={{
- background: 'var(--color-bg)',
- border: '1px solid var(--color-border)',
- borderRadius: 6, padding: 10,
- marginBottom: 8,
- }}
- >
- <div style={{ color: 'var(--color-accent-purple)', fontSize: 10, fontWeight: 600, marginBottom: 3 }}>
- STEP {i + 1}
- </div>
- <div style={{ fontSize: 12, marginBottom: step.url ? 4 : 0 }}>{step.label}</div>
- {step.url && (
- <a
- href={step.url}
- target="_blank"
- rel="noopener noreferrer"
- style={{ color: 'var(--color-accent)', fontSize: 11, textDecoration: 'underline' }}
- >
- {step.urlLabel || 'Open'} &rarr;
- </a>
- )}
- </div>
- ))}
- {meta.inputFields && (
- <InlineConnectForm
- fields={meta.inputFields}
- loading={loading}
- onSubmit={(req) => handleConnect(c.connector_id, req)}
- />
- )}
- </div>
- )}
</div>
);
})}
@@ -744,7 +850,7 @@ function DataSourcesSection() {
</div>
<div className="grid grid-cols-2 gap-2">
{notConnected.map((c) => {
- const meta = SOURCE_CATALOG.find(s => s.connector_id === c.connector_id);
+ const meta = metaFor(c.connector_id);
const isExpanded = expandedId === c.connector_id;
return (
@@ -767,10 +873,10 @@ function DataSourcesSection() {
>
<div style={{ flex: 1, minWidth: 0 }}>
<div className="font-semibold" style={{ fontSize: 14, fontWeight: 600, color: 'var(--color-text)' }}>
- {c.display_name}
+ {meta?.display_name ?? c.display_name}
</div>
<div style={{ fontSize: 11, color: 'var(--color-text-tertiary)', marginTop: 2 }}>
- Not connected
+ {meta?.description ?? 'Not connected'}
</div>
</div>
<span style={{ color: 'var(--color-text-secondary)', fontSize: 12, fontWeight: 500 }}>
@@ -822,6 +928,12 @@ function DataSourcesSection() {
onSubmit={(req) => handleConnect(c.connector_id, req)}
/>
)}
+ {c.connector_id === 'gmail_imap' && (
+ <GmailOAuthAdvanced
+ loading={loading && connectingId === 'gmail'}
+ onConnect={(req) => handleConnect('gmail', req)}
+ />
+ )}
{meta?.troubleshooting && (
<details className="mt-2">
<summary className="text-[11px] cursor-pointer" style={{ color: 'var(--color-text-tertiary)' }}>
diff --git a/frontend/src/types/connectors.ts b/frontend/src/types/connectors.ts
index cb212be6..534a2a80 100644
--- a/frontend/src/types/connectors.ts
+++ b/frontend/src/types/connectors.ts
@@ -36,6 +36,13 @@ export interface SyncStatus {
state: "idle" | "syncing" | "paused" | "error";
items_synced: number;
items_total: number;
+ /** Items processed in the current (or most recent) run only. `null`
+ * when no sync has been triggered through this server session yet. */
+ new_items_synced?: number | null;
+ /** ISO 8601 timestamp of the oldest indexed item, used to label how far
+ * back the corpus reaches ("past 3 months", "past 5 years"). `null`
+ * before anything is indexed. */
+ oldest_item_date?: string | null;
last_sync: string | null;
error: string | null;
}
@@ -73,6 +80,9 @@ export const SOURCE_CATALOG: ConnectorMeta[] = [
},
// ── Communication ──────────────────────────────────────────────────
{
+ // Unified Gmail card. Defaults to the IMAP (app-password) flow because
+ // it needs no Google Cloud setup; the OAuth path is offered as an
+ // "Advanced" disclosure rendered in DataSourcesPage.
connector_id: 'gmail_imap',
display_name: 'Gmail',
auth_type: 'oauth',
@@ -83,23 +93,14 @@ export const SOURCE_CATALOG: ConnectorMeta[] = [
unitLabel: 'emails',
steps: [
{
- label: 'Go to your Google Account \u2192 Security \u2192 2-Step Verification. Make sure it\'s turned ON. App Passwords only work if 2-Step Verification is enabled.',
- url: 'https://myaccount.google.com/signinoptions/two-step-verification',
- urlLabel: 'Open Google Security \u2192',
- },
- {
- label: 'Go to App Passwords. Select app: "Mail", device: "Other" and type "OpenJarvis". Click Generate. This does NOT open a login popup \u2014 you\'ll get a 16-character password to copy (you won\'t see it again).',
+ label: 'Make sure 2-Step Verification is enabled, then generate a 16-character App Password (Mail / Other / "OpenJarvis"). Paste it below \u2014 spaces are fine, and use the app password, not your regular Gmail password.',
url: 'https://myaccount.google.com/apppasswords',
- urlLabel: 'Open App Passwords \u2192',
- },
- {
- label: 'Paste your Gmail address and the 16-character app password below (spaces are fine). Use the app password, NOT your regular Gmail password.',
+ urlLabel: 'How to get an app password \u2192',
},
],
troubleshooting: [
"Don't see App Passwords? Make sure 2-Step Verification is enabled first.",
"Google Workspace user? Your admin may need to enable App Passwords for your organization.",
- "Want OAuth instead? Use the Google Drive connector \u2014 it covers Gmail content too.",
],
inputFields: [
{ name: 'email', placeholder: '[email protected]', type: 'text' },
diff --git a/frontend/src/types/index.ts b/frontend/src/types/index.ts
index 923fe165..4ecbb62d 100644
--- a/frontend/src/types/index.ts
+++ b/frontend/src/types/index.ts
@@ -61,12 +61,69 @@ export interface MessageTelemetry {
suggested_max_tokens?: number;
}
+export interface TimeRange {
+ start?: string;
+ end?: string;
+}
+
+export interface ResearchSource {
+ ref: number;
+ title?: string;
+ sender?: string;
+ date?: string;
+ url?: string;
+}
+
+export interface ResearchSearchTrace {
+ id: string;
+ query: string;
+ person?: string;
+ timeRange?: TimeRange | string;
+ status: 'pending' | 'complete';
+ numHits?: number;
+ topTitles?: string[];
+}
+
+export type ResearchEvent =
+ | {
+ type: 'search_call';
+ arguments: {
+ query: string;
+ person?: string;
+ time_range?: TimeRange | string;
+ };
+ }
+ | {
+ type: 'search_result';
+ num_hits: number;
+ top_titles?: string[];
+ sources?: ResearchSource[];
+ }
+ | { type: 'synthesis'; text: string }
+ | {
+ type: 'system_metrics';
+ power_w: number;
+ energy_j: number;
+ duration_s: number;
+ }
+ | { type: 'done'; usage?: TokenUsage }
+ | { type: 'error'; message: string };
+
+export interface LiveEnergyMetrics {
+ power_w: number;
+ energy_j: number;
+ duration_s: number;
+}
+
export interface ChatMessage {
id: string;
role: 'user' | 'assistant';
content: string;
timestamp: number;
toolCalls?: ToolCallInfo[];
+ researchTraces?: ResearchSearchTrace[];
+ researchSources?: ResearchSource[];
+ isResearch?: boolean;
usage?: TokenUsage;
telemetry?: MessageTelemetry;
audio?: { url: string };
diff --git a/frontend/vite.config.ts b/frontend/vite.config.ts
index cccf520f..2cc44afc 100644
--- a/frontend/vite.config.ts
+++ b/frontend/vite.config.ts
@@ -29,7 +29,7 @@ export default defineConfig({
},
workbox: {
globPatterns: ['**/*.{js,css,html,ico,png,svg}'],
- navigateFallbackDenylist: [/^\/v1\//, /^\/health/, /^\/dashboard/],
+ navigateFallbackDenylist: [/^\/v1\//, /^\/health/, /^\/dashboard/, /^\/api\//],
},
}),
],
@@ -53,6 +53,7 @@ export default defineConfig({
proxy: {
'/v1': process.env.VITE_API_URL || 'http://localhost:8000',
'/health': process.env.VITE_API_URL || 'http://localhost:8000',
+ '/api': process.env.VITE_API_URL || 'http://localhost:8000',
},
},
});
diff --git a/pyproject.toml b/pyproject.toml
index 159f6edf..1bc774b8 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -30,6 +30,7 @@ dependencies = [
"httpx>=0.27",
"openai>=1.30",
"posthog>=3.0",
+ "pynvml>=13.0.1",
"python-telegram-bot>=22.6",
"rich>=13",
"tomli>=2.0; python_version < '3.11'",
@@ -183,6 +184,10 @@ select = ["E", "F", "I", "W"]
# hybrid/ is research code with long prompt strings and paradigm-specific
# config dicts — same relaxation as evals research code above.
"src/openjarvis/agents/hybrid/*.py" = ["E501"]
+# research_loop.py carries the multi-paragraph planner system prompt as
+# inline string literals; line-length wrapping would harm readability of
+# the prompt itself.
+"src/openjarvis/agents/research_loop.py" = ["E501"]
[dependency-groups]
dev = [
diff --git a/src/openjarvis/agents/executor.py b/src/openjarvis/agents/executor.py
index 6a5fe6e0..8d6be76f 100644
--- a/src/openjarvis/agents/executor.py
+++ b/src/openjarvis/agents/executor.py
@@ -93,20 +93,34 @@ class AgentExecutor:
)
return agent.run(input_text)
- def execute_tick(self, agent_id: str) -> None:
+ def execute_tick(
+ self, agent_id: str, *, lock_already_held: bool = False
+ ) -> None:
"""Run one tick for the given agent.
1. Acquire concurrency guard (start_tick)
2. Invoke agent with retry logic
3. Update stats
4. Release guard (end_tick)
+
+ ``lock_already_held`` is set by callers that took the start_tick()
+ lock themselves before spawning the worker (e.g. the HTTP /run route
+ guards against concurrent POSTs by acquiring before threading).
+ Without this flag, the executor would re-acquire and trip its own
+ guard — bailing out with no end_tick(), leaving the agent stuck in
+ ``status='running'`` forever.
"""
- try:
- self._manager.start_tick(agent_id)
+ if lock_already_held:
self._set_activity(agent_id, "Preparing tick...")
- except ValueError:
- logger.warning("Agent %s already running, skipping tick", agent_id)
- return
+ else:
+ try:
+ self._manager.start_tick(agent_id)
+ self._set_activity(agent_id, "Preparing tick...")
+ except ValueError:
+ logger.warning(
+ "Agent %s already running, skipping tick", agent_id
+ )
+ return
agent = self._manager.get_agent(agent_id)
if agent is None:
diff --git a/src/openjarvis/agents/manager.py b/src/openjarvis/agents/manager.py
index 5b03d38d..6eed3302 100644
--- a/src/openjarvis/agents/manager.py
+++ b/src/openjarvis/agents/manager.py
@@ -7,6 +7,7 @@ to the five existing primitives (Intelligence, Agent, Tools, Engine, Learning).
from __future__ import annotations
import json
+import logging
import sqlite3
import time
import uuid
@@ -14,6 +15,8 @@ from pathlib import Path
from typing import Any, Dict, List, Optional
from uuid import uuid4
+logger = logging.getLogger(__name__)
+
_CREATE_AGENTS = """\
CREATE TABLE IF NOT EXISTS managed_agents (
id TEXT PRIMARY KEY,
@@ -123,6 +126,33 @@ class AgentManager:
except sqlite3.OperationalError:
pass # Column already exists
self._conn.commit()
+ self._clear_stale_running_state()
+
+ def _clear_stale_running_state(self) -> None:
+ """Reset any agent stuck in ``status='running'`` on startup.
+
+ Tick worker threads are ``daemon=True`` — when the server process
+ exits (SIGTERM, crash, restart), they die without running the
+ ``finally`` clause that calls :meth:`end_tick`, leaving the DB row
+ in ``running`` forever. The :meth:`start_tick` guard then rejects
+ every subsequent run with "Agent is already running".
+
+ A freshly-started process holds zero tick locks by definition, so
+ any persisted ``running`` is a zombie. Sweep it back to ``idle``
+ and clear the activity string so the UI doesn't show a stale
+ "Preparing tick..." indicator.
+ """
+ cur = self._conn.execute(
+ "UPDATE managed_agents SET status = 'idle', current_activity = '',"
+ " updated_at = ? WHERE status = 'running'",
+ (time.time(),),
+ )
+ self._conn.commit()
+ if cur.rowcount:
+ logger.info(
+ "AgentManager: cleared stale 'running' status on %d agent(s)",
+ cur.rowcount,
+ )
def close(self) -> None:
self._conn.close()
diff --git a/src/openjarvis/agents/research_loop.py b/src/openjarvis/agents/research_loop.py
new file mode 100644
index 00000000..3a1e85ae
--- /dev/null
+++ b/src/openjarvis/agents/research_loop.py
@@ -0,0 +1,683 @@
+"""Agentic research loop over the hybrid-search tool.
+
+A small, self-contained planner-executor loop:
+
+* the planner is a local Ollama chat model (default ``gemma4:31b``),
+* the only tool it can call is :meth:`HybridSearch.search`,
+* it gets up to ``max_iterations`` tool calls,
+* tool results are trimmed before re-entering the context window, and
+* the final reply must cite specific hits.
+
+The loop is deliberately decoupled from the rest of the agent scaffolding
+(`ToolUsingAgent`, `EventBus`, `AgentContext`, etc.) so the surface stays
+small. Anything that wants tracing or registry integration can wrap it.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import sys
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Any, Callable, Dict, List, Optional
+
+from openjarvis.connectors.hybrid_search import HybridSearch, SearchHit
+from openjarvis.core.types import Message, Role, ToolCall
+from openjarvis.engine._base import InferenceEngine
+
+logger = logging.getLogger(__name__)
+
+
+DEFAULT_PLANNER_MODEL = "gemma4:31b"
+
+
+CLARIFY_TOOL_SPEC: Dict[str, Any] = {
+ "type": "function",
+ "function": {
+ "name": "clarify",
+ "description": (
+ "Ask the user a clarifying question and wait for their answer. "
+ "Only use AFTER at least one search has been attempted. Use when "
+ "search results are ambiguous (e.g. three different people share "
+ "a first name), search returned zero results and the query likely "
+ "needs reframing, or the scope is too broad to synthesize "
+ "meaningfully. Never use clarify before searching."
+ ),
+ "parameters": {
+ "type": "object",
+ "properties": {
+ "question": {
+ "type": "string",
+ "description": (
+ "The clarifying question to ask the user. Be specific "
+ "about what you need to know to make progress."
+ ),
+ },
+ },
+ "required": ["question"],
+ },
+ },
+}
+
+
+SEARCH_TOOL_SPEC: Dict[str, Any] = {
+ "type": "function",
+ "function": {
+ "name": "search",
+ "description": (
+ "Hybrid search over the user's personal knowledge corpus (emails, "
+ "notes, calendar events, attachments). Combines BM25 lexical match "
+ "with dense embedding similarity, ranked by reciprocal rank fusion. "
+ "Use structured filters (person, time_range, sources) whenever the "
+ "user names a specific person or time window. Each call returns up "
+ "to 'limit' results with content snippets and thread context."
+ ),
+ "parameters": {
+ "type": "object",
+ "properties": {
+ "query": {
+ "type": "string",
+ "description": (
+ "Natural-language query. Use the topic the user is asking "
+ "about. Can be empty when filtering purely by person or "
+ "time (e.g. 'list all mail from Kelly in May')."
+ ),
+ },
+ "person": {
+ "type": "string",
+ "description": (
+ "Filter to messages involving this person. Matches a "
+ "substring of the name or email address — 'Kelly' or "
+ "'@tldrnewsletter.com' both work."
+ ),
+ },
+ "time_range": {
+ "type": "object",
+ "description": "ISO 8601 datetime range. Either bound may be omitted.",
+ "properties": {
+ "start": {"type": "string", "description": "ISO 8601 start"},
+ "end": {"type": "string", "description": "ISO 8601 end"},
+ },
+ },
+ "sources": {
+ "type": "array",
+ "description": "Restrict to these connectors (e.g. ['gmail']).",
+ "items": {"type": "string"},
+ },
+ "limit": {
+ "type": "integer",
+ "description": "Max results to return (default 20, cap 20).",
+ "default": 20,
+ },
+ },
+ "required": ["query"],
+ },
+ },
+}
+
+
+SYSTEM_PROMPT = """You are a research assistant with access to the user's personal knowledge corpus (their email, notes, calendar). You answer questions by calling two tools:
+
+ search(query, person=None, time_range=None, sources=None, limit=20)
+ clarify(question)
+
+Strategy:
+ 1. If the user names a person, ALWAYS pass `person=` rather than relying on lexical match. Hybrid search will fuzzy-match name or address fragments.
+ 2. When the user mentions ANY time window — "this past week", "recently", "last month", "past few days", "yesterday" — you MUST translate it to a `time_range` parameter. Today is {today}.
+ 3. The `time_range` argument is a JSON object: `{{"start": "<ISO 8601>", "end": "<ISO 8601>"}}`. Either bound may be omitted, but pass at least one whenever the user gave you a temporal cue.
+ 4. If the first structured search returns nothing useful, broaden with a semantic query and drop filters one at a time.
+ 5. You have a clarify tool. Only use it AFTER at least one search attempt. Use it when: you found multiple ambiguous matches (e.g. 3 different people named John), search returned zero results and the query might need reframing, or the scope is too broad to synthesize meaningfully. Never use clarify before searching — always try first.
+ 6. After receiving a clarify response, use the information to construct a precise search with the correct person, time_range, and query parameters. Never send an empty query or a query with no parameters — extract every concrete signal from the user's reply (names, dates, topics) and put it on the call.
+ 7. Tool calls — search AND clarify — share a budget of 5 total. Spend wisely.
+
+Synthesis rules:
+ - Cite sources as individual numbers in square brackets. Always separate — write [4] [7] [20], never [4, 7, 20]. Never format citations as markdown links. Just the number in brackets: [1]. The `ref` field on each hit is the citation number.
+ - Quote sender / date / subject when relevant — the user wants attribution.
+ - If the search returned nothing relevant, say so plainly. Do not invent results.
+ - Only state facts that appear in the retrieved search results. Never supplement with your own knowledge or training data. If you are unsure whether a fact came from the search results, do not include it.
+
+Today's date is {today}.
+"""
+
+
+# ---------------------------------------------------------------------------
+# Tool-result shaping
+# ---------------------------------------------------------------------------
+
+
+def _trim_thread_context(ctx: List[Dict[str, Any]], cap: int) -> List[Dict[str, Any]]:
+ """Keep the first ``cap`` entries; mark elision when trimming."""
+ if len(ctx) <= cap:
+ return ctx
+ trimmed = list(ctx[:cap])
+ trimmed.append({"snippet": f"… {len(ctx) - cap} more chunks in thread …"})
+ return trimmed
+
+
+def shape_results_for_model(
+ hits: List[SearchHit],
+ *,
+ detailed_top: int = 5,
+ thread_ctx_per_hit: int = 3,
+ total_cap: int = 20,
+) -> Dict[str, Any]:
+ """Compact a hit list into a JSON payload the planner can chew through.
+
+ The first ``detailed_top`` rows keep their content snippet and trimmed
+ thread context; the remainder are summarised to title + sender + date so
+ the planner still sees the breadth of what's available without blowing the
+ context window. Each hit gets a 1-indexed numeric ``ref`` so the synthesis
+ can cite it as ``[N]``.
+ """
+ out_hits: List[Dict[str, Any]] = []
+ visible = hits[:total_cap]
+ for i, h in enumerate(visible):
+ sender = h.participants[0] if h.participants else ""
+ base = {
+ "ref": i + 1,
+ "title": h.title,
+ "sender": sender,
+ "timestamp": h.timestamp,
+ "source": h.source,
+ "score": round(h.score, 4),
+ }
+ if i < detailed_top:
+ base["snippet"] = h.content_snippet
+ if h.thread_context:
+ base["thread"] = _trim_thread_context(h.thread_context, thread_ctx_per_hit)
+ out_hits.append(base)
+ return {
+ "num_results": len(hits),
+ "shown": len(visible),
+ "truncated": len(hits) > total_cap,
+ "hits": out_hits,
+ }
+
+
+def _hit_date(timestamp: str) -> str:
+ """Pull a ``YYYY-MM-DD`` date out of a SearchHit timestamp (best effort)."""
+ if not timestamp:
+ return ""
+ try:
+ return datetime.fromisoformat(timestamp.replace("Z", "+00:00")).date().isoformat()
+ except (ValueError, AttributeError):
+ return str(timestamp)[:10]
+
+
+def _bare_doc_id(source: str, document_id: str) -> str:
+ """Strip the connector prefix from a stored ``doc_id``.
+
+ Gmail ingest writes ``doc_id="gmail:<hex_message_id>"`` so that ids stay
+ unique across connectors. The Gmail web UI only resolves the bare hex
+ message id; passing the full prefixed form 404s and bounces the user
+ back to the inbox. Other connectors can be added here as we link out
+ to them.
+ """
+ if not document_id:
+ return ""
+ prefix = f"{source}:"
+ if source and document_id.startswith(prefix):
+ return document_id[len(prefix):]
+ return document_id
+
+
+def _hit_url(source: str, document_id: str) -> str:
+ """Build a clickable URL for a hit, when we know how to link it.
+
+ Gmail is the only source we currently link out for; everything else
+ returns an empty string and the client falls back to non-clickable
+ citation chips.
+
+ Two id flavors land here:
+
+ - **Hex message id** (``19dfa2ccbeff78b0``) — what the OAuth Gmail
+ connector stores. Resolves directly via ``#all/<id>`` permalink.
+ - **RFC822 Message-ID** (``<[email protected]>``) — what the IMAP
+ connector stores, since IMAP doesn't expose Gmail's internal hex
+ id. The permalink form would 404; instead route through Gmail's
+ search URL with the ``rfc822msgid:`` operator, which lands the user
+ on the specific message.
+ """
+ if source == "gmail" and document_id:
+ msg_id = _bare_doc_id(source, document_id)
+ if not msg_id:
+ return ""
+ if "@" in msg_id or "<" in msg_id or ">" in msg_id:
+ rfc_id = msg_id.strip("<>")
+ return f"https://mail.google.com/mail/u/0/#search/rfc822msgid:{rfc_id}"
+ return f"https://mail.google.com/mail/u/0/#all/{msg_id}"
+ return ""
+
+
+def build_sources_for_client(
+ hits: List[SearchHit],
+ *,
+ total_cap: int = 20,
+) -> List[Dict[str, Any]]:
+ """Produce the citation-friendly sources list streamed to the frontend.
+
+ One entry per hit, in the same order the planner sees them — so a
+ ``[N]`` citation in the synthesis maps to ``sources[N - 1]`` on the
+ client. We don't deduplicate by ``document_id``: separate chunks of the
+ same email each get their own citation slot since the planner may quote
+ different parts.
+ """
+ out: List[Dict[str, Any]] = []
+ for i, h in enumerate(hits[:total_cap]):
+ sender = h.participants[0] if h.participants else ""
+ out.append(
+ {
+ "ref": i + 1,
+ "title": h.title,
+ "sender": sender,
+ "date": _hit_date(h.timestamp),
+ "source_id": _bare_doc_id(h.source, h.document_id),
+ "url": _hit_url(h.source, h.document_id),
+ }
+ )
+ return out
+
+
+# ---------------------------------------------------------------------------
+# Agent
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class ToolInvocation:
+ """One tool call together with what the planner asked for and got.
+
+ ``tool_name`` is ``"search"`` or ``"clarify"``. For search calls,
+ ``num_results``, ``top_titles`` and ``raw_hits`` are populated; for
+ clarify calls, ``response`` holds the user's answer.
+ """
+
+ arguments: Dict[str, Any]
+ num_results: int = 0
+ top_titles: List[str] = field(default_factory=list)
+ raw_hits: List[SearchHit] = field(default_factory=list)
+ tool_name: str = "search"
+ response: str = ""
+
+
+def _default_clarify_handler(question: str) -> str:
+ """Prompt the user on stdout and read a one-line answer from stdin.
+
+ Empty answers are echoed back as a sentinel so the planner doesn't think
+ the user was silent because of an upstream error.
+ """
+ print(file=sys.stderr)
+ print(f"\033[1m🤔 Clarification needed:\033[0m {question}", file=sys.stderr)
+ try:
+ answer = input("> ").strip()
+ except EOFError:
+ return "(no answer provided)"
+ return answer or "(user did not provide a clarification)"
+
+
+@dataclass
+class ResearchResult:
+ answer: str
+ iterations: int
+ tool_calls: List[ToolInvocation]
+ usage: Dict[str, int] = field(default_factory=dict)
+
+
+class ResearchAgent:
+ """Planner + executor loop over a single hybrid-search tool.
+
+ Parameters
+ ----------
+ engine:
+ An ``InferenceEngine`` that supports OpenAI-style ``tools`` in
+ ``generate`` (Ollama with a tool-capable model).
+ search:
+ The HybridSearch instance the planner can call.
+ model:
+ Planner model tag (default ``gemma4:31b``).
+ max_iterations:
+ Hard ceiling on tool calls before the loop is forced into synthesis.
+ temperature, max_tokens, num_ctx:
+ Generation parameters passed through to ``engine.generate``.
+ on_event:
+ Optional callback fired at loop milestones so callers (e.g. the SSE
+ research router) can stream progress without rewriting the loop.
+ Receives a dict in one of these shapes:
+ - ``{"type": "search_call", "arguments": {...}}`` — about to call search
+ - ``{"type": "search_result", "num_hits": N, "top_titles": [...], "sources": [{"ref": 1, "title": ..., "sender": ..., "date": ..., "source_id": ..., "url": ...}, ...]}`` — search returned
+ - ``{"type": "clarify_call", "question": "..."}`` — about to ask for clarification
+ - ``{"type": "clarify_response", "response": "..."}`` — clarification received
+ - ``{"type": "final_answer", "text": "..."}`` — synthesis ready
+ The callback runs on the same thread as ``run`` and must be non-blocking.
+ """
+
+ def __init__(
+ self,
+ engine: InferenceEngine,
+ search: HybridSearch,
+ *,
+ model: str = DEFAULT_PLANNER_MODEL,
+ max_iterations: int = 5,
+ temperature: float = 0.3,
+ max_tokens: int = 1500,
+ num_ctx: int = 16384,
+ clarify_handler: Optional[Callable[[str], str]] = None,
+ on_event: Optional[Callable[[Dict[str, Any]], None]] = None,
+ ) -> None:
+ self._engine = engine
+ self._search = search
+ self._model = model
+ self._max_iterations = int(max_iterations)
+ self._temperature = float(temperature)
+ self._max_tokens = int(max_tokens)
+ self._num_ctx = int(num_ctx)
+ self._clarify_handler = clarify_handler or _default_clarify_handler
+ self._on_event = on_event
+
+ def _emit(self, event: Dict[str, Any]) -> None:
+ """Fire ``self._on_event`` if set; swallow callback errors."""
+ if self._on_event is None:
+ return
+ try:
+ self._on_event(event)
+ except Exception as exc: # noqa: BLE001
+ logger.debug("on_event callback raised %s — ignoring", exc)
+
+ # ------------------------------------------------------------------
+ # Argument parsing
+ # ------------------------------------------------------------------
+
+ @staticmethod
+ def _parse_time_range(raw: Any):
+ if not raw or not isinstance(raw, dict):
+ return None
+ def _maybe(v):
+ if not v:
+ return None
+ try:
+ return datetime.fromisoformat(str(v).replace("Z", "+00:00"))
+ except ValueError:
+ return None
+ start = _maybe(raw.get("start"))
+ end = _maybe(raw.get("end"))
+ if start is None and end is None:
+ return None
+ return (start, end)
+
+ def _execute_search(self, args: Dict[str, Any]) -> ToolInvocation:
+ query = str(args.get("query", "") or "")
+ person = args.get("person") or None
+ time_range = self._parse_time_range(args.get("time_range"))
+ sources = args.get("sources") or None
+ if sources and not isinstance(sources, list):
+ sources = [str(sources)]
+ limit = int(args.get("limit", 20) or 20)
+ limit = max(1, min(limit, 20))
+
+ hits = self._search.search(
+ query,
+ person=person,
+ time_range=time_range,
+ sources=sources,
+ limit=limit,
+ )
+ titles = [h.title or (h.content_snippet[:60] + "…") for h in hits[:5]]
+ return ToolInvocation(
+ tool_name="search",
+ arguments={
+ "query": query,
+ "person": person,
+ "time_range": (
+ {"start": time_range[0].isoformat() if time_range and time_range[0] else None,
+ "end": time_range[1].isoformat() if time_range and time_range[1] else None}
+ if time_range else None
+ ),
+ "sources": sources,
+ "limit": limit,
+ },
+ num_results=len(hits),
+ top_titles=titles,
+ raw_hits=hits,
+ )
+
+ def _execute_clarify(self, args: Dict[str, Any]) -> ToolInvocation:
+ question = str(args.get("question", "") or "").strip()
+ if not question:
+ return ToolInvocation(
+ tool_name="clarify",
+ arguments={"question": ""},
+ response="(no question provided by agent — skipping clarify)",
+ )
+ answer = self._clarify_handler(question)
+ return ToolInvocation(
+ tool_name="clarify",
+ arguments={"question": question},
+ response=answer,
+ )
+
+ # ------------------------------------------------------------------
+ # Loop
+ # ------------------------------------------------------------------
+
+ def run(self, query: str) -> ResearchResult:
+ """Run the loop end-to-end and return the synthesis plus a trace."""
+ sys_msg = Message(
+ role=Role.SYSTEM,
+ content=SYSTEM_PROMPT.format(today=datetime.now().isoformat(timespec="minutes")),
+ )
+ messages: List[Message] = [sys_msg, Message(role=Role.USER, content=query)]
+
+ invocations: List[ToolInvocation] = []
+ total_usage = {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}
+
+ iterations = 0
+ for _ in range(self._max_iterations + 1):
+ iterations += 1
+ tools_arg = (
+ [SEARCH_TOOL_SPEC, CLARIFY_TOOL_SPEC]
+ if len(invocations) < self._max_iterations
+ else None
+ )
+ result = self._engine.generate(
+ messages,
+ model=self._model,
+ temperature=self._temperature,
+ max_tokens=self._max_tokens,
+ num_ctx=self._num_ctx,
+ tools=tools_arg,
+ )
+ for k in total_usage:
+ total_usage[k] += int(result.get("usage", {}).get(k, 0))
+
+ content = result.get("content", "") or ""
+ tool_calls_raw = result.get("tool_calls", []) or []
+
+ if not tool_calls_raw:
+ if content.strip():
+ answer = content.strip()
+ self._emit({"type": "final_answer", "text": answer})
+ return ResearchResult(
+ answer=answer,
+ iterations=iterations,
+ tool_calls=invocations,
+ usage=total_usage,
+ )
+ # Empty content with no tool call — push a synthesis prod
+ if invocations:
+ messages.append(Message(role=Role.ASSISTANT, content=content))
+ messages.append(
+ Message(
+ role=Role.USER,
+ content=(
+ "Write your final answer now based on the search "
+ "results above. Cite sources as [1], [2], etc."
+ ),
+ )
+ )
+ continue
+ fallback = "(model returned no content and no tool calls)"
+ self._emit({"type": "final_answer", "text": fallback})
+ return ResearchResult(
+ answer=fallback,
+ iterations=iterations,
+ tool_calls=invocations,
+ usage=total_usage,
+ )
+
+ assistant_msg = Message(
+ role=Role.ASSISTANT,
+ content=content,
+ tool_calls=[
+ ToolCall(
+ id=tc.get("id", f"call_{i}"),
+ name=tc.get("name", "search"),
+ arguments=tc.get("arguments", "{}") or "{}",
+ )
+ for i, tc in enumerate(tool_calls_raw)
+ ],
+ )
+ messages.append(assistant_msg)
+
+ for tc in tool_calls_raw:
+ name = tc.get("name", "")
+ raw_args = tc.get("arguments", "{}") or "{}"
+ try:
+ args = json.loads(raw_args) if isinstance(raw_args, str) else dict(raw_args)
+ except json.JSONDecodeError:
+ args = {}
+
+ if name == "search":
+ # Guard against the planner pre-empting clarify before any
+ # search has run — silently accept; the rule lives in the
+ # system prompt as guidance, not enforcement.
+ self._emit({"type": "search_call", "arguments": args})
+ inv = self._execute_search(args)
+ invocations.append(inv)
+ self._emit(
+ {
+ "type": "search_result",
+ "num_hits": inv.num_results,
+ "top_titles": inv.top_titles,
+ "sources": build_sources_for_client(inv.raw_hits),
+ }
+ )
+ tool_output = json.dumps(
+ shape_results_for_model(inv.raw_hits), ensure_ascii=False
+ )
+ elif name == "clarify":
+ # Enforce the "search first" rule at runtime so we don't
+ # surprise the user with a clarification before showing any
+ # work. If the planner jumps to clarify with no searches
+ # behind it, return an error and let the loop try again.
+ if not any(i.tool_name == "search" for i in invocations):
+ tool_output = json.dumps(
+ {
+ "error": (
+ "clarify is only available after at least "
+ "one search call. Run search first, then "
+ "use clarify if the results are ambiguous "
+ "or empty."
+ )
+ }
+ )
+ else:
+ self._emit(
+ {"type": "clarify_call", "question": str(args.get("question", ""))}
+ )
+ inv = self._execute_clarify(args)
+ invocations.append(inv)
+ self._emit(
+ {"type": "clarify_response", "response": inv.response}
+ )
+ tool_output = json.dumps(
+ {
+ "question": inv.arguments.get("question", ""),
+ "user_response": inv.response,
+ }
+ )
+ else:
+ tool_output = json.dumps(
+ {
+ "error": (
+ f"unknown tool {name!r}; available tools are "
+ "'search' and 'clarify'"
+ )
+ }
+ )
+
+ messages.append(
+ Message(
+ role=Role.TOOL,
+ content=tool_output,
+ tool_call_id=tc.get("id", ""),
+ name=name,
+ )
+ )
+
+ if len(invocations) >= self._max_iterations:
+ messages.append(
+ Message(
+ role=Role.USER,
+ content=(
+ "You have used your tool-call budget (search + "
+ "clarify combined). Write the final synthesis now "
+ "using only the search results and clarifications "
+ "above. Cite sources as [1], [2], etc."
+ ),
+ )
+ )
+
+ # Loop fell through without the model producing a text response.
+ # Force one final tool-less synthesis call so the caller always gets
+ # an answer — bailing out with a sentinel string is never useful to
+ # the user, who already paid for the searches.
+ messages.append(
+ Message(
+ role=Role.USER,
+ content=(
+ "You've used all your search attempts. Synthesize your "
+ "findings now from whatever you've found so far. Do not "
+ "request more tool calls — write the final answer as "
+ "plain text, citing sources as [1], [2], etc. where you can. "
+ "If the searches returned nothing usable, say so plainly."
+ ),
+ )
+ )
+ iterations += 1
+ final = self._engine.generate(
+ messages,
+ model=self._model,
+ temperature=self._temperature,
+ max_tokens=self._max_tokens,
+ num_ctx=self._num_ctx,
+ tools=None,
+ )
+ for k in total_usage:
+ total_usage[k] += int(final.get("usage", {}).get(k, 0))
+ answer = (final.get("content", "") or "").strip()
+ if not answer:
+ answer = (
+ "(no synthesis available — the search budget was exhausted "
+ "and the model returned no text response)"
+ )
+ self._emit({"type": "final_answer", "text": answer})
+ return ResearchResult(
+ answer=answer,
+ iterations=iterations,
+ tool_calls=invocations,
+ usage=total_usage,
+ )
+
+
+__all__ = [
+ "ResearchAgent",
+ "ResearchResult",
+ "ToolInvocation",
+ "SEARCH_TOOL_SPEC",
+ "CLARIFY_TOOL_SPEC",
+ "SYSTEM_PROMPT",
+ "DEFAULT_PLANNER_MODEL",
+ "shape_results_for_model",
+ "build_sources_for_client",
+]
diff --git a/src/openjarvis/analytics/identity.py b/src/openjarvis/analytics/identity.py
index 63f4453f..3a3f6a3b 100644
--- a/src/openjarvis/analytics/identity.py
+++ b/src/openjarvis/analytics/identity.py
@@ -9,6 +9,8 @@ No email, no name, no hardware fingerprint — just an opaque UUID.
from __future__ import annotations
+import os
+import sys
import uuid
from pathlib import Path
@@ -45,5 +47,16 @@ def reset_anon_id(path: Path | str) -> str:
def is_analytics_enabled(cfg: AnalyticsConfig) -> bool:
- """Return True if analytics is enabled in config."""
+ """Return True if analytics is enabled in config.
+
+ Always disabled when running under pytest. The PostHog SDK registers
+ an ``atexit`` hook that synchronously joins its consumer thread; if
+ the host is unreachable (CI runners can't reach the analytics endpoint),
+ each queued batch retries for ``timeout * max_retries`` seconds and the
+ interpreter never exits. Detect pytest via ``PYTEST_CURRENT_TEST``
+ (set per test) and ``"pytest" in sys.modules`` (covers the collection
+ phase before the first test runs).
+ """
+ if os.environ.get("PYTEST_CURRENT_TEST") or "pytest" in sys.modules:
+ return False
return cfg.enabled
diff --git a/src/openjarvis/cli/__init__.py b/src/openjarvis/cli/__init__.py
index 658682f4..50a85fdc 100644
--- a/src/openjarvis/cli/__init__.py
+++ b/src/openjarvis/cli/__init__.py
@@ -60,8 +60,14 @@ def cli(ctx: click.Context, verbose: bool, quiet: bool) -> None:
ctx.obj["quiet"] = quiet
setup_logging(verbose=verbose, quiet=quiet)
- # Check for updates on interactive commands
- if not quiet and ctx.invoked_subcommand:
+ # Check for updates on interactive commands. The banner is noise in
+ # demo recordings of ``jarvis ask --research``, so skip it whenever
+ # the research flag is in argv (cheap argv sniff — Click hasn't
+ # parsed the subcommand's args yet at this point).
+ import sys
+
+ research_mode_active = "--research" in sys.argv
+ if not quiet and ctx.invoked_subcommand and not research_mode_active:
from openjarvis.cli._version_check import check_for_updates
check_for_updates(ctx.invoked_subcommand)
diff --git a/src/openjarvis/cli/_version_check.py b/src/openjarvis/cli/_version_check.py
index d95f4933..a1db0fa4 100644
--- a/src/openjarvis/cli/_version_check.py
+++ b/src/openjarvis/cli/_version_check.py
@@ -4,6 +4,7 @@ from __future__ import annotations
import json
import logging
+import os
import sys
import time
from pathlib import Path
@@ -17,7 +18,13 @@ _CHECK_COMMANDS = {"ask", "chat", "serve"}
def check_for_updates(command_name: str) -> None:
- """Print a message if a newer version is available. Best-effort, never raises."""
+ """Print a message if a newer version is available. Best-effort, never raises.
+
+ Honors ``OPENJARVIS_NO_UPDATE_CHECK`` — set it to any non-empty value
+ (``1``, ``true``, etc.) to skip the GitHub poll and the banner.
+ """
+ if os.environ.get("OPENJARVIS_NO_UPDATE_CHECK", "").strip():
+ return
if command_name not in _CHECK_COMMANDS:
return
try:
diff --git a/src/openjarvis/cli/ask.py b/src/openjarvis/cli/ask.py
index 8c56499d..b1460ae1 100644
--- a/src/openjarvis/cli/ask.py
+++ b/src/openjarvis/cli/ask.py
@@ -32,6 +32,187 @@ from openjarvis.telemetry.store import TelemetryStore
logger = logging.getLogger(__name__)
+def _run_research(
+ *,
+ query_text: str,
+ engine,
+ model_name: str | None,
+ knowledge_db: str | None,
+ output_json: bool,
+ console: Console,
+) -> None:
+ """Run the hybrid-search research loop and print the result to the console.
+
+ Lazy imports keep the cost of this branch off the cold-path of plain
+ ``jarvis ask`` calls.
+ """
+ import re
+
+ from rich.markdown import Markdown
+ from rich.theme import Theme
+
+ from openjarvis.agents.research_loop import DEFAULT_PLANNER_MODEL, ResearchAgent
+ from openjarvis.connectors.embeddings import OllamaEmbedder
+ from openjarvis.connectors.hybrid_search import HybridSearch
+ from openjarvis.connectors.store import KnowledgeStore
+ from openjarvis.engine.ollama import OllamaEngine
+
+ store_kwargs: dict = {}
+ if knowledge_db:
+ store_kwargs["db_path"] = knowledge_db
+ store = KnowledgeStore(**store_kwargs)
+
+ # Research mode is wired specifically to Ollama: the planner prompt
+ # (gemma4:31b) and the function-call schema for search/clarify both
+ # assume Ollama's /api/chat tool semantics. Using the engine returned
+ # by get_engine() here is a foot-gun — discovery can pick any
+ # OpenAI-compatible engine registered on the same port as our own
+ # API server. research_router.py hardcodes OllamaEngine() for the
+ # same reason; mirror that here so CLI and HTTP behave identically.
+ engine = OllamaEngine()
+
+ chunk_count = store._conn.execute(
+ "SELECT COUNT(*) FROM knowledge_chunks"
+ ).fetchone()[0]
+ logger.debug(
+ "research: engine=%s.%s db=%s chunks=%d",
+ type(engine).__module__,
+ type(engine).__name__,
+ store._db_path,
+ chunk_count,
+ )
+
+ embedder = OllamaEmbedder()
+ embedder_available = embedder.is_available()
+ logger.debug("research: embedder available=%s", embedder_available)
+ if not embedder_available:
+ console.print(
+ "[yellow]Ollama embedder unavailable — falling back to BM25-only "
+ "retrieval. Run `ollama pull nomic-embed-text` for hybrid scoring.[/yellow]"
+ )
+ embedder = None
+
+ planner_model = model_name or DEFAULT_PLANNER_MODEL
+ logger.debug("research: planner_model=%s", planner_model)
+
+ # ---- Output styling --------------------------------------------------
+ # Two consoles by design: traces and the timing footer go to stderr
+ # (so ``jarvis ask --research "..." > out.md`` still gives a clean
+ # markdown file), while the rendered synthesis goes to stdout. The
+ # ``markdown.code`` theme override is the cyan-citation hack — see
+ # ``_style_citations`` below.
+ trace = Console(stderr=True, soft_wrap=True, highlight=False)
+ answer_console = Console(
+ theme=Theme({"markdown.code": "cyan bold not italic"}),
+ soft_wrap=True,
+ highlight=False,
+ )
+
+ def _style_citations(text: str) -> str:
+ """Wrap each ``[N]`` in inline code so it renders cyan.
+
+ Rich's Markdown class doesn't expose any hook for styling
+ arbitrary text spans, so we cheat: convert the citation tokens
+ into inline-code markdown (`[1]` → `` `[1]` ``) and override
+ the ``markdown.code`` theme entry above to colour them. Trims
+ the implicit monospace background that some terminal themes
+ give inline code so the result reads as text, not as code.
+ """
+ return re.sub(r"(\[\d+\])", r"`\1`", text)
+
+ def _format_search_call(args: dict) -> str:
+ q = args.get("query", "") or ""
+ extras: list[str] = []
+ person = args.get("person")
+ if person:
+ extras.append(f"person: {person}")
+ time_range = args.get("time_range")
+ if isinstance(time_range, dict):
+ start = time_range.get("start") or ""
+ end = time_range.get("end") or ""
+ if start or end:
+ bounds = f"{start or '…'} → {end or '…'}"
+ extras.append(f"when: {bounds}")
+ suffix = f" ({', '.join(extras)})" if extras else ""
+ return f"'{q}'{suffix}"
+
+ def on_event(event: dict) -> None:
+ etype = event.get("type")
+ if etype == "search_call":
+ args = event.get("arguments", {})
+ trace.print(
+ f" [dim]↳ Searching:[/dim] "
+ f"[dim italic]{_format_search_call(args)}[/dim italic]"
+ )
+ elif etype == "search_result":
+ n = event.get("num_hits", 0)
+ label = "result" if n == 1 else "results"
+ trace.print(f" [dim]↳ Found {n} {label}[/dim]")
+ elif etype == "clarify_call":
+ q = event.get("question", "") or ""
+ trace.print(
+ f" [dim]↳ Clarifying:[/dim] [dim italic]{q}[/dim italic]"
+ )
+ # final_answer and clarify_response are handled outside the loop.
+
+ agent = ResearchAgent(
+ engine=engine,
+ search=HybridSearch(store, embedder),
+ model=planner_model,
+ on_event=on_event,
+ )
+
+ started = time.monotonic()
+ result = agent.run(query_text)
+ elapsed = time.monotonic() - started
+
+ logger.debug(
+ "research: iterations=%d tool_calls=%d usage=%s",
+ result.iterations,
+ len(result.tool_calls),
+ result.usage,
+ )
+
+ if output_json:
+ click.echo(
+ json_mod.dumps(
+ {
+ "answer": result.answer,
+ "iterations": result.iterations,
+ "usage": result.usage,
+ "tool_calls": [
+ {
+ "arguments": inv.arguments,
+ "num_results": inv.num_results,
+ "top_titles": inv.top_titles,
+ }
+ for inv in result.tool_calls
+ ],
+ },
+ indent=2,
+ )
+ )
+ return
+
+ # Visual break between live traces and the synthesis.
+ trace.print()
+
+ # Render the synthesis as Markdown with inline citations coloured cyan.
+ answer_console.print(Markdown(_style_citations(result.answer)))
+
+ # Footer: how long it took + how many distinct sources the model
+ # actually cited. Empty answers (rare; only if the model went silent)
+ # skip the footer entirely.
+ if result.answer:
+ cited = {int(n) for n in re.findall(r"\[(\d+)\]", result.answer)}
+ src_word = "source" if len(cited) == 1 else "sources"
+ trace.print()
+ trace.print(
+ f"[dim]Deep Research · {elapsed:.1f}s · "
+ f"{len(cited)} {src_word} cited[/dim]"
+ )
+
+
def _get_memory_backend(config):
"""Try to instantiate the memory backend.
@@ -368,6 +549,24 @@ def _print_profile(
is_flag=True,
help="Print inference telemetry profile (latency, tokens, energy, IPW).",
)
+ "--research",
+ "research_mode",
+ is_flag=True,
+ help=(
+ "Route the query through the hybrid-search research agent over the "
+ "personal knowledge store (BM25 + dense embeddings, max 5 tool calls)."
+ ),
+)
+ "--knowledge-db",
+ "knowledge_db",
+ default=None,
+ help=(
+ "Override the KnowledgeStore path used by --research "
+ "(default: ~/.openjarvis/knowledge.db)."
+ ),
+)
def ask(
query: tuple[str, ...],
model_name: str | None,
@@ -380,6 +579,8 @@ def ask(
agent_name: str | None,
tool_names: str | None,
enable_profile: bool,
+ research_mode: bool,
+ knowledge_db: str | None,
) -> None:
"""Ask Jarvis a question."""
console = Console(stderr=True)
@@ -445,6 +646,20 @@ def ask(
engine_name, engine = resolved
+ # ------------------------------------------------------------------
+ # Research mode — hybrid search + agentic loop over the knowledge store
+ # ------------------------------------------------------------------
+ if research_mode:
+ _run_research(
+ query_text=query_text,
+ engine=engine,
+ model_name=model_name,
+ knowledge_db=knowledge_db,
+ output_json=output_json,
+ console=console,
+ )
+ return
+
# Apply security guardrails
from openjarvis.security import setup_security
diff --git a/src/openjarvis/connectors/_stubs.py b/src/openjarvis/connectors/_stubs.py
index 6ea4e95a..e956bfe8 100644
--- a/src/openjarvis/connectors/_stubs.py
+++ b/src/openjarvis/connectors/_stubs.py
@@ -26,6 +26,11 @@ class Document:
"""Universal schema for data from any connector.
All connectors normalize their output to this format before ingestion.
+
+ v1 schema fields (``source_id``, ``participants_raw``, ``channel``) default
+ to empty so existing connectors compile without modification; new
+ connectors should populate them. The pipeline derives ``source_id`` from
+ ``doc_id`` by stripping the ``{source}:`` prefix when not set explicitly.
"""
doc_id: str
@@ -40,6 +45,10 @@ class Document:
url: Optional[str] = None
attachments: List[Attachment] = field(default_factory=list)
metadata: Dict[str, Any] = field(default_factory=dict)
+ # v1 schema additions: defaulted empty so legacy connectors keep working.
+ source_id: str = ""
+ participants_raw: List[str] = field(default_factory=list)
+ channel: Optional[str] = None
@dataclass(slots=True)
diff --git a/src/openjarvis/connectors/chunker.py b/src/openjarvis/connectors/chunker.py
index f0515497..b81c6131 100644
--- a/src/openjarvis/connectors/chunker.py
+++ b/src/openjarvis/connectors/chunker.py
@@ -1,20 +1,22 @@
"""Type-aware semantic chunker for Deep Research ingestion.
-Splits text based on document type, never splitting mid-sentence.
-Returns ``ChunkResult`` dataclass objects with section metadata and
-inherited parent metadata.
+Splits text by paragraph → sentence → token/character boundaries while
+enforcing a hard size cap and adding a fixed-size overlap between
+consecutive chunks. The cap is enforced on BOTH token count
+(whitespace-split) AND character count, since marketing-style emails
+frequently contain dense runs without whitespace (zero-width joiners,
+HTML residue) that defeat token-based limits alone.
Splitting strategy by doc_type
-------------------------------
-- ``event``, ``contact`` : Always a single chunk; never split.
+- ``event``, ``contact`` : Always a single chunk; never split, never capped.
- ``email`` : Split on reply boundaries (``On … wrote:``),
- then sentence-split within each part.
-- ``message`` : Split on double-newline boundaries, accumulate
- into chunks up to *max_tokens*.
+ then paragraphs, then sentences, then force-split.
+- ``message`` : Split on double-newline boundaries, then sentences,
+ then force-split.
- ``document``, ``note``,
anything else : Split on ``## Heading`` section boundaries →
- paragraph boundaries (``\\n\\n``) within sections →
- sentence boundaries as a last resort.
+ paragraph boundaries → sentences → force-split.
Token counting uses whitespace splitting: ``len(text.split())``.
"""
@@ -23,15 +25,21 @@ from __future__ import annotations
import re
from dataclasses import dataclass, field
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List, Optional, Tuple
# ---------------------------------------------------------------------------
-# Public types
+# Regexes
# ---------------------------------------------------------------------------
_SENTENCE_SPLIT_RE = re.compile(r'(?<=[.!?])\s+(?=[A-Z"])')
_SECTION_RE = re.compile(r"(?m)^##\s+(.+)$")
_REPLY_BOUNDARY_RE = re.compile(r"(?m)^On .+wrote:\s*$")
+_SENTENCE_END_RE = re.compile(r"[.!?](?:\s|$)")
+
+
+# ---------------------------------------------------------------------------
+# Public types
+# ---------------------------------------------------------------------------
@dataclass(slots=True)
@@ -54,65 +62,118 @@ def _count_tokens(text: str) -> int:
def _split_sentences(text: str) -> List[str]:
- """Split *text* into sentences using the canonical regex.
-
- The regex splits after sentence-ending punctuation (``.``, ``!``, ``?``)
- followed by whitespace and a capital letter or a double-quote.
- """
parts = _SENTENCE_SPLIT_RE.split(text)
return [p.strip() for p in parts if p.strip()]
-def _accumulate(
+def _accumulate_capped(
segments: List[str],
*,
max_tokens: int,
+ max_chars: int,
sep: str = " ",
) -> List[str]:
- """Greedily merge *segments* into chunks up to *max_tokens* tokens.
+ """Greedy-merge segments into chunks bounded by token AND char counts.
- A segment that is already larger than *max_tokens* is placed in its own
- chunk; it is never split further by this function.
+ Segments larger than the bounds pass through as their own chunk —
+ the caller force-splits those in a separate pass.
"""
chunks: List[str] = []
- current_parts: List[str] = []
- current_tokens = 0
+ current: List[str] = []
+ cur_tokens = 0
+ cur_chars = 0
for seg in segments:
seg_tokens = _count_tokens(seg)
- if current_parts and current_tokens + seg_tokens > max_tokens:
- chunks.append(sep.join(current_parts))
- current_parts = [seg]
- current_tokens = seg_tokens
+ seg_chars = len(seg)
+ sep_chars = len(sep) if current else 0
+
+ would_overflow = current and (
+ cur_tokens + seg_tokens > max_tokens
+ or cur_chars + sep_chars + seg_chars > max_chars
+ )
+ if would_overflow:
+ chunks.append(sep.join(current))
+ current = [seg]
+ cur_tokens = seg_tokens
+ cur_chars = seg_chars
else:
- current_parts.append(seg)
- current_tokens += seg_tokens
-
- if current_parts:
- chunks.append(sep.join(current_parts))
+ current.append(seg)
+ cur_tokens += seg_tokens
+ cur_chars += sep_chars + seg_chars
+ if current:
+ chunks.append(sep.join(current))
return chunks
-def _sentence_chunks(text: str, *, max_tokens: int) -> List[str]:
- """Split *text* by sentences and accumulate into max_tokens chunks."""
- sentences = _split_sentences(text)
- if not sentences:
- stripped = text.strip()
- return [stripped] if stripped else []
- return _accumulate(sentences, max_tokens=max_tokens, sep=" ")
+def _best_cut_index(window: str) -> int:
+ """Pick a cut point inside ``window``: sentence end → space → hard cut."""
+ matches = list(_SENTENCE_END_RE.finditer(window))
+ if matches:
+ return matches[-1].end()
+ midpoint = len(window) // 2
+ space = window.rfind(" ", midpoint)
+ if space > 0:
+ return space + 1
+ return len(window)
-def _paragraph_chunks(text: str, *, max_tokens: int) -> List[str]:
- """Split *text* on paragraph breaks (``\\n\\n``), then by sentences if needed."""
- paragraphs = [p.strip() for p in text.split("\n\n") if p.strip()]
- result: List[str] = []
- for para in paragraphs:
- if _count_tokens(para) <= max_tokens:
- result.append(para)
- else:
- result.extend(_sentence_chunks(para, max_tokens=max_tokens))
- return result
+def _force_split(text: str, *, max_chars: int, max_tokens: int) -> List[str]:
+ """Split an oversized run into pieces that fit both caps.
+
+ Walks the text greedily: takes a window up to ``max_chars``, shrinks
+ until the token count fits, then snaps the cut to the last sentence
+ boundary, falling back to a word boundary, then a hard cut.
+ """
+ rest = text.strip()
+ out: List[str] = []
+ while rest:
+ if len(rest) <= max_chars and _count_tokens(rest) <= max_tokens:
+ out.append(rest)
+ break
+
+ end = min(max_chars, len(rest))
+ while end > 1 and _count_tokens(rest[:end]) > max_tokens:
+ end = max(1, int(end * 0.9))
+
+ window = rest[:end]
+ cut = _best_cut_index(window)
+ piece = rest[:cut].strip()
+ if not piece:
+ # Window contained only whitespace before any cuttable boundary.
+ # Force advance to avoid an infinite loop.
+ cut = max(cut + 1, end)
+ piece = rest[:cut].strip()
+ if piece:
+ out.append(piece)
+ rest = rest[cut:].strip()
+ return out
+
+
+def _apply_overlap(chunks: List[str], *, overlap_tokens: int) -> List[str]:
+ """Prepend the last ``overlap_tokens`` tokens of each chunk to the next.
+
+ The tail is taken from the *original* preceding chunk (not the
+ already-augmented output) so overlap doesn't compound, and is also
+ capped by character count — a single whitespace-split "token" can be
+ a 200+ char URL, so a pure-token cap would let tails balloon on
+ content with long opaque strings.
+ """
+ if overlap_tokens <= 0 or len(chunks) < 2:
+ return list(chunks)
+ overlap_chars_cap = overlap_tokens * 4
+ out = [chunks[0]]
+ for i in range(1, len(chunks)):
+ prev_tokens = chunks[i - 1].split()
+ if not prev_tokens:
+ out.append(chunks[i])
+ continue
+ tail = " ".join(prev_tokens[-overlap_tokens:])
+ if len(tail) > overlap_chars_cap:
+ tail = tail[-overlap_chars_cap:]
+ out.append(f"{tail} {chunks[i]}".strip())
+ return out
# ---------------------------------------------------------------------------
@@ -121,18 +182,55 @@ def _paragraph_chunks(text: str, *, max_tokens: int) -> List[str]:
class SemanticChunker:
- """Split text based on document type without breaking mid-sentence.
+ """Split text by document type with a hard size cap and overlap.
Parameters
----------
max_tokens:
- Soft upper limit on chunk size measured in whitespace-delimited tokens
- (i.e. ``len(text.split())``). Single unsplittable segments may exceed
- this limit.
+ Hard upper bound on chunk size in whitespace-delimited tokens.
+ No emitted chunk exceeds this.
+ max_chars:
+ Hard upper bound on chunk size in characters. Defaults to
+ ``max_tokens * 4`` (a typical English chars-per-token estimate).
+ No emitted chunk exceeds this — the chunker force-splits any
+ run that does, even when token count alone would say it fits.
+ overlap_tokens:
+ Token tail copied from each chunk into the head of the next so
+ downstream retrieval doesn't miss context that straddles a chunk
+ boundary. Defaults to ``min(100, max_tokens // 5)`` and is clamped
+ to ``[0, max_tokens - 1]``. Set to ``0`` to disable.
"""
- def __init__(self, max_tokens: int = 512) -> None:
+ def __init__(
+ self,
+ max_tokens: int = 512,
+ *,
+ max_chars: Optional[int] = None,
+ overlap_tokens: Optional[int] = None,
+ ) -> None:
self.max_tokens = max_tokens
+ self.max_chars = max_chars if max_chars is not None else max_tokens * 4
+ if overlap_tokens is None:
+ overlap_tokens = min(100, max(1, max_tokens // 5))
+ self.overlap_tokens = max(0, min(overlap_tokens, max(0, max_tokens - 1)))
+
+ # Content budget per chunk leaves room for the overlap prefix
+ # that gets added in the final pass, so tokens stay within
+ # max_tokens. Char budget intentionally does NOT subtract overlap
+ # — we accept up to ~overlap_tokens*4 chars of headroom over
+ # max_chars after overlap, which still sits under any reasonable
+ # hard ceiling and avoids spurious force-splits of well-behaved
+ # sentences in configurations where the char cap is tight.
+ self._content_tokens = max(1, self.max_tokens - self.overlap_tokens)
+ self._content_chars = self.max_chars
+ # Soft target used to decide when a paragraph is "long enough to
+ # sub-split", and to size sentence-accumulated sub-chunks. At
+ # roughly half the hard cap, typical paragraphs ship as a single
+ # chunk and only the unusually long ones get further split,
+ # which keeps semantic units intact while still pulling the
+ # chunk-length tail down.
+ self._target_tokens = max(1, self._content_tokens // 2)
+ self._target_chars = max(1, self._content_chars // 2)
# ------------------------------------------------------------------
# Public API
@@ -147,16 +245,10 @@ class SemanticChunker:
) -> List[ChunkResult]:
"""Split *text* into ``ChunkResult`` objects.
- Parameters
- ----------
- text: The raw text to split.
- doc_type: Controls the splitting strategy (see module docstring).
- metadata: Parent metadata dict; copied into every chunk's ``metadata``.
-
- Returns
- -------
- A list of ``ChunkResult`` objects with sequential 0-based ``index``
- values. Returns an empty list if *text* is empty or whitespace-only.
+ Returns an empty list if *text* is empty or whitespace-only.
+ Events and contacts are always returned as a single chunk
+ regardless of size; all other types respect the size caps and
+ receive overlap between consecutive chunks.
"""
if not text or not text.strip():
return []
@@ -164,88 +256,115 @@ class SemanticChunker:
parent_meta: Dict[str, Any] = dict(metadata or {})
if doc_type in ("event", "contact"):
- raw_chunks = self._chunk_atomic(text)
- elif doc_type == "email":
+ return [ChunkResult(content=text, index=0, metadata=parent_meta)]
+
+ if doc_type == "email":
raw_chunks = self._chunk_email(text)
elif doc_type == "message":
raw_chunks = self._chunk_message(text)
else:
- # "document", "note", or any unknown type
raw_chunks = self._chunk_document(text)
+ # Apply overlap globally between consecutive chunks.
+ contents = [c for c, _ in raw_chunks]
+ metas = [m for _, m in raw_chunks]
+ overlapped = _apply_overlap(contents, overlap_tokens=self.overlap_tokens)
+
results: List[ChunkResult] = []
- for idx, (content, extra_meta) in enumerate(raw_chunks):
+ for idx, (content, extra_meta) in enumerate(zip(overlapped, metas)):
merged: Dict[str, Any] = dict(parent_meta)
merged.update(extra_meta)
results.append(ChunkResult(content=content, index=idx, metadata=merged))
-
return results
# ------------------------------------------------------------------
# Strategy implementations
# ------------------------------------------------------------------
- def _chunk_atomic(self, text: str) -> List[tuple[str, Dict[str, Any]]]:
- """Return the entire text as a single chunk (event / contact)."""
- return [(text, {})]
+ def _pack_text(self, text: str) -> List[str]:
+ """Emit one chunk per paragraph; sub-split paragraphs over the soft target.
+
+ Paragraphs are the natural chunk unit. A paragraph that fits the
+ soft target ships as one chunk. A paragraph that doesn't is
+ sentence-split and the sentences accumulated up to the soft
+ target; sentences that exceed the hard cap (rare — opaque content
+ with no whitespace) are force-split.
+ """
+ paragraphs = [p.strip() for p in text.split("\n\n") if p.strip()]
+ if not paragraphs:
+ stripped = text.strip()
+ return [stripped] if stripped else []
+
+ out: List[str] = []
+ for para in paragraphs:
+ if (
+ len(para) <= self._target_chars
+ and _count_tokens(para) <= self._target_tokens
+ ):
+ out.append(para)
+ continue
+
+ sents = _split_sentences(para)
+ if not sents:
+ sents = [para]
- def _chunk_email(self, text: str) -> List[tuple[str, Dict[str, Any]]]:
- """Split on reply boundaries; sentence-split each part."""
- # Split the email into parts on "On ... wrote:" lines.
- # re.split with a capturing group keeps the boundary in results,
- # so we re-attach the header to the following segment.
+ accumulated = _accumulate_capped(
+ sents,
+ max_tokens=self._target_tokens,
+ max_chars=self._target_chars,
+ sep=" ",
+ )
+ for c in accumulated:
+ if (
+ len(c) <= self._content_chars
+ and _count_tokens(c) <= self._content_tokens
+ ):
+ out.append(c)
+ else:
+ out.extend(
+ _force_split(
+ c,
+ max_chars=self._content_chars,
+ max_tokens=self._content_tokens,
+ )
+ )
+ return [c for c in out if c]
+
+ def _chunk_email(self, text: str) -> List[Tuple[str, Dict[str, Any]]]:
+ """Split on reply boundaries; pack each part."""
boundaries = _REPLY_BOUNDARY_RE.split(text)
+ headers = _REPLY_BOUNDARY_RE.findall(text)
- # Each boundary match is a separator; reassemble so the "On … wrote:"
- # line stays with the content that follows it (the quoted block).
raw_parts: List[str] = []
if boundaries:
- # The first element is the text before the first boundary (the
- # main reply body).
raw_parts.append(boundaries[0])
- # Subsequent elements alternate: matched boundary, then text after.
- # Because we used split() (not findall), the boundaries themselves
- # are not in the list — only the text segments between them.
- # So boundaries[1:] are the segments after each matched header.
- # We need to re-find the headers to reassemble.
- headers = _REPLY_BOUNDARY_RE.findall(text)
for header, body in zip(headers, boundaries[1:]):
- # We found the header text via findall; reconstruct the part.
part = (header.strip() + "\n" + body).strip()
raw_parts.append(part)
- chunks: List[tuple[str, Dict[str, Any]]] = []
+ chunks: List[Tuple[str, Dict[str, Any]]] = []
for part in raw_parts:
part = part.strip()
if not part:
continue
- if _count_tokens(part) <= self.max_tokens:
- chunks.append((part, {}))
- else:
- for sub in _sentence_chunks(part, max_tokens=self.max_tokens):
- if sub:
- chunks.append((sub, {}))
+ for c in self._pack_text(part):
+ if c:
+ chunks.append((c, {}))
return chunks if chunks else [(text.strip(), {})]
- def _chunk_message(self, text: str) -> List[tuple[str, Dict[str, Any]]]:
- """Split on double-newline boundaries and accumulate up to max_tokens."""
- paragraphs = [p.strip() for p in text.split("\n\n") if p.strip()]
- raw_chunks = _accumulate(paragraphs, max_tokens=self.max_tokens, sep="\n\n")
- return [(c, {}) for c in raw_chunks if c]
+ def _chunk_message(self, text: str) -> List[Tuple[str, Dict[str, Any]]]:
+ """Pack a message via paragraphs → sentences → force-split."""
+ return [(c, {}) for c in self._pack_text(text)]
- def _chunk_document(self, text: str) -> List[tuple[str, Dict[str, Any]]]:
- """Split on ## headings → paragraphs → sentences."""
- # Find all ## heading positions
+ def _chunk_document(self, text: str) -> List[Tuple[str, Dict[str, Any]]]:
+ """Split on ## headings → paragraphs → sentences → force-split."""
section_matches = list(_SECTION_RE.finditer(text))
if not section_matches:
- # No headings — fall back to paragraph/sentence splitting
- raw_chunks = _paragraph_chunks(text, max_tokens=self.max_tokens)
- return [(c, {}) for c in raw_chunks if c]
+ return [(c, {}) for c in self._pack_text(text)]
- # Build (title, body_text) pairs for each section
- sections: List[tuple[str, str]] = []
+ sections: List[Tuple[str, str]] = []
for i, m in enumerate(section_matches):
title = m.group(1).strip()
body_start = m.end()
@@ -257,24 +376,19 @@ class SemanticChunker:
body = text[body_start:body_end].strip()
sections.append((title, body))
- # Check for preamble text before the first heading
+ result: List[Tuple[str, Dict[str, Any]]] = []
preamble = text[: section_matches[0].start()].strip()
- result: List[tuple[str, Dict[str, Any]]] = []
-
if preamble:
- for c in _paragraph_chunks(preamble, max_tokens=self.max_tokens):
+ for c in self._pack_text(preamble):
if c:
result.append((c, {}))
for title, body in sections:
section_meta: Dict[str, Any] = {"section": title}
if not body:
- # Empty section — emit a placeholder chunk with just the title
result.append((title, section_meta))
continue
-
- para_chunks = _paragraph_chunks(body, max_tokens=self.max_tokens)
- for c in para_chunks:
+ for c in self._pack_text(body):
if c:
result.append((c, dict(section_meta)))
diff --git a/src/openjarvis/connectors/embeddings.py b/src/openjarvis/connectors/embeddings.py
new file mode 100644
index 00000000..45b35119
--- /dev/null
+++ b/src/openjarvis/connectors/embeddings.py
@@ -0,0 +1,156 @@
+"""Dense embedding clients for the IngestionPipeline.
+
+A thin HTTP wrapper around a local `Ollama <https://ollama.com>`_ daemon
+running an embedding model (default ``nomic-embed-text``, 768-dim). Embeddings
+are serialised as float32 ``bytes`` for storage in the ``embedding`` BLOB
+column of ``knowledge_chunks``.
+
+The client degrades gracefully when the daemon is unreachable: ``embed``
+returns ``None`` and ``is_available()`` reports ``False`` instead of raising,
+so ingestion never fails because a sidecar service is down.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import List, Optional
+
+import numpy as np
+import requests
+
+logger = logging.getLogger(__name__)
+
+
+DEFAULT_OLLAMA_HOST = "http://localhost:11434"
+DEFAULT_EMBED_MODEL = "nomic-embed-text"
+
+
+class OllamaEmbedder:
+ """Embed text via a local Ollama daemon.
+
+ Parameters
+ ----------
+ model:
+ Ollama model tag (e.g. ``nomic-embed-text``, ``mxbai-embed-large``).
+ host:
+ Base URL for the Ollama HTTP API. Defaults to ``http://localhost:11434``.
+ timeout:
+ Per-request timeout in seconds.
+ """
+
+ def __init__(
+ self,
+ *,
+ model: str = DEFAULT_EMBED_MODEL,
+ host: str = DEFAULT_OLLAMA_HOST,
+ timeout: float = 30.0,
+ ) -> None:
+ self._model = model
+ self._host = host.rstrip("/")
+ self._timeout = timeout
+ self._dim: Optional[int] = None
+
+ # ------------------------------------------------------------------
+ # Identity / capability checks
+ # ------------------------------------------------------------------
+
+ @property
+ def model_version(self) -> str:
+ """Stable identifier persisted alongside each embedding row."""
+ return f"ollama:{self._model}"
+
+ @property
+ def dim(self) -> Optional[int]:
+ """Embedding dimensionality, learned after the first successful call."""
+ return self._dim
+
+ def is_available(self) -> bool:
+ """Return True iff the daemon answers and the model is installed."""
+ try:
+ resp = requests.get(f"{self._host}/api/tags", timeout=2.0)
+ resp.raise_for_status()
+ except requests.RequestException:
+ return False
+
+ try:
+ names = {m.get("name", "") for m in resp.json().get("models", [])}
+ except ValueError:
+ return False
+
+ # Ollama tags include the ":latest" suffix; match either form.
+ return self._model in names or f"{self._model}:latest" in names
+
+ # ------------------------------------------------------------------
+ # Embedding
+ # ------------------------------------------------------------------
+
+ def embed(self, text: str) -> Optional[bytes]:
+ """Embed a single string. Returns float32 bytes or ``None`` on failure."""
+ if not text or not text.strip():
+ return None
+ try:
+ resp = requests.post(
+ f"{self._host}/api/embeddings",
+ json={"model": self._model, "prompt": text},
+ timeout=self._timeout,
+ )
+ resp.raise_for_status()
+ payload = resp.json()
+ except requests.RequestException as exc:
+ logger.warning("OllamaEmbedder.embed: request failed (%s)", exc)
+ return None
+ except ValueError as exc:
+ logger.warning("OllamaEmbedder.embed: bad JSON (%s)", exc)
+ return None
+
+ vec = payload.get("embedding")
+ if not vec:
+ logger.warning(
+ "OllamaEmbedder.embed: empty embedding for %d chars", len(text)
+ )
+ return None
+
+ arr = np.asarray(vec, dtype=np.float32)
+ if self._dim is None:
+ self._dim = int(arr.shape[0])
+ elif arr.shape[0] != self._dim:
+ logger.warning(
+ "OllamaEmbedder.embed: dim drift (expected %d, got %d)",
+ self._dim, arr.shape[0],
+ )
+ return None
+ return arr.tobytes()
+
+ def embed_batch(self, texts: List[str]) -> List[Optional[bytes]]:
+ """Embed a list of strings sequentially.
+
+ Ollama's HTTP API serves one prompt per call; on the same host the
+ round-trip overhead is negligible relative to model inference.
+ """
+ return [self.embed(t) for t in texts]
+
+
+# ---------------------------------------------------------------------------
+# Deserialisation helper (used by verification + future retrieval code)
+# ---------------------------------------------------------------------------
+
+
+def decode_embedding(
+ blob: Optional[bytes], *, dtype: type = np.float32
+) -> Optional[np.ndarray]:
+ """Reconstruct a 1-D vector from a BLOB written by ``OllamaEmbedder.embed``.
+
+ Returns ``None`` when the input is missing or zero-length so callers can
+ treat absent embeddings uniformly.
+ """
+ if not blob:
+ return None
+ return np.frombuffer(blob, dtype=dtype)
+
+
+__all__ = [
+ "OllamaEmbedder",
+ "decode_embedding",
+ "DEFAULT_EMBED_MODEL",
+ "DEFAULT_OLLAMA_HOST",
+]
diff --git a/src/openjarvis/connectors/gcalendar.py b/src/openjarvis/connectors/gcalendar.py
index c2548812..1a0b7735 100644
--- a/src/openjarvis/connectors/gcalendar.py
+++ b/src/openjarvis/connectors/gcalendar.py
@@ -13,6 +13,7 @@ from typing import Any, Dict, Iterator, List, Optional
import httpx
from openjarvis.connectors._stubs import BaseConnector, Document, SyncStatus
+from openjarvis.connectors.google_auth import call_with_refresh
from openjarvis.connectors.oauth import (
GOOGLE_ALL_SCOPES,
build_google_auth_url,
@@ -302,13 +303,15 @@ class GCalendarConnector(BaseConnector):
tokens = load_tokens(self._credentials_path)
if not tokens:
return
-
- token: str = tokens.get("access_token", tokens.get("token", ""))
- if not token:
+ if not tokens.get("access_token") and not tokens.get("token"):
return
- # Fetch list of calendars
- calendars_resp = _gcal_api_calendars_list(token)
+ # Fetch list of calendars. call_with_refresh wraps the token read so
+ # an expired access_token triggers a one-shot refresh + retry instead
+ # of bubbling up a 401.
+ calendars_resp = call_with_refresh(
+ _gcal_api_calendars_list, self._credentials_path
+ )
calendars: List[Dict[str, Any]] = calendars_resp.get("items", [])
# Default to 24 hours ago so we don't dump the entire calendar history
@@ -327,8 +330,9 @@ class GCalendarConnector(BaseConnector):
while True:
try:
- events_resp = _gcal_api_events_list(
- token,
+ events_resp = call_with_refresh(
+ _gcal_api_events_list,
+ self._credentials_path,
calendar_id,
page_token=page_token,
time_min=time_min,
diff --git a/src/openjarvis/connectors/gcontacts.py b/src/openjarvis/connectors/gcontacts.py
index 58852ba9..08659f0d 100644
--- a/src/openjarvis/connectors/gcontacts.py
+++ b/src/openjarvis/connectors/gcontacts.py
@@ -13,6 +13,7 @@ from typing import Any, Dict, Iterator, List, Optional
import httpx
from openjarvis.connectors._stubs import BaseConnector, Document, SyncStatus
+from openjarvis.connectors.google_auth import call_with_refresh
from openjarvis.connectors.oauth import (
GOOGLE_ALL_SCOPES,
build_google_auth_url,
@@ -249,16 +250,16 @@ class GContactsConnector(BaseConnector):
tokens = load_tokens(self._credentials_path)
if not tokens:
return
-
- token: str = tokens.get("access_token", tokens.get("token", ""))
- if not token:
+ if not tokens.get("access_token") and not tokens.get("token"):
return
page_token: Optional[str] = cursor
synced = 0
while True:
- list_resp = _gcontacts_api_list(token, page_token=page_token)
+ list_resp = call_with_refresh(
+ _gcontacts_api_list, self._credentials_path, page_token=page_token
+ )
connections: List[Dict[str, Any]] = list_resp.get("connections", [])
for person in connections:
diff --git a/src/openjarvis/connectors/gdrive.py b/src/openjarvis/connectors/gdrive.py
index 18cf0455..5b7f4cf9 100644
--- a/src/openjarvis/connectors/gdrive.py
+++ b/src/openjarvis/connectors/gdrive.py
@@ -13,6 +13,7 @@ from typing import Any, Dict, Iterator, List, Optional
import httpx
from openjarvis.connectors._stubs import BaseConnector, Document, SyncStatus
+from openjarvis.connectors.google_auth import call_with_refresh
from openjarvis.connectors.oauth import (
GOOGLE_ALL_SCOPES,
build_google_auth_url,
@@ -234,16 +235,16 @@ class GDriveConnector(BaseConnector):
tokens = load_tokens(self._credentials_path)
if not tokens:
return
-
- token: str = tokens.get("access_token", tokens.get("token", ""))
- if not token:
+ if not tokens.get("access_token") and not tokens.get("token"):
return
page_token: Optional[str] = cursor
synced = 0
while True:
- list_resp = _gdrive_api_list_files(token, page_token=page_token)
+ list_resp = call_with_refresh(
+ _gdrive_api_list_files, self._credentials_path, page_token=page_token
+ )
files: List[Dict[str, Any]] = list_resp.get("files", [])
for file_meta in files:
@@ -263,7 +264,12 @@ class GDriveConnector(BaseConnector):
export_mime = _EXPORT_MIME_MAP.get(mime_type)
if export_mime is not None:
try:
- content = _gdrive_api_export(token, file_id, export_mime)
+ content = call_with_refresh(
+ _gdrive_api_export,
+ self._credentials_path,
+ file_id,
+ export_mime,
+ )
except Exception: # noqa: BLE001
content = f"[File: {name}] ({mime_type})"
else:
diff --git a/src/openjarvis/connectors/gmail.py b/src/openjarvis/connectors/gmail.py
index ee157a83..64e87a87 100644
--- a/src/openjarvis/connectors/gmail.py
+++ b/src/openjarvis/connectors/gmail.py
@@ -9,12 +9,21 @@ from __future__ import annotations
import base64
import email.utils
+import logging
+import re
from datetime import datetime
-from typing import Any, Dict, Iterator, List, Optional
+from html.parser import HTMLParser
+from typing import Any, Dict, Iterator, List, Optional, Tuple
import httpx
from openjarvis.connectors._stubs import BaseConnector, Document, SyncStatus
+from openjarvis.connectors.google_auth import (
+ GoogleAuthError,
+)
+from openjarvis.connectors.google_auth import (
+ call_with_refresh as _call_with_refresh,
+)
from openjarvis.connectors.oauth import (
GOOGLE_ALL_SCOPES,
build_google_auth_url,
@@ -27,6 +36,8 @@ from openjarvis.core.config import DEFAULT_CONFIG_DIR
from openjarvis.core.registry import ConnectorRegistry
from openjarvis.tools._stubs import ToolSpec
+logger = logging.getLogger(__name__)
+
# ---------------------------------------------------------------------------
# Constants
# ---------------------------------------------------------------------------
@@ -35,6 +46,13 @@ _GMAIL_API_BASE = "https://gmail.googleapis.com/gmail/v1/users/me"
_GMAIL_SCOPE = "https://www.googleapis.com/auth/gmail.readonly"
_DEFAULT_CREDENTIALS_PATH = str(DEFAULT_CONFIG_DIR / "connectors" / "gmail.json")
+# Token refresh is delegated to the shared google_auth helper so Calendar,
+# Contacts, Drive, and Tasks get the same one-shot 401 retry. ``GmailAuthError``
+# is preserved as a module-level alias for tests/callers that imported it
+# under the historical name.
+GmailAuthError = GoogleAuthError
+
+
# ---------------------------------------------------------------------------
# Module-level API functions (easy to patch in tests)
# ---------------------------------------------------------------------------
@@ -118,21 +136,86 @@ def _extract_header(headers: List[Dict[str, str]], name: str) -> str:
return ""
+class _HTMLTextExtractor(HTMLParser):
+ """Strip HTML tags and return readable text using stdlib only.
+
+ Skips <script>, <style>, and <head> contents entirely so CSS rules and
+ JS payloads don't pollute the text. Inserts a newline at each block-level
+ tag boundary so the downstream chunker still has paragraph-ish breaks
+ to split on; without this, a single <div>-wrapped marketing email
+ becomes one giant unsplittable chunk.
+ """
+
+ _SKIP_TAGS = {"script", "style", "head", "title", "meta", "link"}
+ _BLOCK_TAGS = {
+ "p", "div", "br", "li", "ul", "ol", "tr", "td", "table",
+ "h1", "h2", "h3", "h4", "h5", "h6", "blockquote", "hr",
+ "article", "section", "header", "footer", "pre",
+ }
+
+ def __init__(self) -> None:
+ super().__init__(convert_charrefs=True)
+ self._parts: List[str] = []
+ self._skip_depth = 0
+
+ def handle_starttag(
+ self, tag: str, attrs: List[Tuple[str, Optional[str]]]
+ ) -> None:
+ if tag in self._SKIP_TAGS:
+ self._skip_depth += 1
+ elif tag in self._BLOCK_TAGS and self._skip_depth == 0:
+ self._parts.append("\n")
+
+ def handle_endtag(self, tag: str) -> None:
+ if tag in self._SKIP_TAGS:
+ self._skip_depth = max(0, self._skip_depth - 1)
+
+ def handle_data(self, data: str) -> None:
+ if self._skip_depth == 0:
+ self._parts.append(data)
+
+ def get_text(self) -> str:
+ text = "".join(self._parts)
+ # Collapse runs of horizontal whitespace and excess blank lines so
+ # the chunker sees clean paragraphs rather than walls of \n.
+ text = re.sub(r"[ \t\r\f\v]+", " ", text)
+ text = re.sub(r"\n[ \t]*", "\n", text)
+ text = re.sub(r"\n{3,}", "\n\n", text)
+ return text.strip()
+
+
+def _html_to_text(html: str) -> str:
+ """Convert HTML to plain text. Malformed input returns best-effort output."""
+ extractor = _HTMLTextExtractor()
+ try:
+ extractor.feed(html)
+ extractor.close()
+ except Exception: # noqa: BLE001
+ # Parser exceptions still leave partial output in self._parts.
+ pass
+ return extractor.get_text()
+
+
def _decode_body(payload: Dict[str, Any]) -> str:
"""Decode the message body from a Gmail payload dict.
- Handles both simple payloads (``body.data``) and multipart messages
- by recursively searching for a ``text/plain`` part.
+ Multipart messages prefer text/plain. When only text/html is available
+ (common for marketing emails), the HTML is stripped to plain text so
+ downstream chunkers and embeddings don't ingest raw markup.
"""
mime_type: str = payload.get("mimeType", "")
if mime_type.startswith("multipart/"):
- # Search parts for text/plain first, then any text/* fallback
parts: List[Dict[str, Any]] = payload.get("parts", [])
+ # Prefer text/plain when both alternatives are present.
for part in parts:
if part.get("mimeType", "").startswith("text/plain"):
return _decode_body(part)
- # Fallback: recurse into first part
+ # Fall back to text/html (which gets stripped at the leaf branch).
+ for part in parts:
+ if part.get("mimeType", "").startswith("text/html"):
+ return _decode_body(part)
+ # Last resort: recurse into the first part regardless of type.
if parts:
return _decode_body(parts[0])
return ""
@@ -144,10 +227,14 @@ def _decode_body(payload: Dict[str, Any]) -> str:
# Gmail uses URL-safe base64 without padding
padded = body_data + "=" * (-len(body_data) % 4)
try:
- return base64.urlsafe_b64decode(padded).decode("utf-8", errors="replace")
+ decoded = base64.urlsafe_b64decode(padded).decode("utf-8", errors="replace")
except Exception: # noqa: BLE001
return ""
+ if mime_type.startswith("text/html"):
+ return _html_to_text(decoded)
+ return decoded
+
def _parse_date(date_str: str) -> datetime:
"""Parse an RFC 2822 email date string into a :class:`~datetime.datetime`.
@@ -162,6 +249,31 @@ def _parse_date(date_str: str) -> datetime:
return datetime.now()
+def _normalize_addresses(raw: str) -> List[str]:
+ """Extract lowercase email addresses from a comma-separated header value.
+
+ Uses :func:`email.utils.getaddresses` so multi-recipient ``To``/``Cc``
+ headers are handled correctly. Addresses that fail to parse are dropped.
+ """
+ if not raw:
+ return []
+ return [addr.lower() for _, addr in email.utils.getaddresses([raw]) if addr]
+
+
+# Order matters: a message tagged both SENT and INBOX (rare) reads as SENT,
+# the more specific origin. INBOX is last so it acts as the default.
+_PRIMARY_LABELS = ("SENT", "DRAFT", "SPAM", "TRASH", "INBOX")
+
+
+def _select_channel(label_ids: List[str]) -> Optional[str]:
+ """Pick the most specific Gmail system folder this message lives in."""
+ label_set = set(label_ids)
+ for label in _PRIMARY_LABELS:
+ if label in label_set:
+ return label
+ return None
+
+
# ---------------------------------------------------------------------------
# GmailConnector
# ---------------------------------------------------------------------------
@@ -245,26 +357,34 @@ class GmailConnector(BaseConnector):
cursor:
``nextPageToken`` from a previous sync to resume pagination.
"""
+ # Existence check only — the actual access token is reloaded on every
+ # API call by _call_with_refresh so a mid-sync refresh is picked up
+ # transparently.
tokens = load_tokens(self._credentials_path)
if not tokens:
return
-
- token: str = tokens.get("token", tokens.get("access_token", ""))
- if not token:
+ if not (tokens.get("access_token") or tokens.get("token")):
return
- query = "category:primary"
+ # Default to no filter so SENT, labeled, and category-tabbed mail
+ # all flow in. The previous "category:primary" default excluded
+ # ~95% of a typical mailbox (sent mail, Promotions, Updates, etc.)
+ # which made any C2-style "what did I say to X" query impossible.
+ query_parts: List[str] = []
if since is not None:
# Gmail's after: operator accepts Unix epoch seconds.
- epoch = int(since.timestamp())
- query = f"category:primary after:{epoch}"
+ query_parts.append(f"after:{int(since.timestamp())}")
+ query = " ".join(query_parts)
page_token: Optional[str] = cursor
synced = 0
while True:
- list_resp = _gmail_api_list_messages(
- token, page_token=page_token, query=query
+ list_resp = _call_with_refresh(
+ _gmail_api_list_messages,
+ self._credentials_path,
+ page_token=page_token,
+ query=query,
)
messages: List[Dict[str, Any]] = list_resp.get("messages", [])
@@ -273,39 +393,59 @@ class GmailConnector(BaseConnector):
if not msg_id:
continue
- msg = _gmail_api_get_message(token, msg_id)
+ msg = _call_with_refresh(
+ _gmail_api_get_message,
+ self._credentials_path,
+ msg_id,
+ )
payload: Dict[str, Any] = msg.get("payload", {})
headers: List[Dict[str, str]] = payload.get("headers", [])
from_header = _extract_header(headers, "From")
+ to_header = _extract_header(headers, "To")
+ cc_header = _extract_header(headers, "Cc")
subject = _extract_header(headers, "Subject")
date_str = _extract_header(headers, "Date")
- to_header = _extract_header(headers, "To")
+ rfc_message_id = _extract_header(headers, "Message-ID")
body = _decode_body(payload)
timestamp = _parse_date(date_str)
+ # Raw header values, exactly as Gmail returned them — preserved
+ # so re-normalisation against an updated alias map doesn't need
+ # a re-fetch from the API.
+ participants_raw: List[str] = [
+ h for h in (from_header, to_header, cc_header) if h
+ ]
+ # Lowercase email addresses, multi-recipient-aware.
participants: List[str] = []
- if from_header:
- participants.append(from_header)
- if to_header:
- participants.append(to_header)
+ for header in (from_header, to_header, cc_header):
+ participants.extend(_normalize_addresses(header))
+ label_ids: List[str] = msg.get("labelIds", [])
+ channel = _select_channel(label_ids)
thread_id: Optional[str] = msg.get("threadId")
doc = Document(
doc_id=f"gmail:{msg_id}",
source="gmail",
+ source_id=msg_id,
doc_type="email",
content=body,
title=subject,
author=from_header,
participants=participants,
+ participants_raw=participants_raw,
timestamp=timestamp,
thread_id=thread_id,
+ channel=channel,
metadata={
"message_id": msg_id,
- "labels": msg.get("labelIds", []),
+ "rfc_message_id": rfc_message_id,
+ "labels": label_ids,
+ "snippet": msg.get("snippet", ""),
+ "history_id": msg.get("historyId", ""),
+ "size_estimate": msg.get("sizeEstimate", 0),
},
)
synced += 1
diff --git a/src/openjarvis/connectors/gmail_imap.py b/src/openjarvis/connectors/gmail_imap.py
index 53fbfc98..61836bae 100644
--- a/src/openjarvis/connectors/gmail_imap.py
+++ b/src/openjarvis/connectors/gmail_imap.py
@@ -92,12 +92,14 @@ class GmailIMAPConnector(BaseConnector):
credentials_path: str = "",
*,
imap_host: str = "",
- max_messages: int = 5000,
+ max_messages: Optional[int] = None,
) -> None:
self._email = email_address
self._password = app_password
self._credentials_path = credentials_path or _DEFAULT_CREDENTIALS_PATH
self._imap_host = imap_host or self._default_imap_host
+ # ``None`` means "no cap" — the full inbox is indexed. A positive
+ # value is still honored for tests that want a bounded scan.
self._max_messages = max_messages
self._items_synced = 0
self._items_total = 0
@@ -156,21 +158,27 @@ class GmailIMAPConnector(BaseConnector):
imap.select("INBOX", readonly=True)
- # Build search criteria
- if since:
- date_str = since.strftime("%d-%b-%Y")
- _, data = imap.search(None, f"SINCE {date_str}")
- else:
- _, data = imap.search(None, "ALL")
-
+ # Always SEARCH ALL. IMAP has no native cursor that survives a
+ # server restart, so applying the SyncEngine's ``since`` filter
+ # during a partial backfill would silently skip the older
+ # unprocessed messages. The pipeline-level dedup (_seen_doc_ids
+ # set + INSERT OR IGNORE in KnowledgeStore) makes re-scanning
+ # already-indexed messages cheap, so resume is correct as long
+ # as we keep enumerating the full inbox.
+ _, data = imap.search(None, "ALL")
msg_ids = data[0].split()
self._items_total = len(msg_ids)
- # Take the most recent N messages
- recent = msg_ids[-self._max_messages :]
+ # Newest-first iteration so Deep Research becomes useful while
+ # the long tail of older mail finishes indexing in the
+ # background. IMAP returns sequence numbers in arrival order
+ # (oldest -> newest), so reversing puts the most recent first.
+ ordered = list(reversed(msg_ids))
+ if self._max_messages is not None and self._max_messages > 0:
+ ordered = ordered[: self._max_messages]
synced = 0
- for mid in recent:
+ for mid in ordered:
try:
_, msg_data = imap.fetch(mid, "(RFC822)")
raw = msg_data[0][1]
diff --git a/src/openjarvis/connectors/google_auth.py b/src/openjarvis/connectors/google_auth.py
new file mode 100644
index 00000000..099dc2fe
--- /dev/null
+++ b/src/openjarvis/connectors/google_auth.py
@@ -0,0 +1,130 @@
+"""Shared Google OAuth helpers: access token read + one-shot 401 refresh.
+
+All Google connectors (Gmail, Calendar, Contacts, Drive, Tasks) authenticate
+with the same OAuth flow and store identical token payloads at
+``~/.openjarvis/connectors/*.json`` — typically a shared ``google.json`` file
+plus per-product copies. They all need the same refresh-on-401 behavior, so
+the wrapper lives here instead of being duplicated per connector.
+
+Use ``call_with_refresh(api_fn, credentials_path, *args, **kwargs)`` around
+any token-taking API helper. On a 401 the wrapper exchanges the stored
+``refresh_token`` for a new ``access_token``, updates the credentials file,
+and retries the call once. All other status codes propagate.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Any, Callable, Dict
+
+import httpx
+
+from openjarvis.connectors.oauth import load_tokens, save_tokens
+
+logger = logging.getLogger(__name__)
+
+
+_GOOGLE_TOKEN_ENDPOINT = "https://oauth2.googleapis.com/token"
+
+
+class GoogleAuthError(RuntimeError):
+ """Raised when Google credentials are missing or refresh-token grant fails."""
+
+
+def current_access_token(credentials_path: str) -> str:
+ """Return the current access token from the credentials file (empty if absent)."""
+ tokens = load_tokens(credentials_path) or {}
+ return tokens.get("access_token", tokens.get("token", ""))
+
+
+def refresh_access_token(credentials_path: str) -> str:
+ """Exchange the stored refresh_token for a fresh access_token and persist it.
+
+ Returns the new access_token. Raises :class:`GoogleAuthError` when the
+ credentials file is missing, lacks a refresh_token / client credentials,
+ or when Google rejects the refresh grant (e.g. the refresh_token has been
+ revoked and the user needs to re-authenticate).
+ """
+ tokens = load_tokens(credentials_path)
+ if not tokens:
+ raise GoogleAuthError(
+ f"No credentials at {credentials_path}; re-run the connector OAuth flow."
+ )
+ refresh_token = tokens.get("refresh_token", "")
+ client_id = tokens.get("client_id", "")
+ client_secret = tokens.get("client_secret", "")
+ if not (refresh_token and client_id and client_secret):
+ raise GoogleAuthError(
+ "Stored Google credentials are missing refresh_token / client_id / "
+ "client_secret; re-run the connector OAuth flow to mint a full token."
+ )
+
+ resp = httpx.post(
+ _GOOGLE_TOKEN_ENDPOINT,
+ data={
+ "client_id": client_id,
+ "client_secret": client_secret,
+ "refresh_token": refresh_token,
+ "grant_type": "refresh_token",
+ },
+ timeout=30.0,
+ )
+ if resp.status_code != 200:
+ raise GoogleAuthError(
+ f"Google token refresh failed ({resp.status_code}): {resp.text[:200]}"
+ )
+ payload = resp.json()
+ new_token = payload.get("access_token", "")
+ if not new_token:
+ raise GoogleAuthError(
+ "Google token refresh returned 200 but no access_token in payload."
+ )
+
+ tokens["access_token"] = new_token
+ # Keep the legacy "token" key in sync for older code paths that read it.
+ tokens["token"] = new_token
+ if "expires_in" in payload:
+ tokens["expires_in"] = payload["expires_in"]
+ save_tokens(credentials_path, tokens)
+ logger.info(
+ "Refreshed Google access token (expires_in=%s)", payload.get("expires_in")
+ )
+ return new_token
+
+
+def call_with_refresh(
+ api_fn: Callable[..., Dict[str, Any]],
+ credentials_path: str,
+ *args: Any,
+ **kwargs: Any,
+) -> Dict[str, Any]:
+ """Invoke ``api_fn(token, *args, **kwargs)`` with one-shot 401 auto-refresh.
+
+ Loads the current access token from disk, calls the helper, and if Google
+ returns 401 (the access token has expired or been revoked) uses the stored
+ refresh_token to mint a new access_token, updates the credentials file,
+ and retries the call exactly once.
+
+ Any other ``HTTPStatusError`` is re-raised unchanged — auth-related retries
+ end here; transient 5xx / timeout retries belong further up the stack.
+ """
+ token = current_access_token(credentials_path)
+ try:
+ return api_fn(token, *args, **kwargs)
+ except httpx.HTTPStatusError as exc:
+ if exc.response is None or exc.response.status_code != 401:
+ raise
+ logger.info(
+ "Google returned 401 on %s — refreshing access token and retrying.",
+ getattr(api_fn, "__name__", "<api_fn>"),
+ )
+ new_token = refresh_access_token(credentials_path)
+ return api_fn(new_token, *args, **kwargs)
+
+
+__all__ = [
+ "GoogleAuthError",
+ "current_access_token",
+ "refresh_access_token",
+ "call_with_refresh",
+]
diff --git a/src/openjarvis/connectors/google_tasks.py b/src/openjarvis/connectors/google_tasks.py
index 0f5e0c47..a0f65cc3 100644
--- a/src/openjarvis/connectors/google_tasks.py
+++ b/src/openjarvis/connectors/google_tasks.py
@@ -12,6 +12,7 @@ from typing import Any, Dict, Iterator, Optional
import httpx
from openjarvis.connectors._stubs import BaseConnector, Document, SyncStatus
+from openjarvis.connectors.google_auth import call_with_refresh
from openjarvis.connectors.oauth import load_tokens, resolve_google_credentials
from openjarvis.core.config import DEFAULT_CONFIG_DIR
from openjarvis.core.registry import ConnectorRegistry
@@ -62,10 +63,10 @@ class GoogleTasksConnector(BaseConnector):
def sync(
self, *, since: Optional[datetime] = None, cursor: Optional[str] = None
) -> Iterator[Document]:
- token = self._get_access_token()
-
- # List all task lists first
- task_lists = _tasks_api_get(token, "users/@me/lists")
+ # call_with_refresh handles the access-token read + one-shot 401 retry.
+ task_lists = call_with_refresh(
+ _tasks_api_get, str(self._credentials_path), "users/@me/lists"
+ )
for tl in task_lists.get("items", []):
tl_id = tl["id"]
@@ -79,7 +80,12 @@ class GoogleTasksConnector(BaseConnector):
if since:
params["updatedMin"] = since.isoformat() + "Z"
- tasks = _tasks_api_get(token, f"lists/{tl_id}/tasks", params=params)
+ tasks = call_with_refresh(
+ _tasks_api_get,
+ str(self._credentials_path),
+ f"lists/{tl_id}/tasks",
+ params=params,
+ )
for task in tasks.get("items", []):
due = task.get("due", "")
diff --git a/src/openjarvis/connectors/hybrid_search.py b/src/openjarvis/connectors/hybrid_search.py
new file mode 100644
index 00000000..0465989b
--- /dev/null
+++ b/src/openjarvis/connectors/hybrid_search.py
@@ -0,0 +1,471 @@
+"""Hybrid retrieval over the KnowledgeStore: metadata filter + BM25 + vector cosine.
+
+A single ``search`` entrypoint that the agentic research loop calls as a tool.
+Structured WHERE-clause filters (person, time range, sources) narrow the
+candidate set, then BM25 (FTS5) and dense cosine similarity score the
+survivors. The two ranks are fused with Reciprocal Rank Fusion, which is
+robust to the very different score scales the two signals produce
+(BM25 ~ [0, 20], cosine ~ [0.4, 0.9] for nomic-embed-text).
+
+Each result is enriched with its thread context: when a hit belongs to a
+``thread_id``, the surrounding chunks are attached so the synthesis model
+sees the conversation, not an isolated fragment.
+
+Brute-force vector scan is fine at the current corpus size (~5k chunks ×
+768 dims fits in ~15 MB and matmuls in <50 ms). Swap in an ANN index when
+that stops being true.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Any, Dict, List, Optional, Sequence, Tuple
+
+import numpy as np
+
+from openjarvis.connectors.embeddings import OllamaEmbedder, decode_embedding
+from openjarvis.connectors.store import KnowledgeStore
+
+logger = logging.getLogger(__name__)
+
+
+# ---------------------------------------------------------------------------
+# Result types
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class SearchHit:
+ """A single hybrid-search result with enough context for citation."""
+
+ chunk_id: str
+ document_id: str
+ chunk_idx: int
+ title: str
+ content_snippet: str
+ source: str
+ timestamp: str
+ participants: List[str]
+ score: float
+ bm25_score: float
+ vector_score: float
+ thread_id: str = ""
+ thread_context: List[Dict[str, Any]] = field(default_factory=list)
+
+ def to_dict(self) -> Dict[str, Any]:
+ return {
+ "title": self.title,
+ "content_snippet": self.content_snippet,
+ "source": self.source,
+ "timestamp": self.timestamp,
+ "participants": self.participants,
+ "score": round(self.score, 4),
+ "document_id": self.document_id,
+ "chunk_idx": self.chunk_idx,
+ "thread_id": self.thread_id,
+ "thread_context": self.thread_context,
+ }
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+
+def _iso(ts: Optional[datetime | str]) -> Optional[str]:
+ if ts is None:
+ return None
+ if isinstance(ts, datetime):
+ return ts.isoformat()
+ return str(ts)
+
+
+def _quote_fts(query: str) -> str:
+ """Make a plain user query safe for FTS5 MATCH.
+
+ FTS5 treats characters like ``-``, ``:``, ``"`` as operators; the simplest
+ way to avoid syntax errors on arbitrary user input is to quote each
+ whitespace-delimited token and OR them together.
+ """
+ tokens = [t for t in query.split() if t]
+ if not tokens:
+ return ""
+ return " OR ".join(f'"{t.replace(chr(34), "")}"' for t in tokens)
+
+
+def _parse_participants(raw: Any) -> List[str]:
+ if not raw:
+ return []
+ if isinstance(raw, list):
+ return [str(x) for x in raw]
+ try:
+ parsed = json.loads(raw)
+ return [str(x) for x in parsed] if isinstance(parsed, list) else []
+ except (json.JSONDecodeError, TypeError):
+ return []
+
+
+def _snippet(content: str, max_chars: int = 500) -> str:
+ flat = content.strip()
+ if len(flat) <= max_chars:
+ return flat
+ return flat[:max_chars].rstrip() + "…"
+
+
+# ---------------------------------------------------------------------------
+# HybridSearch
+# ---------------------------------------------------------------------------
+
+
+class HybridSearch:
+ """Hybrid BM25 + dense-cosine retrieval over a ``KnowledgeStore``.
+
+ Parameters
+ ----------
+ store:
+ The store to query.
+ embedder:
+ Embedding client used to encode the query. When ``None``, search
+ falls back to BM25 only and reports ``vector_score=0``.
+ bm25_weight, vector_weight:
+ Weights on the two RRF terms. Defaults to 0.5 / 0.5; raise either to
+ bias retrieval toward lexical or semantic matches.
+ rrf_k:
+ RRF damping constant. Larger values flatten the contribution of
+ deeper ranks; 60 is the canonical value from the original paper.
+ recall_k:
+ How deep each individual ranker recalls before fusion. Should be at
+ least a few times ``limit`` so the fuser has overlap to work with.
+ """
+
+ def __init__(
+ self,
+ store: KnowledgeStore,
+ embedder: Optional[OllamaEmbedder] = None,
+ *,
+ bm25_weight: float = 0.5,
+ vector_weight: float = 0.5,
+ rrf_k: int = 60,
+ recall_k: int = 200,
+ thread_context_cap: int = 20,
+ ) -> None:
+ self._store = store
+ self._embedder = embedder
+ self._bm25_weight = float(bm25_weight)
+ self._vector_weight = float(vector_weight)
+ self._rrf_k = int(rrf_k)
+ self._recall_k = int(recall_k)
+ self._thread_context_cap = int(thread_context_cap)
+
+ # ------------------------------------------------------------------
+ # Filter SQL construction
+ # ------------------------------------------------------------------
+
+ def _build_filters(
+ self,
+ *,
+ person: Optional[str],
+ time_range: Optional[Tuple[Optional[datetime], Optional[datetime]]],
+ sources: Optional[Sequence[str]],
+ alias: str = "",
+ ) -> Tuple[str, List[Any]]:
+ """Return ``(where_fragment, params)`` for the structured filters.
+
+ ``person`` is matched against the participants_raw JSON via LIKE so a
+ substring of a name or email address is enough — handy when the user
+ says "Kelly" rather than "[email protected]".
+
+ ``alias`` qualifies every column reference (e.g. ``kc.`` when joining
+ against the FTS virtual table which also has ``author`` and ``title``
+ columns and would otherwise produce an "ambiguous column" error).
+ """
+ prefix = f"{alias}." if alias else ""
+ clauses: List[str] = [f"{prefix}deleted_at IS NULL"]
+ params: List[Any] = []
+
+ if person:
+ clauses.append(
+ f"({prefix}participants_raw LIKE ? OR {prefix}participants LIKE ? "
+ f"OR {prefix}author LIKE ?)"
+ )
+ needle = f"%{person}%"
+ params.extend([needle, needle, needle])
+
+ if time_range:
+ start, end = time_range
+ if start is not None:
+ clauses.append(f"{prefix}timestamp >= ?")
+ params.append(_iso(start))
+ if end is not None:
+ clauses.append(f"{prefix}timestamp <= ?")
+ params.append(_iso(end))
+
+ if sources:
+ placeholders = ",".join("?" for _ in sources)
+ clauses.append(f"{prefix}source IN ({placeholders})")
+ params.extend(sources)
+
+ return " AND ".join(clauses), params
+
+ # ------------------------------------------------------------------
+ # BM25 leg
+ # ------------------------------------------------------------------
+
+ def _bm25_recall(
+ self, query: str, filter_sql: str, filter_params: List[Any]
+ ) -> List[Tuple[str, float]]:
+ """Return ``[(chunk_id, bm25_score), ...]`` from FTS5."""
+ fts_query = _quote_fts(query)
+ if not fts_query:
+ return []
+ sql = f"""
+ SELECT kc.id, abs(bm25(knowledge_fts)) AS score
+ FROM knowledge_fts
+ JOIN knowledge_chunks kc ON knowledge_fts.rowid = kc.rowid
+ WHERE knowledge_fts MATCH ?
+ AND {filter_sql}
+ ORDER BY score DESC
+ LIMIT ?
+ """
+ try:
+ rows = self._store._conn.execute(
+ sql, [fts_query, *filter_params, self._recall_k]
+ ).fetchall()
+ except Exception as exc: # noqa: BLE001
+ logger.warning("hybrid_search: BM25 leg failed (%s)", exc)
+ return []
+ return [(row["id"], float(row["score"])) for row in rows]
+
+ # ------------------------------------------------------------------
+ # Vector leg
+ # ------------------------------------------------------------------
+
+ def _vector_recall(
+ self, query: str, filter_sql: str, filter_params: List[Any]
+ ) -> List[Tuple[str, float]]:
+ """Return ``[(chunk_id, cosine_score), ...]`` from a brute-force scan."""
+ if self._embedder is None:
+ return []
+ q_blob = self._embedder.embed(query)
+ q_vec = decode_embedding(q_blob)
+ if q_vec is None or q_vec.size == 0:
+ return []
+ q_norm = float(np.linalg.norm(q_vec))
+ if q_norm == 0.0:
+ return []
+ q_unit = q_vec / q_norm
+
+ sql = f"""
+ SELECT id, embedding
+ FROM knowledge_chunks
+ WHERE embedding IS NOT NULL AND {filter_sql}
+ """
+ rows = self._store._conn.execute(sql, filter_params).fetchall()
+ if not rows:
+ return []
+
+ ids: List[str] = []
+ vecs: List[np.ndarray] = []
+ for row in rows:
+ vec = decode_embedding(row["embedding"])
+ if vec is None or vec.size != q_unit.size:
+ continue
+ ids.append(row["id"])
+ vecs.append(vec)
+ if not ids:
+ return []
+
+ mat = np.vstack(vecs).astype(np.float32, copy=False)
+ norms = np.linalg.norm(mat, axis=1)
+ norms[norms == 0.0] = 1.0
+ mat = mat / norms[:, None]
+ scores = mat @ q_unit
+ # Top-recall_k
+ if len(scores) > self._recall_k:
+ top_idx = np.argpartition(-scores, self._recall_k)[: self._recall_k]
+ top_idx = top_idx[np.argsort(-scores[top_idx])]
+ else:
+ top_idx = np.argsort(-scores)
+ return [(ids[int(i)], float(scores[int(i)])) for i in top_idx]
+
+ # ------------------------------------------------------------------
+ # Fusion
+ # ------------------------------------------------------------------
+
+ def _fuse(
+ self,
+ bm25: List[Tuple[str, float]],
+ vector: List[Tuple[str, float]],
+ ) -> List[Tuple[str, float, float, float]]:
+ """Reciprocal Rank Fusion across the two rankers.
+
+ Returns ``[(chunk_id, fused, bm25_score, vector_score), ...]``
+ sorted by ``fused`` descending.
+ """
+ bm25_rank = {cid: i + 1 for i, (cid, _) in enumerate(bm25)}
+ vec_rank = {cid: i + 1 for i, (cid, _) in enumerate(vector)}
+ bm25_scores = {cid: s for cid, s in bm25}
+ vec_scores = {cid: s for cid, s in vector}
+
+ candidates = set(bm25_rank) | set(vec_rank)
+ out: List[Tuple[str, float, float, float]] = []
+ for cid in candidates:
+ fused = 0.0
+ if cid in bm25_rank:
+ fused += self._bm25_weight / (self._rrf_k + bm25_rank[cid])
+ if cid in vec_rank:
+ fused += self._vector_weight / (self._rrf_k + vec_rank[cid])
+ out.append(
+ (cid, fused, bm25_scores.get(cid, 0.0), vec_scores.get(cid, 0.0))
+ )
+ out.sort(key=lambda r: -r[1])
+ return out
+
+ # ------------------------------------------------------------------
+ # Thread enrichment
+ # ------------------------------------------------------------------
+
+ def _thread_context(
+ self, thread_id: str, anchor_chunk_id: str
+ ) -> List[Dict[str, Any]]:
+ """Fetch sibling chunks for ``thread_id`` (capped at ``thread_context_cap``).
+
+ When the thread is longer than the cap, return a centred window around
+ the anchor so the most relevant chunk is always present.
+ """
+ if not thread_id:
+ return []
+ rows = self._store._conn.execute(
+ """
+ SELECT id, chunk_index, content, timestamp, author
+ FROM knowledge_chunks
+ WHERE thread_id = ? AND deleted_at IS NULL
+ ORDER BY timestamp ASC, chunk_index ASC
+ """,
+ (thread_id,),
+ ).fetchall()
+ if not rows:
+ return []
+ cap = self._thread_context_cap
+ if len(rows) > cap:
+ anchor_idx = next(
+ (i for i, r in enumerate(rows) if r["id"] == anchor_chunk_id),
+ len(rows) // 2,
+ )
+ half = cap // 2
+ lo = max(0, anchor_idx - half)
+ hi = min(len(rows), lo + cap)
+ lo = max(0, hi - cap)
+ rows = rows[lo:hi]
+ return [
+ {
+ "chunk_idx": int(r["chunk_index"]),
+ "timestamp": r["timestamp"] or "",
+ "author": r["author"] or "",
+ "snippet": _snippet(r["content"], 240),
+ }
+ for r in rows
+ ]
+
+ # ------------------------------------------------------------------
+ # Public entry point
+ # ------------------------------------------------------------------
+
+ def search(
+ self,
+ query: str,
+ *,
+ person: Optional[str] = None,
+ time_range: Optional[Tuple[Optional[datetime], Optional[datetime]]] = None,
+ sources: Optional[Sequence[str]] = None,
+ limit: int = 20,
+ ) -> List[SearchHit]:
+ """Run the hybrid pipeline and return up to ``limit`` hits.
+
+ See module docstring for ranking semantics. ``query`` may be empty
+ when callers want a pure metadata filter (e.g. "all mail from X in
+ May") — in that case only the vector leg runs (and only if an
+ embedder is configured); if neither leg yields anything the
+ structured filter is applied directly and the most recent rows are
+ returned.
+ """
+ bm25_filter_sql, bm25_filter_params = self._build_filters(
+ person=person, time_range=time_range, sources=sources, alias="kc"
+ )
+ unaliased_filter_sql, unaliased_filter_params = self._build_filters(
+ person=person, time_range=time_range, sources=sources
+ )
+
+ bm25 = (
+ self._bm25_recall(query, bm25_filter_sql, bm25_filter_params)
+ if query.strip()
+ else []
+ )
+ vector = (
+ self._vector_recall(query, unaliased_filter_sql, unaliased_filter_params)
+ if query.strip()
+ else []
+ )
+ fused = self._fuse(bm25, vector)
+
+ # Metadata-only fallback: empty query, or both legs produced nothing
+ # despite a non-empty query. Return the most recent rows matching the
+ # filter so the agent still gets a useful corpus snapshot.
+ if not fused:
+ sql = f"""
+ SELECT id FROM knowledge_chunks
+ WHERE {unaliased_filter_sql}
+ ORDER BY timestamp DESC, created_at DESC
+ LIMIT ?
+ """
+ rows = self._store._conn.execute(
+ sql, [*unaliased_filter_params, limit]
+ ).fetchall()
+ fused = [(row["id"], 0.0, 0.0, 0.0) for row in rows]
+
+ # Materialise the top-N rows in one IN-clause round trip.
+ top = fused[:limit]
+ if not top:
+ return []
+ ids = [cid for cid, *_ in top]
+ placeholders = ",".join("?" for _ in ids)
+ meta_rows = self._store._conn.execute(
+ f"""
+ SELECT id, doc_id, content, source, title, author, participants,
+ timestamp, thread_id, chunk_index
+ FROM knowledge_chunks
+ WHERE id IN ({placeholders})
+ """,
+ ids,
+ ).fetchall()
+ by_id = {r["id"]: r for r in meta_rows}
+
+ hits: List[SearchHit] = []
+ for chunk_id, fused_score, bm25_score, vec_score in top:
+ r = by_id.get(chunk_id)
+ if r is None:
+ continue
+ hits.append(
+ SearchHit(
+ chunk_id=chunk_id,
+ document_id=r["doc_id"],
+ chunk_idx=int(r["chunk_index"]),
+ title=r["title"] or "",
+ content_snippet=_snippet(r["content"]),
+ source=r["source"] or "",
+ timestamp=r["timestamp"] or "",
+ participants=_parse_participants(r["participants"]),
+ score=fused_score,
+ bm25_score=bm25_score,
+ vector_score=vec_score,
+ thread_id=r["thread_id"] or "",
+ thread_context=self._thread_context(r["thread_id"] or "", chunk_id),
+ )
+ )
+ return hits
+
+
+__all__ = ["HybridSearch", "SearchHit"]
diff --git a/src/openjarvis/connectors/pipeline.py b/src/openjarvis/connectors/pipeline.py
index 4df3a030..ea4ae0e7 100644
--- a/src/openjarvis/connectors/pipeline.py
+++ b/src/openjarvis/connectors/pipeline.py
@@ -13,12 +13,49 @@ Typical usage::
from __future__ import annotations
+import hashlib
+import time
from typing import TYPE_CHECKING, Iterable, Optional
from openjarvis.connectors._stubs import Attachment, Document
from openjarvis.connectors.chunker import SemanticChunker
+from openjarvis.connectors.embeddings import OllamaEmbedder
from openjarvis.connectors.store import KnowledgeStore
+
+def _namespace_thread_id(source: str, thread_id: Optional[str]) -> Optional[str]:
+ """Prefix ``thread_id`` with ``{source}:`` so it can't collide across sources.
+
+ Idempotent: if the input already starts with ``{source}:`` it is returned
+ unchanged. Centralised here (rather than per-connector) so a new connector
+ author can't forget to namespace.
+ """
+ if not thread_id:
+ return None
+ prefix = f"{source}:"
+ if thread_id.startswith(prefix):
+ return thread_id
+ return f"{prefix}{thread_id}"
+
+
+def _derive_source_id(doc: Document) -> str:
+ """Return the connector-set ``source_id`` or extract it from ``doc_id``.
+
+ Many existing connectors compose ``doc_id = f"{source}:{native_id}"``;
+ this strips the prefix so storage can index the native ID directly.
+ """
+ if doc.source_id:
+ return doc.source_id
+ prefix = f"{doc.source}:"
+ if doc.doc_id.startswith(prefix):
+ return doc.doc_id[len(prefix):]
+ return doc.doc_id
+
+
+def _content_hash(text: str) -> str:
+ """SHA-256 hex digest of UTF-8-encoded chunk content."""
+ return hashlib.sha256(text.encode("utf-8")).hexdigest()
+
if TYPE_CHECKING:
from openjarvis.connectors.attachment_store import AttachmentStore
@@ -36,6 +73,12 @@ class IngestionPipeline:
Optional ``AttachmentStore`` for persisting attachment blobs and
extracting text from supported MIME types (PDF, plain text, etc.).
When ``None`` (default) attachments are silently ignored.
+ embedder:
+ Optional embedding client (e.g. ``OllamaEmbedder``). When provided,
+ every chunk is embedded at ingest time and the resulting float32
+ vector is written to the ``embedding`` BLOB column alongside
+ ``embedding_model_version``. ``None`` (default) skips embedding so
+ in-memory tests and offline runs don't depend on a sidecar daemon.
"""
def __init__(
@@ -44,10 +87,12 @@ class IngestionPipeline:
*,
max_tokens: int = 512,
attachment_store: Optional[AttachmentStore] = None,
+ embedder: Optional[OllamaEmbedder] = None,
) -> None:
self._store = store
self._chunker = SemanticChunker(max_tokens=max_tokens)
self._attachment_store = attachment_store
+ self._embedder = embedder
self._seen_doc_ids: set[str] = set()
self._load_existing_doc_ids()
@@ -62,6 +107,20 @@ class IngestionPipeline:
).fetchall()
self._seen_doc_ids = {r[0] for r in rows}
+ def _embed_chunk(self, content: str) -> tuple[Optional[bytes], str]:
+ """Return ``(embedding_bytes, model_version)`` for a chunk.
+
+ Returns ``(None, "")`` when no embedder is configured or the embedder
+ fails — ingestion continues with the lexical-only row, so a flaky
+ local daemon never blocks a sync.
+ """
+ if self._embedder is None:
+ return None, ""
+ emb = self._embedder.embed(content)
+ if emb is None:
+ return None, ""
+ return emb, self._embedder.model_version
+
def _extract_attachment_text(self, att: Attachment) -> str:
"""Extract text from an attachment.
@@ -109,15 +168,22 @@ class IngestionPipeline:
if doc.doc_id in self._seen_doc_ids:
continue
+ # Compute v1 provenance fields once per document.
+ namespaced_thread = _namespace_thread_id(doc.source, doc.thread_id)
+ source_id = _derive_source_id(doc)
+ ingest_epoch = time.time()
+
# Build the parent metadata dict that will be inherited by every
# chunk produced from this document.
parent_meta = {
"title": doc.title,
"author": doc.author,
"source": doc.source,
+ "source_id": source_id,
"doc_type": doc.doc_type,
"url": doc.url or "",
- "thread_id": doc.thread_id or "",
+ "thread_id": namespaced_thread or "",
+ "channel": doc.channel or "",
}
# Merge any extra connector-level metadata (without overwriting
# the standard provenance fields set above).
@@ -137,19 +203,27 @@ class IngestionPipeline:
)
for chunk in chunks:
+ embedding_bytes, embedding_version = self._embed_chunk(chunk.content)
self._store.store(
content=chunk.content,
source=doc.source,
+ source_id=source_id,
doc_type=doc.doc_type,
doc_id=doc.doc_id,
title=doc.title,
author=doc.author,
participants=doc.participants,
+ participants_raw=doc.participants_raw,
timestamp=timestamp_str,
- thread_id=doc.thread_id,
+ thread_id=namespaced_thread,
+ channel=doc.channel,
url=doc.url,
metadata=chunk.metadata,
chunk_index=chunk.index,
+ content_hash=_content_hash(chunk.content),
+ embedding=embedding_bytes,
+ embedding_model_version=embedding_version,
+ last_synced=ingest_epoch,
)
chunks_stored += 1
@@ -179,20 +253,35 @@ class IngestionPipeline:
"sha256": sha,
},
)
+ # Synthetic source_id keeps attachment chunks distinct
+ # from body chunks under the UNIQUE(source, source_id,
+ # chunk_index) constraint while still letting them share
+ # a parent doc_id for dedup and blob linkage.
+ att_source_id = f"{source_id}#{att.filename}"
for chunk in att_chunks:
+ embedding_bytes, embedding_version = self._embed_chunk(
+ chunk.content
+ )
self._store.store(
content=chunk.content,
source=doc.source,
+ source_id=att_source_id,
doc_type=doc.doc_type,
doc_id=doc.doc_id,
title=f"{doc.title} [{att.filename}]",
author=doc.author,
participants=doc.participants,
+ participants_raw=doc.participants_raw,
timestamp=timestamp_str,
- thread_id=doc.thread_id,
+ thread_id=namespaced_thread,
+ channel=doc.channel,
url=doc.url,
metadata=chunk.metadata,
chunk_index=chunk.index,
+ content_hash=_content_hash(chunk.content),
+ embedding=embedding_bytes,
+ embedding_model_version=embedding_version,
+ last_synced=ingest_epoch,
)
chunks_stored += 1
diff --git a/src/openjarvis/connectors/store.py b/src/openjarvis/connectors/store.py
index 24df031b..2eb7a4d3 100644
--- a/src/openjarvis/connectors/store.py
+++ b/src/openjarvis/connectors/store.py
@@ -31,19 +31,40 @@ CREATE TABLE IF NOT EXISTS knowledge_chunks (
doc_id TEXT NOT NULL,
content TEXT NOT NULL,
source TEXT NOT NULL DEFAULT '',
+ source_id TEXT NOT NULL DEFAULT '',
doc_type TEXT NOT NULL DEFAULT '',
title TEXT NOT NULL DEFAULT '',
author TEXT NOT NULL DEFAULT '',
participants TEXT NOT NULL DEFAULT '[]',
+ participants_raw TEXT NOT NULL DEFAULT '[]',
timestamp TEXT NOT NULL DEFAULT '',
thread_id TEXT NOT NULL DEFAULT '',
+ channel TEXT NOT NULL DEFAULT '',
url TEXT NOT NULL DEFAULT '',
metadata TEXT NOT NULL DEFAULT '{}',
chunk_index INTEGER NOT NULL DEFAULT 0,
+ embedding BLOB,
+ embedding_model_version TEXT NOT NULL DEFAULT '',
+ content_hash TEXT NOT NULL DEFAULT '',
+ deleted_at REAL,
+ last_synced REAL NOT NULL DEFAULT 0,
created_at REAL NOT NULL
);
"""
+# Additive migrations: existing installs get new columns via ALTER TABLE.
+# Order doesn't matter — each entry is independent and idempotent.
+_V1_COLUMNS: List[tuple[str, str]] = [
+ ("source_id", "TEXT NOT NULL DEFAULT ''"),
+ ("participants_raw", "TEXT NOT NULL DEFAULT '[]'"),
+ ("channel", "TEXT NOT NULL DEFAULT ''"),
+ ("embedding", "BLOB"),
+ ("embedding_model_version", "TEXT NOT NULL DEFAULT ''"),
+ ("content_hash", "TEXT NOT NULL DEFAULT ''"),
+ ("deleted_at", "REAL"),
+ ("last_synced", "REAL NOT NULL DEFAULT 0"),
+]
+
_CREATE_FTS_TABLE = """
CREATE VIRTUAL TABLE IF NOT EXISTS knowledge_fts
USING fts5(
@@ -76,12 +97,17 @@ END;
"""
_CREATE_INDEXES = """
-CREATE INDEX IF NOT EXISTS idx_kc_source ON knowledge_chunks(source);
-CREATE INDEX IF NOT EXISTS idx_kc_doc_type ON knowledge_chunks(doc_type);
-CREATE INDEX IF NOT EXISTS idx_kc_author ON knowledge_chunks(author);
-CREATE INDEX IF NOT EXISTS idx_kc_timestamp ON knowledge_chunks(timestamp);
-CREATE INDEX IF NOT EXISTS idx_kc_thread_id ON knowledge_chunks(thread_id);
-CREATE INDEX IF NOT EXISTS idx_kc_doc_id ON knowledge_chunks(doc_id);
+CREATE INDEX IF NOT EXISTS idx_kc_source ON knowledge_chunks(source);
+CREATE INDEX IF NOT EXISTS idx_kc_doc_type ON knowledge_chunks(doc_type);
+CREATE INDEX IF NOT EXISTS idx_kc_author ON knowledge_chunks(author);
+CREATE INDEX IF NOT EXISTS idx_kc_timestamp ON knowledge_chunks(timestamp);
+CREATE INDEX IF NOT EXISTS idx_kc_thread_id ON knowledge_chunks(thread_id);
+CREATE INDEX IF NOT EXISTS idx_kc_doc_id ON knowledge_chunks(doc_id);
+CREATE INDEX IF NOT EXISTS idx_kc_source_id ON knowledge_chunks(source_id);
+CREATE INDEX IF NOT EXISTS idx_kc_content_hash ON knowledge_chunks(content_hash);
+CREATE UNIQUE INDEX IF NOT EXISTS idx_kc_natural_key
+ ON knowledge_chunks(source, source_id, chunk_index)
+ WHERE source_id != '';
"""
# ---------------------------------------------------------------------------
@@ -100,6 +126,20 @@ def _to_iso(ts: Optional[Union[datetime, str]]) -> str:
return ts.isoformat()
+def _to_epoch(ts: Union[datetime, str, float, int]) -> float:
+ """Normalise a timestamp to Unix epoch seconds (float)."""
+ if isinstance(ts, (int, float)):
+ return float(ts)
+ if isinstance(ts, str):
+ # Treat empty/zero-ish strings as epoch 0; otherwise parse ISO 8601.
+ if not ts:
+ return 0.0
+ return datetime.fromisoformat(ts).timestamp()
+ if ts.tzinfo is None:
+ ts = ts.replace(tzinfo=timezone.utc)
+ return ts.timestamp()
+
+
# ---------------------------------------------------------------------------
# KnowledgeStore
# ---------------------------------------------------------------------------
@@ -147,10 +187,27 @@ class KnowledgeStore(MemoryBackend):
self._conn.execute("PRAGMA journal_mode=WAL;")
self._conn.execute("PRAGMA foreign_keys=ON;")
self._conn.executescript(
- _CREATE_MAIN_TABLE + _CREATE_FTS_TABLE + _CREATE_TRIGGERS + _CREATE_INDEXES
+ _CREATE_MAIN_TABLE + _CREATE_FTS_TABLE + _CREATE_TRIGGERS
)
+ self._migrate_v1_columns()
+ # Indexes reference the new columns, so they run after migrations.
+ self._conn.executescript(_CREATE_INDEXES)
self._conn.commit()
+ def _migrate_v1_columns(self) -> None:
+ """Add v1 schema columns to pre-existing tables (idempotent)."""
+ existing = {
+ row[1]
+ for row in self._conn.execute(
+ "PRAGMA table_info(knowledge_chunks)"
+ ).fetchall()
+ }
+ for col_name, col_def in _V1_COLUMNS:
+ if col_name not in existing:
+ self._conn.execute(
+ f"ALTER TABLE knowledge_chunks ADD COLUMN {col_name} {col_def}"
+ )
+
# ------------------------------------------------------------------
# MemoryBackend interface
# ------------------------------------------------------------------
@@ -170,6 +227,14 @@ class KnowledgeStore(MemoryBackend):
url: Optional[str] = None,
metadata: Optional[Dict[str, Any]] = None,
chunk_index: int = 0,
+ # v1 schema additions
+ source_id: str = "",
+ participants_raw: Optional[List[str]] = None,
+ channel: Optional[str] = None,
+ content_hash: str = "",
+ embedding: Optional[bytes] = None,
+ embedding_model_version: str = "",
+ last_synced: Optional[Union[datetime, str, float]] = None,
) -> str:
"""Persist a content chunk and return its unique chunk id.
@@ -182,50 +247,83 @@ class KnowledgeStore(MemoryBackend):
ts_str = _to_iso(timestamp)
participants_json = json.dumps(participants or [])
+ participants_raw_json = json.dumps(participants_raw or [])
+ last_synced_epoch = (
+ _to_epoch(last_synced) if last_synced is not None else time.time()
+ )
# Merge provenance fields into metadata for easy access in results
combined_meta: Dict[str, Any] = dict(metadata or {})
combined_meta["chunk_id"] = chunk_id
combined_meta["source"] = source
+ combined_meta["source_id"] = source_id
combined_meta["doc_type"] = doc_type
combined_meta["doc_id"] = doc_id
combined_meta["title"] = title
combined_meta["author"] = author
combined_meta["participants"] = participants or []
+ combined_meta["participants_raw"] = participants_raw or []
combined_meta["timestamp"] = ts_str
combined_meta["thread_id"] = thread_id or ""
+ combined_meta["channel"] = channel or ""
combined_meta["url"] = url or ""
combined_meta["chunk_index"] = chunk_index
+ combined_meta["content_hash"] = content_hash
+ combined_meta["embedding_model_version"] = embedding_model_version
meta_json = json.dumps(combined_meta)
- self._conn.execute(
+ cur = self._conn.execute(
"""
- INSERT INTO knowledge_chunks
- (id, doc_id, content, source, doc_type, title, author,
- participants, timestamp, thread_id, url, metadata,
- chunk_index, created_at)
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+ INSERT OR IGNORE INTO knowledge_chunks
+ (id, doc_id, content, source, source_id, doc_type, title, author,
+ participants, participants_raw, timestamp, thread_id, channel,
+ url, metadata, chunk_index, embedding, embedding_model_version,
+ content_hash, last_synced, created_at)
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
(
chunk_id,
doc_id,
content,
source,
+ source_id,
doc_type,
title,
author,
participants_json,
+ participants_raw_json,
ts_str,
thread_id or "",
+ channel or "",
url or "",
meta_json,
chunk_index,
+ embedding,
+ embedding_model_version,
+ content_hash,
+ last_synced_epoch,
time.time(),
),
)
self._conn.commit()
+ # If the natural-key (source, source_id, chunk_index) already existed
+ # SQLite skipped the insert. Re-runs of the dogfood script or a
+ # SyncEngine that re-emits a known document hit this path; return
+ # the existing chunk id so callers don't see two different identities
+ # for the same row, and suppress the MEMORY_STORE event (nothing new
+ # was actually written).
+ if cur.rowcount == 0:
+ existing = self._conn.execute(
+ "SELECT id FROM knowledge_chunks "
+ "WHERE source=? AND source_id=? AND chunk_index=?",
+ (source, source_id, chunk_index),
+ ).fetchone()
+ if existing is not None:
+ return existing["id"]
+ return chunk_id
+
get_event_bus().publish(
EventType.MEMORY_STORE,
{
@@ -288,9 +386,10 @@ class KnowledgeStore(MemoryBackend):
filters.append("kc.timestamp <= ?")
params.append(until_str)
- where_clause = ""
- if filters:
- where_clause = "AND " + " AND ".join(filters)
+ # Always exclude tombstoned rows.
+ filters.append("kc.deleted_at IS NULL")
+
+ where_clause = "AND " + " AND ".join(filters)
# FTS5 bm25() returns negative scores; abs() gives a positive rank
sql = f"""
diff --git a/src/openjarvis/engine/ollama.py b/src/openjarvis/engine/ollama.py
index 830c3a73..5b784120 100644
--- a/src/openjarvis/engine/ollama.py
+++ b/src/openjarvis/engine/ollama.py
@@ -192,6 +192,14 @@ class OllamaEngine(InferenceEngine):
"num_ctx": kwargs.get("num_ctx", 8192),
},
}
+ # Mirror generate()'s default: disable extended thinking unless the
+ # caller opted in. Qwen3/etc. with thinking on can stall the visible
+ # stream for 60+ seconds before any tokens reach the client, which
+ # frontends interpret as a "Load failed" timeout.
+ if "think" not in kwargs:
+ payload["think"] = False
+ elif kwargs["think"] is not None:
+ payload["think"] = kwargs["think"]
try:
with self._client.stream("POST", "/api/chat", json=payload) as resp:
resp.raise_for_status()
diff --git a/src/openjarvis/server/agent_manager_routes.py b/src/openjarvis/server/agent_manager_routes.py
index a0e7e079..5aaea179 100644
--- a/src/openjarvis/server/agent_manager_routes.py
+++ b/src/openjarvis/server/agent_manager_routes.py
@@ -1421,7 +1421,11 @@ def create_agent_manager_router(
server_config,
)
executor.set_system(system)
- executor.execute_tick(agent_id)
+ # The route handler above already called start_tick() to
+ # serialize concurrent POSTs; tell the executor not to
+ # re-acquire, otherwise it bails on its own guard and the
+ # tick never runs.
+ executor.execute_tick(agent_id, lock_already_held=True)
except Exception as exc:
logger.error(
"Run-tick failed for agent %s: %s",
diff --git a/src/openjarvis/server/app.py b/src/openjarvis/server/app.py
index c8a3388c..762abf13 100644
--- a/src/openjarvis/server/app.py
+++ b/src/openjarvis/server/app.py
@@ -16,6 +16,7 @@ from openjarvis.server.comparison import comparison_router
from openjarvis.server.connectors_router import create_connectors_router
from openjarvis.server.dashboard import dashboard_router
from openjarvis.server.digest_routes import create_digest_router
+from openjarvis.server.research_router import router as research_router
from openjarvis.server.routes import router
from openjarvis.server.upload_router import router as upload_router
@@ -287,6 +288,7 @@ def create_app(
app.include_router(create_connectors_router())
app.include_router(create_digest_router())
app.include_router(upload_router)
+ app.include_router(research_router)
app.include_router(analytics_router)
include_all_routes(app)
diff --git a/src/openjarvis/server/connectors_router.py b/src/openjarvis/server/connectors_router.py
index 1738459e..a61bf58b 100644
--- a/src/openjarvis/server/connectors_router.py
+++ b/src/openjarvis/server/connectors_router.py
@@ -125,6 +125,95 @@ def create_connectors_router():
"chunks": chunks,
}
+ # ------------------------------------------------------------------
+ # Background-sync state tracking. Defined here (before the endpoints)
+ # so that POST /connect can fire-and-forget into the same machinery
+ # as POST /sync — a single source of truth for "is this connector
+ # currently syncing", baseline_items, and error translation.
+ # ------------------------------------------------------------------
+
+ _sync_threads: Dict[str, Any] = {}
+ _sync_state: Dict[str, Dict[str, Any]] = {}
+
+ def _translate_sync_error(raw: str) -> str:
+ """Map common backend exceptions to a short user-facing message."""
+ if "401" in raw or "Unauthorized" in raw:
+ return "Authentication failed — credentials may have expired."
+ if "403" in raw or "Forbidden" in raw:
+ return "Permission denied — check API scopes."
+ if "429" in raw or "Too Many Requests" in raw:
+ return "Rate limited — wait a minute and try again."
+ if "timeout" in raw.lower():
+ return "Connection timed out."
+ return raw
+
+ def _start_sync(connector_id: str, instance: Any) -> str:
+ """Spawn a background sync; returns ``"started"`` or ``"already_syncing"``.
+
+ Records baseline items in ``_sync_state`` so GET /{id}/sync polls
+ can compute "new this run" deltas, and translates exceptions into
+ the same compact error strings the POST /sync handler used to do
+ inline. Both POST /connect (auto-ingest) and POST /sync (manual)
+ route through here so they share guard, state, and error handling.
+ """
+ import threading
+
+ existing = _sync_threads.get(connector_id)
+ if existing and existing.is_alive():
+ return "already_syncing"
+
+ # Snapshot the prior checkpoint count so the GET handler can
+ # report "X new this run" without each client tracking it.
+ baseline_items = 0
+ try:
+ from openjarvis.connectors.pipeline import IngestionPipeline
+ from openjarvis.connectors.store import KnowledgeStore
+ from openjarvis.connectors.sync_engine import SyncEngine
+
+ cp = SyncEngine(
+ pipeline=IngestionPipeline(store=KnowledgeStore()),
+ ).get_checkpoint(connector_id)
+ if cp and cp.get("items_synced") is not None:
+ baseline_items = int(cp["items_synced"])
+ except Exception: # noqa: BLE001
+ baseline_items = 0
+
+ _sync_state[connector_id] = {
+ "state": "syncing",
+ "error": None,
+ "baseline_items": baseline_items,
+ }
+
+ def _run_sync() -> None:
+ try:
+ from openjarvis.connectors.pipeline import IngestionPipeline
+ from openjarvis.connectors.store import KnowledgeStore
+ from openjarvis.connectors.sync_engine import SyncEngine
+
+ store = KnowledgeStore()
+ pipeline = IngestionPipeline(store=store)
+ engine = SyncEngine(pipeline=pipeline)
+ engine.sync(instance)
+ logger.info("Sync completed for %s", connector_id)
+ _sync_state[connector_id] = {
+ "state": "complete",
+ "error": None,
+ "baseline_items": baseline_items,
+ }
+ except Exception as exc:
+ error_msg = _translate_sync_error(str(exc))
+ logger.error("Sync failed for %s: %s", connector_id, error_msg)
+ _sync_state[connector_id] = {
+ "state": "error",
+ "error": error_msg,
+ "baseline_items": baseline_items,
+ }
+
+ t = threading.Thread(target=_run_sync, daemon=True)
+ t.start()
+ _sync_threads[connector_id] = t
+ return "started"
+
# ------------------------------------------------------------------
# Endpoints
# ------------------------------------------------------------------
@@ -248,39 +337,19 @@ def create_connectors_router():
except Exception as exc:
raise HTTPException(status_code=400, detail=str(exc))
- # Auto-ingest after successful connection
+ # Auto-trigger a full backfill on a successful connect. Routed
+ # through the same _start_sync helper that POST /sync uses so the
+ # connection's progress is visible to GET /{id}/sync polling
+ # immediately — the user shouldn't have to click "Sync Now".
+ sync_status: Optional[str] = None
if instance.is_connected():
- import threading
-
- def _ingest() -> None:
- try:
- from openjarvis.connectors.pipeline import (
- IngestionPipeline,
- )
- from openjarvis.connectors.store import KnowledgeStore
- from openjarvis.connectors.sync_engine import SyncEngine
-
- with KnowledgeStore() as store:
- pipeline = IngestionPipeline(store)
- engine = SyncEngine(pipeline)
- engine.sync(instance)
- logger.info(
- "Auto-ingested %s after connect",
- connector_id,
- )
- except Exception as exc:
- logger.warning(
- "Auto-ingest failed for %s: %s",
- connector_id,
- exc,
- )
-
- threading.Thread(target=_ingest, daemon=True).start()
+ sync_status = _start_sync(connector_id, instance)
return {
"connector_id": connector_id,
"connected": instance.is_connected(),
"status": "connected" if instance.is_connected() else "pending",
+ "sync_status": sync_status,
}
@router.post("/{connector_id}/disconnect")
@@ -440,15 +509,9 @@ def create_connectors_router():
)
)
- # Track background sync state per connector
- _sync_threads: Dict[str, Any] = {}
- _sync_state: Dict[str, Dict[str, Any]] = {} # {connector_id: {state, error}}
-
@router.post("/{connector_id}/sync")
def trigger_sync(connector_id: str) -> Dict[str, Any]:
"""Trigger a sync in the background and return immediately."""
- import threading
-
_ensure_connectors_registered()
if not ConnectorRegistry.contains(connector_id):
raise HTTPException(
@@ -461,51 +524,8 @@ def create_connectors_router():
status_code=400,
detail=f"Connector '{connector_id}' is not connected",
)
-
- # If already syncing, don't start another
- existing = _sync_threads.get(connector_id)
- if existing and existing.is_alive():
- return {
- "connector_id": connector_id,
- "status": "already_syncing",
- }
-
- # Mark as syncing immediately so the UI picks it up
- _sync_state[connector_id] = {"state": "syncing", "error": None}
-
- def _run_sync() -> None:
- try:
- from openjarvis.connectors.pipeline import IngestionPipeline
- from openjarvis.connectors.store import KnowledgeStore
- from openjarvis.connectors.sync_engine import SyncEngine
-
- with KnowledgeStore() as store:
- pipeline = IngestionPipeline(store=store)
- engine = SyncEngine(pipeline=pipeline)
- engine.sync(inst)
- logger.info("Sync completed for %s", connector_id)
- _sync_state[connector_id] = {"state": "complete", "error": None}
- except Exception as exc:
- error_msg = str(exc)
- if "401" in error_msg or "Unauthorized" in error_msg:
- error_msg = "Authentication failed — credentials may have expired."
- elif "403" in error_msg or "Forbidden" in error_msg:
- error_msg = "Permission denied — check API scopes."
- elif "429" in error_msg or "Too Many Requests" in error_msg:
- error_msg = "Rate limited — wait a minute and try again."
- elif "timeout" in error_msg.lower():
- error_msg = "Connection timed out."
- logger.error("Sync failed for %s: %s", connector_id, error_msg)
- _sync_state[connector_id] = {"state": "error", "error": error_msg}
-
- t = threading.Thread(target=_run_sync, daemon=True)
- t.start()
- _sync_threads[connector_id] = t
-
- return {
- "connector_id": connector_id,
- "status": "started",
- }
+ status = _start_sync(connector_id, inst)
+ return {"connector_id": connector_id, "status": status}
@router.get("/{connector_id}/sync")
async def sync_status(connector_id: str):
@@ -527,6 +547,68 @@ def create_connectors_router():
bg_thread = _sync_threads.get(connector_id)
is_bg_running = bg_thread is not None and bg_thread.is_alive()
+ # Fall back to the SyncEngine's persistent checkpoint for items_synced
+ # and last_sync. The connector's own sync_status() only reflects state
+ # accumulated since the current connector instance was created, so a
+ # fresh process / fresh `_get_or_create` flips back to zeros even when
+ # tens of thousands of items have already been ingested historically.
+ # The checkpoint table is the source of truth.
+ checkpoint: Optional[Dict[str, Any]] = None
+ oldest_item_date: Optional[str] = None
+ try:
+ from openjarvis.connectors.pipeline import IngestionPipeline
+ from openjarvis.connectors.store import KnowledgeStore
+ from openjarvis.connectors.sync_engine import SyncEngine
+
+ store = KnowledgeStore()
+ checkpoint = SyncEngine(
+ pipeline=IngestionPipeline(store=store),
+ ).get_checkpoint(connector_id)
+
+ # Map connector_id → source field as written by the connector.
+ # Most match 1:1, but the IMAP/OAuth Gmail connectors both write
+ # source='gmail' so the unified card pulls a single timeline.
+ _STORE_SOURCE = {"gmail_imap": "gmail"}
+ store_source = _STORE_SOURCE.get(connector_id, connector_id)
+
+ # Oldest indexed item for this connector so the UI can show how
+ # far back the corpus reaches ("synced back to 2024-03-12").
+ # ISO 8601 strings sort correctly under MIN(); skip rows with no
+ # timestamp and any tombstoned chunks so the display reflects
+ # what's actually retrievable.
+ row = store._conn.execute(
+ "SELECT MIN(timestamp) FROM knowledge_chunks "
+ "WHERE source = ? AND timestamp != '' AND deleted_at IS NULL",
+ (store_source,),
+ ).fetchone()
+ if row is not None and row[0]:
+ oldest_item_date = str(row[0])
+ except Exception: # noqa: BLE001
+ checkpoint = None
+
+ # Prefer the checkpoint's items_synced — it is the running cumulative
+ # total across every sync this connector has ever run. The connector
+ # instance's own counter only tracks what *this* in-memory run has
+ # processed, so after a fresh sync of 30 new emails the connector
+ # reports 30 while the checkpoint reflects the correct 8,626 total.
+ if checkpoint and checkpoint.get("items_synced") is not None:
+ items_synced = int(checkpoint["items_synced"])
+ else:
+ items_synced = status.items_synced
+ # items_total reflects the connector's currently-known target for
+ # this sync run (e.g. number of IMAP message IDs matched). Leave it
+ # at 0 when unknown — the UI uses items_synced >= items_total > 0 as
+ # the "complete inbox" signal, which would spuriously fire on every
+ # page load if we masked the zero with items_synced as a fallback.
+ items_total = status.items_total
+ last_sync_str: Optional[str]
+ if status.last_sync:
+ last_sync_str = status.last_sync.isoformat()
+ elif checkpoint and checkpoint.get("last_sync"):
+ last_sync_str = checkpoint["last_sync"]
+ else:
+ last_sync_str = None
+
# Determine effective state
if is_bg_running:
effective_state = "syncing"
@@ -538,14 +620,27 @@ def create_connectors_router():
effective_state = status.state
# Use the bg error if the connector doesn't have one
- effective_error = status.error or bg.get("error")
+ effective_error = (
+ status.error or bg.get("error") or (checkpoint or {}).get("error")
+ )
+
+ # Items processed during the *current* (or just-finished) run = total
+ # minus the baseline captured when this sync was kicked off. This
+ # lets the UI surface "30 new emails (8,623 total indexed)" without
+ # the client having to track its own baseline.
+ baseline_items = bg.get("baseline_items")
+ new_items_synced: Optional[int] = None
+ if isinstance(baseline_items, int):
+ new_items_synced = max(0, items_synced - baseline_items)
return {
"connector_id": connector_id,
"state": effective_state,
- "items_synced": status.items_synced,
- "items_total": status.items_total,
- "last_sync": (status.last_sync.isoformat() if status.last_sync else None),
+ "items_synced": items_synced,
+ "items_total": items_total,
+ "new_items_synced": new_items_synced,
+ "last_sync": last_sync_str,
+ "oldest_item_date": oldest_item_date,
"error": effective_error,
}
diff --git a/src/openjarvis/server/research_router.py b/src/openjarvis/server/research_router.py
new file mode 100644
index 00000000..a81c9057
--- /dev/null
+++ b/src/openjarvis/server/research_router.py
@@ -0,0 +1,478 @@
+"""HTTP route: ``POST /api/research`` — agentic research over the knowledge store.
+
+Drives :class:`openjarvis.agents.research_loop.ResearchAgent` and streams a
+custom SSE event schema back to the client:
+
+* ``search_call`` — about to invoke ``HybridSearch.search`` (with arguments)
+* ``search_result`` — search returned (num_hits, top_titles)
+* ``synthesis`` — final answer, emitted in word-window chunks for an
+ incremental UX (the agent itself returns the full string in one shot;
+ chunking happens in the router so we don't need to rewire the loop)
+* ``done`` — sentinel marking the end of the stream
+
+Clarify is **disabled for the web session** — the agent's clarify_handler is
+overridden to return a fixed "no clarification available" string so the
+loop never blocks waiting for terminal stdin. Bringing real clarify back
+to the browser will require a two-step session protocol; that's a future
+endpoint, not this one.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+import re
+import threading
+import time
+from typing import Any, AsyncGenerator, Callable, Dict, Optional
+
+from fastapi import APIRouter
+from fastapi.responses import StreamingResponse
+from pydantic import BaseModel, Field
+
+from openjarvis.agents.research_loop import (
+ DEFAULT_PLANNER_MODEL,
+ ResearchAgent,
+)
+from openjarvis.connectors.embeddings import OllamaEmbedder
+from openjarvis.connectors.hybrid_search import HybridSearch
+from openjarvis.connectors.store import KnowledgeStore
+from openjarvis.core.config import DEFAULT_CONFIG_DIR
+from openjarvis.core.types import TelemetryRecord
+from openjarvis.engine.ollama import OllamaEngine
+from openjarvis.telemetry.store import TelemetryStore
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(prefix="/api", tags=["research"])
+
+_WEB_CLARIFY_RESPONSE = "no clarification available in web session"
+
+# Sentinel placed on the queue when the agent thread terminates.
+_DONE = object()
+
+
+def _record_research_telemetry(
+ *,
+ model: str,
+ usage: Dict[str, int],
+ latency_seconds: float,
+ energy_joules: float = 0.0,
+ mean_power_watts: float = 0.0,
+ peak_power_watts: float = 0.0,
+ energy_method: str = "",
+) -> None:
+ """Persist a research run into the telemetry DB so /v1/savings includes it.
+
+ GPU energy/power are sampled by ``_LiveGPUSampler`` during ``agent.run()``
+ and passed through here. With these fields populated, the same
+ ``/v1/telemetry/energy`` aggregation that powers the chat System panel
+ rolls research into the same numbers — Power (W) and Energy (kJ) stop
+ reading 0.0 after a research-only session.
+
+ Failures are swallowed — telemetry persistence is best-effort and must
+ never break the user-visible SSE stream.
+ """
+ if not usage:
+ return
+ db_path = DEFAULT_CONFIG_DIR / "telemetry.db"
+ try:
+ store = TelemetryStore(db_path)
+ except Exception as exc: # noqa: BLE001
+ logger.debug("research telemetry: cannot open %s: %s", db_path, exc)
+ return
+ try:
+ rec = TelemetryRecord(
+ timestamp=time.time(),
+ model_id=model,
+ engine="ollama",
+ agent="research",
+ prompt_tokens=int(usage.get("prompt_tokens", 0)),
+ prompt_tokens_evaluated=int(usage.get("prompt_tokens", 0)),
+ completion_tokens=int(usage.get("completion_tokens", 0)),
+ total_tokens=int(usage.get("total_tokens", 0)),
+ latency_seconds=latency_seconds,
+ energy_joules=energy_joules,
+ gpu_energy_joules=energy_joules,
+ power_watts=mean_power_watts,
+ energy_method=energy_method,
+ energy_vendor="nvidia" if energy_method else "",
+ is_streaming=True,
+ metadata={"peak_power_w": peak_power_watts} if peak_power_watts else {},
+ )
+ store.record(rec)
+ except Exception as exc: # noqa: BLE001
+ logger.debug("research telemetry: failed to record: %s", exc)
+ finally:
+ try:
+ store.close()
+ except Exception: # noqa: BLE001
+ pass
+
+
+# ---------------------------------------------------------------------------
+# Live GPU sampler — streams power/energy events during agent.run()
+# ---------------------------------------------------------------------------
+
+
+class _LiveGPUSampler:
+ """Background pynvml poller — emits live power and accumulated energy.
+
+ Runs a small daemon thread for the lifetime of a research query. Every
+ ``interval_s`` seconds it samples instantaneous GPU power across all
+ visible devices, rectangle-integrates it into a running energy total, and
+ invokes ``on_sample`` so the SSE consumer can forward ``system_metrics``
+ frames to the browser in real time.
+
+ On systems without pynvml (or with no visible GPU), ``available`` is
+ False and ``start()`` / ``stop()`` are no-ops — research still runs, the
+ System panel just won't see live metrics for that session.
+ """
+
+ def __init__(
+ self,
+ on_sample: Callable[[float, float, float], None],
+ interval_s: float = 0.75,
+ ) -> None:
+ self._on_sample = on_sample
+ self._interval_s = interval_s
+ self._handles: list = []
+ self._pynvml = None
+ self._available = False
+ self._stop = threading.Event()
+ self._thread: Optional[threading.Thread] = None
+ self._energy_j = 0.0
+ self._peak_w = 0.0
+ self._power_sum = 0.0
+ self._sample_count = 0
+ self._t_start = 0.0
+ self._t_last = 0.0
+ try:
+ import pynvml # type: ignore
+
+ pynvml.nvmlInit()
+ count = pynvml.nvmlDeviceGetCount()
+ self._handles = [
+ pynvml.nvmlDeviceGetHandleByIndex(i) for i in range(count)
+ ]
+ self._pynvml = pynvml
+ self._available = bool(self._handles)
+ if not self._available:
+ pynvml.nvmlShutdown()
+ except Exception as exc: # noqa: BLE001
+ logger.debug("research: pynvml unavailable, no live GPU metrics: %s", exc)
+ self._available = False
+
+ @property
+ def available(self) -> bool:
+ return self._available
+
+ def _poll_power_w(self) -> float:
+ total = 0.0
+ for h in self._handles:
+ try:
+ total += self._pynvml.nvmlDeviceGetPowerUsage(h) / 1000.0
+ except Exception: # noqa: BLE001
+ pass
+ return total
+
+ def _loop(self) -> None:
+ while not self._stop.is_set():
+ now = time.monotonic()
+ p = self._poll_power_w()
+ dt = now - self._t_last
+ # Rectangle integration is fine at sub-second cadence; the chat
+ # path uses the same approach in NvidiaEnergyMonitor's fallback.
+ self._energy_j += p * dt
+ self._t_last = now
+ self._sample_count += 1
+ self._power_sum += p
+ if p > self._peak_w:
+ self._peak_w = p
+ try:
+ self._on_sample(p, self._energy_j, now - self._t_start)
+ except Exception as exc: # noqa: BLE001
+ logger.debug("research: on_sample callback raised: %s", exc)
+ self._stop.wait(self._interval_s)
+
+ def start(self) -> None:
+ if not self._available:
+ return
+ self._t_start = time.monotonic()
+ self._t_last = self._t_start
+ self._thread = threading.Thread(target=self._loop, daemon=True)
+ self._thread.start()
+
+ def stop(self) -> Dict[str, float]:
+ if not self._available:
+ return {
+ "energy_j": 0.0,
+ "mean_power_w": 0.0,
+ "peak_power_w": 0.0,
+ "duration_s": 0.0,
+ }
+ self._stop.set()
+ if self._thread is not None:
+ self._thread.join(timeout=2.0)
+ duration = time.monotonic() - self._t_start
+ mean = self._power_sum / self._sample_count if self._sample_count else 0.0
+ try:
+ self._pynvml.nvmlShutdown()
+ except Exception: # noqa: BLE001
+ pass
+ return {
+ "energy_j": self._energy_j,
+ "mean_power_w": mean,
+ "peak_power_w": self._peak_w,
+ "duration_s": duration,
+ }
+
+
+# ---------------------------------------------------------------------------
+# Request model
+# ---------------------------------------------------------------------------
+
+
+class ResearchRequest(BaseModel):
+ query: str = Field(..., description="Natural-language question to research.")
+ # Deep Research has its own model requirements (function-calling support,
+ # sufficient reasoning capability) that the chat-model selector should not
+ # override. We accept the field for forward-compat with older clients but
+ # ignore it — the planner always runs on DEFAULT_PLANNER_MODEL.
+ model: Optional[str] = Field(
+ default=None, description="Ignored; retained for client compatibility."
+ )
+
+
+# ---------------------------------------------------------------------------
+# SSE helpers
+# ---------------------------------------------------------------------------
+
+
+def _sse(event: Dict[str, Any]) -> str:
+ """Serialize one event dict to an SSE ``data: ...\\n\\n`` frame."""
+ return f"data: {json.dumps(event, ensure_ascii=False)}\n\n"
+
+
+def _chunk_synthesis(text: str, window_chars: int = 40) -> list[str]:
+ """Slice synthesis text into client-streaming-friendly chunks.
+
+ We break on word boundaries so partial deltas always render cleanly in
+ the browser. Each chunk is roughly ``window_chars`` characters long.
+ """
+ if not text:
+ return []
+ tokens = re.findall(r"\S+\s*", text)
+ chunks: list[str] = []
+ buf = ""
+ for tok in tokens:
+ if len(buf) + len(tok) > window_chars and buf:
+ chunks.append(buf)
+ buf = tok
+ else:
+ buf += tok
+ if buf:
+ chunks.append(buf)
+ return chunks
+
+
+# ---------------------------------------------------------------------------
+# Stream generator
+# ---------------------------------------------------------------------------
+
+
+async def _stream_research(query: str, model: str) -> AsyncGenerator[str, None]:
+ """Drive ResearchAgent on a worker thread; yield SSE frames as they land.
+
+ Three error envelopes — setup, worker, consumer — all funnel into the
+ same two-frame contract: ``{"type": "error", ...}`` followed by
+ ``{"type": "done", "usage": {...}}``. The client can rely on always
+ seeing a ``done`` frame, even when the agent never started.
+ """
+ # Phase 1: setup. Failures here (Ollama daemon down, DB locked, etc.)
+ # yield error + done and return — nothing has been emitted yet so the
+ # client gets a clean two-frame stream instead of a dangling connection.
+ try:
+ loop = asyncio.get_event_loop()
+ queue: asyncio.Queue = asyncio.Queue()
+
+ def on_event(event: Dict[str, Any]) -> None:
+ # Called from the agent's worker thread; bounce onto the event loop.
+ loop.call_soon_threadsafe(queue.put_nowait, event)
+
+ # Each request gets its own thin set of connectors. Constructing them
+ # is cheap (SQLite open + HTTP keepalive) and avoids state leaks
+ # between concurrent requests.
+ store = KnowledgeStore()
+ embedder = OllamaEmbedder()
+ if not embedder.is_available():
+ logger.warning(
+ "research: Ollama embedder unavailable; BM25-only retrieval."
+ )
+ embedder = None
+
+ engine = OllamaEngine()
+ agent = ResearchAgent(
+ engine=engine,
+ search=HybridSearch(store, embedder),
+ model=model,
+ clarify_handler=lambda question: _WEB_CLARIFY_RESPONSE,
+ on_event=on_event,
+ )
+ except Exception as exc: # noqa: BLE001
+ logger.exception("research: setup failed before agent could run: %s", exc)
+ yield _sse(
+ {
+ "type": "error",
+ "message": f"Research failed: {type(exc).__name__}: {exc}",
+ }
+ )
+ yield _sse({"type": "done", "usage": {}})
+ return
+
+ def _emit_live_sample(power_w: float, energy_j: float, duration_s: float) -> None:
+ # Called from the sampler's worker thread — bounce onto the asyncio
+ # loop so the SSE consumer sees the same queue ordering as agent
+ # events. The frontend mirrors these into the System panel so Power
+ # (W) and Energy (kJ) update live during the run.
+ loop.call_soon_threadsafe(
+ queue.put_nowait,
+ {
+ "type": "system_metrics",
+ "power_w": round(power_w, 2),
+ "energy_j": round(energy_j, 2),
+ "duration_s": round(duration_s, 2),
+ },
+ )
+
+ sampler = _LiveGPUSampler(on_sample=_emit_live_sample)
+
+ def _run() -> None:
+ t0 = time.time()
+ sampler.start()
+ try:
+ result = agent.run(query)
+ usage_dict = dict(result.usage)
+ totals = sampler.stop()
+ # Persist token usage *and* GPU energy/power so /v1/telemetry/energy
+ # rolls research into the same Power/Energy numbers as chat —
+ # this is what the launch-video System panel reads.
+ _record_research_telemetry(
+ model=model,
+ usage=usage_dict,
+ latency_seconds=time.time() - t0,
+ energy_joules=totals["energy_j"],
+ mean_power_watts=totals["mean_power_w"],
+ peak_power_watts=totals["peak_power_w"],
+ energy_method="polling" if sampler.available else "",
+ )
+ # Forward the aggregated token usage so the consumer can attach it
+ # to the terminal `done` frame. Internal event type — never sent to
+ # the client directly.
+ loop.call_soon_threadsafe(
+ queue.put_nowait,
+ {"type": "_usage", "usage": usage_dict},
+ )
+ except Exception as exc: # noqa: BLE001
+ logger.exception("research agent crashed: %s", exc)
+ # Stop the sampler on failure too so we don't leak the polling thread
+ # past the request lifetime.
+ try:
+ sampler.stop()
+ except Exception: # noqa: BLE001
+ pass
+ loop.call_soon_threadsafe(
+ queue.put_nowait,
+ {"type": "error", "message": f"{type(exc).__name__}: {exc}"},
+ )
+ finally:
+ loop.call_soon_threadsafe(queue.put_nowait, _DONE)
+
+ task = asyncio.create_task(asyncio.to_thread(_run))
+
+ final_answer: Optional[str] = None
+ final_usage: Dict[str, int] = {}
+ try:
+ while True:
+ event = await queue.get()
+ if event is _DONE:
+ break
+ if not isinstance(event, dict):
+ continue
+
+ etype = event.get("type")
+ # Internal usage marker — capture for the done frame, don't forward.
+ if etype == "_usage":
+ final_usage = event.get("usage", {}) or {}
+ continue
+ # We translate the agent's `final_answer` event into a stream of
+ # `synthesis` chunks so the client sees the answer materialize
+ # incrementally rather than as a single blob.
+ if etype == "final_answer":
+ final_answer = event.get("text", "")
+ for piece in _chunk_synthesis(final_answer or ""):
+ yield _sse({"type": "synthesis", "text": piece})
+ continue
+
+ yield _sse(event)
+
+ # If the agent thread crashed before producing a final answer, the
+ # client still gets the error frame (emitted above) followed by done.
+ yield _sse({"type": "done", "usage": final_usage})
+ except Exception as exc: # noqa: BLE001
+ # Consumer loop crashed unexpectedly (e.g. JSON serialization fault,
+ # logic bug). Surface a clean error frame rather than letting the
+ # SSE connection die mid-stream.
+ logger.exception("research: stream consumer crashed: %s", exc)
+ yield _sse(
+ {
+ "type": "error",
+ "message": f"Research failed: {type(exc).__name__}: {exc}",
+ }
+ )
+ yield _sse({"type": "done", "usage": final_usage})
+ finally:
+ # The worker may still be cleaning up (rarely) — make sure we don't
+ # leak a dangling task. Swallow any straggler exception so a worker
+ # failure during teardown doesn't escape the generator after we've
+ # already emitted the terminal done frame.
+ if not task.done():
+ try:
+ await task
+ except Exception as exc: # noqa: BLE001
+ logger.debug("research: worker task ended with %s", exc)
+
+
+# ---------------------------------------------------------------------------
+# Route
+# ---------------------------------------------------------------------------
+
+
[email protected]("/research")
+async def research(req: ResearchRequest) -> StreamingResponse:
+ """Run a research query and stream the agent's trace + synthesis via SSE.
+
+ Response is ``text/event-stream`` with one JSON event per frame. See the
+ module docstring for the schema; a final ``{"type": "done"}`` always
+ terminates the stream so clients can detect end-of-response without
+ parsing the underlying ``[DONE]`` sentinel used by OpenAI-style routes.
+ """
+ if req.model and req.model != DEFAULT_PLANNER_MODEL:
+ logger.info(
+ "research: ignoring client model=%r; using DEFAULT_PLANNER_MODEL=%r",
+ req.model,
+ DEFAULT_PLANNER_MODEL,
+ )
+ return StreamingResponse(
+ _stream_research(req.query, DEFAULT_PLANNER_MODEL),
+ media_type="text/event-stream",
+ headers={
+ "Cache-Control": "no-cache",
+ "Connection": "keep-alive",
+ "X-Accel-Buffering": "no",
+ },
+ )
+
+
+__all__ = ["router", "ResearchRequest"]
diff --git a/tests/agents/test_research_loop.py b/tests/agents/test_research_loop.py
new file mode 100644
index 00000000..33795f6a
--- /dev/null
+++ b/tests/agents/test_research_loop.py
@@ -0,0 +1,186 @@
+"""Tests for the research_loop agent — focused on loop invariants.
+
+The fixtures use a minimal mock engine so these tests are fast and don't
+require a running Ollama daemon.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any, Dict, List, Optional, Sequence
+from unittest.mock import MagicMock
+
+import pytest
+
+from openjarvis.agents.research_loop import ResearchAgent
+
+
+class _MockEngine:
+ """Engine stub that lets a test script the per-call response.
+
+ ``responses`` is a list of dicts in ``OllamaEngine.generate`` shape
+ (``content``, ``tool_calls``, ``usage``); each call to ``generate``
+ consumes the next entry. When the script runs out, the engine repeats
+ the final entry — handy for "loop forever" mock behaviors.
+ """
+
+ def __init__(self, responses: List[Dict[str, Any]]):
+ self._responses = list(responses)
+ self.calls: List[Dict[str, Any]] = []
+
+ def generate(
+ self,
+ messages: Sequence,
+ *,
+ model: str,
+ temperature: float = 0.0,
+ max_tokens: int = 1024,
+ num_ctx: int = 8192,
+ tools: Optional[List[Dict[str, Any]]] = None,
+ **kwargs: Any,
+ ) -> Dict[str, Any]:
+ self.calls.append({"tools": tools, "messages": list(messages)})
+ if self._responses:
+ return (
+ self._responses.pop(0)
+ if len(self._responses) > 1
+ else self._responses[0]
+ )
+ return {"content": "", "tool_calls": [], "usage": {}}
+
+
+def _search_call(call_id: str, query: str = "anything") -> Dict[str, Any]:
+ return {
+ "content": "",
+ "tool_calls": [
+ {
+ "id": call_id,
+ "name": "search",
+ "arguments": json.dumps({"query": query}),
+ }
+ ],
+ "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15},
+ }
+
+
+def _text_response(text: str) -> Dict[str, Any]:
+ return {
+ "content": text,
+ "tool_calls": [],
+ "usage": {"prompt_tokens": 20, "completion_tokens": 10, "total_tokens": 30},
+ }
+
+
+def stub_search() -> MagicMock:
+ """A HybridSearch stand-in whose .search() returns an empty hit list."""
+ s = MagicMock()
+ s.search.return_value = []
+ return s
+
+
+def test_forced_synthesis_when_budget_exhausts(stub_search: MagicMock) -> None:
+ """Loop exits cleanly with a synthesis even when the model keeps tool-calling.
+
+ With max_iterations=1, the budget is exhausted after a single search.
+ The next engine call receives ``tools=None`` and is expected to produce
+ a text answer; the new forced-synthesis fallback then makes one more
+ no-tools call to ensure we always return something usable.
+ """
+ # Force the loop to fall through to the new forced-synthesis path: every
+ # in-loop iteration returns a tool call (so we never take an early text
+ # return), and the final post-loop call returns text.
+ engine = _MockEngine(
+ responses=[
+ _search_call("call-1"),
+ _search_call("call-2"),
+ _text_response("Here is what I found: nothing usable."),
+ ]
+ )
+
+ agent = ResearchAgent(engine, stub_search, model="mock", max_iterations=1)
+ result = agent.run("test query")
+
+ assert "Here is what I found" in result.answer
+ assert "loop exhausted" not in result.answer.lower()
+ # The final call was made with tools=None (the forced-synthesis path).
+ assert engine.calls[-1]["tools"] is None
+ assert all(t.tool_name == "search" for t in result.tool_calls)
+
+
+def test_forced_synthesis_returns_sentinel_when_model_stays_silent(
+ stub_search: MagicMock,
+) -> None:
+ """If even the forced final call returns no text, surface a clear sentinel.
+
+ This is the gracefully-degraded case: the loop did its best but the model
+ refused to emit text. The contract is still that ``answer`` is a non-empty
+ string the caller can show to the user.
+ """
+ engine = _MockEngine(
+ responses=[
+ _search_call("call-1"),
+ _search_call("call-2"),
+ {"content": "", "tool_calls": [], "usage": {}}, # silent forced call
+ ]
+ )
+
+ agent = ResearchAgent(engine, stub_search, model="mock", max_iterations=1)
+ result = agent.run("test query")
+
+ assert result.answer # non-empty
+ assert "no synthesis available" in result.answer.lower()
+
+
+def test_first_turn_text_response_returns_directly(stub_search: MagicMock) -> None:
+ """When the model produces text on the very first turn, return it as-is."""
+ engine = _MockEngine(responses=[_text_response("Quick answer.")])
+ agent = ResearchAgent(engine, stub_search, model="mock", max_iterations=5)
+ result = agent.run("hello")
+ assert result.answer == "Quick answer."
+ assert result.tool_calls == []
+ # Only one engine call needed.
+ assert len(engine.calls) == 1
+
+
+def test_clarify_before_any_search_is_rejected(stub_search: MagicMock) -> None:
+ """clarify() invoked before a search call must return a runtime error result.
+
+ The system prompt asks the planner to search first, but the loop also
+ enforces this at runtime so a non-compliant planner can't surprise the
+ user with a pre-emptive clarification.
+ """
+ clarify_calls: List[str] = []
+
+ def fake_clarify(q: str) -> str:
+ clarify_calls.append(q)
+ return "user response"
+
+ engine = _MockEngine(
+ responses=[
+ {
+ "content": "",
+ "tool_calls": [
+ {
+ "id": "c1",
+ "name": "clarify",
+ "arguments": json.dumps({"question": "who?"}),
+ }
+ ],
+ "usage": {},
+ },
+ _text_response("Final."),
+ ]
+ )
+
+ agent = ResearchAgent(
+ engine, stub_search, model="mock", max_iterations=5,
+ clarify_handler=fake_clarify,
+ )
+ result = agent.run("vague query")
+
+ # Handler must not have been called because the dispatch returned an error.
+ assert clarify_calls == []
+ # No clarify invocation recorded.
+ assert all(t.tool_name != "clarify" for t in result.tool_calls)
+ assert result.answer == "Final."
diff --git a/tests/connectors/test_gmail.py b/tests/connectors/test_gmail.py
index ceb5c358..e232f9dd 100644
--- a/tests/connectors/test_gmail.py
+++ b/tests/connectors/test_gmail.py
@@ -5,6 +5,7 @@ All Gmail API calls are mocked; no network access is required.
from __future__ import annotations
+import base64
import json
from pathlib import Path
from typing import List
@@ -225,7 +226,8 @@ def test_sync_passes_since_as_query(
# Verify the epoch value is correct
expected_epoch = int(since_dt.timestamp())
assert f"after:{expected_epoch}" in call_kwargs["query"]
- assert "category:primary" in call_kwargs["query"]
+ # No category filter — sent mail and all Gmail tabs should be reachable.
+ assert "category:" not in call_kwargs["query"]
# ---------------------------------------------------------------------------
@@ -251,7 +253,8 @@ def test_sync_without_since_passes_empty_query(
mock_list.assert_called_once()
_, call_kwargs = mock_list.call_args
- assert call_kwargs.get("query", "") == "category:primary"
+ # No since= and no hardcoded category filter → empty query.
+ assert call_kwargs.get("query", "") == ""
# ---------------------------------------------------------------------------
@@ -269,3 +272,492 @@ def test_registry() -> None:
assert ConnectorRegistry.contains("gmail")
cls = ConnectorRegistry.get("gmail")
assert cls.connector_id == "gmail"
+
+
+# ---------------------------------------------------------------------------
+# v1 schema tests
+# ---------------------------------------------------------------------------
+
+# base64url("Migration body") == "TWlncmF0aW9uIGJvZHk="
+_MSG_RICH = {
+ "id": "msg-rich-1",
+ "threadId": "thread-rich-1",
+ "labelIds": ["SENT", "CATEGORY_PERSONAL"],
+ "snippet": "Quick note on the migration",
+ "historyId": "1234567",
+ "sizeEstimate": 4096,
+ "payload": {
+ "mimeType": "text/plain",
+ "headers": [
+ {"name": "From", "value": "Alice Bose <[email protected]>"},
+ {
+ "name": "To",
+ },
+ {"name": "Cc", "value": "[email protected]"},
+ {"name": "Subject", "value": "Migration plan"},
+ {"name": "Date", "value": "Mon, 01 Jan 2024 10:00:00 +0000"},
+ {"name": "Message-ID", "value": "<[email protected]>"},
+ ],
+ "body": {"data": "TWlncmF0aW9uIGJvZHk="},
+ },
+}
+
+
+@patch("openjarvis.connectors.gmail._gmail_api_list_messages")
+@patch("openjarvis.connectors.gmail._gmail_api_get_message")
+def test_sync_emits_v1_fields(
+ mock_get,
+ mock_list,
+ connector,
+ tmp_path: Path,
+) -> None:
+ """sync() populates source_id, participants_raw, normalized participants,
+ channel from labels, and Gmail-specific fields in metadata."""
+ creds_path = Path(connector._credentials_path)
+ creds_path.write_text(json.dumps({"token": "fake-access-token"}), encoding="utf-8")
+
+ mock_list.return_value = {"messages": [{"id": "msg-rich-1"}]}
+ mock_get.return_value = _MSG_RICH
+
+ docs: List[Document] = list(connector.sync())
+ assert len(docs) == 1
+ doc = docs[0]
+
+ # source_id is the raw Gmail msg.id (no "gmail:" prefix); doc_id stays
+ # composite for back-compat with legacy callers.
+ assert doc.source_id == "msg-rich-1"
+ assert doc.doc_id == "gmail:msg-rich-1"
+
+ # participants_raw preserves From / To / Cc as Gmail returned them.
+ assert doc.participants_raw == [
+ "Alice Bose <[email protected]>",
+ ]
+ # participants extracts and lowercases all addresses, multi-recipient-aware.
+ assert doc.participants == [
+ ]
+
+ # channel comes from the highest-priority system label.
+ assert doc.channel == "SENT"
+
+ # Gmail-specific richness lives in source_metadata.
+ assert doc.metadata["rfc_message_id"] == "<[email protected]>"
+ assert doc.metadata["snippet"] == "Quick note on the migration"
+ assert doc.metadata["history_id"] == "1234567"
+ assert doc.metadata["size_estimate"] == 4096
+ assert "SENT" in doc.metadata["labels"]
+ assert "CATEGORY_PERSONAL" in doc.metadata["labels"]
+
+
+@patch("openjarvis.connectors.gmail._gmail_api_list_messages")
+@patch("openjarvis.connectors.gmail._gmail_api_get_message")
+def test_sync_channel_falls_back_to_inbox(
+ mock_get,
+ mock_list,
+ connector,
+ tmp_path: Path,
+) -> None:
+ """When only INBOX is present among system labels, channel == 'INBOX'."""
+ creds_path = Path(connector._credentials_path)
+ creds_path.write_text(json.dumps({"token": "fake-access-token"}), encoding="utf-8")
+
+ mock_list.return_value = _LIST_RESPONSE
+ mock_get.side_effect = lambda token, msg_id: _MSG1 if msg_id == "msg1" else _MSG2
+
+ docs = list(connector.sync())
+ assert all(d.channel == "INBOX" for d in docs)
+
+
+@patch("openjarvis.connectors.gmail._gmail_api_list_messages")
+@patch("openjarvis.connectors.gmail._gmail_api_get_message")
+def test_sync_channel_none_when_no_system_label(
+ mock_get,
+ mock_list,
+ connector,
+ tmp_path: Path,
+) -> None:
+ """A message with only user-defined labels has channel=None."""
+ creds_path = Path(connector._credentials_path)
+ creds_path.write_text(json.dumps({"token": "fake-access-token"}), encoding="utf-8")
+
+ msg_userlabel = {
+ **_MSG1,
+ "id": "msg-user",
+ "labelIds": ["Label_1234", "CATEGORY_FORUMS"],
+ }
+ mock_list.return_value = {"messages": [{"id": "msg-user"}]}
+ mock_get.return_value = msg_userlabel
+
+ docs = list(connector.sync())
+ assert len(docs) == 1
+ assert docs[0].channel is None
+
+
+# ---------------------------------------------------------------------------
+# HTML body stripping
+# ---------------------------------------------------------------------------
+
+
+def test_html_to_text_strips_basic_tags() -> None:
+ """_html_to_text() removes tags but preserves visible text content."""
+ from openjarvis.connectors.gmail import _html_to_text # noqa: PLC0415
+
+ html = (
+ "<html><body><p>Hello <b>world</b>!</p>"
+ "<p>Second paragraph.</p></body></html>"
+ )
+ text = _html_to_text(html)
+ assert "Hello" in text
+ assert "world" in text
+ assert "Second paragraph" in text
+ assert "<" not in text and ">" not in text
+
+
+def test_html_to_text_drops_script_and_style() -> None:
+ """Content inside <script>/<style>/<head> is stripped, not rendered."""
+ from openjarvis.connectors.gmail import _html_to_text # noqa: PLC0415
+
+ html = """
+ <html>
+ <head><style>.foo { color: red; }</style><title>Ignore me</title></head>
+ <body>
+ <script>alert('xss')</script>
+ <p>Visible content here.</p>
+ </body>
+ </html>
+ """
+ text = _html_to_text(html)
+ assert "Visible content here" in text
+ assert "color: red" not in text
+ assert "alert" not in text
+ assert "Ignore me" not in text
+
+
+def test_html_to_text_decodes_entities() -> None:
+ """Named and numeric HTML entities are decoded to their characters."""
+ from openjarvis.connectors.gmail import _html_to_text # noqa: PLC0415
+
+ html = "<p>Tom &amp; Jerry &mdash; 50&nbsp;cents</p>"
+ text = _html_to_text(html)
+ assert "Tom & Jerry" in text
+ assert "—" in text # &mdash;
+ assert "50" in text and "cents" in text
+
+
+def test_html_to_text_inserts_paragraph_breaks() -> None:
+ """Block-level tags produce newlines so the chunker sees structure."""
+ from openjarvis.connectors.gmail import _html_to_text # noqa: PLC0415
+
+ html = "<div>line one</div><div>line two</div><div>line three</div>"
+ text = _html_to_text(html)
+ # Each block produces at least one newline boundary.
+ assert text.count("\n") >= 2
+
+
+@patch("openjarvis.connectors.gmail._gmail_api_list_messages")
+@patch("openjarvis.connectors.gmail._gmail_api_get_message")
+def test_sync_strips_html_when_no_text_plain(
+ mock_get,
+ mock_list,
+ connector,
+ tmp_path: Path,
+) -> None:
+ """A multipart message with only text/html yields stripped plain text."""
+ creds_path = Path(connector._credentials_path)
+ creds_path.write_text(json.dumps({"token": "fake-access-token"}), encoding="utf-8")
+
+ html_bytes = (
+ b"<html><body><p>Hello <b>world</b>!</p>"
+ b"<p>Second paragraph.</p></body></html>"
+ )
+ html_b64 = base64.urlsafe_b64encode(html_bytes).decode().rstrip("=")
+
+ msg_html = {
+ "id": "msg-html-1",
+ "threadId": "thread-html-1",
+ "labelIds": ["INBOX"],
+ "payload": {
+ "mimeType": "multipart/alternative",
+ "headers": [
+ {"name": "From", "value": "[email protected]"},
+ {"name": "To", "value": "[email protected]"},
+ {"name": "Subject", "value": "Marketing"},
+ {"name": "Date", "value": "Mon, 01 Jan 2024 10:00:00 +0000"},
+ ],
+ "parts": [
+ # No text/plain alternative — only text/html.
+ {
+ "mimeType": "text/html",
+ "body": {"data": html_b64},
+ },
+ ],
+ },
+ }
+
+ mock_list.return_value = {"messages": [{"id": "msg-html-1"}]}
+ mock_get.return_value = msg_html
+
+ docs = list(connector.sync())
+ assert len(docs) == 1
+ content = docs[0].content
+ assert "Hello" in content
+ assert "world" in content
+ assert "Second paragraph" in content
+ assert "<" not in content and ">" not in content
+
+
+@patch("openjarvis.connectors.gmail._gmail_api_list_messages")
+@patch("openjarvis.connectors.gmail._gmail_api_get_message")
+def test_sync_prefers_text_plain_over_text_html(
+ mock_get,
+ mock_list,
+ connector,
+ tmp_path: Path,
+) -> None:
+ """When both alternatives are present, text/plain wins over text/html."""
+ creds_path = Path(connector._credentials_path)
+ creds_path.write_text(json.dumps({"token": "fake-access-token"}), encoding="utf-8")
+
+ plain_b64 = base64.urlsafe_b64encode(
+ b"Plain text version preferred."
+ ).decode().rstrip("=")
+ html_b64 = base64.urlsafe_b64encode(
+ b"<html><body><p>HTML version</p></body></html>"
+ ).decode().rstrip("=")
+
+ msg_alt = {
+ "id": "msg-alt-1",
+ "threadId": "thread-alt-1",
+ "labelIds": ["INBOX"],
+ "payload": {
+ "mimeType": "multipart/alternative",
+ "headers": [
+ {"name": "From", "value": "[email protected]"},
+ {"name": "To", "value": "[email protected]"},
+ {"name": "Subject", "value": "Both alternatives"},
+ {"name": "Date", "value": "Mon, 01 Jan 2024 10:00:00 +0000"},
+ ],
+ "parts": [
+ {"mimeType": "text/plain", "body": {"data": plain_b64}},
+ {"mimeType": "text/html", "body": {"data": html_b64}},
+ ],
+ },
+ }
+
+ mock_list.return_value = {"messages": [{"id": "msg-alt-1"}]}
+ mock_get.return_value = msg_alt
+
+ docs = list(connector.sync())
+ assert len(docs) == 1
+ assert docs[0].content == "Plain text version preferred."
+
+
+# ---------------------------------------------------------------------------
+# Auto-refresh on 401
+# ---------------------------------------------------------------------------
+
+
+class _FakeResponse:
+ """Minimal stand-in for httpx.Response used by the refresh test."""
+
+ def __init__(
+ self, *, status_code: int, json_data: dict | None = None, text: str = ""
+ ):
+ self.status_code = status_code
+ self._json = json_data or {}
+ self.text = text
+
+ def json(self):
+ return self._json
+
+ def raise_for_status(self):
+ if self.status_code >= 400:
+ import httpx as _httpx
+ raise _httpx.HTTPStatusError(
+ f"HTTP {self.status_code}", request=None, response=self
+ )
+
+
+def _write_full_creds(tmp_path: Path) -> str:
+ """Write a credentials file with a refresh_token and client credentials."""
+ creds_path = tmp_path / "gmail.json"
+ creds_path.write_text(
+ json.dumps(
+ {
+ "access_token": "old-access-token",
+ "refresh_token": "stored-refresh-token",
+ "client_id": "client-id-abc",
+ "client_secret": "client-secret-xyz",
+ }
+ ),
+ encoding="utf-8",
+ )
+ return str(creds_path)
+
+
+def test_401_triggers_refresh_and_retries_with_new_token(tmp_path: Path) -> None:
+ """A 401 on a Gmail API call refreshes the token, persists it, and retries."""
+ from openjarvis.connectors import gmail as gmail_mod
+
+ creds_path = _write_full_creds(tmp_path)
+
+ # The first httpx.get returns 401, the second returns 200.
+ get_calls: list[dict] = []
+
+ def fake_get(url, *, headers, params, timeout):
+ get_calls.append({"url": url, "headers": dict(headers), "params": dict(params)})
+ if len(get_calls) == 1:
+ return _FakeResponse(status_code=401, text="unauthorized")
+ return _FakeResponse(status_code=200, json_data={"id": "msg-1", "payload": {}})
+
+ post_calls: list[dict] = []
+
+ def fake_post(url, *, data, timeout):
+ post_calls.append({"url": url, "data": dict(data)})
+ return _FakeResponse(
+ status_code=200,
+ json_data={"access_token": "fresh-access-token", "expires_in": 3599},
+ )
+
+ with patch.object(gmail_mod.httpx, "get", side_effect=fake_get), \
+ patch.object(gmail_mod.httpx, "post", side_effect=fake_post):
+ result = gmail_mod._call_with_refresh(
+ gmail_mod._gmail_api_get_message, creds_path, "msg-1"
+ )
+
+ # The retried request must carry the fresh token.
+ assert len(get_calls) == 2
+ assert get_calls[0]["headers"]["Authorization"] == "Bearer old-access-token"
+ assert get_calls[1]["headers"]["Authorization"] == "Bearer fresh-access-token"
+
+ # The refresh hit Google's token endpoint with the stored refresh credentials.
+ assert len(post_calls) == 1
+ assert post_calls[0]["url"] == "https://oauth2.googleapis.com/token"
+ assert post_calls[0]["data"] == {
+ "client_id": "client-id-abc",
+ "client_secret": "client-secret-xyz",
+ "refresh_token": "stored-refresh-token",
+ "grant_type": "refresh_token",
+ }
+
+ # The new access_token is persisted to the credentials file.
+ on_disk = json.loads(Path(creds_path).read_text(encoding="utf-8"))
+ assert on_disk["access_token"] == "fresh-access-token"
+ assert on_disk["token"] == "fresh-access-token" # legacy key kept in sync
+ assert on_disk["refresh_token"] == "stored-refresh-token"
+ assert on_disk["expires_in"] == 3599
+
+ # And the caller saw the body of the retried request.
+ assert result == {"id": "msg-1", "payload": {}}
+
+
+def test_non_401_status_is_not_refreshed(tmp_path: Path) -> None:
+ """A 500 from Gmail must propagate — only 401 should trigger refresh."""
+ import httpx as _httpx
+
+ from openjarvis.connectors import gmail as gmail_mod
+
+ creds_path = _write_full_creds(tmp_path)
+
+ def fake_get(url, *, headers, params, timeout):
+ return _FakeResponse(status_code=503, text="service unavailable")
+
+ fake_post = patch.object(gmail_mod.httpx, "post", side_effect=AssertionError(
+ "_call_with_refresh must not refresh on non-401 status"
+ ))
+
+ with patch.object(gmail_mod.httpx, "get", side_effect=fake_get), fake_post:
+ with pytest.raises(_httpx.HTTPStatusError):
+ gmail_mod._call_with_refresh(
+ gmail_mod._gmail_api_get_message, creds_path, "msg-1"
+ )
+
+
+def test_refresh_raises_when_refresh_token_missing(tmp_path: Path) -> None:
+ """Refresh aborts with a clear error when no refresh_token is stored."""
+ from openjarvis.connectors import google_auth
+
+ creds_path = tmp_path / "gmail.json"
+ creds_path.write_text(
+ json.dumps({"access_token": "stale", "client_id": "c", "client_secret": "s"}),
+ encoding="utf-8",
+ )
+
+ with pytest.raises(google_auth.GoogleAuthError, match="refresh_token"):
+ google_auth.refresh_access_token(str(creds_path))
+
+
+def test_sync_recovers_when_list_returns_401(tmp_path: Path) -> None:
+ """An end-to-end sync survives a 401 on messages.list without re-auth.
+
+ Sets up a connector against a creds file with full refresh credentials,
+ has the first list call return 401, the refresh return a new token, and
+ the retried list call return one message stub which is then fetched
+ successfully. Verifies the connector yields the expected Document and
+ that the credentials file is rewritten with the fresh token.
+ """
+ from openjarvis.connectors import gmail as gmail_mod
+
+ creds_path = _write_full_creds(tmp_path)
+ connector = gmail_mod.GmailConnector(credentials_path=creds_path)
+
+ list_calls: list[str] = []
+
+ def fake_get(url, *, headers, params, timeout):
+ auth = headers.get("Authorization", "")
+ if "/messages/" in url:
+ # The single get_message call after the retried list call.
+ return _FakeResponse(
+ status_code=200,
+ json_data={
+ "id": "msg-99",
+ "threadId": "thread-99",
+ "labelIds": ["INBOX"],
+ "payload": {
+ "mimeType": "text/plain",
+ "headers": [
+ {"name": "From", "value": "[email protected]"},
+ {"name": "To", "value": "[email protected]"},
+ {"name": "Subject", "value": "Hi"},
+ {
+ "name": "Date",
+ "value": "Mon, 01 Jan 2024 10:00:00 +0000",
+ },
+ ],
+ "body": {"data": "SGVsbG8="},
+ },
+ },
+ )
+ # /messages list call
+ list_calls.append(auth)
+ if len(list_calls) == 1:
+ return _FakeResponse(status_code=401, text="unauthorized")
+ return _FakeResponse(
+ status_code=200,
+ json_data={"messages": [{"id": "msg-99"}]},
+ )
+
+ def fake_post(url, *, data, timeout):
+ return _FakeResponse(
+ status_code=200,
+ json_data={"access_token": "fresh-token-after-401", "expires_in": 3599},
+ )
+
+ with patch.object(gmail_mod.httpx, "get", side_effect=fake_get), \
+ patch.object(gmail_mod.httpx, "post", side_effect=fake_post):
+ docs: List[Document] = list(connector.sync())
+
+ assert len(docs) == 1
+ assert docs[0].title == "Hi"
+ assert docs[0].author == "[email protected]"
+ # First list call carried the stale token; second carried the fresh one.
+ assert list_calls == ["Bearer old-access-token", "Bearer fresh-token-after-401"]
+ # Creds file persisted the new token.
+ on_disk = json.loads(Path(creds_path).read_text(encoding="utf-8"))
+ assert on_disk["access_token"] == "fresh-token-after-401"
diff --git a/tests/connectors/test_integration.py b/tests/connectors/test_integration.py
index d7489a8e..0e3adc0e 100644
--- a/tests/connectors/test_integration.py
+++ b/tests/connectors/test_integration.py
@@ -55,7 +55,7 @@ def test_full_pipeline_obsidian_to_search(vault: Path, tmp_path: Path) -> None:
assert cp is not None
# SyncEngine checkpoint records total chunks ingested (not document count).
# The two vault files produce 6 chunks via SemanticChunker (note strategy).
- assert cp["items_synced"] == 6 # 6 chunks from 2 md files
+ assert cp["items_synced"] == 6
# 4. Search via knowledge_search tool
tool = KnowledgeSearchTool(store=store)
diff --git a/tests/connectors/test_pipeline.py b/tests/connectors/test_pipeline.py
index fa70efce..0ed74969 100644
--- a/tests/connectors/test_pipeline.py
+++ b/tests/connectors/test_pipeline.py
@@ -292,3 +292,174 @@ def test_ingest_chunk_count_return_value(
assert n2 == 1 # only doc_c is new
assert store.count() == 3
+
+
+# ---------------------------------------------------------------------------
+# v1 schema tests — pipeline-side derivation
+# ---------------------------------------------------------------------------
+
+
+def test_thread_id_namespaced_at_pipeline(
+ pipeline: IngestionPipeline, store: KnowledgeStore
+) -> None:
+ """Pipeline prefixes raw thread_id with '{source}:' so connectors can't forget."""
+ doc = _make_doc(
+ doc_id="gmail:msg-tn1",
+ source="gmail",
+ thread_id="raw-thread-id",
+ content="Thread namespacing should happen here.",
+ )
+ pipeline.ingest([doc])
+
+ rows = store._conn.execute(
+ "SELECT thread_id FROM knowledge_chunks"
+ ).fetchall()
+ assert len(rows) == 1
+ assert rows[0][0] == "gmail:raw-thread-id"
+
+
+def test_thread_id_namespacing_is_idempotent(
+ pipeline: IngestionPipeline, store: KnowledgeStore
+) -> None:
+ """A connector that already namespaced thread_id is not double-prefixed."""
+ doc = _make_doc(
+ doc_id="gmail:msg-tn2",
+ source="gmail",
+ thread_id="gmail:already-prefixed",
+ content="Idempotent namespacing keeps a single prefix.",
+ )
+ pipeline.ingest([doc])
+
+ rows = store._conn.execute(
+ "SELECT thread_id FROM knowledge_chunks"
+ ).fetchall()
+ assert rows[0][0] == "gmail:already-prefixed"
+
+
+def test_source_id_derived_from_doc_id_prefix(
+ pipeline: IngestionPipeline, store: KnowledgeStore
+) -> None:
+ """When source_id isn't set on the Document, the pipeline strips '{source}:'
+ from doc_id so legacy connectors keep working."""
+ doc = _make_doc(
+ doc_id="gmail:msg42",
+ source="gmail",
+ content="Legacy connector with composite doc_id.",
+ )
+ pipeline.ingest([doc])
+
+ rows = store._conn.execute(
+ "SELECT source_id FROM knowledge_chunks"
+ ).fetchall()
+ assert rows[0][0] == "msg42"
+
+
+def test_source_id_uses_explicit_field_when_set(
+ pipeline: IngestionPipeline, store: KnowledgeStore
+) -> None:
+ """An explicit Document.source_id wins over any prefix-stripping heuristic."""
+ doc = _make_doc(
+ doc_id="some-other-shape",
+ source="gmail",
+ content="Explicit source_id should be used verbatim.",
+ )
+ doc.source_id = "explicit-src-id"
+ pipeline.ingest([doc])
+
+ rows = store._conn.execute(
+ "SELECT source_id FROM knowledge_chunks"
+ ).fetchall()
+ assert rows[0][0] == "explicit-src-id"
+
+
+def test_content_hash_computed_per_chunk(
+ pipeline: IngestionPipeline, store: KnowledgeStore
+) -> None:
+ """content_hash equals sha256(chunk.content) for a single-chunk document."""
+ import hashlib as _hashlib
+
+ content = "Hello world this is a test document."
+ doc = _make_doc(doc_id="doc:hash:1", content=content)
+ pipeline.ingest([doc])
+
+ rows = store._conn.execute(
+ "SELECT content, content_hash FROM knowledge_chunks"
+ ).fetchall()
+ assert len(rows) == 1
+ assert rows[0][1] == _hashlib.sha256(rows[0][0].encode("utf-8")).hexdigest()
+
+
+def test_last_synced_set_at_ingest(
+ pipeline: IngestionPipeline, store: KnowledgeStore
+) -> None:
+ """last_synced is populated with the ingest time, not 0 default."""
+ import time as _time
+
+ before = _time.time()
+ pipeline.ingest([_make_doc(doc_id="doc:ls:1", content="Last synced check.")])
+ after = _time.time()
+
+ rows = store._conn.execute(
+ "SELECT last_synced FROM knowledge_chunks"
+ ).fetchall()
+ assert len(rows) == 1
+ assert before <= rows[0][0] <= after
+
+
+# ---------------------------------------------------------------------------
+# Embedding wire-up
+# ---------------------------------------------------------------------------
+
+
+class _StubEmbedder:
+ """Deterministic in-test embedder that mimics the OllamaEmbedder surface."""
+
+ model_version = "stub:test-embedder"
+
+ def __init__(self) -> None:
+ self.calls = 0
+
+ def embed(self, text: str): # type: ignore[no-untyped-def]
+ import numpy as _np
+ self.calls += 1
+ # Map content to a stable 4-d float32 vector for assertion convenience.
+ h = abs(hash(text)) % 10_000
+ return _np.asarray([h, h + 1, h + 2, h + 3], dtype=_np.float32).tobytes()
+
+
+def test_pipeline_populates_embedding_when_embedder_provided(
+ store: KnowledgeStore,
+) -> None:
+ """Pipeline writes float32 embedding bytes + model_version when embedder is set."""
+ import numpy as _np
+
+ embedder = _StubEmbedder()
+ pipeline = IngestionPipeline(store, embedder=embedder)
+ pipeline.ingest([_make_doc(doc_id="doc:emb:1", content="Short embeddable text.")])
+
+ rows = store._conn.execute(
+ "SELECT embedding, embedding_model_version FROM knowledge_chunks"
+ ).fetchall()
+ assert len(rows) == 1
+ blob, version = rows[0]
+ assert blob is not None
+ arr = _np.frombuffer(blob, dtype=_np.float32)
+ assert arr.shape == (4,)
+ assert version == "stub:test-embedder"
+ assert embedder.calls == 1
+
+
+def test_pipeline_skips_embedding_when_no_embedder(
+ pipeline: IngestionPipeline, store: KnowledgeStore,
+) -> None:
+ """Default pipeline leaves embedding NULL and embedding_model_version empty."""
+ pipeline.ingest(
+ [_make_doc(doc_id="doc:emb:none", content="No embedder configured.")]
+ )
+
+ rows = store._conn.execute(
+ "SELECT embedding, embedding_model_version FROM knowledge_chunks"
+ ).fetchall()
+ assert len(rows) == 1
+ assert rows[0][0] is None
+ assert rows[0][1] == ""
diff --git a/tests/connectors/test_store.py b/tests/connectors/test_store.py
index c7f2f354..4d927863 100644
--- a/tests/connectors/test_store.py
+++ b/tests/connectors/test_store.py
@@ -335,6 +335,112 @@ def test_memory_retrieve_event_emitted(tmp_path: Path) -> None:
assert EventType.MEMORY_RETRIEVE in types
+# ---------------------------------------------------------------------------
+# v1 schema tests
+# ---------------------------------------------------------------------------
+
+
+def test_v1_columns_round_trip_via_metadata(ks: KnowledgeStore) -> None:
+ """source_id, channel, content_hash, embedding_model_version, participants_raw
+ survive the store/retrieve round-trip via the metadata payload."""
+ _store(
+ ks,
+ content="Email about partnership review",
+ source="gmail",
+ source_id="msg-abc123",
+ channel="INBOX",
+ content_hash="deadbeefcafe",
+ embedding_model_version="text-embedding-3-small/v1",
+ participants_raw=["Alice Bose <[email protected]>"],
+ )
+ results = ks.retrieve("partnership review", top_k=1)
+ assert len(results) == 1
+ meta = results[0].metadata
+ assert meta.get("source_id") == "msg-abc123"
+ assert meta.get("channel") == "INBOX"
+ assert meta.get("content_hash") == "deadbeefcafe"
+ assert meta.get("embedding_model_version") == "text-embedding-3-small/v1"
+ assert meta.get("participants_raw") == ["Alice Bose <[email protected]>"]
+
+
+def test_deleted_at_filters_retrieve(ks: KnowledgeStore) -> None:
+ """Rows with non-NULL deleted_at are excluded from retrieve()."""
+ import time as _time
+
+ _store(ks, content="Live document about quarterly research")
+ _store(ks, content="Tombstoned document about quarterly research")
+
+ ks._conn.execute(
+ "UPDATE knowledge_chunks SET deleted_at = ? "
+ "WHERE content LIKE 'Tombstoned%'",
+ (_time.time(),),
+ )
+ ks._conn.commit()
+
+ results = ks.retrieve("quarterly research", top_k=10)
+ assert len(results) == 1
+ assert results[0].content.startswith("Live")
+
+
+def test_unique_natural_key_constraint(ks: KnowledgeStore) -> None:
+ """Duplicate (source, source_id, chunk_index) is silently skipped.
+
+ The store uses ``INSERT OR IGNORE`` so re-running a sync or replaying a
+ dogfood script over an already-populated store no longer crashes; the
+ original row stays put and the duplicate is dropped.
+ """
+ first_id = _store(
+ ks,
+ content="First copy",
+ source="gmail",
+ source_id="msg-unique-1",
+ chunk_index=0,
+ )
+ second_id = _store(
+ ks,
+ content="Duplicate copy",
+ source="gmail",
+ source_id="msg-unique-1",
+ chunk_index=0,
+ )
+ # Same identity — the natural key collision returned the existing row's id.
+ assert second_id == first_id
+ # Content of the original row is preserved.
+ row = ks._conn.execute(
+ "SELECT content FROM knowledge_chunks WHERE id = ?", (first_id,)
+ ).fetchone()
+ assert row["content"] == "First copy"
+ # Only one row exists for this natural key.
+ count = ks._conn.execute(
+ "SELECT COUNT(*) FROM knowledge_chunks "
+ "WHERE source = 'gmail' AND source_id = 'msg-unique-1' AND chunk_index = 0"
+ ).fetchone()[0]
+ assert count == 1
+
+
+def test_unique_constraint_skipped_when_source_id_empty(ks: KnowledgeStore) -> None:
+ """Legacy rows with empty source_id can co-exist (partial index excludes them)."""
+ _store(ks, content="Legacy doc one", source="legacy", source_id="")
+ # Should not raise — partial index is WHERE source_id != ''
+ _store(ks, content="Legacy doc two", source="legacy", source_id="")
+ assert ks.count() == 2
+
+
+def test_embedding_blob_round_trip(ks: KnowledgeStore) -> None:
+ """A bytes embedding payload survives storage and reads back unchanged."""
+ payload = b"\x00\x01\x02\x03\x04\xff\xfe\xfd"
+ _store(
+ ks,
+ content="Vector-bearing document for embedding round trip",
+ source="test",
+ embedding=payload,
+ )
+ row = ks._conn.execute(
+ "SELECT embedding FROM knowledge_chunks LIMIT 1"
+ ).fetchone()
+ assert bytes(row[0]) == payload
+
+
def test_context_manager_closes_connection(tmp_path: Path) -> None:
"""Used as a context manager, KnowledgeStore closes its connection on exit."""
import sqlite3
diff --git a/uv.lock b/uv.lock
index 0f66752f..324023e5 100644
--- a/uv.lock
+++ b/uv.lock
@@ -4982,6 +4982,7 @@ dependencies = [
{ name = "httpx" },
{ name = "openai" },
{ name = "posthog" },
+ { name = "pynvml" },
{ name = "python-telegram-bot" },
{ name = "rich" },
{ name = "tomli", marker = "python_full_version < '3.11'" },
@@ -5236,6 +5237,7 @@ requires-dist = [
{ name = "pygemma", marker = "extra == 'inference-gemma'", specifier = ">=0.1.3" },
{ name = "pymessenger", marker = "extra == 'channel-messenger'", specifier = ">=0.0.7" },
{ name = "pynostr", marker = "extra == 'channel-nostr'", specifier = ">=0.6" },
+ { name = "pynvml", specifier = ">=13.0.1" },
{ name = "pynvml", marker = "extra == 'energy-all'", specifier = ">=12.0" },
{ name = "pynvml", marker = "extra == 'gpu-metrics'", specifier = ">=12.0" },
{ name = "pytest", marker = "extra == 'dev'", specifier = ">=8" },