YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(chat): scale tag and speaker filters (#4195)
Ansh Grover7043063e120d
diff --git a/apps/screenpipe-app-tauri/components/standalone-chat.tsx b/apps/screenpipe-app-tauri/components/standalone-chat.tsxindex 864170b3b..b31954666 100644--- a/apps/screenpipe-app-tauri/components/standalone-chat.tsx+++ b/apps/screenpipe-app-tauri/components/standalone-chat.tsx@@ -180,6 +180,8 @@ interface MentionSuggestion {const APP_SUGGESTION_LIMIT = 10;const TAG_SUGGESTION_LIMIT = 10;+const TAG_AUTOCOMPLETE_LIMIT = 50;+const SPEAKER_SUGGESTION_LIMIT = 50;const STREAM_RENDER_THROTTLE_MS = 80;const EMPTY_QUEUED_PROMPTS: PiQueuedPrompt[] = [];const POST_STREAM_SIDE_EFFECT_DELAY_MS = 1_500;@@ -2828,7 +2830,14 @@ export function StandaloneChat({const [selectedMentionIndex, setSelectedMentionIndex] = useState(0);const [speakerSuggestions, setSpeakerSuggestions] = useState<MentionSuggestion[]>([]);const [isLoadingSpeakers, setIsLoadingSpeakers] = useState(false);+ const [tagSearchSuggestions, setTagSearchSuggestions] = useState<MentionSuggestion[]>([]);+ const [isLoadingTagSearch, setIsLoadingTagSearch] = useState(false);const [appFilterOpen, setAppFilterOpen] = useState(false);+ const [filterSearch, setFilterSearch] = useState("");+ const [filterTagResults, setFilterTagResults] = useState<MentionSuggestion[]>([]);+ const [filterSpeakerResults, setFilterSpeakerResults] = useState<MentionSuggestion[]>([]);+ const [isLoadingFilterSearch, setIsLoadingFilterSearch] = useState(false);+ const [selectedFilterResultIndex, setSelectedFilterResultIndex] = useState(0);const [recentSpeakers, setRecentSpeakers] = useState<MentionSuggestion[]>([]);const abortControllerRef = useRef<AbortController | null>(null);const messagesEndRef = useRef<HTMLDivElement>(null);@@ -4337,6 +4346,26 @@ export function StandaloneChat({[activeFilters]);+ const filterSearchGroups = React.useMemo(() => {+ const groups: { label: string; suggestions: MentionSuggestion[] }[] = [];+ if (filterTagResults.length > 0) {+ groups.push({ label: "tags", suggestions: filterTagResults });+ }+ if (filterSpeakerResults.length > 0) {+ groups.push({ label: "speakers", suggestions: filterSpeakerResults });+ }+ return groups;+ }, [filterTagResults, filterSpeakerResults]);++ const filterSearchResults = React.useMemo(+ () => filterSearchGroups.flatMap((group) => group.suggestions),+ [filterSearchGroups]+ );++ useEffect(() => {+ setSelectedFilterResultIndex(0);+ }, [filterSearch, filterSearchResults.length]);+// Remove a specific @mention from inputconst removeFilter = (filterType: "time" | "content" | "app" | "speaker" | "tag", label?: string) => {let newInput = input;@@ -4379,6 +4408,43 @@ export function StandaloneChat({setInput(newInput);};+ const getFilterSuggestionState = (suggestion: MentionSuggestion) => {+ const tagName = suggestion.tag.slice(1);+ const speakerName = suggestion.tag.startsWith('@"')+ ? suggestion.tag.slice(2, -1)+ : tagName;+ const isActive =+ suggestion.category === "tag"+ ? activeFilters.tagNames.includes(tagName)+ : suggestion.category === "speaker"+ ? activeFilters.speakerName === speakerName+ : false;++ return { tagName, speakerName, isActive };+ };++ const applyFilterSuggestion = (suggestion: MentionSuggestion) => {+ const { tagName, speakerName, isActive } = getFilterSuggestionState(suggestion);++ if (suggestion.category === "tag") {+ if (isActive) {+ removeFilter("tag", tagName);+ } else {+ setInput((prev) => `${suggestion.tag} ${prev.trim()}`.trim() + " ");+ }+ } else if (suggestion.category === "speaker") {+ if (isActive) {+ removeFilter("speaker");+ } else {+ if (activeFilters.speakerName) removeFilter("speaker");+ setInput((prev) => `${suggestion.tag} ${prev.trim()}`.trim() + " ");+ }+ }++ setAppFilterOpen(false);+ setFilterSearch("");+ };+// Fetch speakers dynamicallyuseEffect(() => {if (mentionTrigger !== "@") {@@ -4403,13 +4469,12 @@ export function StandaloneChat({setIsLoadingSpeakers(true);try {const response = await localFetch(- `/speakers/search?name=${encodeURIComponent(mentionFilter)}`+ `/speakers/search?name=${encodeURIComponent(mentionFilter)}&limit=${SPEAKER_SUGGESTION_LIMIT}&include_samples=false`);if (response.ok) {const speakers: Speaker[] = await response.json();const suggestions: MentionSuggestion[] = speakers.filter(s => s.name)- .slice(0, 5).map(s => ({tag: s.name.includes(" ") ? `@"${s.name}"` : `@${s.name}`,description: `speaker`,@@ -4428,14 +4493,109 @@ export function StandaloneChat({return () => clearTimeout(debounceTimeout);}, [mentionFilter, mentionTrigger, baseMentionSuggestions]);+ useEffect(() => {+ if (mentionTrigger !== "#" || !mentionFilter.trim()) {+ setTagSearchSuggestions([]);+ return;+ }++ const searchTags = async () => {+ setIsLoadingTagSearch(true);+ try {+ const response = await localFetch(+ `/tags/autocomplete?q=${encodeURIComponent(mentionFilter.trim())}&limit=${TAG_AUTOCOMPLETE_LIMIT}`+ );+ if (response.ok) {+ const tags = await response.json();+ if (Array.isArray(tags)) {+ setTagSearchSuggestions(buildTagMentionSuggestions(tags, TAG_AUTOCOMPLETE_LIMIT));+ }+ }+ } catch (error) {+ console.error("Error searching tags:", error);+ } finally {+ setIsLoadingTagSearch(false);+ }+ };++ const debounceTimeout = setTimeout(searchTags, 200);+ return () => clearTimeout(debounceTimeout);+ }, [mentionFilter, mentionTrigger]);++ useEffect(() => {+ const query = filterSearch.trim();+ if (!appFilterOpen || !query) {+ setFilterTagResults([]);+ setFilterSpeakerResults([]);+ setIsLoadingFilterSearch(false);+ return;+ }++ let cancelled = false;+ const searchFilters = async () => {+ setIsLoadingFilterSearch(true);+ try {+ const [tagResponse, speakerResponse] = await Promise.all([+ localFetch(`/tags/autocomplete?q=${encodeURIComponent(query)}&limit=${TAG_AUTOCOMPLETE_LIMIT}`),+ localFetch(`/speakers/search?name=${encodeURIComponent(query)}&limit=${SPEAKER_SUGGESTION_LIMIT}&include_samples=false`),+ ]);+ if (cancelled) return;++ if (tagResponse.ok) {+ const tags = await tagResponse.json();+ setFilterTagResults(+ Array.isArray(tags)+ ? buildTagMentionSuggestions(tags, TAG_AUTOCOMPLETE_LIMIT)+ : []+ );+ } else {+ setFilterTagResults([]);+ }++ if (speakerResponse.ok) {+ const speakers: Speaker[] = await speakerResponse.json();+ setFilterSpeakerResults(+ Array.isArray(speakers)+ ? speakers+ .filter((speaker) => speaker.name)+ .map((speaker) => ({+ tag: speaker.name.includes(" ") ? `@"${speaker.name}"` : `@${speaker.name}`,+ description: "speaker",+ category: "speaker" as const,+ }))+ : []+ );+ } else {+ setFilterSpeakerResults([]);+ }+ } catch (error) {+ if (!cancelled) {+ setFilterTagResults([]);+ setFilterSpeakerResults([]);+ console.error("Error searching filters:", error);+ }+ } finally {+ if (!cancelled) setIsLoadingFilterSearch(false);+ }+ };++ const debounceTimeout = setTimeout(searchFilters, 200);+ return () => {+ cancelled = true;+ clearTimeout(debounceTimeout);+ };+ }, [appFilterOpen, filterSearch]);+const filteredMentions = React.useMemo(() => {if (mentionTrigger === "#") {const tagSuggestions = !mentionFilter? tagMentionSuggestions- : allTagMentionSuggestions.filter(- s => s.tag.toLowerCase().includes(mentionFilter.toLowerCase()) ||- s.description.toLowerCase().includes(mentionFilter.toLowerCase())- );+ : tagSearchSuggestions.length > 0+ ? tagSearchSuggestions+ : allTagMentionSuggestions.filter(+ s => s.tag.toLowerCase().includes(mentionFilter.toLowerCase()) ||+ s.description.toLowerCase().includes(mentionFilter.toLowerCase())+ );return tagSuggestions;}@@ -4457,6 +4617,7 @@ export function StandaloneChat({appMentionSuggestions,tagMentionSuggestions,allTagMentionSuggestions,+ tagSearchSuggestions,]);const handleInputChange = (e: React.ChangeEvent<HTMLTextAreaElement>) => {@@ -4784,13 +4945,14 @@ export function StandaloneChat({if (!appFilterOpen || recentSpeakers.length > 0) return;(async () => {try {- const response = await localFetch("/speakers/search?name=");+ const response = await localFetch(+ `/speakers/search?name=&limit=${SPEAKER_SUGGESTION_LIMIT}&include_samples=false`+ );if (response.ok) {const speakers: Speaker[] = await response.json();setRecentSpeakers(speakers.filter((s) => s.name)- .slice(0, 5).map((s) => ({tag: s.name.includes(" ") ? `@"${s.name}"` : `@${s.name}`,description: "speaker",@@ -8064,6 +8226,31 @@ export function StandaloneChat({"past hour": "last hour","this morning": "this morning",};+ const filterQuery = filterSearch.trim();++ const renderFilterSearchButton = (suggestion: MentionSuggestion, resultIndex: number) => {+ const { isActive } = getFilterSuggestionState(suggestion);+ const isSelected = resultIndex === selectedFilterResultIndex;+ return (+ <button+ key={`${suggestion.category}-${suggestion.tag}`}+ type="button"+ onMouseEnter={() => setSelectedFilterResultIndex(resultIndex)}+ onClick={() => applyFilterSuggestion(suggestion)}+ className={cn(+ "w-full px-3 py-1.5 text-left text-xs font-mono hover:bg-muted/50 transition-colors flex items-center justify-between gap-2",+ isSelected && "bg-muted/70",+ isActive && "bg-muted"+ )}+ >+ <span className="truncate">{suggestion.tag}</span>+ <span className="text-[10px] text-muted-foreground truncate shrink-0 max-w-[9rem]">+ {isActive ? "selected" : suggestion.description}+ </span>+ </button>+ );+ };+ let filterSearchResultIndex = 0;return (<>@@ -8090,6 +8277,91 @@ export function StandaloneChat({)}</div>+ <div className="sticky top-0 z-10 p-2 border-b border-border/50 bg-background">+ <div className="relative">+ <Search className="absolute left-2 top-1/2 -translate-y-1/2 h-3.5 w-3.5 text-muted-foreground" />+ <Input+ value={filterSearch}+ onChange={(event) => setFilterSearch(event.target.value)}+ onKeyDown={(event) => {+ event.stopPropagation();+ if (event.key === "ArrowDown") {+ event.preventDefault();+ setSelectedFilterResultIndex((index) =>+ filterSearchResults.length === 0+ ? 0+ : (index + 1) % filterSearchResults.length+ );+ } else if (event.key === "ArrowUp") {+ event.preventDefault();+ setSelectedFilterResultIndex((index) =>+ filterSearchResults.length === 0+ ? 0+ : (index - 1 + filterSearchResults.length) % filterSearchResults.length+ );+ } else if (event.key === "Enter") {+ event.preventDefault();+ const selectedSuggestion = filterSearchResults[selectedFilterResultIndex];+ if (selectedSuggestion) applyFilterSuggestion(selectedSuggestion);+ } else if (event.key === "Escape") {+ event.preventDefault();+ if (filterSearch) {+ setFilterSearch("");+ } else {+ setAppFilterOpen(false);+ }+ }+ }}+ placeholder="search tags or speakers"+ className="h-8 pl-7 pr-7 text-xs"+ autoComplete="off"+ />+ {filterSearch && (+ <button+ type="button"+ onClick={() => setFilterSearch("")}+ className="absolute right-2 top-1/2 -translate-y-1/2 text-muted-foreground hover:text-foreground"+ aria-label="Clear filter search"+ >+ <X className="h-3.5 w-3.5" />+ </button>+ )}+ </div>+ </div>++ {filterQuery && (+ <>+ <div className="px-2 py-1 text-[10px] font-medium uppercase tracking-wider text-muted-foreground bg-muted/30 border-b border-border/50 flex items-center gap-1.5">+ <Search className="h-3 w-3" />+ <span>matching filters</span>+ {isLoadingFilterSearch && (+ <Loader2 className="ml-auto h-3 w-3 animate-spin text-muted-foreground" />+ )}+ </div>+ {filterSearchResults.length === 0 && !isLoadingFilterSearch ? (+ <div className="px-3 py-2 text-[10px] text-muted-foreground">+ no matching tags or speakers+ </div>+ ) : (+ <>+ {filterSearchGroups.map((group) => (+ <React.Fragment key={group.label}>+ <div className="px-3 py-1 text-[10px] font-medium uppercase tracking-wider text-muted-foreground/80 bg-muted/20 border-b border-border/40">+ {group.label}+ </div>+ {group.suggestions.map((suggestion) =>+ renderFilterSearchButton(suggestion, filterSearchResultIndex++)+ )}+ </React.Fragment>+ ))}+ </>+ )}+ </>+ )}++ {!filterQuery && (+ <>+<div className="px-2 py-1 text-[10px] font-medium uppercase tracking-wider text-muted-foreground bg-muted/30 border-b border-border/50">time</div>@@ -8294,6 +8566,8 @@ export function StandaloneChat({})}</>)}+ </>+ )}</>);};@@ -9569,6 +9843,12 @@ export function StandaloneChat({<span>Searching speakers...</span></div>)}+ {isLoadingTagSearch && (+ <div className="px-3 py-2 text-[10px] text-muted-foreground flex items-center gap-2">+ <Loader2 className="h-3 w-3 animate-spin" />+ <span>Searching tags...</span>+ </div>+ )}</motion.div>)}</AnimatePresence>@@ -9576,7 +9856,13 @@ export function StandaloneChat({</div>{/* Controls row — sits below the input box, not inside it */}<div className="flex items-center gap-1.5 px-1 pt-2">- <Popover open={appFilterOpen} onOpenChange={setAppFilterOpen}>+ <Popover+ open={appFilterOpen}+ onOpenChange={(open) => {+ setAppFilterOpen(open);+ if (!open) setFilterSearch("");+ }}+ ><PopoverTrigger asChild><Buttontype="button"diff --git a/apps/screenpipe-app-tauri/lib/hooks/use-sql-autocomplete.tsx b/apps/screenpipe-app-tauri/lib/hooks/use-sql-autocomplete.tsxindex b29b8e628..3c84506a3 100644--- a/apps/screenpipe-app-tauri/lib/hooks/use-sql-autocomplete.tsx+++ b/apps/screenpipe-app-tauri/lib/hooks/use-sql-autocomplete.tsx@@ -15,6 +15,7 @@ export interface AutocompleteItem {}const CACHE_DURATION = 5 * 60 * 1000; // 5 minutes+const TAG_AUTOCOMPLETE_LIMIT = 100;const cache: Record<string, { data: AutocompleteItem[]; timestamp: number }> ={};@@ -144,66 +145,15 @@ export function useTagAutocomplete() {setItems(TAG_CACHE.data);return;}- const response = await localFetch("/raw_sql", {- method: "POST",- headers: { "Content-Type": "application/json" },- body: JSON.stringify({- query: `- SELECT- name,- SUM(count) as count,- SUM(frame_count) as frame_count,- SUM(audio_count) as audio_count,- SUM(memory_count) as memory_count- FROM (- SELECT- t.name as name,- COUNT(DISTINCT vt.vision_id) as count,- COUNT(DISTINCT vt.vision_id) as frame_count,- 0 as audio_count,- 0 as memory_count- FROM tags t- JOIN vision_tags vt ON t.id = vt.tag_id- WHERE t.name IS NOT NULL AND t.name != ''- GROUP BY t.name-- UNION ALL-- SELECT- t.name as name,- COUNT(DISTINCT audio_tag_rows.audio_chunk_id) as count,- 0 as frame_count,- COUNT(DISTINCT audio_tag_rows.audio_chunk_id) as audio_count,- 0 as memory_count- FROM tags t- JOIN audio_tags audio_tag_rows ON t.id = audio_tag_rows.tag_id- WHERE t.name IS NOT NULL AND t.name != ''- GROUP BY t.name-- UNION ALL-- SELECT- json_tags.value as name,- COUNT(DISTINCT memories.id) as count,- 0 as frame_count,- 0 as audio_count,- COUNT(DISTINCT memories.id) as memory_count- FROM memories, json_each(memories.tags) json_tags- WHERE json_tags.value IS NOT NULL AND json_tags.value != ''- GROUP BY json_tags.value- )- GROUP BY name- ORDER BY count DESC- LIMIT 100- `,- }),- });+ const response = await localFetch(+ `/tags/autocomplete?limit=${TAG_AUTOCOMPLETE_LIMIT}`+ );if (!response.ok) {throw new Error(`HTTP error! status: ${response.status}`);}const result: AutocompleteItem[] = await response.json();if (!Array.isArray(result)) {- throw new Error("expected array from /raw_sql");+ throw new Error("expected array from /tags/autocomplete");}TAG_CACHE.data = result;TAG_CACHE.ts = Date.now();diff --git a/crates/screenpipe-db/src/db.rs b/crates/screenpipe-db/src/db.rsindex d95d6ff0f..4d268d59b 100644--- a/crates/screenpipe-db/src/db.rs+++ b/crates/screenpipe-db/src/db.rs@@ -36,8 +36,8 @@ use crate::{FrameData, FrameRow, FrameRowLight, FrameWindowData, InsertUiEvent, MeetingRecord,MeetingTranscriptSegment, MemoryRecord, MemorySyncRow, NewDiarizationSegment, OCREntry,OCRResult, OCRResultRaw, OcrEngine, OcrTextBlock, Order, ReplacementAudioTranscription,- SearchMatch, SearchMatchGroup, SearchResult, Speaker, TagContentType, TextBounds, TextPosition,- TimeSeriesChunk, UiContent, UiEventRecord, UiEventRow, VideoMetadata,+ SearchMatch, SearchMatchGroup, SearchResult, Speaker, TagAutocompleteItem, TagContentType,+ TextBounds, TextPosition, TimeSeriesChunk, UiContent, UiEventRecord, UiEventRow, VideoMetadata,MAX_TRANSCRIPTION_ATTEMPTS,};@@ -6137,6 +6137,41 @@ impl DatabaseManager {}pub async fn search_speakers(&self, name_prefix: &str) -> Result<Vec<Speaker>, sqlx::Error> {+ self.search_speakers_limited(name_prefix, 100, 0, true)+ .await+ }++ pub async fn search_speakers_limited(+ &self,+ name_prefix: &str,+ limit: i64,+ offset: i64,+ include_samples: bool,+ ) -> Result<Vec<Speaker>, sqlx::Error> {+ let limit = limit.clamp(1, 100);+ let offset = offset.max(0);++ if !include_samples {+ return sqlx::query_as::<_, Speaker>(+ r#"+ SELECT MIN(id) as id, name, '{}' as metadata+ FROM speakers+ WHERE name LIKE ? || '%' COLLATE NOCASE+ AND hallucination = 0+ AND name IS NOT NULL+ AND name != ''+ GROUP BY name+ ORDER BY name COLLATE NOCASE+ LIMIT ? OFFSET ?+ "#,+ )+ .bind(name_prefix)+ .bind(limit)+ .bind(offset)+ .fetch_all(&self.pool)+ .await;+ }+// Group by name so duplicate names (e.g. multiple "Louis" rows from// separate voice embeddings) appear as a single entry in the dropdown.// Pick the lowest id per name so reassignment targets a stable speaker.@@ -6148,6 +6183,8 @@ impl DatabaseManager {FROM speakersWHERE name LIKE ? || '%' AND hallucination = 0 AND name IS NOT NULL AND name != ''GROUP BY name+ ORDER BY name COLLATE NOCASE+ LIMIT ? OFFSET ?),RecentAudioPaths AS (SELECT DISTINCT@@ -6195,6 +6232,8 @@ impl DatabaseManager {"#,).bind(name_prefix)+ .bind(limit)+ .bind(offset).fetch_all(&self.pool).await}@@ -10522,6 +10561,102 @@ LIMIT ? OFFSET ?Ok(rows.into_iter().map(|r| r.0).collect())}+ pub async fn autocomplete_tags(+ &self,+ query: &str,+ limit: i64,+ offset: i64,+ ) -> Result<Vec<TagAutocompleteItem>, SqlxError> {+ let limit = limit.clamp(1, 100);+ let offset = offset.max(0);+ let candidate_limit = (limit + offset).clamp(1, 200);+ let search = query.trim();++ sqlx::query_as::<_, TagAutocompleteItem>(+ r#"+ WITH candidates AS (+ SELECT name+ FROM (+ SELECT name+ FROM (+ SELECT t.name as name+ FROM tags t+ WHERE t.name IS NOT NULL+ AND t.name != ''+ AND (? = '' OR t.name LIKE '%' || ? || '%' COLLATE NOCASE)+ GROUP BY t.name+ ORDER BY t.name COLLATE NOCASE+ LIMIT ?+ )++ UNION++ SELECT name+ FROM (+ SELECT json_tags.value as name+ FROM memories, json_each(memories.tags) json_tags+ WHERE json_tags.value IS NOT NULL+ AND json_tags.value != ''+ AND (? = '' OR json_tags.value LIKE '%' || ? || '%' COLLATE NOCASE)+ GROUP BY json_tags.value+ ORDER BY json_tags.value COLLATE NOCASE+ LIMIT ?+ )+ )+ GROUP BY name+ ORDER BY name COLLATE NOCASE+ LIMIT ? OFFSET ?+ )+ SELECT+ candidates.name,+ (+ SELECT COUNT(DISTINCT vt.vision_id)+ FROM tags t+ JOIN vision_tags vt ON t.id = vt.tag_id+ WHERE t.name = candidates.name+ ) + (+ SELECT COUNT(DISTINCT audio_tag_rows.audio_chunk_id)+ FROM tags t+ JOIN audio_tags audio_tag_rows ON t.id = audio_tag_rows.tag_id+ WHERE t.name = candidates.name+ ) + (+ SELECT COUNT(DISTINCT memories.id)+ FROM memories, json_each(memories.tags) memory_tags+ WHERE memory_tags.value = candidates.name+ ) as count,+ (+ SELECT COUNT(DISTINCT vt.vision_id)+ FROM tags t+ JOIN vision_tags vt ON t.id = vt.tag_id+ WHERE t.name = candidates.name+ ) as frame_count,+ (+ SELECT COUNT(DISTINCT audio_tag_rows.audio_chunk_id)+ FROM tags t+ JOIN audio_tags audio_tag_rows ON t.id = audio_tag_rows.tag_id+ WHERE t.name = candidates.name+ ) as audio_count,+ (+ SELECT COUNT(DISTINCT memories.id)+ FROM memories, json_each(memories.tags) memory_tags+ WHERE memory_tags.value = candidates.name+ ) as memory_count+ FROM candidates+ ORDER BY count DESC, candidates.name COLLATE NOCASE+ "#,+ )+ .bind(search)+ .bind(search)+ .bind(candidate_limit)+ .bind(search)+ .bind(search)+ .bind(candidate_limit)+ .bind(limit)+ .bind(offset)+ .fetch_all(&self.pool)+ .await+ }+// ========================================================================// Outputs// ========================================================================diff --git a/crates/screenpipe-db/src/migrations/20260616120000_speaker_filter_autocomplete_indexes.sql b/crates/screenpipe-db/src/migrations/20260616120000_speaker_filter_autocomplete_indexes.sqlnew file mode 100644index 000000000..c3c665ee9--- /dev/null+++ b/crates/screenpipe-db/src/migrations/20260616120000_speaker_filter_autocomplete_indexes.sql@@ -0,0 +1,6 @@+-- Speed up chat people filters and speaker autocomplete.+CREATE INDEX IF NOT EXISTS idx_speakers_hallucination_name+ ON speakers(hallucination, name COLLATE NOCASE);++CREATE INDEX IF NOT EXISTS idx_tags_name_nocase+ ON tags(name COLLATE NOCASE);diff --git a/crates/screenpipe-db/src/types.rs b/crates/screenpipe-db/src/types.rsindex 5df30b61d..d5aed8648 100644--- a/crates/screenpipe-db/src/types.rs+++ b/crates/screenpipe-db/src/types.rs@@ -197,6 +197,15 @@ pub struct Speaker {pub metadata: String,}+#[derive(OaSchema, Debug, Serialize, Deserialize, FromRow, Clone)]+pub struct TagAutocompleteItem {+ pub name: String,+ pub count: i64,+ pub frame_count: i64,+ pub audio_count: i64,+ pub memory_count: i64,+}+#[derive(Debug, Clone)]pub struct NewDiarizationSegment {pub provider_speaker_label: String,diff --git a/crates/screenpipe-db/tests/db.rs b/crates/screenpipe-db/tests/db.rsindex 8ebbda294..3c8ae5efe 100644--- a/crates/screenpipe-db/tests/db.rs+++ b/crates/screenpipe-db/tests/db.rs@@ -1551,6 +1551,24 @@ mod tests {assert_eq!(speakers[0].name, "test name");}+ #[tokio::test]+ async fn test_search_speakers_limited_returns_more_than_legacy_picker_cap() {+ let db = setup_test_db().await;++ for i in 0..25 {+ let speaker = db.insert_speaker(&vec![i as f32; 512]).await.unwrap();+ db.update_speaker_name(speaker.id, &format!("person {i:02}"))+ .await+ .unwrap();+ }++ let speakers = db.search_speakers_limited("", 50, 0, false).await.unwrap();++ assert_eq!(speakers.len(), 25);+ assert_eq!(speakers[0].name, "person 00");+ assert!(speakers.iter().all(|speaker| speaker.metadata == "{}"));+ }+#[tokio::test]async fn test_delete_speaker() {let db = setup_test_db().await;diff --git a/crates/screenpipe-db/tests/tag_autocomplete_query_test.rs b/crates/screenpipe-db/tests/tag_autocomplete_query_test.rsindex 4ef02ae5e..a573e44ed 100644--- a/crates/screenpipe-db/tests/tag_autocomplete_query_test.rs+++ b/crates/screenpipe-db/tests/tag_autocomplete_query_test.rs@@ -4,8 +4,8 @@//! Regression coverage for the chat tag autocomplete SQL.//!-//! This mirrors the raw SQL query in:-//! `apps/screenpipe-app-tauri/lib/hooks/use-sql-autocomplete.tsx`.+//! This covers the bounded DB helper used by:+//! `GET /tags/autocomplete`.//!//! The query intentionally spans three tag stores://! - screen/frame tags through `tags` + `vision_tags`@@ -16,55 +16,6 @@ use std::{collections::HashMap, time::Instant};use screenpipe_db::DatabaseManager;-const TAG_AUTOCOMPLETE_QUERY: &str = r#"-SELECT- name,- SUM(count) as count,- SUM(frame_count) as frame_count,- SUM(audio_count) as audio_count,- SUM(memory_count) as memory_count-FROM (- SELECT- t.name as name,- COUNT(DISTINCT vt.vision_id) as count,- COUNT(DISTINCT vt.vision_id) as frame_count,- 0 as audio_count,- 0 as memory_count- FROM tags t- JOIN vision_tags vt ON t.id = vt.tag_id- WHERE t.name IS NOT NULL AND t.name != ''- GROUP BY t.name-- UNION ALL-- SELECT- t.name as name,- COUNT(DISTINCT audio_tag_rows.audio_chunk_id) as count,- 0 as frame_count,- COUNT(DISTINCT audio_tag_rows.audio_chunk_id) as audio_count,- 0 as memory_count- FROM tags t- JOIN audio_tags audio_tag_rows ON t.id = audio_tag_rows.tag_id- WHERE t.name IS NOT NULL AND t.name != ''- GROUP BY t.name-- UNION ALL-- SELECT- json_tags.value as name,- COUNT(DISTINCT memories.id) as count,- 0 as frame_count,- 0 as audio_count,- COUNT(DISTINCT memories.id) as memory_count- FROM memories, json_each(memories.tags) json_tags- WHERE json_tags.value IS NOT NULL AND json_tags.value != ''- GROUP BY json_tags.value-)-GROUP BY name-ORDER BY count DESC-LIMIT 100-"#;-type TagRow = (String, i64, i64, i64, i64);async fn migrated_db() -> DatabaseManager {@@ -86,10 +37,20 @@ async fn exec(db: &DatabaseManager, sql: &str) {}async fn tag_rows(db: &DatabaseManager) -> Vec<TagRow> {- sqlx::query_as::<_, TagRow>(TAG_AUTOCOMPLETE_QUERY)- .fetch_all(&db.pool)+ db.autocomplete_tags("", 100, 0).await.unwrap()+ .into_iter()+ .map(|row| {+ (+ row.name,+ row.count,+ row.frame_count,+ row.audio_count,+ row.memory_count,+ )+ })+ .collect()}fn by_name(rows: Vec<TagRow>) -> HashMap<String, TagRow> {@@ -196,6 +157,33 @@ async fn tag_autocomplete_query_survives_large_mixed_sources() {assert_eq!(rows["bucket:0"].4, 1_000);}+#[tokio::test]+async fn tag_autocomplete_query_is_bounded_and_searchable_at_scale() {+ let db = migrated_db().await;+ seed_large_autocomplete_db(&db, 5_000, 200_000, 60_000, 50_000).await;++ let first_page = db.autocomplete_tags("", 20, 0).await.unwrap();+ assert_eq!(first_page.len(), 20);++ let second_page = db.autocomplete_tags("", 20, 20).await.unwrap();+ assert_eq!(second_page.len(), 20);+ assert_ne!(+ first_page+ .iter()+ .map(|row| row.name.as_str())+ .collect::<Vec<_>>(),+ second_page+ .iter()+ .map(|row| row.name.as_str())+ .collect::<Vec<_>>(),+ );++ let targeted = db.autocomplete_tags("tag:4999", 10, 0).await.unwrap();+ assert_eq!(targeted.len(), 1);+ assert_eq!(targeted[0].name, "tag:4999");+ assert!(targeted[0].count > 0);+}+#[tokio::test]async fn bench_tag_autocomplete_query_large_db() {let db = migrated_db().await;diff --git a/crates/screenpipe-engine/src/routes/content.rs b/crates/screenpipe-engine/src/routes/content.rsindex 4e47bda6f..55b269de1 100644--- a/crates/screenpipe-engine/src/routes/content.rs+++ b/crates/screenpipe-engine/src/routes/content.rs@@ -11,7 +11,7 @@ use oasgen::{oasgen, OaSchema};use chrono::{DateTime, Utc};use screenpipe_audio::core::device::DeviceType;-use screenpipe_db::{Speaker, TagContentType};+use screenpipe_db::{Speaker, TagAutocompleteItem, TagContentType};use image::ImageFormat;use serde::{Deserialize, Serialize};@@ -54,6 +54,13 @@ pub struct MemoryContent {pub updated_at: String,}+#[derive(OaSchema, Serialize, Deserialize, Debug)]+pub(crate) struct TagAutocompleteRequest {+ pub q: Option<String>,+ pub limit: Option<u32>,+ pub offset: Option<u32>,+}+#[derive(OaSchema, Serialize, Deserialize, Debug, Clone)]pub struct OCRContent {pub frame_id: i64,@@ -181,6 +188,29 @@ pub struct RemoveTagsResponse {success: bool,}+#[oasgen]+pub(crate) async fn autocomplete_tags(+ State(state): State<Arc<AppState>>,+ Query(request): Query<TagAutocompleteRequest>,+) -> Result<JsonResponse<Vec<TagAutocompleteItem>>, (StatusCode, JsonResponse<Value>)> {+ let limit = request.limit.unwrap_or(50).clamp(1, 100) as i64;+ let offset = request.offset.unwrap_or(0) as i64;+ let query = request.q.unwrap_or_default();++ state+ .db+ .autocomplete_tags(&query, limit, offset)+ .await+ .map(JsonResponse)+ .map_err(|e| {+ error!("Failed to autocomplete tags: {}", e);+ (+ StatusCode::INTERNAL_SERVER_ERROR,+ JsonResponse(json!({"error": e.to_string()})),+ )+ })+}+#[oasgen]pub(crate) async fn add_tags(State(state): State<Arc<AppState>>,diff --git a/crates/screenpipe-engine/src/routes/speakers.rs b/crates/screenpipe-engine/src/routes/speakers.rsindex b391e0fe4..69220ccd7 100644--- a/crates/screenpipe-engine/src/routes/speakers.rs+++ b/crates/screenpipe-engine/src/routes/speakers.rs@@ -27,6 +27,9 @@ pub struct UpdateSpeakerRequest {#[derive(OaSchema, Serialize, Deserialize, Debug)]pub struct SearchSpeakersRequest {pub name: Option<String>,+ pub limit: Option<u32>,+ pub offset: Option<u32>,+ pub include_samples: Option<bool>,}#[derive(OaSchema, Serialize, Deserialize, Debug)]@@ -182,9 +185,12 @@ pub(crate) async fn search_speakers_handler(Query(request): Query<SearchSpeakersRequest>,) -> Result<JsonResponse<Vec<Speaker>>, (StatusCode, JsonResponse<Value>)> {let search_prefix = request.name.unwrap_or_default();+ let limit = request.limit.unwrap_or(50).clamp(1, 100) as i64;+ let offset = request.offset.unwrap_or(0) as i64;+ let include_samples = request.include_samples.unwrap_or(true);let speakers = state.db- .search_speakers(&search_prefix)+ .search_speakers_limited(&search_prefix, limit, offset, include_samples).await.map_err(|e| {(diff --git a/crates/screenpipe-engine/src/server.rs b/crates/screenpipe-engine/src/server.rsindex 41c580ce7..4800c2b87 100644--- a/crates/screenpipe-engine/src/server.rs+++ b/crates/screenpipe-engine/src/server.rs@@ -23,8 +23,8 @@ use crate::{stop_audio, stop_audio_device,},content::{- add_tags, add_to_database, execute_raw_sql, get_tags_batch, merge_frames_handler,- remove_tags, validate_media_handler,+ add_tags, add_to_database, autocomplete_tags, execute_raw_sql, get_tags_batch,+ merge_frames_handler, remove_tags, validate_media_handler,},data::{backup_handler, checkpoint_handler, delete_device_data_handler,@@ -651,6 +651,7 @@ impl SCServer {.get("/search", search).get("/audio/list", api_list_audio_devices).get("/vision/list", api_list_monitors)+ .get("/tags/autocomplete", autocomplete_tags).post("/tags/vision/batch", get_tags_batch).post("/tags/:content_type/:id", add_tags).delete("/tags/:content_type/:id", remove_tags)