YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(api): add neutral `text` to OCR streaming + remote-frame payloads (#4268)
Louis Beaumontf9650ec9a2da
diff --git a/apps/screenpipe-app-tauri/components/rewind/timeline.tsx b/apps/screenpipe-app-tauri/components/rewind/timeline.tsxindex 8003e5e8b..ca47d345a 100644--- a/apps/screenpipe-app-tauri/components/rewind/timeline.tsx+++ b/apps/screenpipe-app-tauri/components/rewind/timeline.tsx@@ -61,7 +61,10 @@ export interface DeviceMetadata {file_path: string;app_name: string;window_name: string;- ocr_text: string;+ /** Frame text (accessibility-derived for most captures, OCR fallback). */+ text: string;+ /** @deprecated Legacy alias for `text`; the server still sends it but read `text`. */+ ocr_text?: string;timestamp: string;browser_url?: string;}@@ -718,12 +721,13 @@ export default function Timeline({ embedded = false }: { embedded?: boolean }) {contextParts.push(`Apps: ${Array.from(apps).join(", ")}`);}- // Add sample OCR text (first few frames)+ // Add sample frame text (first few frames)const ocrSamples: string[] = [];selectedFrames.slice(0, 3).forEach((frame) => {frame.devices.forEach((device) => {- if (device.metadata.ocr_text && device.metadata.ocr_text.length > 0) {- const sample = device.metadata.ocr_text.slice(0, 200);+ const frameText = device.metadata.text ?? device.metadata.ocr_text;+ if (frameText && frameText.length > 0) {+ const sample = frameText.slice(0, 200);if (sample.trim()) {ocrSamples.push(sample);}diff --git a/crates/screenpipe-engine/src/routes/frames.rs b/crates/screenpipe-engine/src/routes/frames.rsindex 4f3948822..eb36ebcd1 100644--- a/crates/screenpipe-engine/src/routes/frames.rs+++ b/crates/screenpipe-engine/src/routes/frames.rs@@ -123,6 +123,9 @@ pub async fn get_frame_data("error_type": "remote_device","frame_id": frame_id,"timestamp": timestamp,+ // Neutral name for the frame's text, consistent with the+ // rest of the API. `ocr_text` is a deprecated alias.+ "text": acc_text,"ocr_text": ocr_text,"accessibility_text": acc_text,});diff --git a/crates/screenpipe-engine/src/routes/streaming.rs b/crates/screenpipe-engine/src/routes/streaming.rsindex 5d4e9e94b..7986ef41d 100644--- a/crates/screenpipe-engine/src/routes/streaming.rs+++ b/crates/screenpipe-engine/src/routes/streaming.rs@@ -78,6 +78,11 @@ pub struct DeviceMetadata {pub file_path: String,pub app_name: String,pub window_name: String,+ /// Neutral name for the frame's text. Despite living next to `ocr_text`,+ /// the content is accessibility-derived for most captures. Prefer this.+ pub text: String,+ /// @deprecated Legacy alias for `text`, kept for backward compatibility.+ /// Always equal to `text`. Will be removed in a future major version.pub ocr_text: String,pub browser_url: Option<String>,}@@ -115,6 +120,7 @@ impl From<TimeSeriesFrame> for StreamTimeSeriesResponse {file_path: device_frame.metadata.file_path,app_name: device_frame.metadata.app_name,window_name: device_frame.metadata.window_name,+ text: device_frame.metadata.ocr_text.clone(),ocr_text: device_frame.metadata.ocr_text,browser_url: device_frame.metadata.browser_url,},@@ -545,6 +551,7 @@ async fn handle_stream_frames_socket(file_path: hot_frame.snapshot_path.to_string(),app_name: hot_frame.app_name.to_string(),window_name: hot_frame.window_name.to_string(),+ text: hot_frame.ocr_text_preview.to_string(),ocr_text: hot_frame.ocr_text_preview.to_string(),browser_url: hot_frame.browser_url.as_deref().map(String::from),},