Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix(api): add neutral `text` to OCR streaming + remote-frame payloads (#4268)

Louis Beaumontf9650ec9a2da
diff --git a/apps/screenpipe-app-tauri/components/rewind/timeline.tsx b/apps/screenpipe-app-tauri/components/rewind/timeline.tsx
index 8003e5e8b..ca47d345a 100644
--- a/apps/screenpipe-app-tauri/components/rewind/timeline.tsx
+++ b/apps/screenpipe-app-tauri/components/rewind/timeline.tsx
@@ -61,7 +61,10 @@ export interface DeviceMetadata {
file_path: string;
app_name: string;
window_name: string;
- ocr_text: string;
+ /** Frame text (accessibility-derived for most captures, OCR fallback). */
+ text: string;
+ /** @deprecated Legacy alias for `text`; the server still sends it but read `text`. */
+ ocr_text?: string;
timestamp: string;
browser_url?: string;
}
@@ -718,12 +721,13 @@ export default function Timeline({ embedded = false }: { embedded?: boolean }) {
contextParts.push(`Apps: ${Array.from(apps).join(", ")}`);
}
- // Add sample OCR text (first few frames)
+ // Add sample frame text (first few frames)
const ocrSamples: string[] = [];
selectedFrames.slice(0, 3).forEach((frame) => {
frame.devices.forEach((device) => {
- if (device.metadata.ocr_text && device.metadata.ocr_text.length > 0) {
- const sample = device.metadata.ocr_text.slice(0, 200);
+ const frameText = device.metadata.text ?? device.metadata.ocr_text;
+ if (frameText && frameText.length > 0) {
+ const sample = frameText.slice(0, 200);
if (sample.trim()) {
ocrSamples.push(sample);
}
diff --git a/crates/screenpipe-engine/src/routes/frames.rs b/crates/screenpipe-engine/src/routes/frames.rs
index 4f3948822..eb36ebcd1 100644
--- a/crates/screenpipe-engine/src/routes/frames.rs
+++ b/crates/screenpipe-engine/src/routes/frames.rs
@@ -123,6 +123,9 @@ pub async fn get_frame_data(
"error_type": "remote_device",
"frame_id": frame_id,
"timestamp": timestamp,
+ // Neutral name for the frame's text, consistent with the
+ // rest of the API. `ocr_text` is a deprecated alias.
+ "text": acc_text,
"ocr_text": ocr_text,
"accessibility_text": acc_text,
});
diff --git a/crates/screenpipe-engine/src/routes/streaming.rs b/crates/screenpipe-engine/src/routes/streaming.rs
index 5d4e9e94b..7986ef41d 100644
--- a/crates/screenpipe-engine/src/routes/streaming.rs
+++ b/crates/screenpipe-engine/src/routes/streaming.rs
@@ -78,6 +78,11 @@ pub struct DeviceMetadata {
pub file_path: String,
pub app_name: String,
pub window_name: String,
+ /// Neutral name for the frame's text. Despite living next to `ocr_text`,
+ /// the content is accessibility-derived for most captures. Prefer this.
+ pub text: String,
+ /// @deprecated Legacy alias for `text`, kept for backward compatibility.
+ /// Always equal to `text`. Will be removed in a future major version.
pub ocr_text: String,
pub browser_url: Option<String>,
}
@@ -115,6 +120,7 @@ impl From<TimeSeriesFrame> for StreamTimeSeriesResponse {
file_path: device_frame.metadata.file_path,
app_name: device_frame.metadata.app_name,
window_name: device_frame.metadata.window_name,
+ text: device_frame.metadata.ocr_text.clone(),
ocr_text: device_frame.metadata.ocr_text,
browser_url: device_frame.metadata.browser_url,
},
@@ -545,6 +551,7 @@ async fn handle_stream_frames_socket(
file_path: hot_frame.snapshot_path.to_string(),
app_name: hot_frame.app_name.to_string(),
window_name: hot_frame.window_name.to_string(),
+ text: hot_frame.ocr_text_preview.to_string(),
ocr_text: hot_frame.ocr_text_preview.to_string(),
browser_url: hot_frame.browser_url.as_deref().map(String::from),
},