Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix: unblock workflow timing research when star metadata fails

Louis Beaumont48ce117afd79
diff --git a/crates/screenpipe-db/src/db/starred.rs b/crates/screenpipe-db/src/db/starred.rs
index d39291d00..f90e40c7a 100644
--- a/crates/screenpipe-db/src/db/starred.rs
+++ b/crates/screenpipe-db/src/db/starred.rs
@@ -100,6 +100,9 @@ impl DatabaseManager {
/// One bounded query per search page, never one query per capture/frame.
pub async fn starred_timestamps(&self, timestamps: &[String]) -> Result<Vec<bool>, SqlxError> {
+ if timestamps.is_empty() {
+ return Ok(Vec::new());
+ }
let values =
serde_json::to_string(timestamps).map_err(|e| SqlxError::Protocol(e.to_string()))?;
sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM starred_sessions s WHERE s.start <= j.value AND s.end > j.value) FROM json_each(?1) j ORDER BY CAST(j.key AS INTEGER)")
diff --git a/crates/screenpipe-engine/src/routes/search.rs b/crates/screenpipe-engine/src/routes/search.rs
index 422ba0f01..386b1c7be 100644
--- a/crates/screenpipe-engine/src/routes/search.rs
+++ b/crates/screenpipe-engine/src/routes/search.rs
@@ -401,7 +401,9 @@ pub struct StarredSearchHit {
#[serde(flatten)]
pub item: ContentItem,
/// User marked this capture's timestamp as important. Not an accuracy score.
- pub starred: bool,
+ /// Omitted when optional star metadata is unavailable, never guessed false.
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub starred: Option<bool>,
}
fn captured_at(item: &ContentItem) -> String {
@@ -1679,21 +1681,37 @@ pub(crate) async fn search(
};
let timestamps: Vec<_> = content_items.iter().map(captured_at).collect();
- let starred = state
- .db
- .starred_timestamps(&timestamps)
- .await
- .map_err(|_| {
- (
- StatusCode::INTERNAL_SERVER_ERROR,
- JsonResponse(json!({"error":"could not read starred metadata"})),
- )
- })?;
+ // Stars annotate these results; failure must not hide already-retrieved
+ // recordings from chat or workflow evidence research. Explicit star filters
+ // are resolved above and still fail rather than broadening the search.
+ let starred = match tokio::time::timeout(
+ std::time::Duration::from_millis(250),
+ state.db.starred_timestamps(&timestamps),
+ )
+ .await
+ {
+ Ok(Ok(starred)) => Some(starred),
+ Ok(Err(error)) => {
+ warn!(%error, "search returned recordings without optional star metadata");
+ None
+ }
+ Err(_) => {
+ warn!("search star metadata timed out; returning recordings without annotations");
+ None
+ }
+ };
+ let starred_metadata_available = starred.is_some();
let response = SearchResponse {
data: content_items
.into_iter()
- .zip(starred)
- .map(|(item, starred)| StarredSearchHit { item, starred })
+ .enumerate()
+ .map(|(index, item)| StarredSearchHit {
+ item,
+ starred: starred
+ .as_ref()
+ .and_then(|values| values.get(index))
+ .copied(),
+ })
.collect(),
pagination: PaginationInfo {
limit: query.pagination.limit,
@@ -1707,6 +1725,7 @@ pub(crate) async fn search(
capture_direct_api_search_value(&api_client, response.data.len());
let cache_entry = if !history_restricted
+ && starred_metadata_available
&& !query.include_frames
&& cacheable_render
&& !pipe_data_restricted
@@ -2790,7 +2809,7 @@ mod tests {
.into_iter()
.map(|item| StarredSearchHit {
item,
- starred: false,
+ starred: Some(false),
})
.collect(),
pagination: PaginationInfo {
diff --git a/crates/screenpipe-engine/tests/workflow_timing_e2e.rs b/crates/screenpipe-engine/tests/workflow_timing_e2e.rs
index 70f4fb11b..2f8187f5c 100644
--- a/crates/screenpipe-engine/tests/workflow_timing_e2e.rs
+++ b/crates/screenpipe-engine/tests/workflow_timing_e2e.rs
@@ -54,6 +54,15 @@ async fn get(client: &reqwest::Client, base: &str, path: &str) -> Value {
#[tokio::test]
async fn timing_survives_verified_publication_retries_and_disk_reload() {
+ run_timing_case(false).await;
+}
+
+#[tokio::test]
+async fn timing_research_survives_unavailable_optional_star_metadata() {
+ run_timing_case(true).await;
+}
+
+async fn run_timing_case(missing_stars: bool) {
let input: Value = std::env::var("WORKFLOW_TIMING_AGENT_INPUT")
.ok()
.map(|path| serde_json::from_slice(&std::fs::read(path).unwrap()).unwrap())
@@ -130,6 +139,91 @@ async fn timing_survives_verified_publication_retries_and_disk_reload() {
let serving = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() });
let base = format!("http://{address}");
let client = reqwest::Client::new();
+ if missing_stars {
+ // Reproduce the installed compressed-database failure without touching
+ // the user's recorder. Missing annotations must not block source reads.
+ let mut tx = db.begin_immediate_with_retry().await.unwrap();
+ sqlx::query("DROP TABLE starred_sessions")
+ .execute(&mut **tx.conn())
+ .await
+ .unwrap();
+ tx.commit().await.unwrap();
+ let source_dates: Vec<_> = input["rows"]
+ .as_array()
+ .unwrap()
+ .iter()
+ .map(|row| {
+ chrono::DateTime::parse_from_rfc3339(row["timestamp"].as_str().unwrap())
+ .unwrap()
+ .with_timezone(&Utc)
+ })
+ .collect();
+ let query = format!(
+ "/search?content_type=ocr&limit=20&start_time={}&end_time={}",
+ (*source_dates.iter().min().unwrap() - Duration::seconds(1))
+ .to_rfc3339_opts(chrono::SecondsFormat::Millis, true),
+ (*source_dates.iter().max().unwrap() + Duration::seconds(1))
+ .to_rfc3339_opts(chrono::SecondsFormat::Millis, true)
+ );
+ let found = get(&client, &base, &query).await;
+ let hits = found["data"].as_array().unwrap();
+ assert_eq!(hits.len(), input["rows"].as_array().unwrap().len());
+ assert!(hits.iter().all(|hit| hit.get("starred").is_none()));
+ for row in input["rows"].as_array().unwrap() {
+ assert!(hits
+ .iter()
+ .any(|hit| hit["content"]["text"] == row["quote"]));
+ }
+ assert_eq!(found["pagination"]["total"], json!(hits.len()));
+ for filter in ["starred_only=true", "starred_session_id=missing"] {
+ let response = client
+ .get(format!("{base}{query}&{filter}"))
+ .bearer_auth("timing-test-key")
+ .send()
+ .await
+ .unwrap();
+ assert_eq!(
+ response.status().as_u16(),
+ 500,
+ "explicit filters must not widen: {filter}"
+ );
+ }
+ let empty = get(
+ &client,
+ &base,
+ &format!("{query}&app_name=NoSuchFixtureApp"),
+ )
+ .await;
+ assert!(empty["data"].as_array().unwrap().is_empty());
+ // A degraded response must not poison the cache after metadata recovers.
+ let mut tx = db.begin_immediate_with_retry().await.unwrap();
+ sqlx::raw_sql(include_str!(
+ "../../screenpipe-db/src/migrations/20261002190000_starred_sessions.sql"
+ ))
+ .execute(&mut **tx.conn())
+ .await
+ .unwrap();
+ tx.commit().await.unwrap();
+ let recovered = get(&client, &base, &query).await;
+ assert!(recovered["data"]
+ .as_array()
+ .unwrap()
+ .iter()
+ .all(|hit| hit["starred"] == false));
+ if let Ok(output) = std::env::var("WORKFLOW_TIMING_SEARCH_OUTPUT") {
+ tokio::fs::write(output, serde_json::to_vec_pretty(&json!({
+ "status":200,"fixture":"Fictional invoice recordings; starred_sessions unavailable",
+ "response":found,"explicitStarFilters":"500; did not broaden search",
+ "recovery":"Same query immediately restored starred=false after schema repair"
+ })).unwrap()).await.unwrap();
+ }
+ let mut tx = db.begin_immediate_with_retry().await.unwrap();
+ sqlx::query("DROP TABLE starred_sessions")
+ .execute(&mut **tx.conn())
+ .await
+ .unwrap();
+ tx.commit().await.unwrap();
+ }
let path = dir.path().join("workflows/catalog.json");
tokio::fs::create_dir_all(path.parent().unwrap())
.await
diff --git a/docs/pr-assets/workflow-timing-search/after.png b/docs/pr-assets/workflow-timing-search/after.png
new file mode 100644
index 000000000..e960500ec
Binary files /dev/null and b/docs/pr-assets/workflow-timing-search/after.png differ
diff --git a/docs/pr-assets/workflow-timing-search/before.png b/docs/pr-assets/workflow-timing-search/before.png
new file mode 100644
index 000000000..7afc2ebd7
Binary files /dev/null and b/docs/pr-assets/workflow-timing-search/before.png differ