Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix: hide structured output state from artifacts

Louis Beaumonta911e1fbb150
diff --git a/crates/screenpipe-db/src/db/outputs.rs b/crates/screenpipe-db/src/db/outputs.rs
index 8547ef540..683952e96 100644
--- a/crates/screenpipe-db/src/db/outputs.rs
+++ b/crates/screenpipe-db/src/db/outputs.rs
@@ -320,11 +320,32 @@ impl DatabaseManager {
saf_kind: Option<&str>,
limit: u32,
offset: u32,
+ ) -> Result<(Vec<crate::types::OutputRecord>, i64), SqlxError> {
+ // Preserve the existing database API as an unfiltered registry query.
+ // User-facing artifact surfaces use the explicit visibility method.
+ self.search_outputs_for_artifacts(query, source, saf_kind, true, limit, offset)
+ .await
+ }
+
+ pub async fn search_outputs_for_artifacts(
+ &self,
+ query: &str,
+ source: Option<&str>,
+ saf_kind: Option<&str>,
+ include_internal: bool,
+ limit: u32,
+ offset: u32,
) -> Result<(Vec<crate::types::OutputRecord>, i64), SqlxError> {
let fts_query = crate::sanitize_fts5_query(query);
if fts_query.is_empty() {
return self
- .list_outputs_for_artifacts(source, saf_kind, limit, offset)
+ .list_outputs_for_artifacts_with_internal(
+ source,
+ saf_kind,
+ include_internal,
+ limit,
+ offset,
+ )
.await;
}
@@ -337,7 +358,7 @@ impl DatabaseManager {
WHERE output_search_fts MATCH ?1",
);
let mut binds: Vec<String> = vec![fts_query.clone()];
- append_artifact_output_filters(&mut sql, &mut binds, source, saf_kind);
+ append_artifact_output_filters(&mut sql, &mut binds, source, saf_kind, include_internal);
sql.push_str(&format!(
" ORDER BY bm25(output_search_fts), o.updated_at DESC LIMIT ?{} OFFSET ?{}",
binds.len() + 1,
@@ -362,7 +383,13 @@ impl DatabaseManager {
WHERE output_search_fts MATCH ?1",
);
let mut count_binds: Vec<String> = vec![fts_query];
- append_artifact_output_filters(&mut count_sql, &mut count_binds, source, saf_kind);
+ append_artifact_output_filters(
+ &mut count_sql,
+ &mut count_binds,
+ source,
+ saf_kind,
+ include_internal,
+ );
let mut count_query = sqlx::query_scalar::<_, i64>(sqlx::AssertSqlSafe(count_sql));
for b in &count_binds {
count_query = count_query.bind(b);
@@ -378,6 +405,20 @@ impl DatabaseManager {
saf_kind: Option<&str>,
limit: u32,
offset: u32,
+ ) -> Result<(Vec<crate::types::OutputRecord>, i64), SqlxError> {
+ // Preserve compatibility for callers that use this as a raw output
+ // registry. The Artifacts route passes its visibility explicitly.
+ self.list_outputs_for_artifacts_with_internal(source, saf_kind, true, limit, offset)
+ .await
+ }
+
+ pub async fn list_outputs_for_artifacts_with_internal(
+ &self,
+ source: Option<&str>,
+ saf_kind: Option<&str>,
+ include_internal: bool,
+ limit: u32,
+ offset: u32,
) -> Result<(Vec<crate::types::OutputRecord>, i64), SqlxError> {
let mut sql = String::from(
"SELECT id, source, source_type, title, kind, original_path, output_path, \
@@ -386,7 +427,7 @@ impl DatabaseManager {
FROM outputs o WHERE 1=1",
);
let mut binds: Vec<String> = Vec::new();
- append_artifact_output_filters(&mut sql, &mut binds, source, saf_kind);
+ append_artifact_output_filters(&mut sql, &mut binds, source, saf_kind, include_internal);
sql.push_str(&format!(
" ORDER BY updated_at DESC LIMIT ?{} OFFSET ?{}",
binds.len() + 1,
@@ -405,7 +446,13 @@ impl DatabaseManager {
let mut count_sql = String::from("SELECT COUNT(*) FROM outputs o WHERE 1=1");
let mut count_binds: Vec<String> = Vec::new();
- append_artifact_output_filters(&mut count_sql, &mut count_binds, source, saf_kind);
+ append_artifact_output_filters(
+ &mut count_sql,
+ &mut count_binds,
+ source,
+ saf_kind,
+ include_internal,
+ );
let mut count_query = sqlx::query_scalar::<_, i64>(sqlx::AssertSqlSafe(count_sql));
for b in &count_binds {
count_query = count_query.bind(b);
@@ -416,13 +463,27 @@ impl DatabaseManager {
}
pub async fn list_output_sources_for_artifacts(&self) -> Result<Vec<String>, SqlxError> {
- let rows: Vec<(String,)> = sqlx::query_as(
+ // Preserve the historical unfiltered source inventory for direct DB
+ // callers. The Artifacts route excludes internal sources by default.
+ self.list_output_sources_for_artifacts_with_internal(true)
+ .await
+ }
+
+ pub async fn list_output_sources_for_artifacts_with_internal(
+ &self,
+ include_internal: bool,
+ ) -> Result<Vec<String>, SqlxError> {
+ let mut sql = String::from(
"SELECT DISTINCT CASE WHEN source_type = 'chat' THEN 'chat' ELSE source END AS display_source \
- FROM outputs \
- ORDER BY display_source ASC",
- )
- .fetch_all(&self.pool)
- .await?;
+ FROM outputs o WHERE 1=1",
+ );
+ if !include_internal {
+ sql.push_str(" AND COALESCE(o.saf_kind, '') != 'structured-output'");
+ }
+ sql.push_str(" ORDER BY display_source ASC");
+ let rows: Vec<(String,)> = sqlx::query_as(sqlx::AssertSqlSafe(sql))
+ .fetch_all(&self.pool)
+ .await?;
Ok(rows.into_iter().map(|r| r.0).collect())
}
@@ -451,7 +512,11 @@ fn append_artifact_output_filters(
binds: &mut Vec<String>,
source: Option<&str>,
saf_kind: Option<&str>,
+ include_internal: bool,
) {
+ if !include_internal {
+ sql.push_str(" AND COALESCE(o.saf_kind, '') != 'structured-output'");
+ }
if let Some(src) = source.filter(|s| !s.is_empty()) {
binds.push(src.to_string());
let idx = binds.len();
diff --git a/crates/screenpipe-db/tests/output_search_test.rs b/crates/screenpipe-db/tests/output_search_test.rs
index 8057a10bd..544324264 100644
--- a/crates/screenpipe-db/tests/output_search_test.rs
+++ b/crates/screenpipe-db/tests/output_search_test.rs
@@ -463,3 +463,116 @@ async fn sentinel_search_document_marks_non_searchable_output_processed() {
.expect("missing after");
assert!(missing_after.is_empty());
}
+
+#[tokio::test]
+async fn artifact_visibility_hides_structured_output_state_unless_requested() {
+ let db = setup_test_db().await;
+ let token = "SHARED_INTERNAL_OUTPUT_TOKEN";
+
+ let visible_id = db
+ .insert_output(
+ "report-pipe",
+ "pipe",
+ "Weekly Report",
+ "saf",
+ None,
+ "/tmp/screenpipe/outputs/pipe/report/weekly.saf.json",
+ 128,
+ Some("visible report"),
+ None,
+ Some("sop"),
+ Some("weekly-report"),
+ Some(1),
+ )
+ .await
+ .expect("insert visible output");
+ let internal_id = db
+ .insert_output(
+ "time-pipe",
+ "pipe",
+ "Time breakdown block",
+ "saf",
+ None,
+ "/tmp/screenpipe/outputs/pipe/time/structured-output.saf.json",
+ 128,
+ Some("internal Live View state"),
+ None,
+ Some("structured-output"),
+ Some("structured-output-time"),
+ Some(1),
+ )
+ .await
+ .expect("insert internal output");
+
+ for (id, title, source, body, hash) in [
+ (
+ visible_id,
+ "Weekly Report",
+ "report-pipe",
+ format!("visible {token}"),
+ "visible-hash",
+ ),
+ (
+ internal_id,
+ "Time breakdown block",
+ "time-pipe",
+ format!("internal {token}"),
+ "internal-hash",
+ ),
+ ] {
+ db.upsert_output_search_document(
+ id,
+ title,
+ &body,
+ source,
+ "pipe",
+ "saf",
+ hash,
+ body.len() as i64,
+ )
+ .await
+ .expect("index output");
+ }
+
+ let (visible, visible_total) = db
+ .list_outputs_for_artifacts_with_internal(None, None, false, 20, 0)
+ .await
+ .expect("list visible artifacts");
+ assert_eq!(visible_total, 1);
+ assert_eq!(visible.len(), 1);
+ assert_eq!(visible[0].id, visible_id);
+
+ let (all, all_total) = db
+ .list_outputs_for_artifacts_with_internal(None, None, true, 20, 0)
+ .await
+ .expect("list artifacts including internal state");
+ assert_eq!(all_total, 2);
+ assert_eq!(all.len(), 2);
+
+ let (search_visible, search_total) = db
+ .search_outputs_for_artifacts(token, None, None, false, 20, 0)
+ .await
+ .expect("search visible artifacts");
+ assert_eq!(search_total, 1);
+ assert_eq!(search_visible[0].id, visible_id);
+
+ let (search_all, search_all_total) = db
+ .search_outputs_for_artifacts(token, None, None, true, 20, 0)
+ .await
+ .expect("search artifacts including internal state");
+ assert_eq!(search_all_total, 2);
+ assert_eq!(search_all.len(), 2);
+
+ assert_eq!(
+ db.list_output_sources_for_artifacts_with_internal(false)
+ .await
+ .expect("list visible sources"),
+ vec!["report-pipe"]
+ );
+ assert_eq!(
+ db.list_output_sources_for_artifacts_with_internal(true)
+ .await
+ .expect("list all sources"),
+ vec!["report-pipe", "time-pipe"]
+ );
+}
diff --git a/crates/screenpipe-engine/src/routes/artifacts.rs b/crates/screenpipe-engine/src/routes/artifacts.rs
index 5e2fa1b43..b0a0da027 100644
--- a/crates/screenpipe-engine/src/routes/artifacts.rs
+++ b/crates/screenpipe-engine/src/routes/artifacts.rs
@@ -754,6 +754,10 @@ pub(crate) struct ListArtifactsQuery {
/// declarations (newest by mtime).
#[serde(default = "default_per_pipe_limit")]
pub per_pipe_limit: u32,
+ /// Include renderer-facing structured output records. Hidden by default so
+ /// internal Live View JSON does not appear in the user-facing Artifacts UI.
+ #[serde(default)]
+ pub include_internal: bool,
}
#[derive(OaSchema, Serialize)]
@@ -834,7 +838,14 @@ pub(crate) async fn list_artifacts_handler(
let (rows, registered_total) = if let Some(q) = q_filter {
state
.db
- .search_outputs(q, source_filter, saf_kind_filter, registered_fetch_limit, 0)
+ .search_outputs_for_artifacts(
+ q,
+ source_filter,
+ saf_kind_filter,
+ params.include_internal,
+ registered_fetch_limit,
+ 0,
+ )
.await
.map_err(|e| {
(
@@ -845,7 +856,13 @@ pub(crate) async fn list_artifacts_handler(
} else {
state
.db
- .list_outputs_for_artifacts(source_filter, saf_kind_filter, registered_fetch_limit, 0)
+ .list_outputs_for_artifacts_with_internal(
+ source_filter,
+ saf_kind_filter,
+ params.include_internal,
+ registered_fetch_limit,
+ 0,
+ )
.await
.map_err(|e| {
(
@@ -954,7 +971,7 @@ pub(crate) async fn list_artifacts_handler(
};
let mut source_set: std::collections::HashSet<String> = state
.db
- .list_output_sources_for_artifacts()
+ .list_output_sources_for_artifacts_with_internal(params.include_internal)
.await
.map_err(|e| {
(