Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix enterprise sync timestamp-boundary data loss (#5854)

Ezra Ellette900df1bddbb1
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/enterprise/sync.rs b/apps/screenpipe-app-tauri/src-tauri/src/enterprise/sync.rs
index c11ffece9..5be4b3a67 100644
--- a/apps/screenpipe-app-tauri/src-tauri/src/enterprise/sync.rs
+++ b/apps/screenpipe-app-tauri/src-tauri/src/enterprise/sync.rs
@@ -233,6 +233,24 @@ pub struct Cursor {
/// backfill its own bounded window.
#[serde(default)]
pub last_parsed_ts: Option<String>,
+ /// Rows already acknowledged at each stream's current timestamp. Local
+ /// APIs use inclusive timestamp filters, so this durable offset is the
+ /// second half of the pagination key and prevents a 500-row timestamp tie
+ /// from either repeating forever or being skipped after restart.
+ #[serde(default)]
+ boundary: CursorBoundary,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, Default)]
+struct CursorBoundary {
+ frames: u32,
+ audio: u32,
+ ui: u32,
+ memories: u32,
+ parsed: u32,
+ /// Feedback supports a true `(updated_at, id)` keyset cursor because its
+ /// IDs are stable strings and its local route merges DB and legacy rows.
+ feedback_id: Option<String>,
}
impl Cursor {
@@ -282,24 +300,26 @@ impl Cursor {
/// desktop crate against `LocalApiContext`.
#[async_trait::async_trait]
pub trait LocalApiClient: Send + Sync {
- /// Fetch frames + their text since `since_ts` (exclusive), ordered by
- /// timestamp ascending, capped at `limit`.
+ /// Fetch frames + their text at or after `since_ts`, ordered by timestamp
+ /// ascending, skipping `boundary_offset` rows at the boundary timestamp.
async fn fetch_frames_since(
&self,
since_ts: Option<&str>,
+ boundary_offset: u32,
limit: u32,
) -> Result<Vec<FrameRow>, EnterpriseSyncError>;
- /// Fetch audio transcriptions since `since_ts` (exclusive), ordered ASC,
- /// capped at `limit`.
+ /// Fetch audio transcriptions at or after `since_ts`, ordered ascending,
+ /// skipping `boundary_offset` rows at the boundary timestamp.
async fn fetch_audio_since(
&self,
since_ts: Option<&str>,
+ boundary_offset: u32,
limit: u32,
) -> Result<Vec<AudioRow>, EnterpriseSyncError>;
- /// Fetch UI events (clicks, keystrokes, clipboard) since `since_ts`
- /// (exclusive), ordered ASC, capped at `limit`. UI events give the
+ /// Fetch UI events (clicks, keystrokes, clipboard) at or after `since_ts`,
+ /// ordered ascending and skipping `boundary_offset` boundary rows. UI events give the
/// extracted workflows their *verbs* — without them an SOP can only
/// say "the user opened Slack", not "the user clicked Send on the
/// upgrade-confirmed message". Default empty implementation lets
@@ -307,6 +327,7 @@ pub trait LocalApiClient: Send + Sync {
async fn fetch_ui_events_since(
&self,
_since_ts: Option<&str>,
+ _boundary_offset: u32,
_limit: u32,
) -> Result<Vec<UiEventRow>, EnterpriseSyncError> {
Ok(Vec::new())
@@ -330,6 +351,7 @@ pub trait LocalApiClient: Send + Sync {
async fn fetch_memories_since(
&self,
_since_ts: Option<&str>,
+ _boundary_offset: u32,
_limit: u32,
) -> Result<Vec<MemoryRow>, EnterpriseSyncError> {
Ok(Vec::new())
@@ -341,6 +363,7 @@ pub trait LocalApiClient: Send + Sync {
async fn fetch_feedback_since(
&self,
_since_ts: Option<&str>,
+ _after_id: Option<&str>,
_limit: u32,
) -> Result<Vec<FeedbackRow>, EnterpriseSyncError> {
Ok(Vec::new())
@@ -351,6 +374,7 @@ pub trait LocalApiClient: Send + Sync {
async fn fetch_parsed_since(
&self,
_since_ts: Option<&str>,
+ _boundary_offset: u32,
_limit: u32,
) -> Result<Vec<ParsedRow>, EnterpriseSyncError> {
Ok(Vec::new())
@@ -524,6 +548,16 @@ pub async fn run_one_sync(
cursor: &mut Cursor,
local: &dyn LocalApiClient,
http: &reqwest::Client,
+) -> Result<SyncTickReport, EnterpriseSyncError> {
+ run_one_sync_inner(cfg, cursor, local, http, true).await
+}
+
+async fn run_one_sync_inner(
+ cfg: &EnterpriseSyncConfig,
+ cursor: &mut Cursor,
+ local: &dyn LocalApiClient,
+ http: &reqwest::Client,
+ include_snapshot: bool,
) -> Result<SyncTickReport, EnterpriseSyncError> {
if let EnterpriseUploadMode::Blocked(reason) = &cfg.upload_mode {
return Err(EnterpriseSyncError::Configuration(reason.clone()));
@@ -535,26 +569,32 @@ pub async fn run_one_sync(
if cursor.last_frame_ts.is_none() {
let cutoff = chrono::Utc::now() - chrono::Duration::from_std(SAFE_BACKFILL).unwrap();
cursor.last_frame_ts = Some(cutoff.to_rfc3339());
+ cursor.boundary.frames = 0;
}
if cursor.last_audio_ts.is_none() {
let cutoff = chrono::Utc::now() - chrono::Duration::from_std(SAFE_BACKFILL).unwrap();
cursor.last_audio_ts = Some(cutoff.to_rfc3339());
+ cursor.boundary.audio = 0;
}
if cursor.last_ui_ts.is_none() {
let cutoff = chrono::Utc::now() - chrono::Duration::from_std(SAFE_BACKFILL).unwrap();
cursor.last_ui_ts = Some(cutoff.to_rfc3339());
+ cursor.boundary.ui = 0;
}
if cursor.last_memory_ts.is_none() {
let cutoff = chrono::Utc::now() - chrono::Duration::from_std(SAFE_BACKFILL).unwrap();
cursor.last_memory_ts = Some(cutoff.to_rfc3339());
+ cursor.boundary.memories = 0;
}
if cursor.last_feedback_ts.is_none() {
let cutoff = chrono::Utc::now() - chrono::Duration::from_std(SAFE_BACKFILL).unwrap();
cursor.last_feedback_ts = Some(cutoff.to_rfc3339());
+ cursor.boundary.feedback_id = None;
}
if cursor.last_parsed_ts.is_none() {
let cutoff = chrono::Utc::now() - chrono::Duration::from_std(SAFE_BACKFILL).unwrap();
cursor.last_parsed_ts = Some(cutoff.to_rfc3339());
+ cursor.boundary.parsed = 0;
}
// Per-stream sync policy is fetched fresh on every tick — the admin can
@@ -567,14 +607,22 @@ pub async fn run_one_sync(
let frames = if streams.frames {
local
- .fetch_frames_since(cursor.last_frame_ts.as_deref(), PAGE_LIMIT)
+ .fetch_frames_since(
+ cursor.last_frame_ts.as_deref(),
+ cursor.boundary.frames,
+ PAGE_LIMIT,
+ )
.await?
} else {
Vec::new()
};
let audio = if streams.audio {
local
- .fetch_audio_since(cursor.last_audio_ts.as_deref(), PAGE_LIMIT)
+ .fetch_audio_since(
+ cursor.last_audio_ts.as_deref(),
+ cursor.boundary.audio,
+ PAGE_LIMIT,
+ )
.await?
} else {
Vec::new()
@@ -584,7 +632,7 @@ pub async fn run_one_sync(
// The frame + audio paths are the load-bearing ones.
let ui = if streams.ui_events {
match local
- .fetch_ui_events_since(cursor.last_ui_ts.as_deref(), PAGE_LIMIT)
+ .fetch_ui_events_since(cursor.last_ui_ts.as_deref(), cursor.boundary.ui, PAGE_LIMIT)
.await
{
Ok(rows) => rows,
@@ -598,7 +646,7 @@ pub async fn run_one_sync(
};
// One snapshot per tick. Best-effort — failure to encode/fetch
// shouldn't block the rest of the batch.
- let snapshots: Vec<SnapshotRow> = if streams.snapshots {
+ let snapshots: Vec<SnapshotRow> = if streams.snapshots && include_snapshot {
match local.fetch_latest_snapshot().await {
Ok(Some(s)) => vec![s],
Ok(None) => Vec::new(),
@@ -615,7 +663,11 @@ pub async fn run_one_sync(
// the frame+audio path. The default trait impl returns empty.
let memories = if streams.memories {
match local
- .fetch_memories_since(cursor.last_memory_ts.as_deref(), PAGE_LIMIT)
+ .fetch_memories_since(
+ cursor.last_memory_ts.as_deref(),
+ cursor.boundary.memories,
+ PAGE_LIMIT,
+ )
.await
{
Ok(rows) => rows,
@@ -629,7 +681,11 @@ pub async fn run_one_sync(
};
let mut feedback = if streams.feedback != crate::enterprise_policy::FeedbackSyncMode::Off {
match local
- .fetch_feedback_since(cursor.last_feedback_ts.as_deref(), PAGE_LIMIT)
+ .fetch_feedback_since(
+ cursor.last_feedback_ts.as_deref(),
+ cursor.boundary.feedback_id.as_deref(),
+ PAGE_LIMIT,
+ )
.await
{
Ok(rows) => rows,
@@ -656,7 +712,11 @@ pub async fn run_one_sync(
// not expose content_type=parsed.
let parsed = if streams.parsed {
match local
- .fetch_parsed_since(cursor.last_parsed_ts.as_deref(), PAGE_LIMIT)
+ .fetch_parsed_since(
+ cursor.last_parsed_ts.as_deref(),
+ cursor.boundary.parsed,
+ PAGE_LIMIT,
+ )
.await
{
Ok(rows) => rows,
@@ -699,24 +759,40 @@ pub async fn run_one_sync(
let bytes = body.len();
let mut next_cursor = cursor.clone();
- if let Some(latest) = frames.last() {
- next_cursor.last_frame_ts = Some(latest.timestamp.clone());
- }
- if let Some(latest) = audio.last() {
- next_cursor.last_audio_ts = Some(latest.timestamp.clone());
- }
- if let Some(latest) = ui.last() {
- next_cursor.last_ui_ts = Some(latest.timestamp.clone());
- }
- if let Some(latest) = memories.last() {
- next_cursor.last_memory_ts = Some(latest.created_at.clone());
- }
+ advance_timestamp_boundary(
+ &mut next_cursor.last_frame_ts,
+ &mut next_cursor.boundary.frames,
+ &frames,
+ |row| &row.timestamp,
+ );
+ advance_timestamp_boundary(
+ &mut next_cursor.last_audio_ts,
+ &mut next_cursor.boundary.audio,
+ &audio,
+ |row| &row.timestamp,
+ );
+ advance_timestamp_boundary(
+ &mut next_cursor.last_ui_ts,
+ &mut next_cursor.boundary.ui,
+ &ui,
+ |row| &row.timestamp,
+ );
+ advance_timestamp_boundary(
+ &mut next_cursor.last_memory_ts,
+ &mut next_cursor.boundary.memories,
+ &memories,
+ |row| &row.created_at,
+ );
if let Some(latest) = feedback.last() {
next_cursor.last_feedback_ts = Some(latest.updated_at.clone());
+ next_cursor.boundary.feedback_id = Some(latest.feedback_id.clone());
}
- if let Some(latest) = parsed.last() {
- next_cursor.last_parsed_ts = Some(latest.timestamp.clone());
- }
+ advance_timestamp_boundary(
+ &mut next_cursor.last_parsed_ts,
+ &mut next_cursor.boundary.parsed,
+ &parsed,
+ |row| &row.timestamp,
+ );
match &cfg.upload_mode {
EnterpriseUploadMode::HostedIngest => {
@@ -785,6 +861,30 @@ pub async fn run_one_sync(
})
}
+fn advance_timestamp_boundary<T>(
+ cursor_ts: &mut Option<String>,
+ boundary_offset: &mut u32,
+ rows: &[T],
+ timestamp: impl Fn(&T) -> &str,
+) {
+ let Some(last) = rows.last() else {
+ return;
+ };
+ let latest_ts = timestamp(last);
+ let rows_at_latest = rows
+ .iter()
+ .rev()
+ .take_while(|row| timestamp(row) == latest_ts)
+ .count() as u32;
+
+ if cursor_ts.as_deref() == Some(latest_ts) {
+ *boundary_offset = boundary_offset.saturating_add(rows_at_latest);
+ } else {
+ *cursor_ts = Some(latest_ts.to_string());
+ *boundary_offset = rows_at_latest;
+ }
+}
+
#[derive(Debug, Clone, Default, PartialEq)]
pub struct SyncTickReport {
pub frames: usize,
@@ -797,6 +897,69 @@ pub struct SyncTickReport {
pub bytes: usize,
}
+impl SyncTickReport {
+ /// A full cursor-backed stream page means the local API may have more rows
+ /// behind it. Point-in-time snapshots are deliberately excluded.
+ pub fn may_have_more(&self) -> bool {
+ let limit = PAGE_LIMIT as usize;
+ self.frames >= limit
+ || self.parsed >= limit
+ || self.audio >= limit
+ || self.ui >= limit
+ || self.memories >= limit
+ || self.feedback >= limit
+ }
+
+ fn add_assign(&mut self, other: &Self) {
+ self.frames += other.frames;
+ self.parsed += other.parsed;
+ self.audio += other.audio;
+ self.ui += other.ui;
+ self.snapshots += other.snapshots;
+ self.memories += other.memories;
+ self.feedback += other.feedback;
+ self.bytes += other.bytes;
+ }
+}
+
+/// Aggregate result of one scheduled sync plus any immediate catch-up pages.
+#[derive(Debug, Clone, Default, PartialEq)]
+pub struct SyncBurstReport {
+ pub total: SyncTickReport,
+ pub pages: usize,
+}
+
+/// Drain acknowledged pages until every cursor-backed stream returns a
+/// partial page.
+///
+/// `run_one_sync` is intentionally the only mechanism that fetches, uploads,
+/// and advances cursors. Consequently every page is durably checkpointed only
+/// after its own remote acknowledgement; a later-page failure or process exit
+/// resumes from the last successful page without skipping the rest.
+pub async fn run_sync_burst(
+ cfg: &EnterpriseSyncConfig,
+ cursor: &mut Cursor,
+ local: &dyn LocalApiClient,
+ http: &reqwest::Client,
+) -> Result<SyncBurstReport, EnterpriseSyncError> {
+ let mut burst = SyncBurstReport::default();
+ let mut include_snapshot = true;
+
+ loop {
+ let page = run_one_sync_inner(cfg, cursor, local, http, include_snapshot).await?;
+ include_snapshot = false;
+ let more_pending = page.may_have_more();
+ burst.total.add_assign(&page);
+ burst.pages += 1;
+
+ if !more_pending {
+ break;
+ }
+ }
+
+ Ok(burst)
+}
+
// ─── On-demand frame fulfillment (P3) ───────────────────────────────────────
//
// The org's cloud pipes cite frame_ids they want as images (SOP step
@@ -1470,7 +1633,7 @@ pub async fn run(
// hosted-to-customer-storage policy change without requiring an app
// restart, and preserves the last safe mode on lookup failure.
let result = match cfg.resolve_upload_mode().await {
- Ok(()) => run_one_sync(&cfg, &mut cursor, local.as_ref(), &http).await,
+ Ok(()) => run_sync_burst(&cfg, &mut cursor, local.as_ref(), &http).await,
Err(error) => Err(error),
};
@@ -1498,22 +1661,25 @@ pub async fn run(
match result {
Ok(report) => {
- if report.frames > 0
- || report.audio > 0
- || report.ui > 0
- || report.snapshots > 0
- || report.memories > 0
- || report.feedback > 0
+ let total = &report.total;
+ if total.frames > 0
+ || total.audio > 0
+ || total.ui > 0
+ || total.snapshots > 0
+ || total.memories > 0
+ || total.feedback > 0
{
info!(
- "enterprise sync: pushed {} frames, {} audio, {} ui, {} snapshots, {} memories, {} feedback ({} bytes)",
- report.frames,
- report.audio,
- report.ui,
- report.snapshots,
- report.memories,
- report.feedback,
- report.bytes
+ "enterprise sync: pushed {} frames, {} parsed, {} audio, {} ui, {} snapshots, {} memories, {} feedback across {} page(s) ({} bytes)",
+ total.frames,
+ total.parsed,
+ total.audio,
+ total.ui,
+ total.snapshots,
+ total.memories,
+ total.feedback,
+ report.pages,
+ total.bytes
);
}
backoff = BACKOFF_INITIAL;
@@ -1827,6 +1993,23 @@ mod tests {
}
}
+ fn frame_page(first_id: i64, count: usize) -> Vec<FrameRow> {
+ let base = chrono::DateTime::parse_from_rfc3339("2026-07-27T23:26:23Z")
+ .unwrap()
+ .with_timezone(&chrono::Utc);
+ (0..count)
+ .map(|offset| {
+ let id = first_id + offset as i64;
+ frame(
+ id,
+ &(base + chrono::Duration::seconds(id)).to_rfc3339(),
+ "Arc",
+ "recorded locally during upload outage",
+ )
+ })
+ .collect()
+ }
+
fn audio(id: i64, ts: &str, text: &str) -> AudioRow {
AudioRow {
transcription_id: id,
@@ -2157,12 +2340,37 @@ mod tests {
last_memory_ts: Some("2026-05-07T09:15:00Z".to_string()),
last_feedback_ts: None,
last_parsed_ts: Some("2026-05-07T09:15:00Z".to_string()),
+ boundary: CursorBoundary {
+ frames: 500,
+ feedback_id: Some("feedback-0500".to_string()),
+ ..CursorBoundary::default()
+ },
};
c.save(&p).unwrap();
let loaded = Cursor::load(&p);
assert_eq!(loaded.last_frame_ts, c.last_frame_ts);
assert_eq!(loaded.last_audio_ts, c.last_audio_ts);
assert_eq!(loaded.last_ui_ts, c.last_ui_ts);
+ assert_eq!(loaded.boundary.frames, 500);
+ assert_eq!(
+ loaded.boundary.feedback_id.as_deref(),
+ Some("feedback-0500")
+ );
+ }
+
+ #[test]
+ fn cursor_from_older_app_defaults_boundary_progress() {
+ let cursor: Cursor = serde_json::from_str(
+ r#"{"last_frame_ts":"2026-05-07T10:00:00Z","last_audio_ts":null}"#,
+ )
+ .unwrap();
+
+ assert_eq!(
+ cursor.last_frame_ts.as_deref(),
+ Some("2026-05-07T10:00:00Z")
+ );
+ assert_eq!(cursor.boundary.frames, 0);
+ assert!(cursor.boundary.feedback_id.is_none());
}
#[test]
@@ -2176,6 +2384,7 @@ mod tests {
last_memory_ts: None,
last_feedback_ts: None,
last_parsed_ts: None,
+ boundary: CursorBoundary::default(),
}
.save(&p)
.unwrap();
@@ -2365,11 +2574,115 @@ mod tests {
}
}
+ /// Models the local `/search` and `/memories` pagination contract: the
+ /// timestamp boundary is inclusive and the durable boundary offset skips
+ /// rows already acknowledged at that exact timestamp.
+ struct InclusiveTimestampLocal {
+ frames: Vec<FrameRow>,
+ }
+
+ #[async_trait::async_trait]
+ impl LocalApiClient for InclusiveTimestampLocal {
+ async fn fetch_frames_since(
+ &self,
+ since_ts: Option<&str>,
+ boundary_offset: u32,
+ limit: u32,
+ ) -> Result<Vec<FrameRow>, EnterpriseSyncError> {
+ Ok(self
+ .frames
+ .iter()
+ .filter(|row| since_ts.is_none_or(|since| row.timestamp.as_str() >= since))
+ .skip(boundary_offset as usize)
+ .take(limit as usize)
+ .cloned()
+ .collect())
+ }
+
+ async fn fetch_audio_since(
+ &self,
+ _since_ts: Option<&str>,
+ _boundary_offset: u32,
+ _limit: u32,
+ ) -> Result<Vec<AudioRow>, EnterpriseSyncError> {
+ Ok(Vec::new())
+ }
+ }
+
+ /// Models the corrected feedback keyset contract. The regression test
+ /// below proves why the old exclusive `updated_at > cursor` filter was
+ /// unsafe when a full page shared one timestamp.
+ struct ExclusiveTimestampLocal {
+ feedback: Vec<FeedbackRow>,
+ }
+
+ #[async_trait::async_trait]
+ impl LocalApiClient for ExclusiveTimestampLocal {
+ async fn fetch_frames_since(
+ &self,
+ _since_ts: Option<&str>,
+ _boundary_offset: u32,
+ _limit: u32,
+ ) -> Result<Vec<FrameRow>, EnterpriseSyncError> {
+ Ok(Vec::new())
+ }
+
+ async fn fetch_audio_since(
+ &self,
+ _since_ts: Option<&str>,
+ _boundary_offset: u32,
+ _limit: u32,
+ ) -> Result<Vec<AudioRow>, EnterpriseSyncError> {
+ Ok(Vec::new())
+ }
+
+ async fn fetch_feedback_since(
+ &self,
+ since_ts: Option<&str>,
+ after_id: Option<&str>,
+ limit: u32,
+ ) -> Result<Vec<FeedbackRow>, EnterpriseSyncError> {
+ Ok(self
+ .feedback
+ .iter()
+ .filter(|row| {
+ since_ts.is_none_or(|since| {
+ row.updated_at.as_str() > since
+ || (row.updated_at.as_str() == since
+ && after_id.is_none_or(|id| row.feedback_id.as_str() > id))
+ })
+ })
+ .take(limit as usize)
+ .cloned()
+ .collect())
+ }
+ }
+
+ /// Restore the process-global stream policy even when an intentionally
+ /// red regression test panics on its assertion.
+ struct RestoreDefaultSyncStreams;
+
+ impl Drop for RestoreDefaultSyncStreams {
+ fn drop(&mut self) {
+ crate::enterprise_policy::set_sync_streams(
+ true,
+ false,
+ true,
+ true,
+ true,
+ true,
+ "off".to_string(),
+ "off".to_string(),
+ );
+ }
+ }
+
#[async_trait::async_trait]
impl LocalApiClient for MockLocal {
async fn fetch_frames_since(
&self,
since_ts: Option<&str>,
+ _boundary_offset: u32,
_limit: u32,
) -> Result<Vec<FrameRow>, EnterpriseSyncError> {
*self.last_frames_since.lock().unwrap() = since_ts.map(|s| s.to_string());
@@ -2384,6 +2697,7 @@ mod tests {
async fn fetch_audio_since(
&self,
since_ts: Option<&str>,
+ _boundary_offset: u32,
_limit: u32,
) -> Result<Vec<AudioRow>, EnterpriseSyncError> {
*self.last_audio_since.lock().unwrap() = since_ts.map(|s| s.to_string());
@@ -2398,6 +2712,7 @@ mod tests {
async fn fetch_memories_since(
&self,
since_ts: Option<&str>,
+ _boundary_offset: u32,
_limit: u32,
) -> Result<Vec<MemoryRow>, EnterpriseSyncError> {
*self.last_memories_since.lock().unwrap() = since_ts.map(|s| s.to_string());
@@ -2412,6 +2727,7 @@ mod tests {
async fn fetch_feedback_since(
&self,
since_ts: Option<&str>,
+ _after_id: Option<&str>,
_limit: u32,
) -> Result<Vec<FeedbackRow>, EnterpriseSyncError> {
*self.last_feedback_since.lock().unwrap() = since_ts.map(str::to_string);
@@ -2426,6 +2742,7 @@ mod tests {
async fn fetch_parsed_since(
&self,
since_ts: Option<&str>,
+ _boundary_offset: u32,
_limit: u32,
) -> Result<Vec<ParsedRow>, EnterpriseSyncError> {
*self.last_parsed_since.lock().unwrap() = since_ts.map(|s| s.to_string());
@@ -2527,6 +2844,7 @@ mod tests {
last_memory_ts: Some("2026-05-07T10:00:00Z".to_string()),
last_feedback_ts: None,
last_parsed_ts: Some("2026-05-07T10:00:00Z".to_string()),
+ boundary: CursorBoundary::default(),
};
let local = MockLocal::new(vec![vec![]], vec![vec![]]);
let http = reqwest::Client::new();
@@ -2582,6 +2900,7 @@ mod tests {
last_memory_ts: None,
last_feedback_ts: None,
last_parsed_ts: None,
+ boundary: CursorBoundary::default(),
};
let local = MockLocal::new(
vec![vec![
@@ -2609,6 +2928,225 @@ mod tests {
assert_eq!(loaded.last_frame_ts, cursor.last_frame_ts);
}
+ #[tokio::test]
+ async fn old_cursor_drains_every_page_without_waiting_for_normal_ticks() {
+ let server = wiremock::MockServer::start().await;
+ wiremock::Mock::given(wiremock::matchers::method("POST"))
+ .and(wiremock::matchers::path("/ingest"))
+ .respond_with(wiremock::ResponseTemplate::new(200))
+ .expect(3)
+ .mount(&server)
+ .await;
+
+ let first = frame_page(1, PAGE_LIMIT as usize);
+ let second = frame_page(501, PAGE_LIMIT as usize);
+ let final_page = frame_page(1001, 37);
+ let expected_last = final_page.last().unwrap().timestamp.clone();
+
+ let dir = TempDir::new().unwrap();
+ let cfg = test_cfg(&dir, format!("{}/ingest", server.uri()));
+ let mut cursor = Cursor {
+ last_frame_ts: Some("2026-07-27T23:26:23Z".to_string()),
+ last_audio_ts: Some("2026-07-27T23:26:23Z".to_string()),
+ last_ui_ts: Some("2026-07-27T23:26:23Z".to_string()),
+ last_memory_ts: Some("2026-07-27T23:26:23Z".to_string()),
+ last_feedback_ts: Some("2026-07-27T23:26:23Z".to_string()),
+ last_parsed_ts: Some("2026-07-27T23:26:23Z".to_string()),
+ boundary: CursorBoundary::default(),
+ };
+ // MockLocal pops from the end: yield first → second → final.
+ let local = MockLocal::new(vec![final_page, second, first], vec![]);
+
+ let report = run_sync_burst(&cfg, &mut cursor, &local, &reqwest::Client::new())
+ .await
+ .unwrap();
+
+ assert_eq!(report.pages, 3);
+ assert_eq!(report.total.frames, 1_037);
+ assert_eq!(
+ cursor.last_frame_ts.as_deref(),
+ Some(expected_last.as_str())
+ );
+ assert_eq!(
+ Cursor::load(&cfg.cursor_path).last_frame_ts.as_deref(),
+ Some(expected_last.as_str()),
+ "the final acknowledged page is durable"
+ );
+ }
+
+ #[tokio::test]
+ async fn timestamp_gte_cursor_must_progress_past_a_full_boundary_tie() {
+ let server = wiremock::MockServer::start().await;
+ wiremock::Mock::given(wiremock::matchers::method("POST"))
+ .and(wiremock::matchers::path("/ingest"))
+ .respond_with(wiremock::ResponseTemplate::new(200))
+ .mount(&server)
+ .await;
+
+ let tied_at = "2026-08-01T00:00:00Z";
+ let local = InclusiveTimestampLocal {
+ frames: (1..=PAGE_LIMIT as i64 + 1)
+ .map(|id| frame(id, tied_at, "Arc", "same timestamp boundary"))
+ .collect(),
+ };
+ let dir = TempDir::new().unwrap();
+ let cfg = test_cfg(&dir, format!("{}/ingest", server.uri()));
+ let mut cursor = Cursor {
+ last_frame_ts: Some("2026-07-31T23:59:59Z".to_string()),
+ last_audio_ts: Some("2026-07-31T23:59:59Z".to_string()),
+ last_ui_ts: Some("2026-07-31T23:59:59Z".to_string()),
+ last_memory_ts: Some("2026-07-31T23:59:59Z".to_string()),
+ last_feedback_ts: Some("2026-07-31T23:59:59Z".to_string()),
+ last_parsed_ts: Some("2026-07-31T23:59:59Z".to_string()),
+ boundary: CursorBoundary::default(),
+ };
+
+ let first = run_one_sync(&cfg, &mut cursor, &local, &reqwest::Client::new())
+ .await
+ .unwrap();
+ cursor = Cursor::load(&cfg.cursor_path);
+ let second = run_one_sync(&cfg, &mut cursor, &local, &reqwest::Client::new())
+ .await
+ .unwrap();
+
+ assert_eq!(first.frames, PAGE_LIMIT as usize);
+ assert_eq!(
+ second.frames, 1,
+ "an inclusive timestamp-only cursor repeats the first 500 tied rows instead of reaching row 501"
+ );
+ }
+
+ #[tokio::test]
+ async fn timestamp_gt_cursor_must_not_skip_the_rest_of_a_full_boundary_tie() {
+ let _policy_guard = crate::enterprise_policy::sync_streams_test_lock();
+ let _restore_policy = RestoreDefaultSyncStreams;
+ crate::enterprise_policy::set_sync_streams(
+ true,
+ false,
+ true,
+ true,
+ true,
+ true,
+ "full".to_string(),
+ "off".to_string(),
+ );
+
+ let server = wiremock::MockServer::start().await;
+ wiremock::Mock::given(wiremock::matchers::method("POST"))
+ .and(wiremock::matchers::path("/ingest"))
+ .respond_with(wiremock::ResponseTemplate::new(200))
+ .mount(&server)
+ .await;
+
+ let tied_at = "2026-08-01T00:00:00Z";
+ let local = ExclusiveTimestampLocal {
+ feedback: (1..=PAGE_LIMIT + 1)
+ .map(|id| feedback(&format!("feedback-{id:04}"), tied_at))
+ .collect(),
+ };
+ let dir = TempDir::new().unwrap();
+ let cfg = test_cfg(&dir, format!("{}/ingest", server.uri()));
+ let mut cursor = Cursor {
+ last_frame_ts: Some("2026-07-31T23:59:59Z".to_string()),
+ last_audio_ts: Some("2026-07-31T23:59:59Z".to_string()),
+ last_ui_ts: Some("2026-07-31T23:59:59Z".to_string()),
+ last_memory_ts: Some("2026-07-31T23:59:59Z".to_string()),
+ last_feedback_ts: Some("2026-07-31T23:59:59Z".to_string()),
+ last_parsed_ts: Some("2026-07-31T23:59:59Z".to_string()),
+ boundary: CursorBoundary::default(),
+ };
+
+ let first = run_one_sync(&cfg, &mut cursor, &local, &reqwest::Client::new())
+ .await
+ .unwrap();
+ cursor = Cursor::load(&cfg.cursor_path);
+ let second = run_one_sync(&cfg, &mut cursor, &local, &reqwest::Client::new())
+ .await
+ .unwrap();
+
+ assert_eq!(first.feedback, PAGE_LIMIT as usize);
+ assert_eq!(
+ second.feedback, 1,
+ "an exclusive timestamp-only cursor skips row 501 because it shares the acknowledged page's timestamp"
+ );
+ }
+
+ #[tokio::test]
+ async fn catch_up_failure_keeps_last_acknowledged_page_for_restart() {
+ use std::sync::atomic::{AtomicUsize, Ordering};
+
+ let calls = Arc::new(AtomicUsize::new(0));
+ let calls_for_responder = calls.clone();
+ let failing_server = wiremock::MockServer::start().await;
+ wiremock::Mock::given(wiremock::matchers::method("POST"))
+ .and(wiremock::matchers::path("/ingest"))
+ .respond_with(move |_request: &wiremock::Request| {
+ if calls_for_responder.fetch_add(1, Ordering::SeqCst) == 0 {
+ wiremock::ResponseTemplate::new(200)
+ } else {
+ wiremock::ResponseTemplate::new(503)
+ }
+ })
+ .expect(2)
+ .mount(&failing_server)
+ .await;
+
+ let first = frame_page(1, PAGE_LIMIT as usize);
+ let second = frame_page(501, 20);
+ let first_last = first.last().unwrap().timestamp.clone();
+ let second_last = second.last().unwrap().timestamp.clone();
+ let dir = TempDir::new().unwrap();
+ let cfg = test_cfg(&dir, format!("{}/ingest", failing_server.uri()));
+ let mut cursor = Cursor {
+ last_frame_ts: Some("2026-07-27T23:26:23Z".to_string()),
+ last_audio_ts: Some("2026-07-27T23:26:23Z".to_string()),
+ last_ui_ts: Some("2026-07-27T23:26:23Z".to_string()),
+ last_memory_ts: Some("2026-07-27T23:26:23Z".to_string()),
+ last_feedback_ts: Some("2026-07-27T23:26:23Z".to_string()),
+ last_parsed_ts: Some("2026-07-27T23:26:23Z".to_string()),
+ boundary: CursorBoundary::default(),
+ };
+ let local = MockLocal::new(vec![second.clone(), first], vec![]);
+
+ let error = run_sync_burst(&cfg, &mut cursor, &local, &reqwest::Client::new())
+ .await
+ .unwrap_err();
+ assert!(matches!(error, EnterpriseSyncError::IngestServerError(503)));
+ assert_eq!(cursor.last_frame_ts.as_deref(), Some(first_last.as_str()));
+ assert_eq!(
+ Cursor::load(&cfg.cursor_path).last_frame_ts.as_deref(),
+ Some(first_last.as_str())
+ );
+
+ // Simulate an app restart: reload the durable cursor and refetch the
+ // unacknowledged page from the local database.
+ let healthy_server = wiremock::MockServer::start().await;
+ wiremock::Mock::given(wiremock::matchers::method("POST"))
+ .and(wiremock::matchers::path("/ingest"))
+ .respond_with(wiremock::ResponseTemplate::new(200))
+ .expect(1)
+ .mount(&healthy_server)
+ .await;
+ let resumed_cfg = test_cfg(&dir, format!("{}/ingest", healthy_server.uri()));
+ let mut resumed_cursor = Cursor::load(&resumed_cfg.cursor_path);
+ let resumed_local = MockLocal::new(vec![second], vec![]);
+ let resumed = run_sync_burst(
+ &resumed_cfg,
+ &mut resumed_cursor,
+ &resumed_local,
+ &reqwest::Client::new(),
+ )
+ .await
+ .unwrap();
+
+ assert_eq!(resumed.pages, 1);
+ assert_eq!(resumed.total.frames, 20);
+ assert_eq!(
+ resumed_cursor.last_frame_ts.as_deref(),
+ Some(second_last.as_str())
+ );
+ }
+
#[tokio::test]
async fn hosted_ingest_sends_large_payload_across_multiple_requests() {
let server = wiremock::MockServer::start().await;
@@ -2628,6 +3166,7 @@ mod tests {
last_memory_ts: None,
last_feedback_ts: None,
last_parsed_ts: None,
+ boundary: CursorBoundary::default(),
};
let mut cursor = original_cursor.clone();
let large_text = "x".repeat(2 * 1024 * 1024);
@@ -2690,6 +3229,7 @@ mod tests {
last_memory_ts: None,
last_feedback_ts: None,
last_parsed_ts: None,
+ boundary: CursorBoundary::default(),
};
let before = cursor.clone();
let large_text = "x".repeat(2 * 1024 * 1024);
@@ -2734,6 +3274,7 @@ mod tests {
last_memory_ts: Some("2026-05-07T09:00:00Z".to_string()),
last_feedback_ts: None,
last_parsed_ts: Some("2026-05-07T09:00:00Z".to_string()),
+ boundary: CursorBoundary::default(),
};
let local = MockLocal::new(vec![vec![]], vec![vec![]]).with_memories(vec![vec![
memory(1, "2026-05-07T10:00:00Z", "first"),
@@ -2788,6 +3329,7 @@ mod tests {
last_memory_ts: Some("2026-05-07T09:00:00Z".to_string()),
last_feedback_ts: Some("2026-05-07T09:00:00Z".to_string()),
last_parsed_ts: None,
+ boundary: CursorBoundary::default(),
};
let local = MockLocal::new(vec![], vec![])
.with_feedback(vec![vec![feedback("feedback-1", "2026-05-07T10:00:00Z")]]);
@@ -2851,6 +3393,7 @@ mod tests {
last_memory_ts: Some("2026-05-07T09:00:00Z".to_string()),
last_feedback_ts: None,
last_parsed_ts: Some("2026-05-07T09:00:00Z".to_string()),
+ boundary: CursorBoundary::default(),
};
let local = MockLocal::new(vec![], vec![]).with_parsed(vec![vec![parsed(
42,
@@ -2952,6 +3495,7 @@ mod tests {
last_memory_ts: None,
last_feedback_ts: None,
last_parsed_ts: None,
+ boundary: CursorBoundary::default(),
};
let local = MockLocal::new(
vec![vec![frame(1, "2026-05-07T10:00:00Z", "Arc", "secret")]],
@@ -3025,6 +3569,7 @@ mod tests {
last_memory_ts: None,
last_feedback_ts: None,
last_parsed_ts: None,
+ boundary: CursorBoundary::default(),
};
let local = MockLocal::new(
vec![vec![frame(
@@ -3089,6 +3634,7 @@ mod tests {
last_memory_ts: None,
last_feedback_ts: None,
last_parsed_ts: None,
+ boundary: CursorBoundary::default(),
};
let local = MockLocal::new(
vec![vec![frame(1, "2026-05-07T10:00:00Z", "Arc", "secret")]],
@@ -3124,6 +3670,7 @@ mod tests {
last_memory_ts: None,
last_feedback_ts: None,
last_parsed_ts: None,
+ boundary: CursorBoundary::default(),
};
let local = MockLocal::new(
vec![vec![frame(1, "2026-05-07T10:00:00Z", "Arc", "x")]],
@@ -3158,6 +3705,7 @@ mod tests {
last_memory_ts: None,
last_feedback_ts: None,
last_parsed_ts: None,
+ boundary: CursorBoundary::default(),
};
let local = MockLocal::new(
vec![vec![frame(1, "2026-05-07T10:00:00Z", "Arc", "x")]],
@@ -3198,6 +3746,7 @@ mod tests {
last_memory_ts: None,
last_feedback_ts: None,
last_parsed_ts: None,
+ boundary: CursorBoundary::default(),
};
let local = MockLocal::new(
vec![vec![frame(1, "2026-05-07T10:00:00Z", "Arc", "x")]],
@@ -3246,6 +3795,7 @@ mod tests {
async fn fetch_frames_since(
&self,
_since: Option<&str>,
+ _boundary_offset: u32,
_limit: u32,
) -> Result<Vec<FrameRow>, EnterpriseSyncError> {
*self.frames_calls.lock().unwrap() += 1;
@@ -3255,6 +3805,7 @@ mod tests {
async fn fetch_audio_since(
&self,
_since: Option<&str>,
+ _boundary_offset: u32,
_limit: u32,
) -> Result<Vec<AudioRow>, EnterpriseSyncError> {
*self.audio_calls.lock().unwrap() += 1;
@@ -3264,6 +3815,7 @@ mod tests {
async fn fetch_ui_events_since(
&self,
_since: Option<&str>,
+ _boundary_offset: u32,
_limit: u32,
) -> Result<Vec<UiEventRow>, EnterpriseSyncError> {
*self.ui_calls.lock().unwrap() += 1;
@@ -3278,6 +3830,7 @@ mod tests {
async fn fetch_memories_since(
&self,
_since: Option<&str>,
+ _boundary_offset: u32,
_limit: u32,
) -> Result<Vec<MemoryRow>, EnterpriseSyncError> {
*self.memories_calls.lock().unwrap() += 1;
@@ -3342,6 +3895,7 @@ mod tests {
last_memory_ts: Some("2026-05-07T09:00:00Z".to_string()),
last_feedback_ts: None,
last_parsed_ts: Some("2026-05-07T09:00:00Z".to_string()),
+ boundary: CursorBoundary::default(),
};
let local = CallCountingLocal::new();
let http = reqwest::Client::new();
@@ -3440,6 +3994,7 @@ mod tests {
&self,
_: Option<&str>,
_: u32,
+ _: u32,
) -> Result<Vec<FrameRow>, EnterpriseSyncError> {
Ok(Vec::new())
}
@@ -3447,6 +4002,7 @@ mod tests {
&self,
_: Option<&str>,
_: u32,
+ _: u32,
) -> Result<Vec<AudioRow>, EnterpriseSyncError> {
Ok(Vec::new())
}
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/enterprise_sync.rs b/apps/screenpipe-app-tauri/src-tauri/src/enterprise_sync.rs
index e04e53f33..76a635b14 100644
--- a/apps/screenpipe-app-tauri/src-tauri/src/enterprise_sync.rs
+++ b/apps/screenpipe-app-tauri/src-tauri/src/enterprise_sync.rs
@@ -198,14 +198,14 @@ mod imp {
async fn fetch_frames_since(
&self,
since_ts: Option<&str>,
+ boundary_offset: u32,
limit: u32,
) -> Result<Vec<FrameRow>, EnterpriseSyncError> {
- // /search takes start_time as ISO. We pass `since` (exclusive in
- // spirit) — server returns >= start_time; one duplicate row per
- // tick is acceptable since server-side dedups by (device_id, frame_id).
+ // /search returns rows whose timestamp is >= `since`. The durable
+ // boundary offset skips rows already acknowledged at that timestamp.
let mut url = format!(
- "{}/search?content_type=ocr&limit={}&order=ascending",
- self.api_url_base, limit
+ "{}/search?content_type=ocr&limit={}&offset={}&order=ascending",
+ self.api_url_base, limit, boundary_offset
);
if let Some(ts) = since_ts {
url.push_str(&format!("&start_time={}", urlencoding::encode(ts)));
@@ -248,11 +248,12 @@ mod imp {
async fn fetch_audio_since(
&self,
since_ts: Option<&str>,
+ boundary_offset: u32,
limit: u32,
) -> Result<Vec<AudioRow>, EnterpriseSyncError> {
let mut url = format!(
- "{}/search?content_type=audio&limit={}&order=ascending",
- self.api_url_base, limit
+ "{}/search?content_type=audio&limit={}&offset={}&order=ascending",
+ self.api_url_base, limit, boundary_offset
);
if let Some(ts) = since_ts {
url.push_str(&format!("&start_time={}", urlencoding::encode(ts)));
@@ -292,11 +293,12 @@ mod imp {
async fn fetch_parsed_since(
&self,
since_ts: Option<&str>,
+ boundary_offset: u32,
limit: u32,
) -> Result<Vec<ParsedRow>, EnterpriseSyncError> {
let mut url = format!(
- "{}/search?content_type=parsed&limit={}&order=ascending",
- self.api_url_base, limit
+ "{}/search?content_type=parsed&limit={}&offset={}&order=ascending",
+ self.api_url_base, limit, boundary_offset
);
if let Some(ts) = since_ts {
url.push_str(&format!("&start_time={}", urlencoding::encode(ts)));
@@ -332,6 +334,7 @@ mod imp {
async fn fetch_ui_events_since(
&self,
since_ts: Option<&str>,
+ boundary_offset: u32,
limit: u32,
) -> Result<Vec<UiEventRow>, EnterpriseSyncError> {
// Local /search content_type=input returns rows from the
@@ -342,8 +345,8 @@ mod imp {
// targets) — keystroke noise without element context isn't
// useful for SOP synthesis and bloats the corpus.
let mut url = format!(
- "{}/search?content_type=input&limit={}&order=ascending&input_context_only=true",
- self.api_url_base, limit
+ "{}/search?content_type=input&limit={}&offset={}&order=ascending&input_context_only=true",
+ self.api_url_base, limit, boundary_offset
);
if let Some(ts) = since_ts {
url.push_str(&format!("&start_time={}", urlencoding::encode(ts)));
@@ -558,15 +561,14 @@ mod imp {
async fn fetch_memories_since(
&self,
since_ts: Option<&str>,
+ boundary_offset: u32,
limit: u32,
) -> Result<Vec<MemoryRow>, EnterpriseSyncError> {
- // /memories filters by created_at >= start_time; ascending order
- // means the cursor advances monotonically. Server-side dedup is
- // on (device_id, memory_id), so a single-row overlap per tick is
- // acceptable (same convention as /search-backed fetches above).
+ // /memories returns created_at >= start_time. The durable boundary
+ // offset skips rows already acknowledged at that timestamp.
let mut url = format!(
- "{}/memories?limit={}&order_by=created_at&order_dir=asc",
- self.api_url_base, limit
+ "{}/memories?limit={}&offset={}&order_by=created_at&order_dir=asc",
+ self.api_url_base, limit, boundary_offset
);
if let Some(ts) = since_ts {
url.push_str(&format!("&start_time={}", urlencoding::encode(ts)));
@@ -607,12 +609,19 @@ mod imp {
async fn fetch_feedback_since(
&self,
since_ts: Option<&str>,
+ after_id: Option<&str>,
limit: u32,
) -> Result<Vec<FeedbackRow>, EnterpriseSyncError> {
- let mut url = format!("{}/feedback?limit={}&order=asc", self.api_url_base, limit);
+ let mut url = format!(
+ "{}/feedback?limit={}&order=asc&since_inclusive=true",
+ self.api_url_base, limit
+ );
if let Some(ts) = since_ts {
url.push_str(&format!("&since={}", urlencoding::encode(ts)));
}
+ if let Some(id) = after_id {
+ url.push_str(&format!("&after_id={}", urlencoding::encode(id)));
+ }
let resp = self
.auth(self.http.get(&url))
.send()
diff --git a/crates/screenpipe-db/src/db/feedback.rs b/crates/screenpipe-db/src/db/feedback.rs
index 9fc77d19c..ec4bea9ab 100644
--- a/crates/screenpipe-db/src/db/feedback.rs
+++ b/crates/screenpipe-db/src/db/feedback.rs
@@ -151,6 +151,8 @@ impl DatabaseManager {
rating: Option<&str>,
query: Option<&str>,
since: Option<&str>,
+ since_inclusive: bool,
+ after_id: Option<&str>,
ascending: bool,
limit: i64,
) -> Result<Vec<FeedbackRecord>, SqlxError> {
@@ -165,14 +167,16 @@ impl DatabaseManager {
AND (?2 IS NULL OR target_id = ?2)
AND (?3 IS NULL OR producer_ref = ?3)
AND (?4 IS NULL OR rating = ?4)
- AND (?5 IS NULL OR updated_at > ?5)
- AND (?6 IS NULL OR target_id LIKE '%' || ?6 || '%'
- OR COALESCE(producer_ref, '') LIKE '%' || ?6 || '%'
- OR COALESCE(comment, '') LIKE '%' || ?6 || '%'
- OR COALESCE(snapshot, '') LIKE '%' || ?6 || '%'
- OR context LIKE '%' || ?6 || '%')
- ORDER BY updated_at {order}
- LIMIT ?7"#,
+ AND (?5 IS NULL OR updated_at > ?5
+ OR (?6 = 1 AND updated_at = ?5
+ AND (?7 IS NULL OR id > ?7)))
+ AND (?8 IS NULL OR target_id LIKE '%' || ?8 || '%'
+ OR COALESCE(producer_ref, '') LIKE '%' || ?8 || '%'
+ OR COALESCE(comment, '') LIKE '%' || ?8 || '%'
+ OR COALESCE(snapshot, '') LIKE '%' || ?8 || '%'
+ OR context LIKE '%' || ?8 || '%')
+ ORDER BY updated_at {order}, id {order}
+ LIMIT ?9"#,
);
sqlx::query_as::<_, FeedbackRecord>(sqlx::AssertSqlSafe(sql))
.bind(target_kind)
@@ -180,6 +184,8 @@ impl DatabaseManager {
.bind(producer_ref)
.bind(rating)
.bind(since)
+ .bind(since_inclusive)
+ .bind(after_id)
.bind(query)
.bind(limit.clamp(1, 500))
.fetch_all(&self.pool)
@@ -239,14 +245,57 @@ mod tests {
assert_eq!(second.rating, "up");
let all = db
- .list_feedback(None, None, None, None, None, None, false, 50)
+ .list_feedback(None, None, None, None, None, None, false, None, false, 50)
.await
.unwrap();
assert_eq!(all.len(), 1);
let by_snapshot = db
- .list_feedback(None, None, None, None, Some("today"), None, false, 50)
+ .list_feedback(
+ None,
+ None,
+ None,
+ None,
+ Some("today"),
+ None,
+ false,
+ None,
+ false,
+ 50,
+ )
.await
.unwrap();
assert_eq!(by_snapshot.len(), 1);
}
+
+ #[tokio::test]
+ async fn inclusive_feedback_cursor_uses_id_to_cross_timestamp_ties() {
+ let dir = tempfile::tempdir().unwrap();
+ let db = test_db(&dir).await;
+ let tied_at = "2026-07-30T10:00:00Z";
+ for id in 1..=3 {
+ let mut item = record("down", None, tied_at);
+ item.id = format!("feedback-{id:04}");
+ item.target_id = format!("daily-recap-{id:04}");
+ db.upsert_feedback(&item).await.unwrap();
+ }
+
+ let page = db
+ .list_feedback(
+ None,
+ None,
+ None,
+ None,
+ None,
+ Some(tied_at),
+ true,
+ Some("feedback-0002"),
+ true,
+ 50,
+ )
+ .await
+ .unwrap();
+
+ assert_eq!(page.len(), 1);
+ assert_eq!(page[0].id, "feedback-0003");
+ }
}
diff --git a/crates/screenpipe-db/src/db/memories.rs b/crates/screenpipe-db/src/db/memories.rs
index 853102cca..5d8b42a43 100644
--- a/crates/screenpipe-db/src/db/memories.rs
+++ b/crates/screenpipe-db/src/db/memories.rs
@@ -378,8 +378,8 @@ impl DatabaseManager {
_ => "DESC",
};
sql.push_str(&format!(
- " ORDER BY {} {} LIMIT ?7 OFFSET ?8",
- order_col, order_direction
+ " ORDER BY {} {}, id {} LIMIT ?7 OFFSET ?8",
+ order_col, order_direction, order_direction
));
let fts_query = query.map(crate::text_normalizer::sanitize_fts5_query);
diff --git a/crates/screenpipe-engine/src/routes/ai_feedback.rs b/crates/screenpipe-engine/src/routes/ai_feedback.rs
index 172abfbd7..b93b86f0b 100644
--- a/crates/screenpipe-engine/src/routes/ai_feedback.rs
+++ b/crates/screenpipe-engine/src/routes/ai_feedback.rs
@@ -163,6 +163,12 @@ pub(crate) struct ListFeedbackQuery {
pub q: Option<String>,
#[serde(default)]
pub since: Option<String>,
+ /// Enterprise pagination opts into an inclusive timestamp boundary and
+ /// supplies the last acknowledged id to continue safely through ties.
+ #[serde(default)]
+ pub since_inclusive: bool,
+ #[serde(default)]
+ pub after_id: Option<String>,
#[serde(default)]
pub order: Option<String>,
#[serde(default = "default_limit")]
@@ -361,6 +367,7 @@ pub(crate) async fn list_ai_feedback_handler(
}
let q = clean_optional(query.q);
let since = clean_optional(query.since);
+ let after_id = clean_optional(query.after_id);
let ascending = query.order.as_deref() == Some("asc");
let mut records = state
.db
@@ -371,6 +378,8 @@ pub(crate) async fn list_ai_feedback_handler(
rating.as_deref(),
q.as_deref(),
since.as_deref(),
+ query.since_inclusive,
+ after_id.as_deref(),
ascending,
query.limit.clamp(1, 500) as i64,
)
@@ -428,11 +437,19 @@ pub(crate) async fn list_ai_feedback_handler(
{
return false;
}
- if since
- .as_deref()
- .is_some_and(|since| record.updated_at.as_str() <= since)
- {
- return false;
+ if let Some(since) = since.as_deref() {
+ if query.since_inclusive {
+ if record.updated_at.as_str() < since
+ || (record.updated_at.as_str() == since
+ && after_id
+ .as_deref()
+ .is_some_and(|id| record.id.as_str() <= id))
+ {
+ return false;
+ }
+ } else if record.updated_at.as_str() <= since {
+ return false;
+ }
}
if let Some(q) = q.as_deref() {
let haystack = serde_json::to_string(record)
@@ -446,9 +463,18 @@ pub(crate) async fn list_ai_feedback_handler(
});
records.extend(legacy);
if ascending {
- records.sort_by(|left, right| left.updated_at.cmp(&right.updated_at));
+ records.sort_by(|left, right| {
+ left.updated_at
+ .cmp(&right.updated_at)
+ .then_with(|| left.id.cmp(&right.id))
+ });
} else {
- records.sort_by(|left, right| right.updated_at.cmp(&left.updated_at));
+ records.sort_by(|left, right| {
+ right
+ .updated_at
+ .cmp(&left.updated_at)
+ .then_with(|| right.id.cmp(&left.id))
+ });
}
records.truncate(query.limit.clamp(1, 500));
diff --git a/crates/screenpipe-gateway/src/ingest.rs b/crates/screenpipe-gateway/src/ingest.rs
index 80173ecad..483a846a8 100644
--- a/crates/screenpipe-gateway/src/ingest.rs
+++ b/crates/screenpipe-gateway/src/ingest.rs
@@ -1111,6 +1111,8 @@ mod tests {
Some("project names"),
None,
false,
+ None,
+ false,
20,
)
.await