Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

perf(desktop-gpui): skip re-parsing unchanged recordings in the upload reconcile loop

Khushal Bhardwaj9ee9bb460ab3
diff --git a/apps/desktop-gpui/src/upload/queue.rs b/apps/desktop-gpui/src/upload/queue.rs
index 6cf371979..040f0544a 100644
--- a/apps/desktop-gpui/src/upload/queue.rs
+++ b/apps/desktop-gpui/src/upload/queue.rs
@@ -3,7 +3,9 @@ use cap_project::{InstantRecordingMeta, RecordingMetaInner};
use cap_recording::upload_resume::{UploadLock, UploadLockError, collect_segment_events};
use cap_recording::upload_verification::{UploadVerification, VerifiedUploadReceipt};
use futures_util::FutureExt as _;
+use std::ffi::OsString;
use std::io::Write;
+use std::time::SystemTime;
static STATE_CHANGED: tokio::sync::Notify = tokio::sync::Notify::const_new();
@@ -536,6 +538,7 @@ struct Manager<B> {
wake: tokio::sync::Notify,
confirmation_timeout: Duration,
pending_rechecks: Mutex<HashMap<String, u32>>,
+ skipped_meta: Mutex<SkippedMeta>,
}
impl<B: UploadBackend> Manager<B> {
@@ -547,6 +550,7 @@ impl<B: UploadBackend> Manager<B> {
wake: tokio::sync::Notify::new(),
confirmation_timeout,
pending_rechecks: Mutex::new(HashMap::new()),
+ skipped_meta: Mutex::new(SkippedMeta::default()),
}
}
@@ -564,6 +568,12 @@ impl<B: UploadBackend> Manager<B> {
}
}
+ fn skipped_meta(&self) -> std::sync::MutexGuard<'_, SkippedMeta> {
+ self.skipped_meta
+ .lock()
+ .unwrap_or_else(|error| error.into_inner())
+ }
+
fn forget_pending_rechecks(&self, video_id: &str) {
self.pending_rechecks
.lock()
@@ -901,9 +911,90 @@ fn legacy_state(meta: &RecordingMeta) -> Option<UploadState> {
})
}
+fn upload_candidate(meta: &RecordingMeta) -> bool {
+ matches!(
+ meta.inner,
+ RecordingMetaInner::Instant(InstantRecordingMeta::Complete { .. })
+ ) && !matches!(meta.upload, None | Some(UploadMeta::Complete))
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+struct MetaStamp {
+ modified: SystemTime,
+ len: u64,
+ #[cfg(unix)]
+ inode: u64,
+ #[cfg(unix)]
+ changed: (i64, i64),
+}
+
+impl MetaStamp {
+ fn read(project: &Path) -> Option<Self> {
+ #[cfg(unix)]
+ use std::os::unix::fs::MetadataExt;
+ let metadata = std::fs::metadata(project.join("recording-meta.json")).ok()?;
+ Some(Self {
+ modified: metadata.modified().ok()?,
+ len: metadata.len(),
+ #[cfg(unix)]
+ inode: metadata.ino(),
+ #[cfg(unix)]
+ changed: (metadata.ctime(), metadata.ctime_nsec()),
+ })
+ }
+}
+
+// Only "not a candidate" verdicts are cached: that verdict depends solely on
+// recording-meta.json, while candidates still go through the upload lock and
+// instant-upload.json (where auth changes apply) on every pass. Every writer
+// replaces the file by rename, so a save also changes the inode on unix.
+#[derive(Default)]
+struct SkippedMeta {
+ stamps: HashMap<OsString, (MetaStamp, u64)>,
+ pass: u64,
+}
+
+impl SkippedMeta {
+ fn begin_pass(&mut self) {
+ self.pass = self.pass.wrapping_add(1);
+ }
+
+ fn is_candidate(
+ &mut self,
+ project: &Path,
+ load: impl FnOnce(&Path) -> Option<RecordingMeta>,
+ ) -> bool {
+ let stamp = MetaStamp::read(project);
+ if let Some(stamp) = stamp
+ && let Some((cached, seen)) = self.stamps.get_mut(project.as_os_str())
+ && *cached == stamp
+ {
+ *seen = self.pass;
+ return false;
+ }
+ let candidate = load(project).is_some_and(|meta| upload_candidate(&meta));
+ match stamp {
+ Some(stamp) if !candidate => {
+ self.stamps
+ .insert(project.as_os_str().to_os_string(), (stamp, self.pass));
+ }
+ _ => {
+ self.stamps.remove(project.as_os_str());
+ }
+ }
+ candidate
+ }
+
+ fn finish_pass(&mut self) {
+ let pass = self.pass;
+ self.stamps.retain(|_, (_, seen)| *seen == pass);
+ }
+}
+
impl Manager<LiveBackend> {
async fn reconcile(self: &Arc<Self>, auth_changed: bool) {
self.reap().await;
+ self.skipped_meta().begin_pass();
for directory in crate::library::known_recordings_dirs() {
let Ok(entries) = std::fs::read_dir(directory) else {
continue;
@@ -913,16 +1004,9 @@ impl Manager<LiveBackend> {
return;
}
let project = entry.path();
- let Ok(meta) = RecordingMeta::load_for_project(&project) else {
- continue;
- };
- if !matches!(
- meta.inner,
- RecordingMetaInner::Instant(InstantRecordingMeta::Complete { .. })
- ) {
- continue;
- }
- if matches!(meta.upload, None | Some(UploadMeta::Complete)) {
+ if !self.skipped_meta().is_candidate(&project, |project| {
+ RecordingMeta::load_for_project(project).ok()
+ }) {
continue;
}
let ownership = match UploadLock::acquire(&project) {
@@ -951,6 +1035,7 @@ impl Manager<LiveBackend> {
}
}
}
+ self.skipped_meta().finish_pass();
}
async fn resume(
@@ -2285,4 +2370,130 @@ mod tests {
assert_eq!(state.phase, UploadPhase::Retrying);
fixture.assert_retained();
}
+
+ fn save_library_meta(project: &Path, upload: Option<UploadMeta>) {
+ std::fs::create_dir_all(project).unwrap();
+ RecordingMeta {
+ platform: None,
+ project_path: project.to_path_buf(),
+ pretty_name: "Library recording".into(),
+ sharing: None,
+ inner: RecordingMetaInner::Instant(InstantRecordingMeta::Complete {
+ fps: 30,
+ sample_rate: None,
+ }),
+ upload,
+ }
+ .save_for_project()
+ .unwrap();
+ }
+
+ fn scan_library(
+ skipped: &mut SkippedMeta,
+ library: &Path,
+ parses: &AtomicUsize,
+ ) -> Vec<PathBuf> {
+ skipped.begin_pass();
+ let mut candidates = std::fs::read_dir(library)
+ .unwrap()
+ .flatten()
+ .map(|entry| entry.path())
+ .filter(|project| {
+ skipped.is_candidate(project, |project| {
+ parses.fetch_add(1, Ordering::Relaxed);
+ RecordingMeta::load_for_project(project).ok()
+ })
+ })
+ .collect::<Vec<_>>();
+ skipped.finish_pass();
+ candidates.sort();
+ candidates
+ }
+
+ fn library_with_skipped_projects(skipped_projects: usize) -> Fixture {
+ let fixture = Fixture::new();
+ for index in 0..skipped_projects {
+ let upload = (index % 2 == 0).then_some(UploadMeta::Complete);
+ save_library_meta(&fixture.0.join(format!("skip-{index}.cap")), upload);
+ }
+ std::fs::write(fixture.0.join("notes.txt"), b"not a recording").unwrap();
+ fixture
+ }
+
+ #[test]
+ fn unchanged_skipped_recordings_are_not_parsed_again() {
+ let fixture = library_with_skipped_projects(6);
+ let mut skipped = SkippedMeta::default();
+ let parses = AtomicUsize::new(0);
+
+ assert_eq!(
+ scan_library(&mut skipped, &fixture.0, &parses),
+ vec![fixture.project()]
+ );
+ assert_eq!(parses.swap(0, Ordering::Relaxed), 8);
+ assert_eq!(skipped.stamps.len(), 6);
+
+ assert_eq!(
+ scan_library(&mut skipped, &fixture.0, &parses),
+ vec![fixture.project()]
+ );
+ assert_eq!(parses.load(Ordering::Relaxed), 2);
+ assert_eq!(
+ read_state(&fixture.project()).unwrap().unwrap().phase,
+ UploadPhase::Pending
+ );
+ }
+
+ #[test]
+ fn rewritten_recording_meta_is_reclassified_on_the_next_pass() {
+ let fixture = library_with_skipped_projects(2);
+ let mut skipped = SkippedMeta::default();
+ let parses = AtomicUsize::new(0);
+ let flipped = fixture.0.join("skip-1.cap");
+ scan_library(&mut skipped, &fixture.0, &parses);
+
+ save_library_meta(
+ &flipped,
+ Some(UploadMeta::SinglePartUpload {
+ video_id: "flipped-video".into(),
+ file_path: flipped.join("content/output.mp4"),
+ screenshot_path: flipped.join("screenshots/display.jpg"),
+ recording_dir: flipped.clone(),
+ }),
+ );
+ assert_eq!(
+ scan_library(&mut skipped, &fixture.0, &parses),
+ vec![fixture.project(), flipped.clone()]
+ );
+ assert!(!skipped.stamps.contains_key(flipped.as_os_str()));
+
+ save_library_meta(&flipped, Some(UploadMeta::Complete));
+ assert_eq!(
+ scan_library(&mut skipped, &fixture.0, &parses),
+ vec![fixture.project()]
+ );
+ parses.store(0, Ordering::Relaxed);
+ scan_library(&mut skipped, &fixture.0, &parses);
+ assert_eq!(parses.load(Ordering::Relaxed), 2);
+ }
+
+ #[test]
+ fn deleted_recordings_leave_the_skip_cache() {
+ let fixture = library_with_skipped_projects(4);
+ let mut skipped = SkippedMeta::default();
+ let parses = AtomicUsize::new(0);
+ let deleted = fixture.0.join("skip-2.cap");
+ scan_library(&mut skipped, &fixture.0, &parses);
+ assert!(skipped.stamps.contains_key(deleted.as_os_str()));
+
+ std::fs::remove_dir_all(&deleted).unwrap();
+ scan_library(&mut skipped, &fixture.0, &parses);
+ assert_eq!(skipped.stamps.len(), 3);
+ assert!(!skipped.stamps.contains_key(deleted.as_os_str()));
+
+ std::fs::remove_dir_all(&fixture.0).unwrap();
+ std::fs::create_dir_all(&fixture.0).unwrap();
+ scan_library(&mut skipped, &fixture.0, &parses);
+ assert!(skipped.stamps.is_empty());
+ }
}