Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
perf(desktop-gpui): reuse the presigned manifest URL during Instant uploads
Richie McIlroy472b5ed05480
diff --git a/apps/desktop-gpui/src/upload.rs b/apps/desktop-gpui/src/upload.rsindex 27b874a8b..435e86049 100644--- a/apps/desktop-gpui/src/upload.rs+++ b/apps/desktop-gpui/src/upload.rs@@ -34,11 +34,16 @@ const SEGMENT_RETRY_WINDOW: Duration = Duration::from_secs(5 * 60);const SEGMENT_RETRY_MIN_DELAY: Duration = Duration::from_secs(2);const SEGMENT_RETRY_MAX_DELAY: Duration = Duration::from_secs(30);const MANIFEST_UPLOAD_INTERVAL: Duration = Duration::from_secs(1);+const MANIFEST_SUBPATH: &str = "segments/manifest.json";+// Presigned S3 PUT URLs are signed for 30 minutes and accept repeated PUTs, so+// the manifest reuses one well inside that window instead of signing a new URL+// every second. Google Drive upload sessions are single use and never reused.+const MANIFEST_URL_REUSE: Duration = Duration::from_secs(10 * 60);const UPLOAD_CONNECT_TIMEOUT: Duration = Duration::from_secs(15);const UPLOAD_TCP_KEEPALIVE: Duration = Duration::from_secs(30);const UPLOAD_POOL_IDLE_TIMEOUT: Duration = Duration::from_secs(30);-#[derive(Debug, Deserialize)]+#[derive(Debug, Clone, Deserialize)]struct SignedUploadTarget {url: String,#[serde(default)]@@ -572,7 +577,7 @@ async fn run_segment_upload(authorization: Option<CompletionRequirement>,) -> Result<(), String> {let result = upload_segments(- &LiveSegmentTransport,+ &LiveSegmentTransport::default(),&video.id,events,cancel.clone(),@@ -676,7 +681,43 @@ trait SegmentTransport: Send + Sync {) -> impl Future<Output = Result<(), String>> + Send;}-struct LiveSegmentTransport;+struct ReusableUploadTarget {+ target: SignedUploadTarget,+ signed_at: tokio::time::Instant,+}++impl ReusableUploadTarget {+ fn new(target: SignedUploadTarget, signed_at: tokio::time::Instant) -> Option<Self> {+ (!is_google_drive_resumable_url(&target.url)).then_some(Self { target, signed_at })+ }++ fn is_fresh(&self) -> bool {+ self.signed_at.elapsed() < MANIFEST_URL_REUSE+ }+}++#[derive(Default)]+struct LiveSegmentTransport {+ manifest_target: Mutex<Option<ReusableUploadTarget>>,+}++impl LiveSegmentTransport {+ fn reusable_manifest_target(&self) -> Option<ReusableUploadTarget> {+ self.manifest_target+ .lock()+ .unwrap_or_else(|error| error.into_inner())+ .take()+ .filter(ReusableUploadTarget::is_fresh)+ }++ fn keep_manifest_target(&self, target: Option<ReusableUploadTarget>) {+ *self+ .manifest_target+ .lock()+ .unwrap_or_else(|error| error.into_inner()) = target;+ }+}+impl SegmentTransport for LiveSegmentTransport {async fn prepare(&self,@@ -719,9 +760,20 @@ impl SegmentTransport for LiveSegmentTransport {start: u32,count: u32,) -> Result<HashMap<String, String>, String> {- prefetch_segment_urls(video_id, start, count)+ let signed_at = tokio::time::Instant::now();+ let mut urls = prefetch_segment_urls(video_id, start, count).await- .map_err(|error| error.to_string())+ .map_err(|error| error.to_string())?;+ if let Some(url) = urls.remove(MANIFEST_SUBPATH) {+ self.keep_manifest_target(ReusableUploadTarget::new(+ SignedUploadTarget {+ url,+ headers: HashMap::new(),+ },+ signed_at,+ ));+ }+ Ok(urls)}async fn segment(&self,@@ -738,7 +790,7 @@ impl SegmentTransport for LiveSegmentTransport {manifest: &SegmentUploadManifest,cancel: &Arc<AtomicBool>,) -> Result<(), String> {- upload_segment_manifest_with_retry(video_id, manifest, cancel).await+ upload_segment_manifest_with_retry(self, video_id, manifest, cancel).await}async fn complete(&self, video_id: &str, cancel: &Arc<AtomicBool>) -> Result<(), String> {signal_recording_complete_with_retry(video_id, cancel).await@@ -983,7 +1035,7 @@ fn prefetched_segment_paths(start: u32, count: u32) -> Vec<String> {if start == 1 {subpaths.push("segments/video/init.mp4".to_string());subpaths.push("segments/audio/init.mp4".to_string());- subpaths.push("segments/manifest.json".to_string());+ subpaths.push(MANIFEST_SUBPATH.to_string());}for index in start..start.saturating_add(count) {subpaths.push(format!("segments/video/segment_{index:03}.m4s"));@@ -1138,18 +1190,46 @@ fn segment_subpath(event: &SegmentCompletedEvent) -> String {}async fn upload_segment_manifest(+ transport: &LiveSegmentTransport,video_id: &str,manifest: &SegmentUploadManifest,cancel: &AtomicBool,) -> Result<(), String> {let bytes = serde_json::to_vec(manifest).map_err(|error| format!("Failed to serialize instant upload manifest: {error}"))?;- presigned_put_bytes_inner(video_id, "segments/manifest.json", bytes, Some(cancel))+ if let Some(reusable) = transport.reusable_manifest_target() {+ match upload_signed_bytes_inner(+ reusable.target.clone(),+ MANIFEST_SUBPATH,+ bytes.clone(),+ Some(cancel),+ )+ .await+ {+ Ok(()) => {+ transport.keep_manifest_target(Some(reusable));+ return Ok(());+ }+ Err(error) => {+ check_segment_cancelled(cancel)?;+ tracing::debug!("Reused instant manifest URL failed; signing a new one: {error}");+ }+ }+ }+ let signed_at = tokio::time::Instant::now();+ let target = presign_put_target(video_id, MANIFEST_SUBPATH, Some(cancel))+ .await+ .map_err(|error| format!("Failed to upload instant recording manifest: {error}"))?;+ let reusable = ReusableUploadTarget::new(target.clone(), signed_at);+ upload_signed_bytes_inner(target, MANIFEST_SUBPATH, bytes, Some(cancel)).await- .map_err(|error| format!("Failed to upload instant recording manifest: {error}"))+ .map_err(|error| format!("Failed to upload instant recording manifest: {error}"))?;+ transport.keep_manifest_target(reusable);+ Ok(())}async fn upload_segment_manifest_with_retry(+ transport: &LiveSegmentTransport,video_id: &str,manifest: &SegmentUploadManifest,cancel: &AtomicBool,@@ -1159,7 +1239,7 @@ async fn upload_segment_manifest_with_retry(return Err("Instant recording upload cancelled".to_string());}match checked_segment_step(cancel, || {- upload_segment_manifest(video_id, manifest, cancel)+ upload_segment_manifest(transport, video_id, manifest, cancel)}).await{@@ -2288,6 +2368,15 @@ async fn presigned_put_bytes_inner(bytes: Vec<u8>,cancel: Option<&AtomicBool>,) -> Result<(), AuthApiError> {+ let target = presign_put_target(video_id, subpath, cancel).await?;+ upload_signed_bytes_inner(target, subpath, bytes, cancel).await+}++async fn presign_put_target(+ video_id: &str,+ subpath: &str,+ cancel: Option<&AtomicBool>,+) -> Result<SignedUploadTarget, AuthApiError> {#[derive(Deserialize)]#[serde(rename_all = "camelCase")]struct Response {@@ -2313,12 +2402,11 @@ async fn presigned_put_bytes_inner(format!("api/upload_signed/{}", status.as_u16()),));}- let target = response+ response.json::<Response>().await- .map_err(|error| AuthApiError::Other(format!("api/upload_signed/response: {error}")))?- .presigned_put_data;- upload_signed_bytes_inner(target, subpath, bytes, cancel).await+ .map(|response| response.presigned_put_data)+ .map_err(|error| AuthApiError::Other(format!("api/upload_signed/response: {error}")))}async fn upload_signed_bytes_inner(diff --git a/apps/desktop-gpui/src/upload/verify_tests.rs b/apps/desktop-gpui/src/upload/verify_tests.rsindex 5f998870d..5b0f4602d 100644--- a/apps/desktop-gpui/src/upload/verify_tests.rs+++ b/apps/desktop-gpui/src/upload/verify_tests.rs@@ -251,8 +251,10 @@ pub(super) struct Budget {////// Studio: a 26 MiB export in six parts (five of 5 MiB, one of 1 MiB + 12345/// bytes), each part a presign and a PUT. Instant: 2 init segments and 8-/// audio/video pairs on URLs from one batch presign, progress manifests (a-/// presign and a PUT each) and the completion signal.+/// audio/video pairs on URLs from one batch presign, progress manifests and+/// the completion signal. Manifests are PUT to the URL the batch presign+/// returned for them, so Instant request budgets no longer count a presign+/// per manifest.pub(super) mod budget {use super::{Budget, PART, STUDIO_LAST_PART, STUDIO_SIZE, VIDEO_SEGMENT};@@ -300,29 +302,31 @@ pub(super) mod budget {budget(26, 10, 0, 3 * PART, 1, 12);pub(in crate::upload) const INSTANT_LIVE_CLEAN: Budget =- budget(38, 18, 9, 0, 0, 13 + POOL_RACE);+ budget(29, 18, 9, 0, 0, 13 + POOL_RACE);pub(in crate::upload) const INSTANT_LIVE_SEGMENT_503: Budget =- budget(42, 19, 10, SEGMENT_3, 0, 15 + POOL_RACE);+ budget(32, 19, 10, SEGMENT_3, 0, 15 + POOL_RACE);pub(in crate::upload) const INSTANT_LIVE_SEGMENT_503X3: Budget =- budget(44, 21, 9, 3 * SEGMENT_3, 0, 16 + POOL_RACE);+ budget(35, 21, 9, 3 * SEGMENT_3, 0, 16 + POOL_RACE);pub(in crate::upload) const INSTANT_LIVE_SEGMENT_RESET: Budget =- budget(44, 21, 9, 3 * (SEGMENT_4 / 2), 0, 18 + POOL_RACE);+ budget(35, 21, 9, 3 * (SEGMENT_4 / 2), 0, 18 + POOL_RACE);pub(in crate::upload) const INSTANT_LIVE_OUTAGE: Budget =- budget(41, 19, 7, SEGMENT_3 / 2, 0, 22 + POOL_RACE);+ budget(34, 19, 7, SEGMENT_3 / 2, 0, 22 + POOL_RACE);pub(in crate::upload) const INSTANT_LIVE_SLOW_SEGMENT: Budget =- budget(38, 18, 9, 0, 0, 13 + POOL_RACE);+ budget(29, 18, 9, 0, 0, 13 + POOL_RACE);pub(in crate::upload) const INSTANT_LIVE_OUTAGE_PAST_WINDOW: Budget =- budget(15, 8, 3, SEGMENT_3 / 2, 0, 6 + POOL_RACE);+ budget(12, 8, 3, SEGMENT_3 / 2, 0, 6 + POOL_RACE);pub(in crate::upload) const INSTANT_LIVE_PERMANENT_403: Budget =- budget(22, 11, 4, 3 * SEGMENT_3, 0, 9 + POOL_RACE);+ budget(18, 11, 4, 3 * SEGMENT_3, 0, 9 + POOL_RACE);pub(in crate::upload) const INSTANT_LIVE_CANCEL: Budget =- budget(19, 10, 4, 0, 0, 7 + POOL_RACE);+ budget(15, 10, 4, 0, 0, 7 + POOL_RACE);+ pub(in crate::upload) const INSTANT_LIVE_MANIFEST_403: Budget =+ budget(31, 18, 10, 0, 0, 13 + POOL_RACE);pub(in crate::upload) const INSTANT_LIVE_COMPLETION_500: Budget =- budget(40, 18, 9, 0, 0, 15 + POOL_RACE);+ budget(31, 18, 9, 0, 0, 15 + POOL_RACE);/// The relaunch session only: every segment and the thumbnail once.pub(in crate::upload) const QUEUE_RESUME_AFTER_QUIT_MID_TRANSFER: Budget =- budget(24, 15, 2, 1_795_340, 0, 12 + POOL_RACE);+ budget(22, 15, 2, 1_795_340, 0, 12 + POOL_RACE);pub(in crate::upload) const QUEUE_RESUME_AFTER_QUIT_DURING_VERIFICATION: Budget =budget(1, 0, 0, 0, 0, 1);}@@ -1565,7 +1569,7 @@ fn instant_case(let feeder = tokio::spawn(feed(video.clone(), audio.clone(), sender));let started = tokio::time::Instant::now();let result = upload_segments(- &LiveSegmentTransport,+ &LiveSegmentTransport::default(),INSTANT_VIDEO_ID,events,mock.cancel_flag(),@@ -1609,6 +1613,106 @@ fn instant_live_clean() {);}+fn manifest_presigns(mock: &MockCap) -> usize {+ mock.lock()+ .requests+ .iter()+ .filter(|(method, path, ..)| method == "POST" && path == "/api/upload/signed")+ .count()+}++#[test]+fn instant_live_manifests_reuse_the_batch_presigned_url() {+ instant_case(+ "instant_live_manifest_reuse",+ Vec::new(),+ budget::INSTANT_LIVE_CLEAN,+ |mock, run| {+ assert_instant_complete(mock, &run);+ assert!(mock.count("PUT", "segments/manifest.json") > 1);+ assert_eq!(manifest_presigns(mock), 0, "{}", mock.timeline());+ },+ );+}++#[test]+fn instant_live_manifest_signs_a_new_url_when_the_reused_one_fails() {+ instant_case(+ "instant_live_manifest_reuse_rejected",+ vec![rule(+ Route::Object("segments/manifest.json"),+ Fault::Status(403),+ )],+ budget::INSTANT_LIVE_MANIFEST_403,+ |mock, run| {+ assert_instant_complete(mock, &run);+ assert_eq!(manifest_presigns(mock), 1, "{}", mock.timeline());+ },+ );+}++#[test]+fn instant_live_manifest_url_is_resigned_before_it_can_expire() {+ let scratch = Scratch::new("instant_live_manifest_url_expiry");+ virtual_time(async {+ let mock = MockCap::start(Vec::new()).await;+ scratch.sign_in(&mock.url);+ let (video, audio) = instant_tracks(&scratch.0, INSTANT_PAIRS);+ let (sender, events) = flume::unbounded();+ let feeder = tokio::spawn({+ let (video, audio) = (video.clone(), audio.clone());+ async move {+ let start = tokio::time::Instant::now();+ let _ = sender.send(video[0].clone());+ let _ = sender.send(audio[0].clone());+ for k in 1..video.len() {+ tokio::time::sleep_until(start + Duration::from_secs(100 * k as u64)).await;+ let _ = sender.send(audio[k].clone());+ let _ = sender.send(video[k].clone());+ }+ }+ });+ let result = upload_segments(+ &LiveSegmentTransport::default(),+ INSTANT_VIDEO_ID,+ events,+ mock.cancel_flag(),+ None,+ )+ .await;+ feeder.await.unwrap();+ assert_eq!(result, Ok(()));+ assert_segments_stored(&mock, INSTANT_VIDEO_ID, &video, &audio);+ assert_eq!(manifest_presigns(&mock), 1, "{}", mock.timeline());+ mock.assert_clean();+ });+}++#[test]+fn only_object_store_urls_are_reused_for_manifests() {+ let target = |url: &str| SignedUploadTarget {+ url: url.into(),+ headers: HashMap::new(),+ };+ let now = tokio::time::Instant::now();+ assert!(+ ReusableUploadTarget::new(+ target("https://bucket.s3.us-east-1.amazonaws.com/o/v/segments/manifest.json?X-Amz-Signature=1"),+ now,+ )+ .is_some()+ );+ assert!(+ ReusableUploadTarget::new(+ target(+ "https://www.googleapis.com/upload/drive/v3/files?uploadType=resumable&upload_id=1"+ ),+ now,+ )+ .is_none()+ );+}+#[test]fn instant_live_segment_503_retries_in_call() {instant_case(@@ -1673,7 +1777,7 @@ fn instant_live_deferred_retries_back_off_after_recording_ends() {let (sender, events) = flume::unbounded();let feeder = tokio::spawn(feed(video.clone(), audio.clone(), sender));let result = upload_segments(- &LiveSegmentTransport,+ &LiveSegmentTransport::default(),INSTANT_VIDEO_ID,events,mock.cancel_flag(),