Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
perf(rendering): precompute cursor spring for preview and thumbnail
Kyzer43938fa11922
diff --git a/crates/editor/src/thumbnail.rs b/crates/editor/src/thumbnail.rsindex 2c078355f..9b1abe68a 100644--- a/crates/editor/src/thumbnail.rs+++ b/crates/editor/src/thumbnail.rs@@ -1,6 +1,8 @@use crate::{EditorInstance, editor::RendererTransitionInput};use cap_project::{ProjectConfiguration, XY};-use cap_rendering::{ProjectUniforms, RenderedFrame, ZoomTransformTimeline};+use cap_rendering::{+ PrecomputedCursorTimeline, ProjectUniforms, RenderedFrame, ZoomTransformTimeline,+};use std::{io::Write, path::Path, sync::Arc};const THUMBNAIL_SIZE: XY<u32> = XY { x: 640, y: 640 };@@ -85,7 +87,8 @@ impl EditorInstance {segment.recording_clip,);zoom.ensure_precomputed_until(1.0 / THUMBNAIL_FPS as f32);- let uniforms = ProjectUniforms::new(+ let cursor_timeline = PrecomputedCursorTimeline::from_project(&media.cursor, &project);+ let uniforms = ProjectUniforms::new_with_precomputed_cursor(&self.render_constants,&project,0,@@ -95,6 +98,7 @@ impl EditorInstance {&frames,duration,&zoom,+ &cursor_timeline,);let frame = self.rendererdiff --git a/crates/export/src/preview.rs b/crates/export/src/preview.rsindex 09e1172af..918bd05dd 100644--- a/crates/export/src/preview.rs+++ b/crates/export/src/preview.rs@@ -4,8 +4,9 @@ use base64::{Engine, engine::general_purpose::STANDARD};use cap_editor::EditorInstance;use cap_project::{CursorEvents, ProjectConfiguration, RecordingMeta, TimelineFrameMapping, XY};use cap_rendering::{- FrameRenderer, ProjectUniforms, RecordingSegmentDecoders, RenderVideoConstants, RenderedFrame,- RendererLayers, TransitionRenderInput, ZoomTransformTimeline,+ FrameRenderer, PrecomputedCursorTimeline, ProjectUniforms, RecordingSegmentDecoders,+ RenderVideoConstants, RenderedFrame, RendererLayers, TransitionRenderInput,+ ZoomTransformTimeline,};use image::{Rgba,@@ -277,7 +278,9 @@ async fn render_preview_frame();zoom_timeline.ensure_precomputed_until((frame_number as f32 + 1.0) / settings.fps as f32);- let uniforms = ProjectUniforms::new(+ let cursor_timeline =+ PrecomputedCursorTimeline::from_project(segment_media.cursor, project_config);+ let uniforms = ProjectUniforms::new_with_precomputed_cursor(render_constants,project_config,frame_number,@@ -287,6 +290,7 @@ async fn render_preview_frame(&segment_frames,total_duration,&zoom_timeline,+ &cursor_timeline,);let mut frame_renderer = FrameRenderer::new(render_constants);@@ -341,7 +345,16 @@ async fn render_preview_frame(outgoing.segment.recording_clip,);outgoing_zoom.ensure_precomputed_until((frame_number as f32 + 1.0) / settings.fps as f32);- let outgoing_uniforms = ProjectUniforms::new(+ let outgoing_cursor_timeline = if outgoing.segment.recording_clip == segment.recording_clip+ {+ None+ } else {+ Some(PrecomputedCursorTimeline::from_project(+ outgoing_media.cursor,+ project_config,+ ))+ };+ let outgoing_uniforms = ProjectUniforms::new_with_precomputed_cursor(render_constants,project_config,frame_number,@@ -351,6 +364,9 @@ async fn render_preview_frame(&outgoing_frames,total_duration,&outgoing_zoom,+ outgoing_cursor_timeline+ .as_ref()+ .unwrap_or(&cursor_timeline),);frame_rendererdiff --git a/crates/rendering/src/cursor_interpolation.rs b/crates/rendering/src/cursor_interpolation.rsindex 002283bb6..5696c2d47 100644--- a/crates/rendering/src/cursor_interpolation.rs+++ b/crates/rendering/src/cursor_interpolation.rs@@ -3,7 +3,9 @@ use std::{sync::{Arc, Mutex},};-use cap_project::{ClickSpringConfig, CursorClickEvent, CursorEvents, CursorMoveEvent, XY};+use cap_project::{+ ClickSpringConfig, CursorClickEvent, CursorEvents, CursorMoveEvent, ProjectConfiguration, XY,+};use crate::{Coord, RawDisplayUVSpace,@@ -270,7 +272,7 @@ pub fn interpolate_cursor_with_click_spring(smoothing_config,click_spring,);- interpolate_timeline(&timeline, time_ms)+ interpolate_timeline(&timeline.events, &timeline.cursor_ids, time_ms)} else {interpolate_raw_cursor(cursor, time_ms, false)}@@ -348,6 +350,7 @@ pub(crate) struct CursorEventOrder {pub struct PrecomputedCursorTimeline {timeline: Vec<SmoothedCursorEvent>,+ cursor_ids: Vec<String>,raw_cursor: CursorEvents,raw_moves_are_ordered: bool,clicks_are_ordered: bool,@@ -372,6 +375,7 @@ impl PrecomputedCursorTimeline {if cursor.moves.is_empty() || smoothing.is_none() {return Self {timeline: vec![],+ cursor_ids: Vec::new(),raw_cursor: cursor.clone(),raw_moves_are_ordered,clicks_are_ordered,@@ -383,7 +387,7 @@ impl PrecomputedCursorTimeline {let smoothing_config = smoothing.unwrap();let filtered_moves = filter_cursor_shake(&cursor.moves);let prepared_moves = decimate_cursor_moves(filtered_moves.as_ref());- let timeline = build_smoothed_timeline(+ let built = build_smoothed_timeline(cursor,prepared_moves.as_ref(),smoothing_config,@@ -391,7 +395,8 @@ impl PrecomputedCursorTimeline {);Self {- timeline,+ timeline: built.events,+ cursor_ids: built.cursor_ids,raw_cursor: CursorEvents::default(),raw_moves_are_ordered,clicks_are_ordered,@@ -400,6 +405,21 @@ impl PrecomputedCursorTimeline {}}+ /// Spring settings match playback and export: the base cursor, with style+ /// overrides applied later by `ProjectUniforms::new_with_precomputed_cursor`.+ pub fn from_project(cursor: &CursorEvents, project: &ProjectConfiguration) -> Self {+ let smoothing = (!project.cursor.raw).then_some(SpringMassDamperSimulationConfig {+ tension: project.cursor.tension,+ mass: project.cursor.mass,+ friction: project.cursor.friction,+ });+ Self::new(+ cursor,+ smoothing,+ Some(project.cursor.click_spring_config()),+ )+ }+pub(crate) fn cached_variant(&self,cursor: &CursorEvents,@@ -443,7 +463,7 @@ impl PrecomputedCursorTimeline {pub fn interpolate(&self, time_secs: f32) -> Option<InterpolatedCursorPosition> {let time_ms = (time_secs * 1000.0) as f64;if self.has_smoothing {- interpolate_timeline(&self.timeline, time_ms)+ interpolate_timeline(&self.timeline, &self.cursor_ids, time_ms)} else {interpolate_raw_cursor(&self.raw_cursor, time_ms, self.raw_moves_are_ordered)}@@ -463,14 +483,59 @@ impl PrecomputedCursorTimeline {}}+struct SmoothedTimeline {+ events: Vec<SmoothedCursorEvent>,+ cursor_ids: Vec<String>,+}++/// Unique cursor images for one spring build. Steps store an index so a long+/// recording does not allocate a `String` at every 60 Hz sample.+struct CursorIdIndex {+ ids: Vec<String>,+ last: Option<u32>,+}++impl CursorIdIndex {+ fn new() -> Self {+ Self {+ ids: Vec::new(),+ last: None,+ }+ }++ fn intern(&mut self, id: &str) -> u32 {+ if let Some(last) = self.last+ && self.ids[last as usize] == id+ {+ return last;+ }+ if let Some(index) = self.ids.iter().position(|existing| existing == id) {+ let index = index as u32;+ self.last = Some(index);+ return index;+ }+ let index = self.ids.len() as u32;+ self.ids.push(id.to_owned());+ self.last = Some(index);+ index+ }+}++fn cursor_id_string(cursor_ids: &[String], index: u32) -> String {+ cursor_ids[index as usize].clone()+}+fn build_smoothed_timeline(cursor: &CursorEvents,moves: &[CursorMoveEvent],smoothing_config: SpringMassDamperSimulationConfig,click_spring: Option<ClickSpringConfig>,-) -> Vec<SmoothedCursorEvent> {+) -> SmoothedTimeline {if moves.is_empty() {- return vec![];+ return SmoothedTimeline {+ events: vec![],+ cursor_ids: Vec::new(),+ };}let moves_are_ordered =@@ -491,12 +556,13 @@ fn build_smoothed_timeline(let mut events = Vec::with_capacity(capacity);let mut target_hint: usize = 0;let mut cid_hint: usize = 0;+ let mut cursor_ids = CursorIdIndex::new();events.push(SmoothedCursorEvent {time: 0.0,position: start_pos,velocity: XY::new(0.0, 0.0),- cursor_id: moves[0].cursor_id.clone(),+ cursor_id: cursor_ids.intern(&moves[0].cursor_id),});let mut t_ms = SIMULATION_STEP_MS;@@ -516,7 +582,7 @@ fn build_smoothed_timeline(let lead_t = (clamped_t + lead_ms).min(end_time_ms);let (cx, cy) = position_at_time_hinted(moves, lead_t, &mut target_hint);let _ = position_at_time_hinted(moves, clamped_t, &mut cid_hint);- let cid = cursor_id_at_time(moves, clamped_t, cid_hint).to_string();+ let cursor_id = cursor_ids.intern(cursor_id_at_time(moves, clamped_t, cid_hint));let target = if let Some(click) =next_click_within(&cursor.clicks, t_ms, CLICK_LOOKAHEAD_TARGET_MS)@@ -536,7 +602,7 @@ fn build_smoothed_timeline(time: t_ms as f32,position: sim.position,velocity: sim.velocity,- cursor_id: cid,+ cursor_id,});if t_ms >= end_time_ms && sim.is_at_rest() {@@ -546,11 +612,15 @@ fn build_smoothed_timeline(t_ms += SIMULATION_STEP_MS;}- events+ SmoothedTimeline {+ events,+ cursor_ids: cursor_ids.ids,+ }}fn interpolate_timeline(events: &[SmoothedCursorEvent],+ cursor_ids: &[String],query_ms: f64,) -> Option<InterpolatedCursorPosition> {if events.is_empty() {@@ -564,7 +634,7 @@ fn interpolate_timeline(return Some(InterpolatedCursorPosition {position: Coord::new(XY::new(e.position.x as f64, e.position.y as f64)),velocity: e.velocity,- cursor_id: e.cursor_id.clone(),+ cursor_id: cursor_id_string(cursor_ids, e.cursor_id),});}@@ -573,7 +643,7 @@ fn interpolate_timeline(return Some(InterpolatedCursorPosition {position: Coord::new(XY::new(e.position.x as f64, e.position.y as f64)),velocity: e.velocity,- cursor_id: e.cursor_id.clone(),+ cursor_id: cursor_id_string(cursor_ids, e.cursor_id),});}@@ -603,7 +673,7 @@ fn interpolate_timeline(return Some(InterpolatedCursorPosition {position: Coord::new(XY::new(e.position.x as f64, e.position.y as f64)),velocity: e.velocity,- cursor_id: e.cursor_id.clone(),+ cursor_id: cursor_id_string(cursor_ids, e.cursor_id),});}}@@ -626,7 +696,7 @@ fn interpolate_timeline(a.velocity.x * inv + b.velocity.x * t,a.velocity.y * inv + b.velocity.y * t,),- cursor_id: a.cursor_id.clone(),+ cursor_id: cursor_id_string(cursor_ids, a.cursor_id),})}@@ -732,7 +802,7 @@ struct SmoothedCursorEvent {time: f32,position: XY<f32>,velocity: XY<f32>,- cursor_id: String,+ cursor_id: u32,}#[cfg(test)]@@ -1001,12 +1071,19 @@ mod tests {})}+ struct OracleCursorEvent {+ time: f32,+ position: XY<f32>,+ velocity: XY<f32>,+ cursor_id: String,+ }+fn linear_build_smoothed_timeline(cursor: &CursorEvents,moves: &[CursorMoveEvent],smoothing_config: SpringMassDamperSimulationConfig,click_spring: Option<ClickSpringConfig>,- ) -> Vec<SmoothedCursorEvent> {+ ) -> Vec<OracleCursorEvent> {if moves.is_empty() {return vec![];}@@ -1028,7 +1105,7 @@ mod tests {let mut target_hint: usize = 0;let mut cid_hint: usize = 0;- events.push(SmoothedCursorEvent {+ events.push(OracleCursorEvent {time: 0.0,position: start_pos,velocity: XY::new(0.0, 0.0),@@ -1049,7 +1126,7 @@ mod tests {let lead_t = (clamped_t + lead_ms).min(end_time_ms);let (cx, cy) = position_at_time_hinted(moves, lead_t, &mut target_hint);let _ = position_at_time_hinted(moves, clamped_t, &mut cid_hint);- let cid = cursor_id_at_time(moves, clamped_t, cid_hint).to_string();+ let cursor_id = cursor_id_at_time(moves, clamped_t, cid_hint).to_string();let target = if let Some(click) =next_click_within(&cursor.clicks, t_ms, CLICK_LOOKAHEAD_TARGET_MS)@@ -1064,11 +1141,11 @@ mod tests {sim.run(SIMULATION_STEP_MS as f32);- events.push(SmoothedCursorEvent {+ events.push(OracleCursorEvent {time: t_ms as f32,position: sim.position,velocity: sim.velocity,- cursor_id: cid,+ cursor_id,});if t_ms >= end_time_ms && sim.is_at_rest() {@@ -1081,8 +1158,8 @@ mod tests {events}- fn linear_interpolate_timeline(- events: &[SmoothedCursorEvent],+ fn oracle_interpolate_timeline(+ events: &[OracleCursorEvent],query_ms: f64,) -> Option<InterpolatedCursorPosition> {if events.is_empty() {@@ -1109,6 +1186,70 @@ mod tests {});}+ let end = events.partition_point(|event| event.time <= query);+ let Some(window) = end+ .checked_sub(1)+ .and_then(|start| events.get(start..=end))+ .filter(|window| window[0].time <= query && query < window[1].time)+ else {+ let e = events.last().unwrap();+ return Some(InterpolatedCursorPosition {+ position: Coord::new(XY::new(e.position.x as f64, e.position.y as f64)),+ velocity: e.velocity,+ cursor_id: e.cursor_id.clone(),+ });+ };+ let (a, b) = (&window[0], &window[1]);+ let dt = b.time - a.time;+ let t = if dt.abs() < 1e-6 {+ 0.0+ } else {+ ((query - a.time) / dt).clamp(0.0, 1.0)+ };+ let inv = 1.0 - t;++ Some(InterpolatedCursorPosition {+ position: Coord::new(XY::new(+ (a.position.x * inv + b.position.x * t) as f64,+ (a.position.y * inv + b.position.y * t) as f64,+ )),+ velocity: XY::new(+ a.velocity.x * inv + b.velocity.x * t,+ a.velocity.y * inv + b.velocity.y * t,+ ),+ cursor_id: a.cursor_id.clone(),+ })+ }++ fn linear_interpolate_timeline(+ events: &[SmoothedCursorEvent],+ cursor_ids: &[String],+ query_ms: f64,+ ) -> Option<InterpolatedCursorPosition> {+ if events.is_empty() {+ return None;+ }++ let query = query_ms as f32;++ if query <= events[0].time {+ let e = &events[0];+ return Some(InterpolatedCursorPosition {+ position: Coord::new(XY::new(e.position.x as f64, e.position.y as f64)),+ velocity: e.velocity,+ cursor_id: cursor_id_string(cursor_ids, e.cursor_id),+ });+ }++ if query >= events.last().unwrap().time {+ let e = events.last().unwrap();+ return Some(InterpolatedCursorPosition {+ position: Coord::new(XY::new(e.position.x as f64, e.position.y as f64)),+ velocity: e.velocity,+ cursor_id: cursor_id_string(cursor_ids, e.cursor_id),+ });+ }+let first_time = events[0].time;let step = if events.len() > 1 {events[1].time - events[0].time@@ -1135,7 +1276,7 @@ mod tests {return Some(InterpolatedCursorPosition {position: Coord::new(XY::new(e.position.x as f64, e.position.y as f64)),velocity: e.velocity,- cursor_id: e.cursor_id.clone(),+ cursor_id: cursor_id_string(cursor_ids, e.cursor_id),});}}@@ -1158,7 +1299,7 @@ mod tests {a.velocity.x * inv + b.velocity.x * t,a.velocity.y * inv + b.velocity.y * t,),- cursor_id: a.cursor_id.clone(),+ cursor_id: cursor_id_string(cursor_ids, a.cursor_id),})}@@ -1185,7 +1326,11 @@ mod tests {}}- fn assert_timeline_bits(expected: &[SmoothedCursorEvent], actual: &[SmoothedCursorEvent]) {+ fn assert_timeline_bits(+ expected: &[OracleCursorEvent],+ actual: &[SmoothedCursorEvent],+ cursor_ids: &[String],+ ) {assert_eq!(expected.len(), actual.len());for (index, (expected, actual)) in expected.iter().zip(actual).enumerate() {assert_eq!(@@ -1213,16 +1358,23 @@ mod tests {actual.velocity.y.to_bits(),"velocity.y at {index}");- assert_eq!(expected.cursor_id, actual.cursor_id, "cursor_id at {index}");+ assert_eq!(+ expected.cursor_id, cursor_ids[actual.cursor_id as usize],+ "cursor_id at {index}"+ );}}+ fn sample_cursor_ids() -> Vec<String> {+ vec!["0".into(), "1".into(), "2".into()]+ }+fn sampled_event(time: f32, index: usize) -> SmoothedCursorEvent {SmoothedCursorEvent {time,position: XY::new(index as f32 * 0.013, index as f32 * -0.007),velocity: XY::new(index as f32 * -0.002, index as f32 * 0.003),- cursor_id: (index % 3).to_string(),+ cursor_id: (index % 3) as u32,}}@@ -1265,7 +1417,9 @@ mod tests {assert!(timeline.has_smoothing);assert!(timeline.raw_cursor.moves.is_empty());assert!(timeline.raw_cursor.clicks.is_empty());- assert_timeline_bits(&expected, &timeline.timeline);+ assert!(timeline.cursor_ids.len() <= 3);+ assert!(timeline.timeline.len() > timeline.cursor_ids.len());+ assert_timeline_bits(&expected, &timeline.timeline, &timeline.cursor_ids);drop(moves);drop(filtered);drop(cursor);@@ -1273,7 +1427,7 @@ mod tests {for index in 0..=500 {let time_secs = index as f32 * 0.017_137;assert_position_bits(- linear_interpolate_timeline(&expected, (time_secs * 1000.0) as f64),+ oracle_interpolate_timeline(&expected, (time_secs * 1000.0) as f64),timeline.interpolate(time_secs),);}@@ -1676,12 +1830,12 @@ mod tests {let expected =linear_build_smoothed_timeline(&cursor, &moves, config, click_spring);let actual = build_smoothed_timeline(&cursor, &moves, config, click_spring);- assert_timeline_bits(&expected, &actual);+ assert_timeline_bits(&expected, &actual.events, &actual.cursor_ids);for index in 0..=500 {let query = index as f64 * 17.137;assert_position_bits(- linear_interpolate_timeline(&expected, query),- interpolate_timeline(&actual, query),+ oracle_interpolate_timeline(&expected, query),+ interpolate_timeline(&actual.events, &actual.cursor_ids, query),);}}@@ -1692,9 +1846,10 @@ mod tests {#[test]fn timeline_lookup_matches_linear_reference_at_long_times_and_duplicate_timestamps() {assert_position_bits(- linear_interpolate_timeline(&[], 0.0),- interpolate_timeline(&[], 0.0),+ linear_interpolate_timeline(&[], &[], 0.0),+ interpolate_timeline(&[], &[], 0.0),);+ let cursor_ids = sample_cursor_ids();for duration_hours in [0.0, 1.0, 20.0] {for interval_ms in [1.0, SIMULATION_STEP_MS] {let origin_ms = duration_hours * 3_600_000.0;@@ -1717,8 +1872,8 @@ mod tests {origin_ms + 10_000.0,] {assert_position_bits(- linear_interpolate_timeline(&events, query),- interpolate_timeline(&events, query),+ linear_interpolate_timeline(&events, &cursor_ids, query),+ interpolate_timeline(&events, &cursor_ids, query),);}for event in &events {@@ -1729,8 +1884,8 @@ mod tests {event.time as f64 + interval_ms * 0.5,] {assert_position_bits(- linear_interpolate_timeline(&events, query),- interpolate_timeline(&events, query),+ linear_interpolate_timeline(&events, &cursor_ids, query),+ interpolate_timeline(&events, &cursor_ids, query),);}}@@ -1743,8 +1898,8 @@ mod tests {.collect();for query in [-0.0, 0.0, 0.01, 8.0, 16.0, 16.01, 24.0, 32.0] {assert_position_bits(- linear_interpolate_timeline(&duplicate_origin, query),- interpolate_timeline(&duplicate_origin, query),+ linear_interpolate_timeline(&duplicate_origin, &cursor_ids, query),+ interpolate_timeline(&duplicate_origin, &cursor_ids, query),);}}@@ -1765,12 +1920,16 @@ mod tests {let started = Instant::now();let actual = build_smoothed_timeline(&cursor, &moves, DEFAULT_CLICK_SPRING, None);let optimized_time = started.elapsed();- assert_timeline_bits(black_box(&expected), black_box(&actual));+ assert_timeline_bits(+ black_box(&expected),+ black_box(&actual.events),+ black_box(&actual.cursor_ids),+ );println!("cursor_precompute duration_secs={duration_secs} moves={} clicks={} samples={} baseline_ms={:.3} optimized_ms={:.3}",moves.len(),cursor.clicks.len(),- actual.len(),+ actual.events.len(),baseline_time.as_secs_f64() * 1000.0,optimized_time.as_secs_f64() * 1000.0);@@ -1779,23 +1938,26 @@ mod tests {let events: Vec<_> = (0..=20 * 60 * 60 * 60).map(|index| {let mut event = sampled_event((index as f64 * SIMULATION_STEP_MS) as f32, index);- event.cursor_id = String::new();+ event.cursor_id = 0;event}).collect();+ let cursor_ids = [String::new()];let queries: Vec<_> = (0..128).map(|index| 72_000_000.0 - index as f64 * 171.13 - 1.0).collect();let started = Instant::now();let expected: Vec<_> = queries.iter()- .map(|&query| linear_interpolate_timeline(black_box(&events), black_box(query)))+ .map(|&query| {+ linear_interpolate_timeline(black_box(&events), &cursor_ids, black_box(query))+ }).collect();let baseline_time = started.elapsed();let started = Instant::now();let actual: Vec<_> = queries.iter()- .map(|&query| interpolate_timeline(black_box(&events), black_box(query)))+ .map(|&query| interpolate_timeline(black_box(&events), &cursor_ids, black_box(query))).collect();let optimized_time = started.elapsed();for (expected, actual) in expected.into_iter().zip(actual) {