YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
test(macos): verify typing privacy across native virtual displays
Louis Beaumontb8b875bf91f0
diff --git a/crates/screenpipe-a11y/src/platform/macos_click_attribution_e2e.rs b/crates/screenpipe-a11y/src/platform/macos_click_attribution_e2e.rsindex 36fa9f45e..12dbab4ff 100644--- a/crates/screenpipe-a11y/src/platform/macos_click_attribution_e2e.rs+++ b/crates/screenpipe-a11y/src/platform/macos_click_attribution_e2e.rs@@ -362,6 +362,32 @@ fn live_recorder_does_not_capture_excluded_app_after_allowed_app() {exercise_live_typing(true);}+fn run_display_fixture(path: &str, args: &[&str]) -> String {+ let mut child = Command::new(path)+ .args(args)+ .stdout(std::process::Stdio::piped())+ .spawn()+ .unwrap();+ let deadline = Instant::now() + Duration::from_secs(8);+ loop {+ if child.try_wait().unwrap().is_some() {+ break;+ }+ if Instant::now() >= deadline {+ let _ = child.kill();+ let _ = child.wait();+ panic!("native display command timed out");+ }+ std::thread::sleep(Duration::from_millis(20));+ }+ let output = child.wait_with_output().unwrap();+ assert!(+ output.status.success(),+ "native display command failed: {args:?}"+ );+ String::from_utf8(output.stdout).unwrap()+}+fn exercise_live_typing(exclude_terminal: bool) {assert_eq!(std::env::var("SCREENPIPE_ISOLATED_VM_TEST").as_deref(),@@ -401,6 +427,57 @@ fn exercise_live_typing(exclude_terminal: bool) {).unwrap();wait_for_window_snapshot(terminal.0, Duration::from_secs(10)).unwrap();+ // Optional native topology acceptance; exact placement and distinct display+ // membership are checked before posting any synthetic input.+ let display_fixture = std::env::var("SCREENPIPE_DISPLAY_FIXTURE").ok();+ let origins = std::env::var("SCREENPIPE_E2E_DISPLAY_ORIGINS")+ .ok()+ .map(|raw| {+ raw.split(',')+ .map(|x| x.parse::<f64>().unwrap())+ .collect::<Vec<_>>()+ });+ if let Some(points) = &origins {+ assert_eq!(points.len(), 4);+ let mut displays = HashSet::new();+ for (index, pid) in [textedit.0, terminal.0].iter().enumerate() {+ let x = points[index * 2] + 40.;+ let y = points[index * 2 + 1] + 80.;+ let id = run_display_fixture(+ display_fixture.as_ref().unwrap(),+ &[+ "place-window",+ &pid.to_string(),+ &x.to_string(),+ &y.to_string(),+ ],+ );+ displays.insert(id.trim().parse::<u32>().unwrap());+ let deadline = Instant::now() + Duration::from_secs(5);+ loop {+ let (_, bounds) = wait_for_window_snapshot(*pid, Duration::from_secs(1)).unwrap();+ if (bounds.origin.x - x).abs() < 2. && (bounds.origin.y - y).abs() < 2. {+ eprintln!(+ "MULTIMON fixture pid={pid} native_display={} window_origin=({}, {})",+ id.trim(),+ bounds.origin.x,+ bounds.origin.y+ );+ break;+ }+ assert!(+ Instant::now() < deadline,+ "window did not move to requested display: {bounds:?}"+ );+ std::thread::sleep(Duration::from_millis(50));+ }+ }+ assert_eq!(+ displays.len(),+ 2,+ "typing apps must occupy distinct native displays"+ );+ }let handle = recorder.start().unwrap();let rx = handle.receiver().clone();std::thread::sleep(Duration::from_secs(1));@@ -414,6 +491,15 @@ fn exercise_live_typing(exclude_terminal: bool) {let activated: bool = unsafe { objc2::msg_send![object, activateWithOptions: 2usize] };assert!(activated, "failed to activate fixture pid {pid}");wait_for_focus(pid, Duration::from_secs(5)).unwrap();+ if let Some(points) = &origins {+ let index = if pid == textedit.0 { 0 } else { 1 };+ let x = points[index * 2] + 100.;+ let y = points[index * 2 + 1] + 120.;+ run_display_fixture(+ display_fixture.as_ref().unwrap(),+ &["warp-to", &x.to_string(), &y.to_string()],+ );+ }// Shorter than the text debounce: a flush-time identity read fails// this test by attributing the earlier app's text to the later app.std::thread::sleep(Duration::from_millis(150));diff --git a/scripts/native-test-fixtures/macos/CHROMIUM-LICENSE b/scripts/native-test-fixtures/macos/CHROMIUM-LICENSEnew file mode 100644index 000000000..2249a2865--- /dev/null+++ b/scripts/native-test-fixtures/macos/CHROMIUM-LICENSE@@ -0,0 +1,27 @@+// Copyright 2015 The Chromium Authors+//+// Redistribution and use in source and binary forms, with or without+// modification, are permitted provided that the following conditions are+// met:+//+// * Redistributions of source code must retain the above copyright+// notice, this list of conditions and the following disclaimer.+// * Redistributions in binary form must reproduce the above+// copyright notice, this list of conditions and the following disclaimer+// in the documentation and/or other materials provided with the+// distribution.+// * Neither the name of Google LLC nor the names of its+// contributors may be used to endorse or promote products derived from+// this software without specific prior written permission.+//+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.diff --git a/scripts/native-test-fixtures/macos/README.md b/scripts/native-test-fixtures/macos/README.mdnew file mode 100644index 000000000..de7508340--- /dev/null+++ b/scripts/native-test-fixtures/macos/README.md@@ -0,0 +1,64 @@+# Isolated macOS multi-display acceptance++Run only in a disposable, logged-in Mac VM. This fixture creates real WindowServer+virtual displays with private CGVirtualDisplay APIs, changes their arrangement,+warps the guest cursor, and places synthetic red/blue marker windows. It must+never run on a working user's desktop. It requires the exact opt-in+`SCREENPIPE_ISOLATED_VM_TEST=1` and removes its display when its process exits.+The private API declarations originate from Chromium; see `CHROMIUM-LICENSE`.++Build the fixture on macOS with ARC (required for display ownership/cleanup):++```sh+clang++ -fobjc-arc -framework Cocoa -framework CoreGraphics \+ -framework ApplicationServices -mmacosx-version-min=15.0 \+ scripts/native-test-fixtures/macos/display-fixture.mm \+ -o display-fixture+cargo test -p screenpipe-engine --lib --no-run+cargo test -p screenpipe-a11y --lib --no-run+```++Transfer the corresponding native test binaries and helper to the guest. Grant+Screen Recording and Accessibility/Input Monitoring to the guest test processes+using the disposable VM's test-permission setup. No host permission changes.++```sh+SCREENPIPE_ISOLATED_VM_TEST=1 \+SCREENPIPE_DISPLAY_FIXTURE=/absolute/path/display-fixture \+SCREENPIPE_VM_EVIDENCE_DIR=/absolute/path/frames \+./engine-tests --ignored native_multi_monitor_focus_capture_gap_and_hotplug --nocapture+```++The engine test asserts actual display geometry (right, negative-left, above,+offset), cursor positions, production tracker/controller attribution, distinct+red/blue capture content, stale-topology grace/recovery, focused-secondary removal,+Retina backing scale and display cleanup. It reports an OS-clamped literal+cursor-gap attempt as unsupported, never as a passing gap reproduction. The+stale-topology case rearranges the actual display while intentionally withholding+discovery, waits through the real grace, and verifies recovery after fresh+production enumeration without moving the cursor. It calls fresh production+monitor enumeration explicitly, so it does not test the automatic monitor-watcher+loop.+It checks the health predicate, not the live HTTP endpoint or packaged UI.++For cross-display typing, keep the helper running and send `layout X Y` through+its stdin. Each line returns JSON topology. Verify the requested layout and two+unmirrored displays, then run each native recorder acceptance with:++```sh+SCREENPIPE_ISOLATED_VM_TEST=1 \+SCREENPIPE_DISPLAY_FIXTURE=/absolute/path/display-fixture \+SCREENPIPE_E2E_DISPLAY_ORIGINS=0,0,1024,400 \+./a11y-tests --ignored live_recorder_keeps --nocapture+# Repeat with filter live_recorder_does_not to verify excluded-app handling.+```++Choose the origins from the actual topology; do not assume the example fits+another guest. The recorder test verifies actual AX window positions and distinct+native display membership. It explicitly controls synthetic modifiers and bounds+helper execution. Native input acceptance still allows 150ms observer settling.++Stop and reap the fixture process and verify only the base display remains.+Destroy the disposable guest and task-owned tunnels after collecting evidence.+These tests do not establish physical cable/dock behavior, persisted-frame app+attribution, packaged UI warnings or the customer's exact hardware incident.diff --git a/scripts/native-test-fixtures/macos/display-fixture.mm b/scripts/native-test-fixtures/macos/display-fixture.mmnew file mode 100644index 000000000..77e314376--- /dev/null+++ b/scripts/native-test-fixtures/macos/display-fixture.mm@@ -0,0 +1,182 @@+// screenpipe — AI that knows everything you have seen, said, or heard+// https://screenpipe.com+// Test-only private API declarations from Chromium, BSD-3-Clause.+// https://chromium.googlesource.com/chromium/src/+/HEAD/ui/display/mac/test/virtual_display_util_mac.mm+#import <Cocoa/Cocoa.h>+#import <CoreGraphics/CoreGraphics.h>+@interface CGVirtualDisplay : NSObject++@property(readonly, nonatomic) unsigned int vendorID;+@property(readonly, nonatomic) unsigned int productID;+@property(readonly, nonatomic) unsigned int serialNum;+@property(readonly, nonatomic) NSString* name;+@property(readonly, nonatomic) struct CGSize sizeInMillimeters;+@property(readonly, nonatomic) unsigned int maxPixelsWide;+@property(readonly, nonatomic) unsigned int maxPixelsHigh;+@property(readonly, nonatomic) struct CGPoint redPrimary;+@property(readonly, nonatomic) struct CGPoint greenPrimary;+@property(readonly, nonatomic) struct CGPoint bluePrimary;+@property(readonly, nonatomic) struct CGPoint whitePoint;+@property(readonly, nonatomic) id queue;+@property(readonly, nonatomic) id terminationHandler;+@property(readonly, nonatomic) unsigned int displayID;+@property(readonly, nonatomic) unsigned int hiDPI;+@property(readonly, nonatomic) NSArray* modes;+@property(readonly, nonatomic) unsigned int serialNumber;+@property(readonly, nonatomic) unsigned int rotation;+- (BOOL)applySettings:(id)arg1;+- (void)dealloc;+- (id)initWithDescriptor:(id)arg1;++@end++// These interfaces were generated from CoreGraphics binaries.+@interface CGVirtualDisplayDescriptor : NSObject++@property(nonatomic) unsigned int vendorID;+@property(nonatomic) unsigned int productID;+@property(nonatomic) unsigned int serialNum;+@property(strong, nonatomic) NSString* name;+@property(nonatomic) struct CGSize sizeInMillimeters;+@property(nonatomic) unsigned int maxPixelsWide;+@property(nonatomic) unsigned int maxPixelsHigh;+@property(nonatomic) struct CGPoint redPrimary;+@property(nonatomic) struct CGPoint greenPrimary;+@property(nonatomic) struct CGPoint bluePrimary;+@property(nonatomic) struct CGPoint whitePoint;+@property(strong, nonatomic) id queue;+@property(copy, nonatomic) id terminationHandler;+@property(nonatomic) unsigned int serialNumber;+- (void)dealloc;+- (id)init;+- (id)dispatchQueue;+- (void)setDispatchQueue:(id)arg1;++@end++// These interfaces were generated from CoreGraphics binaries.+@interface CGVirtualDisplayMode : NSObject++@property(readonly, nonatomic) unsigned int width;+@property(readonly, nonatomic) unsigned int height;+@property(readonly, nonatomic) double refreshRate;+@property(copy, nonatomic) id transferFunction API_AVAILABLE(macos(13.3));+- (id)initWithWidth:(unsigned int)arg1+ height:(unsigned int)arg2+ refreshRate:(double)arg3;+- (id)initWithWidth:(unsigned int)arg1+ height:(unsigned int)arg2+ refreshRate:(double)arg3+ transferFunction:(id)arg4 API_AVAILABLE(macos(13.3));++@end++// These interfaces were generated from CoreGraphics binaries.+@interface CGVirtualDisplaySettings : NSObject++@property(strong, nonatomic) NSArray* modes;+@property(nonatomic) unsigned int hiDPI;+@property(nonatomic) unsigned int rotation;+- (void)dealloc;+- (id)init;++@end++// Copyright 2022 The Chromium Authors (private API declarations above).+// BSD-3-Clause: https://chromium.googlesource.com/chromium/src/+/HEAD/LICENSE+static CGVirtualDisplay *extra;+static NSMutableDictionary<NSNumber *, NSWindow *> *markers;+static unsigned serial = 20261017;+static void addDisplay(BOOL retina) {+ CGVirtualDisplayDescriptor *d = [CGVirtualDisplayDescriptor new];+ d.name = @"Screenpipe disposable acceptance display";+ d.vendorID = 505; d.productID = 1; d.serialNum = ++serial; d.serialNumber = serial;+ d.maxPixelsWide = 2560; d.maxPixelsHigh = 1440;+ d.sizeInMillimeters = CGSizeMake(340, 190);+ d.whitePoint = CGPointMake(0.3125,0.3291);+ d.redPrimary = CGPointMake(0.64,0.33); d.greenPrimary = CGPointMake(0.3,0.6); d.bluePrimary = CGPointMake(0.15,0.06);+ d.queue = dispatch_get_global_queue(QOS_CLASS_USER_INITIATED,0);+ extra = [[CGVirtualDisplay alloc] initWithDescriptor:d];+ if (!extra) { fprintf(stderr,"virtual display creation failed\n"); exit(2); }+ CGVirtualDisplaySettings *settings = [CGVirtualDisplaySettings new];+ settings.hiDPI = retina;+ settings.modes = @[[[CGVirtualDisplayMode alloc] initWithWidth:1280 height:720 refreshRate:60]];+ if (![extra applySettings:settings]) exit(3);+}+static void settle(void) {+ NSDate *end=[NSDate dateWithTimeIntervalSinceNow:0.6];+ while ([end timeIntervalSinceNow]>0) [[NSRunLoop currentRunLoop] runUntilDate:[NSDate dateWithTimeIntervalSinceNow:0.05]];+}+static void report(void) {+ CGDirectDisplayID ids[16]; uint32_t count=0;+ CGGetOnlineDisplayList(16,ids,&count);+ NSMutableArray *items=[NSMutableArray new];+ if (!markers) markers=[NSMutableDictionary new];+ NSMutableSet *live=[NSMutableSet new];+ CGFloat mainHeight=CGDisplayBounds(CGMainDisplayID()).size.height;+ for (uint32_t i=0;i<count;i++) {+ CGRect r=CGDisplayBounds(ids[i]);+ NSNumber *key=@(ids[i]);[live addObject:key];+ NSWindow *marker=markers[key];+ if(!marker) {+ marker=[[NSWindow alloc] initWithContentRect:NSMakeRect(0,0,200,200) styleMask:NSWindowStyleMaskBorderless backing:NSBackingStoreBuffered defer:NO];+ marker.level=NSFloatingWindowLevel;marker.ignoresMouseEvents=YES;marker.releasedWhenClosed=NO;+ marker.backgroundColor=CGDisplayIsMain(ids[i]) ? [NSColor colorWithRed:0.9 green:0.1 blue:0.1 alpha:1] : [NSColor colorWithRed:0.1 green:0.1 blue:0.9 alpha:1];+ markers[key]=marker;+ }+ [marker setFrame:NSMakeRect(r.origin.x+20,mainHeight-r.origin.y-220,200,200) display:YES];[marker orderFrontRegardless];+ CGDisplayModeRef mode=CGDisplayCopyDisplayMode(ids[i]);+ [items addObject:@{@"id":@(ids[i]),@"x":@(r.origin.x),@"y":@(r.origin.y),@"w":@(r.size.width),@"h":@(r.size.height),@"pixels_w":@(CGDisplayModeGetPixelWidth(mode)),@"pixels_h":@(CGDisplayModeGetPixelHeight(mode)),@"main":@(CGDisplayIsMain(ids[i])),@"mirror":@(CGDisplayMirrorsDisplay(ids[i]))}];+ CGDisplayModeRelease(mode);+ }+ for(NSNumber *key in [markers.allKeys copy]) if(![live containsObject:key]) { [markers[key] close];[markers removeObjectForKey:key]; }+ CGEventRef ev=CGEventCreate(NULL); CGPoint p=CGEventGetLocation(ev); CFRelease(ev);+ NSData *json=[NSJSONSerialization dataWithJSONObject:@{@"displays":items,@"extra":@(extra ? extra.displayID:0),@"cursor_x":@(p.x),@"cursor_y":@(p.y)} options:0 error:nil];+ fwrite(json.bytes,1,json.length,stdout);puts("");fflush(stdout);+}+static int placeWindow(pid_t pid,double x,double y) {+ CGDirectDisplayID target=0;uint32_t targets=0;+ CGGetDisplaysWithPoint(CGPointMake(x,y),1,&target,&targets);+ if(targets!=1 || !CGRectContainsRect(CGDisplayBounds(target),CGRectMake(x,y,600,400))) return 22;+ AXUIElementRef app=AXUIElementCreateApplication(pid);AXUIElementSetMessagingTimeout(app,0.3);CFTypeRef raw=NULL;+ AXError e=AXUIElementCopyAttributeValue(app,kAXFocusedWindowAttribute,&raw);+ if(e || !raw) { CFRelease(app);return 20; }+ CGPoint position=CGPointMake(x,y);CGSize size=CGSizeMake(600,400);+ AXValueRef pos=AXValueCreate((AXValueType)kAXValueCGPointType,&position), dim=AXValueCreate((AXValueType)kAXValueCGSizeType,&size);+ AXUIElementSetAttributeValue((AXUIElementRef)raw,kAXSizeAttribute,dim);+ e=AXUIElementSetAttributeValue((AXUIElementRef)raw,kAXPositionAttribute,pos);+ CFRelease(pos);CFRelease(dim);CFRelease(raw);CFRelease(app);+ if(!e) printf("%u\n",target);+ return e ? 21 : 0;+}+int main(int argc,char **argv) { @autoreleasepool {+ const char *opt=getenv("SCREENPIPE_ISOLATED_VM_TEST");+ if (!opt || strcmp(opt,"1")!=0) return 10;+ [NSApplication sharedApplication];+ [NSApp setActivationPolicy:NSApplicationActivationPolicyAccessory];+ if(argc==2 && strcmp(argv[1],"--list-only")==0) {+ CGDirectDisplayID ids[16];uint32_t count=0;CGGetOnlineDisplayList(16,ids,&count);+ printf("{\"count\":%u}\n",count);return 0;+ }+ if(argc==5 && strcmp(argv[1],"place-window")==0) return placeWindow(atoi(argv[2]),atof(argv[3]),atof(argv[4]));+ if(argc==4 && strcmp(argv[1],"warp-to")==0) return CGWarpMouseCursorPosition(CGPointMake(atof(argv[2]),atof(argv[3])));+ addDisplay(NO); settle(); report();+ char *line=NULL; size_t length=0;+ while (getline(&line,&length,stdin)>0) { @autoreleasepool {+ double x=0,y=0;+ if (sscanf(line,"layout %lf %lf",&x,&y)==2) {+ CGDisplayConfigRef cfg; CGBeginDisplayConfiguration(&cfg);+ CGConfigureDisplayOrigin(cfg,extra.displayID,(int32_t)x,(int32_t)y);+ CGError e=CGCompleteDisplayConfiguration(cfg,kCGConfigureForSession);+ if(e) fprintf(stderr,"layout error %d\n",e);+ } else if(sscanf(line,"warp %lf %lf",&x,&y)==2) {+ CGError e=CGWarpMouseCursorPosition(CGPointMake(x,y));+ if(e) fprintf(stderr,"warp error %d\n",e);+ } else if(strncmp(line,"remove",6)==0) { extra=nil;+ } else if(strncmp(line,"add-retina",10)==0) { extra=nil;settle();addDisplay(YES);+ } else if(strncmp(line,"add",3)==0) { extra=nil;settle();addDisplay(NO);+ } else if(strncmp(line,"quit",4)==0) { break; }+ settle();report();+ }}+ free(line);extra=nil;return 0;+}}