Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

test(macos): verify typing privacy across native virtual displays

Louis Beaumontb8b875bf91f0
diff --git a/crates/screenpipe-a11y/src/platform/macos_click_attribution_e2e.rs b/crates/screenpipe-a11y/src/platform/macos_click_attribution_e2e.rs
index 36fa9f45e..12dbab4ff 100644
--- a/crates/screenpipe-a11y/src/platform/macos_click_attribution_e2e.rs
+++ b/crates/screenpipe-a11y/src/platform/macos_click_attribution_e2e.rs
@@ -362,6 +362,32 @@ fn live_recorder_does_not_capture_excluded_app_after_allowed_app() {
exercise_live_typing(true);
}
+fn run_display_fixture(path: &str, args: &[&str]) -> String {
+ let mut child = Command::new(path)
+ .args(args)
+ .stdout(std::process::Stdio::piped())
+ .spawn()
+ .unwrap();
+ let deadline = Instant::now() + Duration::from_secs(8);
+ loop {
+ if child.try_wait().unwrap().is_some() {
+ break;
+ }
+ if Instant::now() >= deadline {
+ let _ = child.kill();
+ let _ = child.wait();
+ panic!("native display command timed out");
+ }
+ std::thread::sleep(Duration::from_millis(20));
+ }
+ let output = child.wait_with_output().unwrap();
+ assert!(
+ output.status.success(),
+ "native display command failed: {args:?}"
+ );
+ String::from_utf8(output.stdout).unwrap()
+}
+
fn exercise_live_typing(exclude_terminal: bool) {
assert_eq!(
std::env::var("SCREENPIPE_ISOLATED_VM_TEST").as_deref(),
@@ -401,6 +427,57 @@ fn exercise_live_typing(exclude_terminal: bool) {
)
.unwrap();
wait_for_window_snapshot(terminal.0, Duration::from_secs(10)).unwrap();
+ // Optional native topology acceptance; exact placement and distinct display
+ // membership are checked before posting any synthetic input.
+ let display_fixture = std::env::var("SCREENPIPE_DISPLAY_FIXTURE").ok();
+ let origins = std::env::var("SCREENPIPE_E2E_DISPLAY_ORIGINS")
+ .ok()
+ .map(|raw| {
+ raw.split(',')
+ .map(|x| x.parse::<f64>().unwrap())
+ .collect::<Vec<_>>()
+ });
+ if let Some(points) = &origins {
+ assert_eq!(points.len(), 4);
+ let mut displays = HashSet::new();
+ for (index, pid) in [textedit.0, terminal.0].iter().enumerate() {
+ let x = points[index * 2] + 40.;
+ let y = points[index * 2 + 1] + 80.;
+ let id = run_display_fixture(
+ display_fixture.as_ref().unwrap(),
+ &[
+ "place-window",
+ &pid.to_string(),
+ &x.to_string(),
+ &y.to_string(),
+ ],
+ );
+ displays.insert(id.trim().parse::<u32>().unwrap());
+ let deadline = Instant::now() + Duration::from_secs(5);
+ loop {
+ let (_, bounds) = wait_for_window_snapshot(*pid, Duration::from_secs(1)).unwrap();
+ if (bounds.origin.x - x).abs() < 2. && (bounds.origin.y - y).abs() < 2. {
+ eprintln!(
+ "MULTIMON fixture pid={pid} native_display={} window_origin=({}, {})",
+ id.trim(),
+ bounds.origin.x,
+ bounds.origin.y
+ );
+ break;
+ }
+ assert!(
+ Instant::now() < deadline,
+ "window did not move to requested display: {bounds:?}"
+ );
+ std::thread::sleep(Duration::from_millis(50));
+ }
+ }
+ assert_eq!(
+ displays.len(),
+ 2,
+ "typing apps must occupy distinct native displays"
+ );
+ }
let handle = recorder.start().unwrap();
let rx = handle.receiver().clone();
std::thread::sleep(Duration::from_secs(1));
@@ -414,6 +491,15 @@ fn exercise_live_typing(exclude_terminal: bool) {
let activated: bool = unsafe { objc2::msg_send![object, activateWithOptions: 2usize] };
assert!(activated, "failed to activate fixture pid {pid}");
wait_for_focus(pid, Duration::from_secs(5)).unwrap();
+ if let Some(points) = &origins {
+ let index = if pid == textedit.0 { 0 } else { 1 };
+ let x = points[index * 2] + 100.;
+ let y = points[index * 2 + 1] + 120.;
+ run_display_fixture(
+ display_fixture.as_ref().unwrap(),
+ &["warp-to", &x.to_string(), &y.to_string()],
+ );
+ }
// Shorter than the text debounce: a flush-time identity read fails
// this test by attributing the earlier app's text to the later app.
std::thread::sleep(Duration::from_millis(150));
diff --git a/scripts/native-test-fixtures/macos/CHROMIUM-LICENSE b/scripts/native-test-fixtures/macos/CHROMIUM-LICENSE
new file mode 100644
index 000000000..2249a2865
--- /dev/null
+++ b/scripts/native-test-fixtures/macos/CHROMIUM-LICENSE
@@ -0,0 +1,27 @@
+// Copyright 2015 The Chromium Authors
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+// * Neither the name of Google LLC nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/scripts/native-test-fixtures/macos/README.md b/scripts/native-test-fixtures/macos/README.md
new file mode 100644
index 000000000..de7508340
--- /dev/null
+++ b/scripts/native-test-fixtures/macos/README.md
@@ -0,0 +1,64 @@
+# Isolated macOS multi-display acceptance
+
+Run only in a disposable, logged-in Mac VM. This fixture creates real WindowServer
+virtual displays with private CGVirtualDisplay APIs, changes their arrangement,
+warps the guest cursor, and places synthetic red/blue marker windows. It must
+never run on a working user's desktop. It requires the exact opt-in
+`SCREENPIPE_ISOLATED_VM_TEST=1` and removes its display when its process exits.
+The private API declarations originate from Chromium; see `CHROMIUM-LICENSE`.
+
+Build the fixture on macOS with ARC (required for display ownership/cleanup):
+
+```sh
+clang++ -fobjc-arc -framework Cocoa -framework CoreGraphics \
+ -framework ApplicationServices -mmacosx-version-min=15.0 \
+ scripts/native-test-fixtures/macos/display-fixture.mm \
+ -o display-fixture
+cargo test -p screenpipe-engine --lib --no-run
+cargo test -p screenpipe-a11y --lib --no-run
+```
+
+Transfer the corresponding native test binaries and helper to the guest. Grant
+Screen Recording and Accessibility/Input Monitoring to the guest test processes
+using the disposable VM's test-permission setup. No host permission changes.
+
+```sh
+SCREENPIPE_ISOLATED_VM_TEST=1 \
+SCREENPIPE_DISPLAY_FIXTURE=/absolute/path/display-fixture \
+SCREENPIPE_VM_EVIDENCE_DIR=/absolute/path/frames \
+./engine-tests --ignored native_multi_monitor_focus_capture_gap_and_hotplug --nocapture
+```
+
+The engine test asserts actual display geometry (right, negative-left, above,
+offset), cursor positions, production tracker/controller attribution, distinct
+red/blue capture content, stale-topology grace/recovery, focused-secondary removal,
+Retina backing scale and display cleanup. It reports an OS-clamped literal
+cursor-gap attempt as unsupported, never as a passing gap reproduction. The
+stale-topology case rearranges the actual display while intentionally withholding
+discovery, waits through the real grace, and verifies recovery after fresh
+production enumeration without moving the cursor. It calls fresh production
+monitor enumeration explicitly, so it does not test the automatic monitor-watcher
+loop.
+It checks the health predicate, not the live HTTP endpoint or packaged UI.
+
+For cross-display typing, keep the helper running and send `layout X Y` through
+its stdin. Each line returns JSON topology. Verify the requested layout and two
+unmirrored displays, then run each native recorder acceptance with:
+
+```sh
+SCREENPIPE_ISOLATED_VM_TEST=1 \
+SCREENPIPE_DISPLAY_FIXTURE=/absolute/path/display-fixture \
+SCREENPIPE_E2E_DISPLAY_ORIGINS=0,0,1024,400 \
+./a11y-tests --ignored live_recorder_keeps --nocapture
+# Repeat with filter live_recorder_does_not to verify excluded-app handling.
+```
+
+Choose the origins from the actual topology; do not assume the example fits
+another guest. The recorder test verifies actual AX window positions and distinct
+native display membership. It explicitly controls synthetic modifiers and bounds
+helper execution. Native input acceptance still allows 150ms observer settling.
+
+Stop and reap the fixture process and verify only the base display remains.
+Destroy the disposable guest and task-owned tunnels after collecting evidence.
+These tests do not establish physical cable/dock behavior, persisted-frame app
+attribution, packaged UI warnings or the customer's exact hardware incident.
diff --git a/scripts/native-test-fixtures/macos/display-fixture.mm b/scripts/native-test-fixtures/macos/display-fixture.mm
new file mode 100644
index 000000000..77e314376
--- /dev/null
+++ b/scripts/native-test-fixtures/macos/display-fixture.mm
@@ -0,0 +1,182 @@
+// screenpipe — AI that knows everything you have seen, said, or heard
+// https://screenpipe.com
+// Test-only private API declarations from Chromium, BSD-3-Clause.
+// https://chromium.googlesource.com/chromium/src/+/HEAD/ui/display/mac/test/virtual_display_util_mac.mm
+#import <Cocoa/Cocoa.h>
+#import <CoreGraphics/CoreGraphics.h>
+@interface CGVirtualDisplay : NSObject
+
+@property(readonly, nonatomic) unsigned int vendorID;
+@property(readonly, nonatomic) unsigned int productID;
+@property(readonly, nonatomic) unsigned int serialNum;
+@property(readonly, nonatomic) NSString* name;
+@property(readonly, nonatomic) struct CGSize sizeInMillimeters;
+@property(readonly, nonatomic) unsigned int maxPixelsWide;
+@property(readonly, nonatomic) unsigned int maxPixelsHigh;
+@property(readonly, nonatomic) struct CGPoint redPrimary;
+@property(readonly, nonatomic) struct CGPoint greenPrimary;
+@property(readonly, nonatomic) struct CGPoint bluePrimary;
+@property(readonly, nonatomic) struct CGPoint whitePoint;
+@property(readonly, nonatomic) id queue;
+@property(readonly, nonatomic) id terminationHandler;
+@property(readonly, nonatomic) unsigned int displayID;
+@property(readonly, nonatomic) unsigned int hiDPI;
+@property(readonly, nonatomic) NSArray* modes;
+@property(readonly, nonatomic) unsigned int serialNumber;
+@property(readonly, nonatomic) unsigned int rotation;
+- (BOOL)applySettings:(id)arg1;
+- (void)dealloc;
+- (id)initWithDescriptor:(id)arg1;
+
+@end
+
+// These interfaces were generated from CoreGraphics binaries.
+@interface CGVirtualDisplayDescriptor : NSObject
+
+@property(nonatomic) unsigned int vendorID;
+@property(nonatomic) unsigned int productID;
+@property(nonatomic) unsigned int serialNum;
+@property(strong, nonatomic) NSString* name;
+@property(nonatomic) struct CGSize sizeInMillimeters;
+@property(nonatomic) unsigned int maxPixelsWide;
+@property(nonatomic) unsigned int maxPixelsHigh;
+@property(nonatomic) struct CGPoint redPrimary;
+@property(nonatomic) struct CGPoint greenPrimary;
+@property(nonatomic) struct CGPoint bluePrimary;
+@property(nonatomic) struct CGPoint whitePoint;
+@property(strong, nonatomic) id queue;
+@property(copy, nonatomic) id terminationHandler;
+@property(nonatomic) unsigned int serialNumber;
+- (void)dealloc;
+- (id)init;
+- (id)dispatchQueue;
+- (void)setDispatchQueue:(id)arg1;
+
+@end
+
+// These interfaces were generated from CoreGraphics binaries.
+@interface CGVirtualDisplayMode : NSObject
+
+@property(readonly, nonatomic) unsigned int width;
+@property(readonly, nonatomic) unsigned int height;
+@property(readonly, nonatomic) double refreshRate;
+@property(copy, nonatomic) id transferFunction API_AVAILABLE(macos(13.3));
+- (id)initWithWidth:(unsigned int)arg1
+ height:(unsigned int)arg2
+ refreshRate:(double)arg3;
+- (id)initWithWidth:(unsigned int)arg1
+ height:(unsigned int)arg2
+ refreshRate:(double)arg3
+ transferFunction:(id)arg4 API_AVAILABLE(macos(13.3));
+
+@end
+
+// These interfaces were generated from CoreGraphics binaries.
+@interface CGVirtualDisplaySettings : NSObject
+
+@property(strong, nonatomic) NSArray* modes;
+@property(nonatomic) unsigned int hiDPI;
+@property(nonatomic) unsigned int rotation;
+- (void)dealloc;
+- (id)init;
+
+@end
+
+// Copyright 2022 The Chromium Authors (private API declarations above).
+// BSD-3-Clause: https://chromium.googlesource.com/chromium/src/+/HEAD/LICENSE
+static CGVirtualDisplay *extra;
+static NSMutableDictionary<NSNumber *, NSWindow *> *markers;
+static unsigned serial = 20261017;
+static void addDisplay(BOOL retina) {
+ CGVirtualDisplayDescriptor *d = [CGVirtualDisplayDescriptor new];
+ d.name = @"Screenpipe disposable acceptance display";
+ d.vendorID = 505; d.productID = 1; d.serialNum = ++serial; d.serialNumber = serial;
+ d.maxPixelsWide = 2560; d.maxPixelsHigh = 1440;
+ d.sizeInMillimeters = CGSizeMake(340, 190);
+ d.whitePoint = CGPointMake(0.3125,0.3291);
+ d.redPrimary = CGPointMake(0.64,0.33); d.greenPrimary = CGPointMake(0.3,0.6); d.bluePrimary = CGPointMake(0.15,0.06);
+ d.queue = dispatch_get_global_queue(QOS_CLASS_USER_INITIATED,0);
+ extra = [[CGVirtualDisplay alloc] initWithDescriptor:d];
+ if (!extra) { fprintf(stderr,"virtual display creation failed\n"); exit(2); }
+ CGVirtualDisplaySettings *settings = [CGVirtualDisplaySettings new];
+ settings.hiDPI = retina;
+ settings.modes = @[[[CGVirtualDisplayMode alloc] initWithWidth:1280 height:720 refreshRate:60]];
+ if (![extra applySettings:settings]) exit(3);
+}
+static void settle(void) {
+ NSDate *end=[NSDate dateWithTimeIntervalSinceNow:0.6];
+ while ([end timeIntervalSinceNow]>0) [[NSRunLoop currentRunLoop] runUntilDate:[NSDate dateWithTimeIntervalSinceNow:0.05]];
+}
+static void report(void) {
+ CGDirectDisplayID ids[16]; uint32_t count=0;
+ CGGetOnlineDisplayList(16,ids,&count);
+ NSMutableArray *items=[NSMutableArray new];
+ if (!markers) markers=[NSMutableDictionary new];
+ NSMutableSet *live=[NSMutableSet new];
+ CGFloat mainHeight=CGDisplayBounds(CGMainDisplayID()).size.height;
+ for (uint32_t i=0;i<count;i++) {
+ CGRect r=CGDisplayBounds(ids[i]);
+ NSNumber *key=@(ids[i]);[live addObject:key];
+ NSWindow *marker=markers[key];
+ if(!marker) {
+ marker=[[NSWindow alloc] initWithContentRect:NSMakeRect(0,0,200,200) styleMask:NSWindowStyleMaskBorderless backing:NSBackingStoreBuffered defer:NO];
+ marker.level=NSFloatingWindowLevel;marker.ignoresMouseEvents=YES;marker.releasedWhenClosed=NO;
+ marker.backgroundColor=CGDisplayIsMain(ids[i]) ? [NSColor colorWithRed:0.9 green:0.1 blue:0.1 alpha:1] : [NSColor colorWithRed:0.1 green:0.1 blue:0.9 alpha:1];
+ markers[key]=marker;
+ }
+ [marker setFrame:NSMakeRect(r.origin.x+20,mainHeight-r.origin.y-220,200,200) display:YES];[marker orderFrontRegardless];
+ CGDisplayModeRef mode=CGDisplayCopyDisplayMode(ids[i]);
+ [items addObject:@{@"id":@(ids[i]),@"x":@(r.origin.x),@"y":@(r.origin.y),@"w":@(r.size.width),@"h":@(r.size.height),@"pixels_w":@(CGDisplayModeGetPixelWidth(mode)),@"pixels_h":@(CGDisplayModeGetPixelHeight(mode)),@"main":@(CGDisplayIsMain(ids[i])),@"mirror":@(CGDisplayMirrorsDisplay(ids[i]))}];
+ CGDisplayModeRelease(mode);
+ }
+ for(NSNumber *key in [markers.allKeys copy]) if(![live containsObject:key]) { [markers[key] close];[markers removeObjectForKey:key]; }
+ CGEventRef ev=CGEventCreate(NULL); CGPoint p=CGEventGetLocation(ev); CFRelease(ev);
+ NSData *json=[NSJSONSerialization dataWithJSONObject:@{@"displays":items,@"extra":@(extra ? extra.displayID:0),@"cursor_x":@(p.x),@"cursor_y":@(p.y)} options:0 error:nil];
+ fwrite(json.bytes,1,json.length,stdout);puts("");fflush(stdout);
+}
+static int placeWindow(pid_t pid,double x,double y) {
+ CGDirectDisplayID target=0;uint32_t targets=0;
+ CGGetDisplaysWithPoint(CGPointMake(x,y),1,&target,&targets);
+ if(targets!=1 || !CGRectContainsRect(CGDisplayBounds(target),CGRectMake(x,y,600,400))) return 22;
+ AXUIElementRef app=AXUIElementCreateApplication(pid);AXUIElementSetMessagingTimeout(app,0.3);CFTypeRef raw=NULL;
+ AXError e=AXUIElementCopyAttributeValue(app,kAXFocusedWindowAttribute,&raw);
+ if(e || !raw) { CFRelease(app);return 20; }
+ CGPoint position=CGPointMake(x,y);CGSize size=CGSizeMake(600,400);
+ AXValueRef pos=AXValueCreate((AXValueType)kAXValueCGPointType,&position), dim=AXValueCreate((AXValueType)kAXValueCGSizeType,&size);
+ AXUIElementSetAttributeValue((AXUIElementRef)raw,kAXSizeAttribute,dim);
+ e=AXUIElementSetAttributeValue((AXUIElementRef)raw,kAXPositionAttribute,pos);
+ CFRelease(pos);CFRelease(dim);CFRelease(raw);CFRelease(app);
+ if(!e) printf("%u\n",target);
+ return e ? 21 : 0;
+}
+int main(int argc,char **argv) { @autoreleasepool {
+ const char *opt=getenv("SCREENPIPE_ISOLATED_VM_TEST");
+ if (!opt || strcmp(opt,"1")!=0) return 10;
+ [NSApplication sharedApplication];
+ [NSApp setActivationPolicy:NSApplicationActivationPolicyAccessory];
+ if(argc==2 && strcmp(argv[1],"--list-only")==0) {
+ CGDirectDisplayID ids[16];uint32_t count=0;CGGetOnlineDisplayList(16,ids,&count);
+ printf("{\"count\":%u}\n",count);return 0;
+ }
+ if(argc==5 && strcmp(argv[1],"place-window")==0) return placeWindow(atoi(argv[2]),atof(argv[3]),atof(argv[4]));
+ if(argc==4 && strcmp(argv[1],"warp-to")==0) return CGWarpMouseCursorPosition(CGPointMake(atof(argv[2]),atof(argv[3])));
+ addDisplay(NO); settle(); report();
+ char *line=NULL; size_t length=0;
+ while (getline(&line,&length,stdin)>0) { @autoreleasepool {
+ double x=0,y=0;
+ if (sscanf(line,"layout %lf %lf",&x,&y)==2) {
+ CGDisplayConfigRef cfg; CGBeginDisplayConfiguration(&cfg);
+ CGConfigureDisplayOrigin(cfg,extra.displayID,(int32_t)x,(int32_t)y);
+ CGError e=CGCompleteDisplayConfiguration(cfg,kCGConfigureForSession);
+ if(e) fprintf(stderr,"layout error %d\n",e);
+ } else if(sscanf(line,"warp %lf %lf",&x,&y)==2) {
+ CGError e=CGWarpMouseCursorPosition(CGPointMake(x,y));
+ if(e) fprintf(stderr,"warp error %d\n",e);
+ } else if(strncmp(line,"remove",6)==0) { extra=nil;
+ } else if(strncmp(line,"add-retina",10)==0) { extra=nil;settle();addDisplay(YES);
+ } else if(strncmp(line,"add",3)==0) { extra=nil;settle();addDisplay(NO);
+ } else if(strncmp(line,"quit",4)==0) { break; }
+ settle();report();
+ }}
+ free(line);extra=nil;return 0;
+}}