Run a virtual iPhone on an Apple Silicon Mac.
Swift
feat: expose native AX parent-child hierarchy snapshots
Yanni Pangc3d81c7ca2c0
diff --git a/Research/0_binary_patch_comparison.md b/Research/0_binary_patch_comparison.mdindex 8315ca8a..2c12cffc 100644--- a/Research/0_binary_patch_comparison.md+++ b/Research/0_binary_patch_comparison.md@@ -2122,3 +2122,25 @@ do not establish fresh-client readiness. The three keys were testedtogether; each key's individual necessity was not isolated. Upstream2.2.3 has passed the profile-preservation assertion but has not beenlive-tested with this change.++## Opt-in native accessibility hierarchy (2026-09-30)++The local native bridge behind ui.tree with nested:true follows private iOS+attribute 5001 (immediate children) and checks each edge against attribute 5002+(parent) using CFEqual identity. The existing flat interface continues to use+icli's visible-element query; this bridge does not infer ancestry from frames,+labels, identifiers, or the visible-element array. Primary attribute mappings+are in [MCPAXNodeSource.m](https://github.com/witchan/ios-mcp/blob/8f46b68/MCPAXNodeSource.m)+and [MCPAXAttributeBridge.m](https://github.com/witchan/ios-mcp/blob/8f46b68/MCPAXAttributeBridge.m).++An isolated entitled helper on pcc-research-01, iOS 27, returned 176 Settings+nodes with 173 matching parent back-references and 3 mismatches, 44 Calculator+nodes with 42 matches and 1 mismatch, and 66 Clock nodes with 65 matches and+no mismatch. Settings and Calculator are explicitly partial. These counts+include unnamed, offscreen, zero-frame, and frameless structure; repeated+objects use snapshot-local references. Node, depth, query, and deadline limits+are explicit, and a missing or failed messaging timeout blocks the AX query.+Foreground PID stayed unchanged and accessibility switches were restored for+each helper run. Temporary services were unloaded and their directories removed;+pcc-research-02 was untouched. Helper evidence is not an integrated daemon+release or cold-start readiness claim.diff --git a/VPhoneDaemon/Daemon/GuestAPI+Hierarchy.swift b/VPhoneDaemon/Daemon/GuestAPI+Hierarchy.swiftnew file mode 100644index 00000000..1ad9f7fe--- /dev/null+++ b/VPhoneDaemon/Daemon/GuestAPI+Hierarchy.swift@@ -0,0 +1,60 @@+import CoreFoundation+import Foundation+import IcliKit+import VphonedNative++extension GuestAPI {+ // AX enable/restore is process-wide. Interface operations share this lock,+ // so a flat query cannot race this opt-in native snapshot's restoration.+ static let interfaceLock = NSRecursiveLock()++ private static func hierarchyInteger(+ _ params: [String: Any], _ key: String, default fallback: Int, range: ClosedRange<Int>,+ ) throws -> Int {+ guard let value = params[key] else { return fallback }+ guard let number = value as? NSNumber,+ CFGetTypeID(number) != CFBooleanGetTypeID(),+ number.doubleValue.isFinite,+ number.doubleValue.rounded(.towardZero) == number.doubleValue,+ range.contains(number.intValue)+ else { throw GuestAPIError.invalidRequest("\(key) must be an integer in \(range)") }+ return number.intValue+ }++ private static func hierarchyForeground() throws -> [String: Any] {+ let front = frontmostApp()+ guard front["verified"] as? Bool == true,+ let bundle = front["bundle_id"] as? String, !bundle.isEmpty,+ let running = try runningApps()["apps"] as? [[String: Any]],+ let pid = running.first(where: { $0["bundle_id"] as? String == bundle })?["pid"] as? Int,+ pid > 1, pid <= Int(Int32.max)+ else { throw GuestAPIError.operationFailed("Cannot verify foreground application and PID") }+ return ["bundle_id": bundle, "pid": pid, "verified": true, "source": front["source"] ?? ""]+ }++ static func nestedHierarchy(_ params: [String: Any]) throws -> [String: Any] {+ guard params["limit"] == nil,+ !bool(params, "clickable_only"), params["visible_only"] == nil+ else {+ throw GuestAPIError.invalidRequest("Nested snapshots preserve all structural nodes; filtering and limit are unsupported")+ }+ let count = try hierarchyInteger(params, "max_elements", default: 500, range: 1 ... 2000)+ let depth = try hierarchyInteger(params, "max_depth", default: 32, range: 1 ... 32)+ let duration = try hierarchyInteger(params, "timeout_ms", default: 5000, range: 100 ... 10000)+ let before = try hierarchyForeground()+ let pid = before["pid"] as! Int+ let native = vp_ax_hierarchy(Int32(pid), Int32(count), Int32(depth), Int32(duration))+ guard let report = native as? [String: Any] else {+ throw GuestAPIError.operationFailed("Native hierarchy returned no diagnostic snapshot")+ }+ let after = try hierarchyForeground()+ guard NSDictionary(dictionary: before).isEqual(to: after) else {+ throw GuestAPIError.operationFailed("Foreground application changed during hierarchy snapshot")+ }+ return report.merging([+ "foreground_before": before, "foreground_after": after,+ "action_eligible": false, "filtering_applied": false,+ "runtime": ["daemon_binary_hash": binaryHash, "pid": ProcessInfo.processInfo.processIdentifier],+ ], uniquingKeysWith: { _, value in value })+ }+}diff --git a/VPhoneDaemon/Daemon/GuestAPI+Input.swift b/VPhoneDaemon/Daemon/GuestAPI+Input.swiftindex 8badfe21..5f097ad0 100644--- a/VPhoneDaemon/Daemon/GuestAPI+Input.swift+++ b/VPhoneDaemon/Daemon/GuestAPI+Input.swift@@ -1,3 +1,4 @@+import CoreFoundationimport Foundationimport IcliKit@@ -67,33 +68,41 @@ extension GuestAPI {// MARK: - Accessibility and OCRstatic func executeInterface(_ method: String, _ params: [String: Any]) throws -> [String: Any]? {+ interfaceLock.lock()+ defer { interfaceLock.unlock() }switch method {case "ui.tree", "accessibility.tree":- try uiElements(+ if let value = params["nested"] {+ guard let number = value as? NSNumber, CFGetTypeID(number) == CFBooleanGetTypeID() else {+ throw GuestAPIError.invalidRequest("nested must be a boolean")+ }+ }+ if bool(params, "nested") { return try nestedHierarchy(params) }+ return try uiElements(maxElements: (params["max_elements"] as? NSNumber)?.intValue ?? 500,visibleOnly: bool(params, "visible_only", default: true),clickableOnly: bool(params, "clickable_only"),limit: (params["limit"] as? NSNumber)?.intValue,)case "ui.element_at":- try elementAt(x: requiredNumber(params, "x"), y: requiredNumber(params, "y"))+ return try elementAt(x: requiredNumber(params, "x"), y: requiredNumber(params, "y"))case "ui.tap_element":- try tapElement(selector(params))+ return try tapElement(selector(params))case "ui.wait", "ui.wait_gone":- try waitForElement(+ return try waitForElement(selector(params),appear: method == "ui.wait",timeout: number(params, "timeout", default: 10),)case "ui.ocr":- try recognizeScreen(+ return try recognizeScreen(languages: params["languages"] as? [String] ?? ["en-US"],minConfidence: Float(number(params, "min_confidence", default: 0.3)),)case "ui.describe":- try describeScreen()+ return try describeScreen()default:- nil+ return nil}}diff --git a/VPhoneDaemon/Native/Include/VphonedNative.h b/VPhoneDaemon/Native/Include/VphonedNative.hindex 512cc617..b28566b1 100644--- a/VPhoneDaemon/Native/Include/VphonedNative.h+++ b/VPhoneDaemon/Native/Include/VphonedNative.h@@ -47,3 +47,8 @@ char *vp_usb_set_serial(const char *serial, bool force, bool *changed);/// The guest's own USB serial, built from `/chosen` `chip-id` and/// `unique-chip-id`. malloc-owned, or NULL when the device tree lacks either.char *vp_usb_own_serial(void);++#ifdef __OBJC__+/// Snapshot-local nested accessibility tree, using actual private iOS child links.+NSDictionary *vp_ax_hierarchy(int pid, int maxElements, int maxDepth, int timeoutMS);+#endifdiff --git a/VPhoneDaemon/Native/vphoned_ax_hierarchy.h b/VPhoneDaemon/Native/vphoned_ax_hierarchy.hnew file mode 100644index 00000000..3927b2f3--- /dev/null+++ b/VPhoneDaemon/Native/vphoned_ax_hierarchy.h@@ -0,0 +1,12 @@+#pragma once+#import <Foundation/Foundation.h>+typedef struct {+ CFTypeRef (*createApp)(pid_t);+ int (*copy)(CFTypeRef,CFStringRef,CFTypeRef *);+ int (*setTimeout)(CFTypeRef,float);+ CFTypeID (*elementTypeID)(void);+ CFTypeID (*valueTypeID)(void);+ Boolean (*getValue)(CFTypeRef,int,void *);+} VPAxFunctions;+NSDictionary *vp_ax_walk(VPAxFunctions functions, CFTypeRef root, pid_t pid, int maxElements, int maxDepth, int timeoutMS);+NSDictionary *vp_ax_hierarchy(int pid, int maxElements, int maxDepth, int timeoutMS);diff --git a/VPhoneDaemon/Native/vphoned_ax_hierarchy.m b/VPhoneDaemon/Native/vphoned_ax_hierarchy.mnew file mode 100644index 00000000..cee479a9--- /dev/null+++ b/VPhoneDaemon/Native/vphoned_ax_hierarchy.m@@ -0,0 +1,168 @@+#import "vphoned_ax_hierarchy.h"+#import <CoreGraphics/CoreGraphics.h>+#import <dlfcn.h>+#import <math.h>+#import <time.h>++// These private iOS attribute numbers are NOT macOS AX string attributes.+// Primary mapping: witchan/ios-mcp, commit 8f46b68, MCPAXNodeSource.m.+// Only 5001 supplies immediate children; visible/explorer arrays are never roots.+static double milliseconds(void) {+ struct timespec value; clock_gettime(CLOCK_MONOTONIC,&value);+ return value.tv_sec*1000.0+value.tv_nsec/1e6;+}+@interface VPAxWalker : NSObject {+@public+ VPAxFunctions f;+ int maxElements,maxDepth,maxQueries,timeoutMS,queries,verified,mismatch,unavailable;+ double start;+ NSMutableArray *seen;+ NSMutableSet *active,*reasons;+ BOOL incomplete;+}+- (NSDictionary *)node:(CFTypeRef)element parent:(CFTypeRef)parent depth:(int)depth;+@end+@implementation VPAxWalker+- (BOOL)budget {+ if(queries>=maxQueries) {[reasons addObject:@"max_queries"];return NO;}+ if(milliseconds()-start>=timeoutMS) {[reasons addObject:@"timeout"];return NO;}+ return YES;+}+- (id)attribute:(uint32_t)attribute element:(CFTypeRef)element errors:(NSMutableArray *)errors {+ if(![self budget]) return nil;+ double remaining=timeoutMS-(milliseconds()-start);+ int timeoutError=f.setTimeout?f.setTimeout(element,(float)(MIN(100.0,remaining)/1000.0)):-25208;+ if(timeoutError) {+ [errors addObject:@{@"attribute":@(attribute),@"error_code":@(timeoutError),@"value_present":@NO,@"timeout_configuration_failed":@YES}];+ incomplete=YES; return nil;+ }+ CFTypeRef value=NULL; ++queries;+ int error=f.copy(element,(CFStringRef)(uintptr_t)attribute,&value);+ if(error || !value) {+ [errors addObject:@{@"attribute":@(attribute),@"error_code":@(error),@"value_present":@(value!=NULL)}];+ if(value) CFRelease(value);+ return nil;+ }+ return CFBridgingRelease(value);+}+- (NSString *)text:(id)value {+ if(!value || CFGetTypeID((__bridge CFTypeRef)value)!=CFStringGetTypeID())return @"";+ NSString *string=value;+ return string.length>1024?[string substringToIndex:1024]:string;+}+- (id)frame:(id)value {+ if(!value || !f.valueTypeID || !f.getValue || CFGetTypeID((__bridge CFTypeRef)value)!=f.valueTypeID()) return NSNull.null;+ CGRect rect;+ if(!f.getValue((__bridge CFTypeRef)value,3,&rect) || !isfinite(rect.origin.x) || !isfinite(rect.origin.y) ||+ !isfinite(rect.size.width) || !isfinite(rect.size.height)) return NSNull.null;+ return @{@"x":@(rect.origin.x),@"y":@(rect.origin.y),@"width":@(rect.size.width),@"height":@(rect.size.height)};+}+- (id)parentProof:(CFTypeRef)element parent:(CFTypeRef)parent errors:(NSMutableArray *)errors {+ if(!parent)return NSNull.null;+ id actual=[self attribute:5002 element:element errors:errors];+ if(!actual || CFGetTypeID((__bridge CFTypeRef)actual)!=f.elementTypeID()) {++unavailable;incomplete=YES;return @NO;}+ BOOL matches=CFEqual((__bridge CFTypeRef)actual,parent);+ if(matches) ++verified; else {++mismatch;incomplete=YES;}+ return @(matches);+}+- (NSDictionary *)node:(CFTypeRef)element parent:(CFTypeRef)parent depth:(int)depth {+ if(![self budget]) return nil;+ NSUInteger existing=NSNotFound;+ for(NSUInteger index=0;index<seen.count;++index) if(CFEqual(element,(__bridge CFTypeRef)seen[index])) {existing=index;break;}+ if(existing!=NSNotFound) {+ NSMutableArray *errors=[NSMutableArray array];+ id proof=[self parentProof:element parent:parent errors:errors];+ NSString *reference=[NSString stringWithFormat:@"ax-%lu",(unsigned long)existing];+ BOOL cycle=[active containsObject:reference]; if(cycle) incomplete=YES;+ return @{@"ref":reference,@"cycle":@(cycle),@"parent_verified":proof,@"query_errors":errors.copy};+ }+ if(seen.count>=(NSUInteger)maxElements) {[reasons addObject:@"max_elements"];return nil;}+ NSString *identity=[NSString stringWithFormat:@"ax-%lu",(unsigned long)seen.count];+ [seen addObject:(__bridge id)element];[active addObject:identity];+ NSMutableArray *errors=[NSMutableArray array];+ id proof=[self parentProof:element parent:parent errors:errors];+ NSString *label=[self text:[self attribute:2001 element:element errors:errors]];+ NSString *identifier=[self text:[self attribute:5019 element:element errors:errors]];+ id type=[self attribute:5003 element:element errors:errors];+ NSString *role=[self text:type];+ id frame=[self frame:[self attribute:2003 element:element errors:errors]];+ id raw=[self attribute:5001 element:element errors:errors];+ NSMutableArray *children=[NSMutableArray array];+ NSString *status=@"complete";+ if(!raw || CFGetTypeID((__bridge CFTypeRef)raw)!=CFArrayGetTypeID()) {status=@"unavailable";incomplete=YES;+ if(raw) [errors addObject:@{@"attribute":@5001,@"error_code":@0,@"value_present":@YES,@"invalid_type":@YES}];}+ else {+ CFArrayRef array=(__bridge CFArrayRef)raw;+ CFIndex count=CFArrayGetCount(array);+ if(depth>=maxDepth && count) {[reasons addObject:@"max_depth"];status=@"truncated";}+ else for(CFIndex i=0;i<count;++i) {+ if(![self budget]) {status=@"truncated";break;}+ CFTypeRef child=CFArrayGetValueAtIndex(array,i);+ if(!child || CFGetTypeID(child)!=f.elementTypeID()) {status=@"unavailable";incomplete=YES;+ [errors addObject:@{@"attribute":@5001,@"error_code":@0,@"value_present":@(child!=NULL),@"invalid_child_type":@YES}];continue;}+ NSDictionary *node=[self node:child parent:element depth:depth+1];+ if(!node) {status=@"truncated";break;}+ [children addObject:node];+ }+ }+ if(![self budget] && [status isEqual:@"unavailable"]) status=@"truncated";+ [active removeObject:identity];+ return @{@"id":identity,@"label":label,@"identifier":identifier,@"role":role,@"frame":frame,+ @"children":children.copy,@"children_status":status,@"query_errors":errors.copy,@"parent_verified":proof,@"depth":@(depth)};+}+@end++NSDictionary *vp_ax_walk(VPAxFunctions functions, CFTypeRef root, pid_t pid, int elements, int depth, int duration) {+ VPAxWalker *walker=[VPAxWalker new];walker->f=functions;+ walker->maxElements=MAX(1,MIN(elements,2000));walker->maxDepth=MAX(0,MIN(depth,32));+ walker->timeoutMS=MAX(100,MIN(duration,10000));walker->maxQueries=walker->maxElements*8;+ walker->seen=[NSMutableArray array];walker->active=[NSMutableSet set];walker->reasons=[NSMutableSet set];walker->start=milliseconds();+ NSDictionary *node=root && functions.copy && functions.elementTypeID?[walker node:root parent:NULL depth:0]:nil;+ BOOL truncated=walker->reasons.count>0;+ NSString *status=!node?@"unavailable":truncated || walker->incomplete?@"partial":@"complete";+ return @{@"source":@"ax",@"format":@"nested",@"relationship_source":@"AXUIElementCopyAttributeValue:5001",+ @"pid":@(pid),@"roots":node?@[node]:@[],@"count":@(walker->seen.count),@"status":status,+ @"truncated":@(truncated),@"truncation_reasons":[walker->reasons.allObjects sortedArrayUsingSelector:@selector(compare:)],+ @"limits":@{@"max_elements":@(walker->maxElements),@"max_depth":@(walker->maxDepth),@"max_queries":@(walker->maxQueries),@"timeout_ms":@(walker->timeoutMS)},+ @"queries":@(walker->queries),@"elapsed_ms":@(milliseconds()-walker->start),+ @"parent_checks":@{@"verified":@(walker->verified),@"mismatch":@(walker->mismatch),@"unavailable":@(walker->unavailable)}};+}+NSDictionary *vp_ax_hierarchy(int pid,int maxElements,int maxDepth,int timeoutMS) {+ void *ax=dlopen("/System/Library/PrivateFrameworks/AXRuntime.framework/AXRuntime",RTLD_NOW);+ VPAxFunctions f={0};+#define LOAD(M,N) f.M=(typeof(f.M))(ax?dlsym(ax,N):NULL)+ LOAD(createApp,"_AXUIElementCreateAppElementWithPid");LOAD(copy,"AXUIElementCopyAttributeValue");+ LOAD(setTimeout,"AXUIElementSetMessagingTimeout");LOAD(elementTypeID,"AXUIElementGetTypeID");+ LOAD(valueTypeID,"AXValueGetTypeID");LOAD(getValue,"AXValueGetValue");+#undef LOAD+ CFTypeRef root=f.createApp && pid>1?f.createApp(pid):NULL;+ void *library=dlopen("/usr/lib/libAccessibility.dylib",RTLD_NOW);+ Boolean (*appEnabled)(void)=library?dlsym(library,"_AXSApplicationAccessibilityEnabled"):NULL;+ void (*setApp)(Boolean)=library?dlsym(library,"_AXSApplicationAccessibilitySetEnabled"):NULL;+ Boolean (*autoEnabled)(void)=library?dlsym(library,"_AXSAutomationEnabled"):NULL;+ void (*setAuto)(Boolean)=library?dlsym(library,"_AXSSetAutomationEnabled"):NULL;+ void (*client)(uint32_t)=dlsym(RTLD_DEFAULT,"__AXSetRequestingClient");+ uint64_t (*override)(uint64_t)=dlsym(RTLD_DEFAULT,"_AXOverrideRequestingClientType");+ if(client) client(2); if(override) override(2);+ BOOL beforeApp=appEnabled?appEnabled():NO,beforeAuto=autoEnabled?autoEnabled():NO;+ BOOL changedApp=appEnabled && setApp && !beforeApp, changedAuto=autoEnabled && setAuto && !beforeAuto;+ NSDictionary *result=nil;+ @try {+ if(changedApp) setApp(true); if(changedAuto) setAuto(true);+ result=vp_ax_walk(f,root,pid,maxElements,maxDepth,timeoutMS);+ } @catch(NSException *exception) {+ NSMutableDictionary *failure=[vp_ax_walk(f,NULL,pid,maxElements,maxDepth,timeoutMS) mutableCopy];+ failure[@"native_exception"]=exception.name; result=failure.copy;+ } @finally {+ if(changedAuto)setAuto(false);if(changedApp)setApp(false);+ }+ NSMutableDictionary *report=[result mutableCopy];+ report[@"symbols"]=@{@"create_application":@(f.createApp!=NULL),@"copy_attribute":@(f.copy!=NULL),+ @"set_timeout":@(f.setTimeout!=NULL),@"element_type_id":@(f.elementTypeID!=NULL),+ @"requesting_client":@(client!=NULL),@"requesting_client_override":@(override!=NULL)};+ report[@"switches_before"]=@{@"application":appEnabled?@(beforeApp):NSNull.null,@"automation":autoEnabled?@(beforeAuto):NSNull.null};+ report[@"switches_after_restore"]=@{@"application":appEnabled?@(appEnabled()):NSNull.null,@"automation":autoEnabled?@(autoEnabled()):NSNull.null};+ result=report.copy;+ if(root) CFRelease(root);+ return result;+}