Skip to content

Run a virtual iPhone on an Apple Silicon Mac.

Swift

feat: expose native AX parent-child hierarchy snapshots

Yanni Pangc3d81c7ca2c0
diff --git a/Research/0_binary_patch_comparison.md b/Research/0_binary_patch_comparison.md
index 8315ca8a..2c12cffc 100644
--- a/Research/0_binary_patch_comparison.md
+++ b/Research/0_binary_patch_comparison.md
@@ -2122,3 +2122,25 @@ do not establish fresh-client readiness. The three keys were tested
together; each key's individual necessity was not isolated. Upstream
2.2.3 has passed the profile-preservation assertion but has not been
live-tested with this change.
+
+## Opt-in native accessibility hierarchy (2026-09-30)
+
+The local native bridge behind ui.tree with nested:true follows private iOS
+attribute 5001 (immediate children) and checks each edge against attribute 5002
+(parent) using CFEqual identity. The existing flat interface continues to use
+icli's visible-element query; this bridge does not infer ancestry from frames,
+labels, identifiers, or the visible-element array. Primary attribute mappings
+are in [MCPAXNodeSource.m](https://github.com/witchan/ios-mcp/blob/8f46b68/MCPAXNodeSource.m)
+and [MCPAXAttributeBridge.m](https://github.com/witchan/ios-mcp/blob/8f46b68/MCPAXAttributeBridge.m).
+
+An isolated entitled helper on pcc-research-01, iOS 27, returned 176 Settings
+nodes with 173 matching parent back-references and 3 mismatches, 44 Calculator
+nodes with 42 matches and 1 mismatch, and 66 Clock nodes with 65 matches and
+no mismatch. Settings and Calculator are explicitly partial. These counts
+include unnamed, offscreen, zero-frame, and frameless structure; repeated
+objects use snapshot-local references. Node, depth, query, and deadline limits
+are explicit, and a missing or failed messaging timeout blocks the AX query.
+Foreground PID stayed unchanged and accessibility switches were restored for
+each helper run. Temporary services were unloaded and their directories removed;
+pcc-research-02 was untouched. Helper evidence is not an integrated daemon
+release or cold-start readiness claim.
diff --git a/VPhoneDaemon/Daemon/GuestAPI+Hierarchy.swift b/VPhoneDaemon/Daemon/GuestAPI+Hierarchy.swift
new file mode 100644
index 00000000..1ad9f7fe
--- /dev/null
+++ b/VPhoneDaemon/Daemon/GuestAPI+Hierarchy.swift
@@ -0,0 +1,60 @@
+import CoreFoundation
+import Foundation
+import IcliKit
+import VphonedNative
+
+extension GuestAPI {
+ // AX enable/restore is process-wide. Interface operations share this lock,
+ // so a flat query cannot race this opt-in native snapshot's restoration.
+ static let interfaceLock = NSRecursiveLock()
+
+ private static func hierarchyInteger(
+ _ params: [String: Any], _ key: String, default fallback: Int, range: ClosedRange<Int>,
+ ) throws -> Int {
+ guard let value = params[key] else { return fallback }
+ guard let number = value as? NSNumber,
+ CFGetTypeID(number) != CFBooleanGetTypeID(),
+ number.doubleValue.isFinite,
+ number.doubleValue.rounded(.towardZero) == number.doubleValue,
+ range.contains(number.intValue)
+ else { throw GuestAPIError.invalidRequest("\(key) must be an integer in \(range)") }
+ return number.intValue
+ }
+
+ private static func hierarchyForeground() throws -> [String: Any] {
+ let front = frontmostApp()
+ guard front["verified"] as? Bool == true,
+ let bundle = front["bundle_id"] as? String, !bundle.isEmpty,
+ let running = try runningApps()["apps"] as? [[String: Any]],
+ let pid = running.first(where: { $0["bundle_id"] as? String == bundle })?["pid"] as? Int,
+ pid > 1, pid <= Int(Int32.max)
+ else { throw GuestAPIError.operationFailed("Cannot verify foreground application and PID") }
+ return ["bundle_id": bundle, "pid": pid, "verified": true, "source": front["source"] ?? ""]
+ }
+
+ static func nestedHierarchy(_ params: [String: Any]) throws -> [String: Any] {
+ guard params["limit"] == nil,
+ !bool(params, "clickable_only"), params["visible_only"] == nil
+ else {
+ throw GuestAPIError.invalidRequest("Nested snapshots preserve all structural nodes; filtering and limit are unsupported")
+ }
+ let count = try hierarchyInteger(params, "max_elements", default: 500, range: 1 ... 2000)
+ let depth = try hierarchyInteger(params, "max_depth", default: 32, range: 1 ... 32)
+ let duration = try hierarchyInteger(params, "timeout_ms", default: 5000, range: 100 ... 10000)
+ let before = try hierarchyForeground()
+ let pid = before["pid"] as! Int
+ let native = vp_ax_hierarchy(Int32(pid), Int32(count), Int32(depth), Int32(duration))
+ guard let report = native as? [String: Any] else {
+ throw GuestAPIError.operationFailed("Native hierarchy returned no diagnostic snapshot")
+ }
+ let after = try hierarchyForeground()
+ guard NSDictionary(dictionary: before).isEqual(to: after) else {
+ throw GuestAPIError.operationFailed("Foreground application changed during hierarchy snapshot")
+ }
+ return report.merging([
+ "foreground_before": before, "foreground_after": after,
+ "action_eligible": false, "filtering_applied": false,
+ "runtime": ["daemon_binary_hash": binaryHash, "pid": ProcessInfo.processInfo.processIdentifier],
+ ], uniquingKeysWith: { _, value in value })
+ }
+}
diff --git a/VPhoneDaemon/Daemon/GuestAPI+Input.swift b/VPhoneDaemon/Daemon/GuestAPI+Input.swift
index 8badfe21..5f097ad0 100644
--- a/VPhoneDaemon/Daemon/GuestAPI+Input.swift
+++ b/VPhoneDaemon/Daemon/GuestAPI+Input.swift
@@ -1,3 +1,4 @@
+import CoreFoundation
import Foundation
import IcliKit
@@ -67,33 +68,41 @@ extension GuestAPI {
// MARK: - Accessibility and OCR
static func executeInterface(_ method: String, _ params: [String: Any]) throws -> [String: Any]? {
+ interfaceLock.lock()
+ defer { interfaceLock.unlock() }
switch method {
case "ui.tree", "accessibility.tree":
- try uiElements(
+ if let value = params["nested"] {
+ guard let number = value as? NSNumber, CFGetTypeID(number) == CFBooleanGetTypeID() else {
+ throw GuestAPIError.invalidRequest("nested must be a boolean")
+ }
+ }
+ if bool(params, "nested") { return try nestedHierarchy(params) }
+ return try uiElements(
maxElements: (params["max_elements"] as? NSNumber)?.intValue ?? 500,
visibleOnly: bool(params, "visible_only", default: true),
clickableOnly: bool(params, "clickable_only"),
limit: (params["limit"] as? NSNumber)?.intValue,
)
case "ui.element_at":
- try elementAt(x: requiredNumber(params, "x"), y: requiredNumber(params, "y"))
+ return try elementAt(x: requiredNumber(params, "x"), y: requiredNumber(params, "y"))
case "ui.tap_element":
- try tapElement(selector(params))
+ return try tapElement(selector(params))
case "ui.wait", "ui.wait_gone":
- try waitForElement(
+ return try waitForElement(
selector(params),
appear: method == "ui.wait",
timeout: number(params, "timeout", default: 10),
)
case "ui.ocr":
- try recognizeScreen(
+ return try recognizeScreen(
languages: params["languages"] as? [String] ?? ["en-US"],
minConfidence: Float(number(params, "min_confidence", default: 0.3)),
)
case "ui.describe":
- try describeScreen()
+ return try describeScreen()
default:
- nil
+ return nil
}
}
diff --git a/VPhoneDaemon/Native/Include/VphonedNative.h b/VPhoneDaemon/Native/Include/VphonedNative.h
index 512cc617..b28566b1 100644
--- a/VPhoneDaemon/Native/Include/VphonedNative.h
+++ b/VPhoneDaemon/Native/Include/VphonedNative.h
@@ -47,3 +47,8 @@ char *vp_usb_set_serial(const char *serial, bool force, bool *changed);
/// The guest's own USB serial, built from `/chosen` `chip-id` and
/// `unique-chip-id`. malloc-owned, or NULL when the device tree lacks either.
char *vp_usb_own_serial(void);
+
+#ifdef __OBJC__
+/// Snapshot-local nested accessibility tree, using actual private iOS child links.
+NSDictionary *vp_ax_hierarchy(int pid, int maxElements, int maxDepth, int timeoutMS);
+#endif
diff --git a/VPhoneDaemon/Native/vphoned_ax_hierarchy.h b/VPhoneDaemon/Native/vphoned_ax_hierarchy.h
new file mode 100644
index 00000000..3927b2f3
--- /dev/null
+++ b/VPhoneDaemon/Native/vphoned_ax_hierarchy.h
@@ -0,0 +1,12 @@
+#pragma once
+#import <Foundation/Foundation.h>
+typedef struct {
+ CFTypeRef (*createApp)(pid_t);
+ int (*copy)(CFTypeRef,CFStringRef,CFTypeRef *);
+ int (*setTimeout)(CFTypeRef,float);
+ CFTypeID (*elementTypeID)(void);
+ CFTypeID (*valueTypeID)(void);
+ Boolean (*getValue)(CFTypeRef,int,void *);
+} VPAxFunctions;
+NSDictionary *vp_ax_walk(VPAxFunctions functions, CFTypeRef root, pid_t pid, int maxElements, int maxDepth, int timeoutMS);
+NSDictionary *vp_ax_hierarchy(int pid, int maxElements, int maxDepth, int timeoutMS);
diff --git a/VPhoneDaemon/Native/vphoned_ax_hierarchy.m b/VPhoneDaemon/Native/vphoned_ax_hierarchy.m
new file mode 100644
index 00000000..cee479a9
--- /dev/null
+++ b/VPhoneDaemon/Native/vphoned_ax_hierarchy.m
@@ -0,0 +1,168 @@
+#import "vphoned_ax_hierarchy.h"
+#import <CoreGraphics/CoreGraphics.h>
+#import <dlfcn.h>
+#import <math.h>
+#import <time.h>
+
+// These private iOS attribute numbers are NOT macOS AX string attributes.
+// Primary mapping: witchan/ios-mcp, commit 8f46b68, MCPAXNodeSource.m.
+// Only 5001 supplies immediate children; visible/explorer arrays are never roots.
+static double milliseconds(void) {
+ struct timespec value; clock_gettime(CLOCK_MONOTONIC,&value);
+ return value.tv_sec*1000.0+value.tv_nsec/1e6;
+}
+@interface VPAxWalker : NSObject {
+@public
+ VPAxFunctions f;
+ int maxElements,maxDepth,maxQueries,timeoutMS,queries,verified,mismatch,unavailable;
+ double start;
+ NSMutableArray *seen;
+ NSMutableSet *active,*reasons;
+ BOOL incomplete;
+}
+- (NSDictionary *)node:(CFTypeRef)element parent:(CFTypeRef)parent depth:(int)depth;
+@end
+@implementation VPAxWalker
+- (BOOL)budget {
+ if(queries>=maxQueries) {[reasons addObject:@"max_queries"];return NO;}
+ if(milliseconds()-start>=timeoutMS) {[reasons addObject:@"timeout"];return NO;}
+ return YES;
+}
+- (id)attribute:(uint32_t)attribute element:(CFTypeRef)element errors:(NSMutableArray *)errors {
+ if(![self budget]) return nil;
+ double remaining=timeoutMS-(milliseconds()-start);
+ int timeoutError=f.setTimeout?f.setTimeout(element,(float)(MIN(100.0,remaining)/1000.0)):-25208;
+ if(timeoutError) {
+ [errors addObject:@{@"attribute":@(attribute),@"error_code":@(timeoutError),@"value_present":@NO,@"timeout_configuration_failed":@YES}];
+ incomplete=YES; return nil;
+ }
+ CFTypeRef value=NULL; ++queries;
+ int error=f.copy(element,(CFStringRef)(uintptr_t)attribute,&value);
+ if(error || !value) {
+ [errors addObject:@{@"attribute":@(attribute),@"error_code":@(error),@"value_present":@(value!=NULL)}];
+ if(value) CFRelease(value);
+ return nil;
+ }
+ return CFBridgingRelease(value);
+}
+- (NSString *)text:(id)value {
+ if(!value || CFGetTypeID((__bridge CFTypeRef)value)!=CFStringGetTypeID())return @"";
+ NSString *string=value;
+ return string.length>1024?[string substringToIndex:1024]:string;
+}
+- (id)frame:(id)value {
+ if(!value || !f.valueTypeID || !f.getValue || CFGetTypeID((__bridge CFTypeRef)value)!=f.valueTypeID()) return NSNull.null;
+ CGRect rect;
+ if(!f.getValue((__bridge CFTypeRef)value,3,&rect) || !isfinite(rect.origin.x) || !isfinite(rect.origin.y) ||
+ !isfinite(rect.size.width) || !isfinite(rect.size.height)) return NSNull.null;
+ return @{@"x":@(rect.origin.x),@"y":@(rect.origin.y),@"width":@(rect.size.width),@"height":@(rect.size.height)};
+}
+- (id)parentProof:(CFTypeRef)element parent:(CFTypeRef)parent errors:(NSMutableArray *)errors {
+ if(!parent)return NSNull.null;
+ id actual=[self attribute:5002 element:element errors:errors];
+ if(!actual || CFGetTypeID((__bridge CFTypeRef)actual)!=f.elementTypeID()) {++unavailable;incomplete=YES;return @NO;}
+ BOOL matches=CFEqual((__bridge CFTypeRef)actual,parent);
+ if(matches) ++verified; else {++mismatch;incomplete=YES;}
+ return @(matches);
+}
+- (NSDictionary *)node:(CFTypeRef)element parent:(CFTypeRef)parent depth:(int)depth {
+ if(![self budget]) return nil;
+ NSUInteger existing=NSNotFound;
+ for(NSUInteger index=0;index<seen.count;++index) if(CFEqual(element,(__bridge CFTypeRef)seen[index])) {existing=index;break;}
+ if(existing!=NSNotFound) {
+ NSMutableArray *errors=[NSMutableArray array];
+ id proof=[self parentProof:element parent:parent errors:errors];
+ NSString *reference=[NSString stringWithFormat:@"ax-%lu",(unsigned long)existing];
+ BOOL cycle=[active containsObject:reference]; if(cycle) incomplete=YES;
+ return @{@"ref":reference,@"cycle":@(cycle),@"parent_verified":proof,@"query_errors":errors.copy};
+ }
+ if(seen.count>=(NSUInteger)maxElements) {[reasons addObject:@"max_elements"];return nil;}
+ NSString *identity=[NSString stringWithFormat:@"ax-%lu",(unsigned long)seen.count];
+ [seen addObject:(__bridge id)element];[active addObject:identity];
+ NSMutableArray *errors=[NSMutableArray array];
+ id proof=[self parentProof:element parent:parent errors:errors];
+ NSString *label=[self text:[self attribute:2001 element:element errors:errors]];
+ NSString *identifier=[self text:[self attribute:5019 element:element errors:errors]];
+ id type=[self attribute:5003 element:element errors:errors];
+ NSString *role=[self text:type];
+ id frame=[self frame:[self attribute:2003 element:element errors:errors]];
+ id raw=[self attribute:5001 element:element errors:errors];
+ NSMutableArray *children=[NSMutableArray array];
+ NSString *status=@"complete";
+ if(!raw || CFGetTypeID((__bridge CFTypeRef)raw)!=CFArrayGetTypeID()) {status=@"unavailable";incomplete=YES;
+ if(raw) [errors addObject:@{@"attribute":@5001,@"error_code":@0,@"value_present":@YES,@"invalid_type":@YES}];}
+ else {
+ CFArrayRef array=(__bridge CFArrayRef)raw;
+ CFIndex count=CFArrayGetCount(array);
+ if(depth>=maxDepth && count) {[reasons addObject:@"max_depth"];status=@"truncated";}
+ else for(CFIndex i=0;i<count;++i) {
+ if(![self budget]) {status=@"truncated";break;}
+ CFTypeRef child=CFArrayGetValueAtIndex(array,i);
+ if(!child || CFGetTypeID(child)!=f.elementTypeID()) {status=@"unavailable";incomplete=YES;
+ [errors addObject:@{@"attribute":@5001,@"error_code":@0,@"value_present":@(child!=NULL),@"invalid_child_type":@YES}];continue;}
+ NSDictionary *node=[self node:child parent:element depth:depth+1];
+ if(!node) {status=@"truncated";break;}
+ [children addObject:node];
+ }
+ }
+ if(![self budget] && [status isEqual:@"unavailable"]) status=@"truncated";
+ [active removeObject:identity];
+ return @{@"id":identity,@"label":label,@"identifier":identifier,@"role":role,@"frame":frame,
+ @"children":children.copy,@"children_status":status,@"query_errors":errors.copy,@"parent_verified":proof,@"depth":@(depth)};
+}
+@end
+
+NSDictionary *vp_ax_walk(VPAxFunctions functions, CFTypeRef root, pid_t pid, int elements, int depth, int duration) {
+ VPAxWalker *walker=[VPAxWalker new];walker->f=functions;
+ walker->maxElements=MAX(1,MIN(elements,2000));walker->maxDepth=MAX(0,MIN(depth,32));
+ walker->timeoutMS=MAX(100,MIN(duration,10000));walker->maxQueries=walker->maxElements*8;
+ walker->seen=[NSMutableArray array];walker->active=[NSMutableSet set];walker->reasons=[NSMutableSet set];walker->start=milliseconds();
+ NSDictionary *node=root && functions.copy && functions.elementTypeID?[walker node:root parent:NULL depth:0]:nil;
+ BOOL truncated=walker->reasons.count>0;
+ NSString *status=!node?@"unavailable":truncated || walker->incomplete?@"partial":@"complete";
+ return @{@"source":@"ax",@"format":@"nested",@"relationship_source":@"AXUIElementCopyAttributeValue:5001",
+ @"pid":@(pid),@"roots":node?@[node]:@[],@"count":@(walker->seen.count),@"status":status,
+ @"truncated":@(truncated),@"truncation_reasons":[walker->reasons.allObjects sortedArrayUsingSelector:@selector(compare:)],
+ @"limits":@{@"max_elements":@(walker->maxElements),@"max_depth":@(walker->maxDepth),@"max_queries":@(walker->maxQueries),@"timeout_ms":@(walker->timeoutMS)},
+ @"queries":@(walker->queries),@"elapsed_ms":@(milliseconds()-walker->start),
+ @"parent_checks":@{@"verified":@(walker->verified),@"mismatch":@(walker->mismatch),@"unavailable":@(walker->unavailable)}};
+}
+NSDictionary *vp_ax_hierarchy(int pid,int maxElements,int maxDepth,int timeoutMS) {
+ void *ax=dlopen("/System/Library/PrivateFrameworks/AXRuntime.framework/AXRuntime",RTLD_NOW);
+ VPAxFunctions f={0};
+#define LOAD(M,N) f.M=(typeof(f.M))(ax?dlsym(ax,N):NULL)
+ LOAD(createApp,"_AXUIElementCreateAppElementWithPid");LOAD(copy,"AXUIElementCopyAttributeValue");
+ LOAD(setTimeout,"AXUIElementSetMessagingTimeout");LOAD(elementTypeID,"AXUIElementGetTypeID");
+ LOAD(valueTypeID,"AXValueGetTypeID");LOAD(getValue,"AXValueGetValue");
+#undef LOAD
+ CFTypeRef root=f.createApp && pid>1?f.createApp(pid):NULL;
+ void *library=dlopen("/usr/lib/libAccessibility.dylib",RTLD_NOW);
+ Boolean (*appEnabled)(void)=library?dlsym(library,"_AXSApplicationAccessibilityEnabled"):NULL;
+ void (*setApp)(Boolean)=library?dlsym(library,"_AXSApplicationAccessibilitySetEnabled"):NULL;
+ Boolean (*autoEnabled)(void)=library?dlsym(library,"_AXSAutomationEnabled"):NULL;
+ void (*setAuto)(Boolean)=library?dlsym(library,"_AXSSetAutomationEnabled"):NULL;
+ void (*client)(uint32_t)=dlsym(RTLD_DEFAULT,"__AXSetRequestingClient");
+ uint64_t (*override)(uint64_t)=dlsym(RTLD_DEFAULT,"_AXOverrideRequestingClientType");
+ if(client) client(2); if(override) override(2);
+ BOOL beforeApp=appEnabled?appEnabled():NO,beforeAuto=autoEnabled?autoEnabled():NO;
+ BOOL changedApp=appEnabled && setApp && !beforeApp, changedAuto=autoEnabled && setAuto && !beforeAuto;
+ NSDictionary *result=nil;
+ @try {
+ if(changedApp) setApp(true); if(changedAuto) setAuto(true);
+ result=vp_ax_walk(f,root,pid,maxElements,maxDepth,timeoutMS);
+ } @catch(NSException *exception) {
+ NSMutableDictionary *failure=[vp_ax_walk(f,NULL,pid,maxElements,maxDepth,timeoutMS) mutableCopy];
+ failure[@"native_exception"]=exception.name; result=failure.copy;
+ } @finally {
+ if(changedAuto)setAuto(false);if(changedApp)setApp(false);
+ }
+ NSMutableDictionary *report=[result mutableCopy];
+ report[@"symbols"]=@{@"create_application":@(f.createApp!=NULL),@"copy_attribute":@(f.copy!=NULL),
+ @"set_timeout":@(f.setTimeout!=NULL),@"element_type_id":@(f.elementTypeID!=NULL),
+ @"requesting_client":@(client!=NULL),@"requesting_client_override":@(override!=NULL)};
+ report[@"switches_before"]=@{@"application":appEnabled?@(beforeApp):NSNull.null,@"automation":autoEnabled?@(beforeAuto):NSNull.null};
+ report[@"switches_after_restore"]=@{@"application":appEnabled?@(appEnabled()):NSNull.null,@"automation":autoEnabled?@(autoEnabled()):NSNull.null};
+ result=report.copy;
+ if(root) CFRelease(root);
+ return result;
+}