Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: preserve unknown FaceTime state on unreadable accessibility data

Adam Eivy4ab78322bbde
diff --git a/server/native/facetime-ax/main.swift b/server/native/facetime-ax/main.swift
index 3cfd92f425..4aaf7efd7a 100644
--- a/server/native/facetime-ax/main.swift
+++ b/server/native/facetime-ax/main.swift
@@ -74,23 +74,42 @@ let semanticAttributes: [CFString] = [
kAXIdentifierAttribute as CFString,
]
-func attribute(_ element: AXUIElement, _ name: CFString) -> CFTypeRef? {
+enum SnapshotReadError: Error { case unavailable }
+
+func attribute(_ element: AXUIElement, _ name: CFString) throws -> CFTypeRef? {
var value: CFTypeRef?
- return AXUIElementCopyAttributeValue(element, name, &value) == .success ? value : nil
+ switch AXUIElementCopyAttributeValue(element, name, &value) {
+ case .success:
+ guard let value else { throw SnapshotReadError.unavailable }
+ return value
+ case .attributeUnsupported, .noValue:
+ return nil
+ default:
+ // A vanished element or failed IPC is not an absent attribute. Treating
+ // either as empty can falsely confirm that an active call disconnected.
+ throw SnapshotReadError.unavailable
+ }
}
-func stringAttribute(_ element: AXUIElement, _ name: CFString) -> String? {
- attribute(element, name) as? String
+func stringAttribute(_ element: AXUIElement, _ name: CFString) throws -> String? {
+ try attribute(element, name) as? String
}
-func childElements(_ element: AXUIElement) -> [AXUIElement] {
- attribute(element, kAXChildrenAttribute as CFString) as? [AXUIElement] ?? []
+func childElements(_ element: AXUIElement) throws -> [AXUIElement] {
+ guard let value = try attribute(element, kAXChildrenAttribute as CFString) else { return [] }
+ // Swift's conditional array cast does not check CoreFoundation element
+ // identities. Prove each CF type before accepting the typed collection.
+ guard CFGetTypeID(value) == CFArrayGetTypeID(),
+ let values = value as? [CFTypeRef],
+ values.allSatisfy({ CFGetTypeID($0) == AXUIElementGetTypeID() }),
+ let children = value as? [AXUIElement] else { throw SnapshotReadError.unavailable }
+ return children
}
-func actionNames(_ element: AXUIElement) -> Set<String> {
+func actionNames(_ element: AXUIElement) throws -> Set<String> {
var names: CFArray?
guard AXUIElementCopyActionNames(element, &names) == .success,
- let values = names as? [String] else { return [] }
+ let values = names as? [String] else { throw SnapshotReadError.unavailable }
return Set(values)
}
@@ -103,23 +122,27 @@ func snapshot(_ application: NSRunningApplication) -> Snapshot? {
var cursor = 0
let maximumNodes = 2_000
- while cursor < pending.count && nodes.count < maximumNodes {
- let (element, parent) = pending[cursor]
- cursor += 1
- let index = nodes.count
- let values = semanticAttributes.compactMap { stringAttribute(element, $0) }
- .map { $0.trimmingCharacters(in: .whitespacesAndNewlines) }
- .filter { !$0.isEmpty }
- let role = stringAttribute(element, kAXRoleAttribute as CFString) ?? ""
- let interactiveRoles = [kAXButtonRole as String, kAXMenuButtonRole as String, "AXLink"]
- nodes.append(Node(
- element: element,
- parent: parent,
- role: role,
- text: values,
- actions: interactiveRoles.contains(role) ? actionNames(element) : []
- ))
- pending.append(contentsOf: childElements(element).map { ($0, index) })
+ do {
+ while cursor < pending.count && nodes.count < maximumNodes {
+ let (element, parent) = pending[cursor]
+ cursor += 1
+ let index = nodes.count
+ let values = try semanticAttributes.compactMap { try stringAttribute(element, $0) }
+ .map { $0.trimmingCharacters(in: .whitespacesAndNewlines) }
+ .filter { !$0.isEmpty }
+ guard let role = try stringAttribute(element, kAXRoleAttribute as CFString), !role.isEmpty else { return nil }
+ let interactiveRoles = [kAXButtonRole as String, kAXMenuButtonRole as String, "AXLink"]
+ nodes.append(Node(
+ element: element,
+ parent: parent,
+ role: role,
+ text: values,
+ actions: interactiveRoles.contains(role) ? try actionNames(element) : []
+ ))
+ pending.append(contentsOf: try childElements(element).map { ($0, index) })
+ }
+ } catch {
+ return nil
}
return Snapshot(nodes: nodes, truncated: cursor < pending.count)
}
diff --git a/server/services/voice/facetimeBridge.test.js b/server/services/voice/facetimeBridge.test.js
index d662bb0c85..528d13a6eb 100644
--- a/server/services/voice/facetimeBridge.test.js
+++ b/server/services/voice/facetimeBridge.test.js
@@ -128,6 +128,90 @@ completeAction(command: CommandLine.arguments[1], confirmedState: state, action:
}
}, 60_000);
+ it.runIf(process.platform === 'darwin')('keeps unreadable AX snapshots unknown instead of confirming a disconnect', () => {
+ const sourceDir = join(import.meta.dirname, '..', '..', 'native', 'facetime-ax');
+ const tempDir = mkdtempSync(join(process.env.PORTOS_TEST_TMPDIR || tmpdir(), 'portos-facetime-snapshot-'));
+ const runner = join(tempDir, 'main.swift');
+ const binary = join(tempDir, 'snapshot-test');
+ try {
+ // Run the native state/JSON boundary with synthetic AppKit/AX responses.
+ // No live app state is read and no controls are pressed.
+ const source = readFileSync(join(sourceDir, 'main.swift'), 'utf8');
+ const declarations = source.slice(0, source.indexOf('let arguments = CommandLine.arguments'));
+ writeFileSync(runner, `${declarations}
+
+// These local symbols shadow AppKit/AX entrypoints so no live AX reads occur.
+struct NSRunningApplication {
+ let processIdentifier: pid_t
+ let bundleIdentifier: String? = "com.apple.FaceTime"
+ let localizedName: String? = nil
+ let executableURL: URL? = nil
+ static func runningApplications(withBundleIdentifier bundle: String) -> [NSRunningApplication] {
+ bundle == "com.apple.FaceTime" ? [NSRunningApplication(processIdentifier: 100)] : []
+ }
+}
+struct NSWorkspace {
+ static let shared = NSWorkspace()
+ let runningApplications: [NSRunningApplication] = []
+}
+let scenario = CommandLine.arguments[1]
+func AXUIElementCopyAttributeNames(_ element: AXUIElement, _ names: UnsafeMutablePointer<CFArray?>) -> AXError {
+ names.pointee = [] as CFArray
+ return .success
+}
+func AXUIElementCopyActionNames(_ element: AXUIElement, _ names: UnsafeMutablePointer<CFArray?>) -> AXError {
+ if scenario == "actions-error" { return .cannotComplete }
+ names.pointee = [kAXPressAction] as CFArray
+ return .success
+}
+func AXUIElementCopyAttributeValue(_ element: AXUIElement, _ attr: CFString, _ output: UnsafeMutablePointer<CFTypeRef?>) -> AXError {
+ var pid: pid_t = 0
+ guard AXUIElementGetPid(element, &pid) == .success else { return .illegalArgument }
+ let name = attr as String
+ if name == kAXChildrenAttribute as String {
+ if pid == 100 { output.pointee = [AXUIElementCreateApplication(101)] as CFArray; return .success }
+ if pid == 101 {
+ if scenario == "children-error" { return .cannotComplete }
+ if scenario == "children-malformed" { output.pointee = ["wrong"] as CFArray; return .success }
+ if scenario == "children-not-array" { output.pointee = "wrong" as CFString; return .success }
+ output.pointee = [AXUIElementCreateApplication(102), AXUIElementCreateApplication(103)] as CFArray
+ return .success
+ }
+ return scenario == "leaf-no-value" ? .noValue : .attributeUnsupported
+ }
+ if name == kAXRoleAttribute as String {
+ if scenario == "role-error" && pid == 102 { return .cannotComplete }
+ if scenario == "role-missing" && pid == 102 { return .noValue }
+ let roles: [pid_t: String] = [100: kAXApplicationRole as String, 101: kAXGroupRole as String, 102: kAXStaticTextRole as String, 103: kAXButtonRole as String]
+ output.pointee = roles[pid]! as CFString
+ return .success
+ }
+ if name == kAXTitleAttribute as String && pid == 102 { output.pointee = "Example Caller" as CFString; return .success }
+ if name == kAXTitleAttribute as String && pid == 103 { output.pointee = "Hang up" as CFString; return .success }
+ return .noValue
+}
+let matcher = IdentityMatcher(handle: "+15551234567", identity: "Example Caller")
+let observed = probe(matcher: matcher)
+emit(Result(ok: true, command: "probe", state: observed, authorized: true, action: "probe", message: "synthetic", errorCode: nil))
+`);
+ const compiled = spawnSync('swiftc', ['-warnings-as-errors', join(sourceDir, 'identityMatcher.swift'), runner, '-o', binary], { encoding: 'utf8' });
+ expect(compiled.status, compiled.stderr).toBe(0);
+ for (const [scenario, state] of [
+ ['connected', 'connected'], ['leaf-no-value', 'connected'],
+ ['children-error', 'unknown'], ['children-malformed', 'unknown'],
+ ['children-not-array', 'unknown'], ['actions-error', 'unknown'],
+ ['role-error', 'unknown'], ['role-missing', 'unknown'],
+ ]) {
+ const result = spawnSync(binary, [scenario], { encoding: 'utf8' });
+ expect(result.status, `${scenario}: ${result.stderr}`).toBe(0);
+ expect(facetimeControlResultSchema.parse(JSON.parse(result.stdout)), scenario)
+ .toMatchObject({ ok: true, command: 'probe', state });
+ }
+ } finally {
+ rmSync(tempDir, { recursive: true, force: true });
+ }
+ }, 60_000);
+
it('accepts only the strict helper result contract', () => {
const result = facetimeControlResultSchema.safeParse({
ok: true, command: 'probe', state: 'idle', authorized: true,