Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: preserve unknown FaceTime state on unreadable accessibility data
Adam Eivy4ab78322bbde
diff --git a/server/native/facetime-ax/main.swift b/server/native/facetime-ax/main.swiftindex 3cfd92f425..4aaf7efd7a 100644--- a/server/native/facetime-ax/main.swift+++ b/server/native/facetime-ax/main.swift@@ -74,23 +74,42 @@ let semanticAttributes: [CFString] = [kAXIdentifierAttribute as CFString,]-func attribute(_ element: AXUIElement, _ name: CFString) -> CFTypeRef? {+enum SnapshotReadError: Error { case unavailable }++func attribute(_ element: AXUIElement, _ name: CFString) throws -> CFTypeRef? {var value: CFTypeRef?- return AXUIElementCopyAttributeValue(element, name, &value) == .success ? value : nil+ switch AXUIElementCopyAttributeValue(element, name, &value) {+ case .success:+ guard let value else { throw SnapshotReadError.unavailable }+ return value+ case .attributeUnsupported, .noValue:+ return nil+ default:+ // A vanished element or failed IPC is not an absent attribute. Treating+ // either as empty can falsely confirm that an active call disconnected.+ throw SnapshotReadError.unavailable+ }}-func stringAttribute(_ element: AXUIElement, _ name: CFString) -> String? {- attribute(element, name) as? String+func stringAttribute(_ element: AXUIElement, _ name: CFString) throws -> String? {+ try attribute(element, name) as? String}-func childElements(_ element: AXUIElement) -> [AXUIElement] {- attribute(element, kAXChildrenAttribute as CFString) as? [AXUIElement] ?? []+func childElements(_ element: AXUIElement) throws -> [AXUIElement] {+ guard let value = try attribute(element, kAXChildrenAttribute as CFString) else { return [] }+ // Swift's conditional array cast does not check CoreFoundation element+ // identities. Prove each CF type before accepting the typed collection.+ guard CFGetTypeID(value) == CFArrayGetTypeID(),+ let values = value as? [CFTypeRef],+ values.allSatisfy({ CFGetTypeID($0) == AXUIElementGetTypeID() }),+ let children = value as? [AXUIElement] else { throw SnapshotReadError.unavailable }+ return children}-func actionNames(_ element: AXUIElement) -> Set<String> {+func actionNames(_ element: AXUIElement) throws -> Set<String> {var names: CFArray?guard AXUIElementCopyActionNames(element, &names) == .success,- let values = names as? [String] else { return [] }+ let values = names as? [String] else { throw SnapshotReadError.unavailable }return Set(values)}@@ -103,23 +122,27 @@ func snapshot(_ application: NSRunningApplication) -> Snapshot? {var cursor = 0let maximumNodes = 2_000- while cursor < pending.count && nodes.count < maximumNodes {- let (element, parent) = pending[cursor]- cursor += 1- let index = nodes.count- let values = semanticAttributes.compactMap { stringAttribute(element, $0) }- .map { $0.trimmingCharacters(in: .whitespacesAndNewlines) }- .filter { !$0.isEmpty }- let role = stringAttribute(element, kAXRoleAttribute as CFString) ?? ""- let interactiveRoles = [kAXButtonRole as String, kAXMenuButtonRole as String, "AXLink"]- nodes.append(Node(- element: element,- parent: parent,- role: role,- text: values,- actions: interactiveRoles.contains(role) ? actionNames(element) : []- ))- pending.append(contentsOf: childElements(element).map { ($0, index) })+ do {+ while cursor < pending.count && nodes.count < maximumNodes {+ let (element, parent) = pending[cursor]+ cursor += 1+ let index = nodes.count+ let values = try semanticAttributes.compactMap { try stringAttribute(element, $0) }+ .map { $0.trimmingCharacters(in: .whitespacesAndNewlines) }+ .filter { !$0.isEmpty }+ guard let role = try stringAttribute(element, kAXRoleAttribute as CFString), !role.isEmpty else { return nil }+ let interactiveRoles = [kAXButtonRole as String, kAXMenuButtonRole as String, "AXLink"]+ nodes.append(Node(+ element: element,+ parent: parent,+ role: role,+ text: values,+ actions: interactiveRoles.contains(role) ? try actionNames(element) : []+ ))+ pending.append(contentsOf: try childElements(element).map { ($0, index) })+ }+ } catch {+ return nil}return Snapshot(nodes: nodes, truncated: cursor < pending.count)}diff --git a/server/services/voice/facetimeBridge.test.js b/server/services/voice/facetimeBridge.test.jsindex d662bb0c85..528d13a6eb 100644--- a/server/services/voice/facetimeBridge.test.js+++ b/server/services/voice/facetimeBridge.test.js@@ -128,6 +128,90 @@ completeAction(command: CommandLine.arguments[1], confirmedState: state, action:}}, 60_000);+ it.runIf(process.platform === 'darwin')('keeps unreadable AX snapshots unknown instead of confirming a disconnect', () => {+ const sourceDir = join(import.meta.dirname, '..', '..', 'native', 'facetime-ax');+ const tempDir = mkdtempSync(join(process.env.PORTOS_TEST_TMPDIR || tmpdir(), 'portos-facetime-snapshot-'));+ const runner = join(tempDir, 'main.swift');+ const binary = join(tempDir, 'snapshot-test');+ try {+ // Run the native state/JSON boundary with synthetic AppKit/AX responses.+ // No live app state is read and no controls are pressed.+ const source = readFileSync(join(sourceDir, 'main.swift'), 'utf8');+ const declarations = source.slice(0, source.indexOf('let arguments = CommandLine.arguments'));+ writeFileSync(runner, `${declarations}++// These local symbols shadow AppKit/AX entrypoints so no live AX reads occur.+struct NSRunningApplication {+ let processIdentifier: pid_t+ let bundleIdentifier: String? = "com.apple.FaceTime"+ let localizedName: String? = nil+ let executableURL: URL? = nil+ static func runningApplications(withBundleIdentifier bundle: String) -> [NSRunningApplication] {+ bundle == "com.apple.FaceTime" ? [NSRunningApplication(processIdentifier: 100)] : []+ }+}+struct NSWorkspace {+ static let shared = NSWorkspace()+ let runningApplications: [NSRunningApplication] = []+}+let scenario = CommandLine.arguments[1]+func AXUIElementCopyAttributeNames(_ element: AXUIElement, _ names: UnsafeMutablePointer<CFArray?>) -> AXError {+ names.pointee = [] as CFArray+ return .success+}+func AXUIElementCopyActionNames(_ element: AXUIElement, _ names: UnsafeMutablePointer<CFArray?>) -> AXError {+ if scenario == "actions-error" { return .cannotComplete }+ names.pointee = [kAXPressAction] as CFArray+ return .success+}+func AXUIElementCopyAttributeValue(_ element: AXUIElement, _ attr: CFString, _ output: UnsafeMutablePointer<CFTypeRef?>) -> AXError {+ var pid: pid_t = 0+ guard AXUIElementGetPid(element, &pid) == .success else { return .illegalArgument }+ let name = attr as String+ if name == kAXChildrenAttribute as String {+ if pid == 100 { output.pointee = [AXUIElementCreateApplication(101)] as CFArray; return .success }+ if pid == 101 {+ if scenario == "children-error" { return .cannotComplete }+ if scenario == "children-malformed" { output.pointee = ["wrong"] as CFArray; return .success }+ if scenario == "children-not-array" { output.pointee = "wrong" as CFString; return .success }+ output.pointee = [AXUIElementCreateApplication(102), AXUIElementCreateApplication(103)] as CFArray+ return .success+ }+ return scenario == "leaf-no-value" ? .noValue : .attributeUnsupported+ }+ if name == kAXRoleAttribute as String {+ if scenario == "role-error" && pid == 102 { return .cannotComplete }+ if scenario == "role-missing" && pid == 102 { return .noValue }+ let roles: [pid_t: String] = [100: kAXApplicationRole as String, 101: kAXGroupRole as String, 102: kAXStaticTextRole as String, 103: kAXButtonRole as String]+ output.pointee = roles[pid]! as CFString+ return .success+ }+ if name == kAXTitleAttribute as String && pid == 102 { output.pointee = "Example Caller" as CFString; return .success }+ if name == kAXTitleAttribute as String && pid == 103 { output.pointee = "Hang up" as CFString; return .success }+ return .noValue+}+let matcher = IdentityMatcher(handle: "+15551234567", identity: "Example Caller")+let observed = probe(matcher: matcher)+emit(Result(ok: true, command: "probe", state: observed, authorized: true, action: "probe", message: "synthetic", errorCode: nil))+`);+ const compiled = spawnSync('swiftc', ['-warnings-as-errors', join(sourceDir, 'identityMatcher.swift'), runner, '-o', binary], { encoding: 'utf8' });+ expect(compiled.status, compiled.stderr).toBe(0);+ for (const [scenario, state] of [+ ['connected', 'connected'], ['leaf-no-value', 'connected'],+ ['children-error', 'unknown'], ['children-malformed', 'unknown'],+ ['children-not-array', 'unknown'], ['actions-error', 'unknown'],+ ['role-error', 'unknown'], ['role-missing', 'unknown'],+ ]) {+ const result = spawnSync(binary, [scenario], { encoding: 'utf8' });+ expect(result.status, `${scenario}: ${result.stderr}`).toBe(0);+ expect(facetimeControlResultSchema.parse(JSON.parse(result.stdout)), scenario)+ .toMatchObject({ ok: true, command: 'probe', state });+ }+ } finally {+ rmSync(tempDir, { recursive: true, force: true });+ }+ }, 60_000);+it('accepts only the strict helper result contract', () => {const result = facetimeControlResultSchema.safeParse({ok: true, command: 'probe', state: 'idle', authorized: true,