Skip to content

Capture iPhone LiDAR and photos, reconstruct editable Three.js rooms, and explore them in a browser.

TypeScript
18.5 KB
Raw Blame
import Foundation
import RoomPlan
import simd
import UIKit

/// One capture session inside a house on disk
/// (Documents/Houses/<house>/sessions/<id>/, see HouseProject).
///
///     manifest.json          what's in the bundle (rewritten after every room, final at the end)
///     structure.json         all rooms merged by RoomPlan's StructureBuilder
///     structure.usdz         same, for Quick Look
///     rooms/room-N.json      each CapturedRoom as RoomPlan encodes it
///     rooms/room-N.usdz
///     rooms/room-N-data.json RoomPlan's raw CapturedRoomData (re-run RoomBuilder on it later)
///     worldmap.arworldmap    ARWorldMap at the end of the session (NSKeyedArchiver, secure coding)
///     mesh/room-N.ply        LiDAR mesh after room N's detail pass (world space; per-vertex
///                            normal + photo coverage score, per-face ARKit classification)
///     photos/photos.json     camera pose + intrinsics for every photo
///     photos/NNNN.jpg
///     depth/frames.json      LiDAR depth frames: pose + intrinsics (scaled to the depth map)
///     depth/NNNNN.f16/.conf  256×192 Float16 depth (meters) + UInt8 confidence (0–2)
///
/// Everything shares one coordinate system: the ARKit world of the capture
/// session (meters, +Y up against gravity, right-handed). house.json says how that
/// frame sits in the house (`HouseProject.toHouse(session)`).
struct ScanBundle {
  let url: URL

  static let subfolders = ["rooms", "mesh", "photos", "depth"]

  /// A new session folder inside a house: <house>/sessions/<id>/
  static func create(in house: HouseProject, id: String) throws -> ScanBundle {
    let url = house.url.appendingPathComponent("sessions", isDirectory: true).appendingPathComponent(id, isDirectory: true)
    guard !FileManager.default.fileExists(atPath: url.path) else {
      throw CocoaError(.fileWriteFileExists, userInfo: [NSFilePathErrorKey: url.path])
    }
    let bundle = ScanBundle(url: url)
    for dir in subfolders {
      try FileManager.default.createDirectory(at: url.appendingPathComponent(dir), withIntermediateDirectories: true)
    }
    return bundle
  }

  var name: String { url.lastPathComponent }

  func file(_ path: String) -> URL { url.appendingPathComponent(path) }

  // MARK: rooms + structure

  func saveRoom(_ room: CapturedRoom, index: Int) throws {
    try encoder.encode(room).write(to: file("rooms/room-\(index).json"))
    try? room.export(to: file("rooms/room-\(index).usdz"), exportOptions: .parametric)
    // RoomPlan's detailed object shapes (legs, backs, arms) instead of boxes
    try? room.export(to: file("rooms/room-\(index)-mesh.usdz"), exportOptions: .mesh)
  }

  func deleteRoom(index: Int) {
    for name in ["room-\(index).json", "room-\(index).usdz", "room-\(index)-mesh.usdz", "room-\(index)-data.json"] {
      try? FileManager.default.removeItem(at: file("rooms/\(name)"))
    }
  }

  /// RoomPlan's raw capture data, so RoomBuilder can be re-run on it later (Codable since iOS 17)
  static func encodeRoomData(_ data: CapturedRoomData) throws -> Data {
    try JSONEncoder().encode(data)
  }

  /// photos/photos.json, or [] if there isn't one
  var photos: [PhotoRecord] {
    guard let data = try? Data(contentsOf: file("photos/photos.json")) else { return [] }
    return (try? JSONDecoder().decode([PhotoRecord].self, from: data)) ?? []
  }

  /// depth/frames.json, or [] if there isn't one
  var depthFrames: [DepthRecorder.Record] {
    guard let data = try? Data(contentsOf: file("depth/frames.json")) else { return [] }
    return (try? JSONDecoder().decode([DepthRecorder.Record].self, from: data)) ?? []
  }

  func saveStructure(_ structure: CapturedStructure) throws {
    try Self.saveStructure(structure, in: url)
  }

  /// structure.json + structure.usdz in a folder (a session bundle, or a zone's folder)
  static func saveStructure(_ structure: CapturedStructure, in folder: URL) throws {
    try FileManager.default.createDirectory(at: folder, withIntermediateDirectories: true)
    let e = JSONEncoder()
    e.outputFormatting = [.prettyPrinted, .sortedKeys]
    e.dateEncodingStrategy = .iso8601
    try e.encode(structure).write(to: folder.appendingPathComponent("structure.json"), options: .atomic)
    try? structure.export(to: folder.appendingPathComponent("structure.usdz"), exportOptions: .parametric)
  }

  // MARK: photos

  func savePhotos(_ photos: [PhotoRecord]) throws {
    try encoder.encode(photos).write(to: file("photos/photos.json"))
  }

  // MARK: manifest

  func writeManifest(_ manifest: Manifest) throws {
    try encoder.encode(manifest).write(to: file("manifest.json"), options: .atomic)
  }

  var manifest: Manifest? {
    guard let data = try? Data(contentsOf: file("manifest.json")) else { return nil }
    let decoder = JSONDecoder()
    decoder.dateDecodingStrategy = .iso8601 // matches the encoder
    return try? decoder.decode(Manifest.self, from: data)
  }

  func delete() throws {
    try FileManager.default.removeItem(at: url)
  }

  /// Zips the bundle (NSFileCoordinator does this for directories) for sharing.
  func zipped(as zipName: String? = nil) throws -> URL {
    try FolderZipper.zip(url, name: zipName ?? name)
  }

  /// Export the selected scan as a standalone bundle, retaining its original room index
  /// and coordinate frame. Never copy the session-wide structure or tracking map.
  func zippedRoom(index: Int, name: String) throws -> URL {
    let staging = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString, isDirectory: true)
    defer { try? FileManager.default.removeItem(at: staging) }
    let folder = staging.appendingPathComponent("Room", isDirectory: true)
    try exportRoom(index: index, name: name, to: folder)
    return try FolderZipper.zip(folder, name: name)
  }

  func exportRoom(index: Int, name: String, to destination: URL) throws {
    let fm = FileManager.default
    guard var manifest, var room = manifest.rooms.first(where: { $0.index == index }) else {
      throw CocoaError(.fileReadCorruptFile)
    }
    // Imported bundles are untrusted: no manifest path may escape its selected folder.
    func checkedSource(_ path: String) throws -> URL {
      let root = url.resolvingSymlinksInPath().standardizedFileURL.path + "/"
      let source = file(path).resolvingSymlinksInPath().standardizedFileURL
      guard !path.isEmpty, !path.hasPrefix("/"), !path.split(separator: "/").contains(".."), source.path.hasPrefix(root) else {
        throw CocoaError(.fileReadInvalidFileName)
      }
      return source
    }
    // Decode explicitly: a damaged index must not silently export an empty recording.
    func records<T: Decodable>(_ type: T.Type, path: String) throws -> T {
      try JSONDecoder().decode(type, from: Data(contentsOf: checkedSource(path)))
    }
    let selectedPhotos = try records([PhotoRecord].self, path: manifest.photos).filter { $0.room == index }
    let selectedDepth: [DepthRecorder.Record]
    if let path = manifest.depthFrames {
      selectedDepth = try records([DepthRecorder.Record].self, path: path).filter { $0.room == index }
    } else {
      selectedDepth = []
    }
    for folder in Self.subfolders {
      try fm.createDirectory(at: destination.appendingPathComponent(folder), withIntermediateDirectories: true)
    }
    let output = ScanBundle(url: destination)
    func copy(_ path: String) throws {
      let source = try checkedSource(path)
      let target = output.file(path)
      try fm.createDirectory(at: target.deletingLastPathComponent(), withIntermediateDirectories: true)
      try fm.copyItem(at: source, to: target)
    }
    if !room.file.isEmpty { try copy(room.file) }
    if !room.usdz.isEmpty {
      if fm.fileExists(atPath: file(room.usdz).path) { try copy(room.usdz) }
      else { room.usdz = "" } // Quick Look export is optional during capture.
    }
    if let path = room.data { try copy(path) }
    if let mesh = room.mesh { try copy(mesh.file) }
    for path in ["rooms/room-\(index)-mesh.usdz", "mesh/room-\(index)-colors.bin"] {
      if fm.fileExists(atPath: file(path).path) { try copy(path) }
    }
    for photo in selectedPhotos { try copy(photo.file) }
    for frame in selectedDepth {
      try copy(frame.file)
      try copy(frame.confidence)
      if let rgb = frame.rgb { try copy(rgb) }
    }
    try output.savePhotos(selectedPhotos)
    try encoder.encode(selectedDepth).write(to: output.file("depth/frames.json"))
    room.name = name
    manifest.rooms = [room]
    manifest.photos = "photos/photos.json"
    manifest.photoCount = selectedPhotos.count
    manifest.depthFrames = "depth/frames.json"
    manifest.depthFrameCount = selectedDepth.count
    manifest.mesh = room.mesh
    manifest.structure = nil
    manifest.structureError = nil
    manifest.worldMap = nil
    manifest.worldMapStatus = nil
    manifest.diagnostics = nil
    try output.writeManifest(manifest)
  }

  private var encoder: JSONEncoder {
    let e = JSONEncoder()
    e.outputFormatting = [.prettyPrinted, .sortedKeys]
    e.dateEncodingStrategy = .iso8601
    return e
  }
}

/// Zips a folder for sharing (AirDrop to the Mac)
enum FolderZipper {
  static func zip(_ folder: URL, name: String) throws -> URL {
    let fm = FileManager.default
    let root = fm.temporaryDirectory.appendingPathComponent("share", isDirectory: true)
    try fm.createDirectory(at: root, withIntermediateDirectories: true)
    // Unique directories keep two simultaneous exports from deleting each other's files.
    // Recover abandoned shares after a crash without touching recent active shares.
    for item in (try? fm.contentsOfDirectory(at: root, includingPropertiesForKeys: [.contentModificationDateKey])) ?? [] {
      if let date = try? item.resourceValues(forKeys: [.contentModificationDateKey]).contentModificationDate,
         date < Date().addingTimeInterval(-86_400) { try? fm.removeItem(at: item) }
    }
    let dir = root.appendingPathComponent(UUID().uuidString, isDirectory: true)
    try fm.createDirectory(at: dir, withIntermediateDirectories: true)
    var succeeded = false
    defer { if !succeeded { try? fm.removeItem(at: dir) } }
    let safe = name.replacingOccurrences(of: "/", with: "-").replacingOccurrences(of: ":", with: "-")
    let dest = dir.appendingPathComponent("\(safe).zip")
    var result: Result<URL, Error> = .failure(CocoaError(.fileReadUnknown))
    var coordError: NSError?
    NSFileCoordinator().coordinate(readingItemAt: folder, options: [.forUploading], error: &coordError) { zipURL in
      // the coordinator's zip is deleted when this block returns: move it out (no second copy
      // of a multi-GB file), fall back to copying
      result = Result {
        do { try fm.moveItem(at: zipURL, to: dest) } catch { try fm.copyItem(at: zipURL, to: dest) }
        return dest
      }
    }
    if let coordError { throw coordError }
    let archive = try result.get()
    succeeded = true
    return archive
  }

  static func removeArchive(_ archive: URL) {
    let root = FileManager.default.temporaryDirectory.appendingPathComponent("share").standardizedFileURL
    let directory = archive.deletingLastPathComponent().standardizedFileURL
    guard directory.deletingLastPathComponent() == root else { return }
    try? FileManager.default.removeItem(at: directory)
  }
}

// MARK: - Bundle file formats

struct Manifest: Codable {
  static let currentFormat = "house-capture/5"
  var format = Manifest.currentFormat
  var createdAt: Date
  var device: String
  var systemVersion: String
  var coordinateSystem = "ARKit world space of this capture session: meters, +Y up (gravity), right-handed. Rooms, structure, mesh and photo poses all share it. For house sessions, the house's house.json maps it into the house frame: zone.toHouse × session.alignment.toZone."
  /// The house this session belongs to
  var house: String
  var houseName: String?
  var session: String
  var zone: String
  /// same as house.json's session.alignment at the time of writing (house.json is authoritative)
  var alignment: HouseProject.Alignment?
  /// "worldmap.arworldmap" once saved
  var worldMap: String?
  /// the ARKit world-mapping status when the map was saved ("mapped", "extending") or why it wasn't
  var worldMapStatus: String?
  var endedAt: Date?
  var rooms: [RoomRecord]
  var structure: String?
  var structureError: String?
  var mesh: MeshRecord?
  /// what happened when the app tried to turn on ARKit scene reconstruction
  var sceneReconstruction: String?
  var photos: String
  var photoCount: Int
  var depthFrames: String?
  var depthFrameCount: Int?
  var diagnostics: Diagnostics?
}

/// What the capture session actually did, for debugging bundles after the fact
struct Diagnostics: Codable {
  struct Jump: Codable {
    var time: TimeInterval
    var meters: Float
    var phase: String
  }
  /// the ARConfiguration RoomPlan ran (class, reconstruction, depth, video format)
  var roomPlanConfiguration: String?
  /// the configuration of the detail (coverage) pass
  var detailConfiguration: String?
  /// frame-to-frame camera jumps > 0.5 m: world re-localized; rooms/photos across it may not line up
  var worldJumps: [Jump] = []
  /// white balance locked after the first few seconds of scanning (gains in the photos' camera state)
  var whiteBalance: String?
  /// house-capture/5: tracking state / world-mapping status changes, pauses, map saves (capped)
  var events: [Event]?
  /// Continue sessions: how relocalizing against the zone's saved map went
  var relocalization: Relocalization?
  /// periodic live-mesh ↔ saved-mesh checks after relocalizing (capped)
  var alignmentChecks: [AlignmentCheck]?

  /// Structured framework errors, including the underlying NSError chain.
  var failures: [Failure]?
  /// Bounded samples, persisted during capture as diagnostics.json as well as in the manifest.
  var resourceSamples: [ResourceSample]?
  var memoryWarnings: Int?
  var configurationChanges: Int?

  struct Failure: Codable, Sendable {
    var t: TimeInterval
    var source: String
    var chain: [ErrorDetail]
  }

  struct ErrorDetail: Codable, Sendable {
    var domain: String
    var code: Int
    var description: String
    var reason: String?
    var recovery: String?
  }

  struct ResourceSample: Codable {
    var t: TimeInterval
    var phase: String
    var thermal: String
    var memoryMB: Double?
    var frameAgeSeconds: Double?
    var tracking: String?
    var mapping: String?
    var featurePoints: Int
    var meshAnchors: Int
    var meshVertices: Int
    var photos: Int
    var depthFrames: Int
    var photosInFlight: Int
    var coverageQueueDelayMS: Double?
    var pendingMeshUpdates: Int?
    var coalescedMeshUpdates: Int?
  }

  struct Event: Codable {
    /// seconds since the session started
    var t: TimeInterval
    var what: String
  }

  struct Relocalization: Codable {
    var worldMapBytes: Int?
    var referenceVertices: Int?
    var ghostTriangles: Int?
    var secondsToNormal: TimeInterval?
    var secondsToVerified: TimeInterval?
    var residualMeters: Float?
    var inlierFraction: Float?
    /// "relocalized", "unaligned: <why>", "cancelled"
    var outcome: String?
  }

  struct AlignmentCheck: Codable {
    var t: TimeInterval
    /// live points near the saved surfaces / of those, within 8 cm / their median distance (m)
    var known: Int
    var matched: Int
    var median: Float?
  }

  mutating func log(_ what: String, since start: Date) {
    var list = events ?? []
    list.append(Event(t: (Date().timeIntervalSince(start) * 10).rounded() / 10, what: what))
    if list.count > 400 { list.removeFirst(list.count - 400) }
    events = list
  }
}

struct RoomRecord: Codable, Identifiable {
  var index: Int
  var name: String
  var file: String
  var usdz: String
  var capturedAt: Date
  var walls: Int
  var doors: Int
  var windows: Int
  var openings: Int
  var objects: Int
  /// detail pass results
  var mesh: MeshRecord?
  /// fraction of the in-room mesh area that's well photographed
  var coverage: Float?
  /// Photo detail on observed mesh. Older exports used a mixed heuristic here;
  /// prefer `coverage` for display so old scans also get an honest metric.
  var photoCoverage: Int?
  var photographedPercent: Int? {
    guard let coverage, coverage.isFinite else { return nil }
    return Int((min(1, max(0, coverage)) * 100).rounded())
  }
  var detailSeconds: TimeInterval?
  var detailPhotos: Int?
  /// house-capture/5: RoomPlan's raw CapturedRoomData ("rooms/room-N-data.json")
  var data: String?
  var id: Int { index }
}

struct MeshRecord: Codable {
  var file: String
  var anchors: Int
  var vertices: Int
  var faces: Int
}

/// One reference photo. The JPEG's pixels are the raw camera image in sensor
/// orientation (landscape); `intrinsics` match those pixels. It carries EXIF
/// orientation 6 so viewers show it upright; ignore that when projecting.
struct PhotoRecord: Codable {
  var file: String
  /// index of the room being scanned when this was taken
  var room: Int
  var timestamp: TimeInterval
  var width: Int
  var height: Int
  /// camera-to-world, column-major 4x4. ARKit camera looks down -Z, +Y up, +X right (in sensor orientation)
  var transform: [Float]
  /// column-major 3x3: fx, 0, 0, 0, fy, 0, cx, cy, 1
  var intrinsics: [Float]
  var tracking: String
  var manual: Bool
  /// shutter time in seconds
  var exposure: Double?
  /// predicted motion blur in pixels from camera motion during the exposure (auto photos)
  var blurPixels: Float?
  /// "layout" (RoomPlan pass) or "detail" (coverage pass)
  var pass: String?
  /// a full-resolution still rather than a video frame
  var highRes: Bool?
  /// camera state when the photo was taken, for normalizing brightness/color across photos
  var camera: CameraState?
}

/// Exposure + white balance + ARKit's light estimate for one frame
struct CameraState: Codable {
  /// sensor ISO
  var iso: Float?
  /// shutter time in seconds
  var shutter: Double?
  /// ARKit exposure offset (EV)
  var exposureOffset: Float?
  /// white-balance gains (red, green, blue), constant once white balance is locked
  var whiteBalance: [Float]?
  var whiteBalanceLocked: Bool?
  /// ARKit ambient light estimate: intensity (lumen, ~1000 = neutral) and color temperature (K)
  var ambientIntensity: Float?
  var colorTemperature: Float?
}

extension simd_float4x4 {
  var columnMajor: [Float] {
    [columns.0, columns.1, columns.2, columns.3].flatMap { [$0.x, $0.y, $0.z, $0.w] }
  }
}

extension simd_float3x3 {
  var columnMajor: [Float] {
    [columns.0, columns.1, columns.2].flatMap { [$0.x, $0.y, $0.z] }
  }
}

extension UIDevice {
  /// e.g. "iPhone16,1"
  static var modelIdentifier: String {
    var info = utsname()
    uname(&info)
    return withUnsafeBytes(of: &info.machine) { buf in
      String(decoding: buf.prefix { $0 != 0 }, as: UTF8.self)
    }
  }
}