Skip to content

Capture iPhone LiDAR and photos, reconstruct editable Three.js rooms, and explore them in a browser.

TypeScript
19.6 KB
Raw Blame
import ARKit
import ImageIO
import RoomPlan
import SceneKit

/// The scanning view: camera feed with ARKit's LiDAR mesh drawn over it, each
/// surface tinted by how well the photos cover it, plus RoomPlan's live walls,
/// doors and windows as outlines. Renders whatever configuration the session runs.
@MainActor
final class ScanView: NSObject, ARSCNViewDelegate {
  let view: ARSCNView
  let engine: CoverageEngine
  var onSessionFailure: (([Diagnostics.ErrorDetail]) -> Void)?
  var onSessionEvent: ((String) -> Void)?
  private var nodes: [UUID: SCNNode] = [:]
  /// meshes that arrived before their node was registered
  private var waiting: [UUID: CoverageEngine.AnchorMesh] = [:]
  private let outline = SCNNode()
  private var lastOutline: CFTimeInterval = 0

  /// Red facets need photos; green facets have good photo coverage.
  /// Keep the camera visible through the fill and trace the actual LiDAR triangles on top.
  private let materials: [SCNMaterial] = [
    (0xFF3939, 0.28), (0xF94B4B, 0.23), (0xE82B2B, 0.25),
    (0xFF3939, 0.28), (0xF94B4B, 0.23), (0xE82B2B, 0.25),
    (0x39ED75, 0.24), (0x51F589, 0.22), (0x26D966, 0.23),
    (0xFFFFFF, 0.0), (0xFFFFFF, 0.0), (0xFFFFFF, 0.0),
  ].map { hex, alpha in
    let m = SCNMaterial()
    m.lightingModel = .constant
    m.diffuse.contents = UIColor(red: CGFloat((hex >> 16) & 255) / 255,
                                green: CGFloat((hex >> 8) & 255) / 255,
                                blue: CGFloat(hex & 255) / 255, alpha: 1)
    m.transparency = alpha
    m.blendMode = .alpha
    m.isDoubleSided = false
    m.writesToDepthBuffer = true
    m.readsFromDepthBuffer = true
    return m
  }
  private let triangleLine: SCNMaterial = {
    let m = SCNMaterial()
    m.lightingModel = .constant
    m.diffuse.contents = UIColor.white
    m.transparency = 0.42
    m.fillMode = .lines
    m.blendMode = .alpha
    m.isDoubleSided = false
    m.readsFromDepthBuffer = true
    m.writesToDepthBuffer = false
    return m
  }()
  private let wallLine: SCNMaterial = {
    let m = SCNMaterial()
    m.lightingModel = .constant
    m.diffuse.contents = UIColor.white
    m.transparency = 0.9
    m.readsFromDepthBuffer = false // always visible, like RoomPlan's own outlines
    return m
  }()
  private let openingLine: SCNMaterial = {
    let m = SCNMaterial()
    m.lightingModel = .constant
    m.diffuse.contents = UIColor.cyan
    m.readsFromDepthBuffer = false
    return m
  }()

  /// Whether live mesh anchors go to the coverage engine. Off while relocalizing (the world
  /// isn't the zone's yet, and the engine keeps every anchor it's given) and while paused.
  /// Read on SceneKit's render thread.
  nonisolated let feed = FeedGate()
  /// earlier scans of the zone (Continue sessions), drawn under the live mesh
  private let ghost = SCNNode()
  private let ghostEdges = SCNNode()
  private let ghostMaterial: SCNMaterial = {
    let m = SCNMaterial()
    m.lightingModel = .constant
    m.diffuse.contents = UIColor(red: 0.2, green: 1, blue: 0.45, alpha: 1)
    m.blendMode = .alpha
    m.isDoubleSided = true
    // nearest ghost surface only (don't stack the rooms behind a wall); the ghost sits 1.5 cm
    // behind the saved surface so the live mesh draws over it without z-fighting
    m.writesToDepthBuffer = true
    m.readsFromDepthBuffer = true
    return m
  }()

  init(session: ARSession, engine: CoverageEngine) {
    view = ARSCNView(frame: .zero)
    self.engine = engine
    super.init()
    view.session = session
    view.delegate = self
    view.automaticallyUpdatesLighting = false
    view.rendersCameraGrain = false
    view.rendersMotionBlur = false
    view.antialiasingMode = .multisampling4X
    view.scene = SCNScene()
    outline.renderingOrder = 10
    view.scene.rootNode.addChildNode(outline)
    ghost.renderingOrder = -5 // before the live coverage mesh
    ghost.isHidden = true
    view.scene.rootNode.addChildNode(ghost)
    ghostEdges.renderingOrder = -4
    ghost.addChildNode(ghostEdges)
    engine.onMeshes = { [weak self] meshes in self?.apply(meshes) }
  }

  // MARK: ghost of earlier scans

  var hasGhost: Bool { ghost.geometry != nil }

  /// Zone frame = this session's world once relocalized, so the ghost sits at the origin
  func setGhost(_ g: ZoneReference.Ghost?) {
    guard let g, g.triangleCount > 0 else {
      ghost.geometry = nil
      ghostEdges.geometry = nil
      return
    }
    let positions = g.positions.withUnsafeBufferPointer { Data(buffer: $0) }
    let normals = g.normals.withUnsafeBufferPointer { Data(buffer: $0) }
    let stride = MemoryLayout<SIMD3<Float>>.stride
    let vs = SCNGeometrySource(data: positions, semantic: .vertex, vectorCount: g.positions.count,
                               usesFloatComponents: true, componentsPerVector: 3, bytesPerComponent: 4, dataOffset: 0, dataStride: stride)
    let ns = SCNGeometrySource(data: normals, semantic: .normal, vectorCount: g.normals.count,
                               usesFloatComponents: true, componentsPerVector: 3, bytesPerComponent: 4, dataOffset: 0, dataStride: stride)
    let idx = g.indices.withUnsafeBufferPointer { Data(buffer: $0) }
    let el = SCNGeometryElement(data: idx, primitiveType: .triangles, primitiveCount: g.triangleCount, bytesPerIndex: 4)
    let geometry = SCNGeometry(sources: [vs, ns], elements: [el])
    geometry.materials = [ghostMaterial]
    ghost.geometry = geometry
    let edges = SCNGeometry(sources: [vs], elements: [el])
    let material = triangleLine.copy() as! SCNMaterial
    material.diffuse.contents = UIColor.systemGreen
    edges.materials = [material]
    ghostEdges.geometry = edges
  }

  /// 0 hides it
  func setGhostOpacity(_ alpha: CGFloat) {
    ghost.isHidden = alpha <= 0 || ghost.geometry == nil
    ghostMaterial.transparency = alpha
    ghostEdges.opacity = min(1, alpha * 2)
  }

  /// Hand every live mesh anchor to the engine again (after it wasn't fed for a while).
  /// Copying the buffers is heavy, so it happens off main; holding the anchors keeps them valid.
  func refeedEngine(from session: ARSession) {
    let anchors = session.currentFrame?.anchors.compactMap { $0 as? ARMeshAnchor } ?? []
    let engine = engine
    DispatchQueue.global(qos: .userInitiated).async {
      for a in anchors { engine.update(a) }
    }
  }

  /// RoomPlan's current walls (white) and doors/windows/openings (cyan), as outlines
  func showRoom(_ room: CapturedRoom?) {
    let now = CACurrentMediaTime()
    guard room == nil || now - lastOutline > 0.4 else { return } // RoomPlan updates a lot
    lastOutline = now
    outline.childNodes.forEach { $0.removeFromParentNode() }
    guard let room else { return }
    for w in room.walls { outline.addChildNode(rectangle(w.transform, w.dimensions, wallLine, inset: 0)) }
    for o in room.doors + room.windows + room.openings { outline.addChildNode(rectangle(o.transform, o.dimensions, openingLine, inset: 0.01)) }
  }

  private func rectangle(_ t: simd_float4x4, _ d: SIMD3<Float>, _ material: SCNMaterial, inset: Float) -> SCNNode {
    let hw = d.x / 2 - inset, hh = d.y / 2 - inset
    let pts = [SIMD3<Float>(-hw, -hh, 0.005), SIMD3(hw, -hh, 0.005), SIMD3(hw, hh, 0.005), SIMD3(-hw, hh, 0.005)]
    let src = SCNGeometrySource(vertices: pts.map { SCNVector3($0.x, $0.y, $0.z) })
    let idx: [UInt16] = [0, 1, 1, 2, 2, 3, 3, 0]
    let el = SCNGeometryElement(indices: idx, primitiveType: .line)
    let g = SCNGeometry(sources: [src], elements: [el])
    g.materials = [material]
    let node = SCNNode(geometry: g)
    node.simdTransform = t
    node.renderingOrder = 10
    return node
  }

  // MARK: ARSCNViewDelegate (render thread)

  /// Clip anything within 30 cm of the lens: it only blocks the view, and it's too close
  /// to photograph usefully anyway
  nonisolated func renderer(_ renderer: SCNSceneRenderer, updateAtTime time: TimeInterval) {
    if let cam = renderer.pointOfView?.camera, cam.zNear != 0.3 { cam.zNear = 0.3 }
  }

  /// Relocalize after an interruption (a phone call, the app in the background) instead of
  /// starting a new world
  nonisolated func sessionShouldAttemptRelocalization(_ session: ARSession) -> Bool { true }

  // Observe through ARSCNView, without replacing RoomPlan's ARSession delegate.
  nonisolated func session(_ session: ARSession, didFailWithError error: Error) {
    let chain = Diagnostics.errorChain(error)
    Task { @MainActor in self.onSessionFailure?(chain) }
  }

  nonisolated func sessionWasInterrupted(_ session: ARSession) {
    Task { @MainActor in self.onSessionEvent?("ARKit session interrupted") }
  }

  nonisolated func sessionInterruptionEnded(_ session: ARSession) {
    Task { @MainActor in self.onSessionEvent?("ARKit session interruption ended") }
  }


  nonisolated func renderer(_ renderer: SCNSceneRenderer, nodeFor anchor: ARAnchor) -> SCNNode? {
    guard let mesh = anchor as? ARMeshAnchor else { return nil }
    if feed.isOn { engine.update(mesh) }
    let node = SCNNode()
    let id = mesh.identifier
    Task { @MainActor in
      self.nodes[id] = node
      if let m = self.waiting.removeValue(forKey: id) { self.apply([m]) }
    }
    return node
  }

  nonisolated func renderer(_ renderer: SCNSceneRenderer, didUpdate node: SCNNode, for anchor: ARAnchor) {
    guard let mesh = anchor as? ARMeshAnchor, feed.isOn else { return }
    engine.update(mesh)
  }

  nonisolated func renderer(_ renderer: SCNSceneRenderer, didRemove node: SCNNode, for anchor: ARAnchor) {
    guard anchor is ARMeshAnchor else { return }
    let id = anchor.identifier
    engine.remove(id)
    Task { @MainActor in
      self.nodes[id] = nil
      self.waiting[id] = nil
    }
  }

  // MARK: geometry

  private func apply(_ meshes: [CoverageEngine.AnchorMesh]) {
    for m in meshes {
      guard m.vertexCount > 0, m.bandCounts.reduce(0, +) > 0 else { continue }
      guard let node = nodes[m.id] else {
        waiting[m.id] = m
        continue
      }
      // SIMD3<Float> is padded to 16 bytes in Swift arrays
      let stride = MemoryLayout<SIMD3<Float>>.stride
      let vertices = SCNGeometrySource(
        data: m.positions, semantic: .vertex, vectorCount: m.vertexCount,
        usesFloatComponents: true, componentsPerVector: 3, bytesPerComponent: 4, dataOffset: 0, dataStride: stride)
      let normals = SCNGeometrySource(
        data: m.normals, semantic: .normal, vectorCount: m.vertexCount,
        usesFloatComponents: true, componentsPerVector: 3, bytesPerComponent: 4, dataOffset: 0, dataStride: stride)
      var elements: [SCNGeometryElement] = []
      var mats: [SCNMaterial] = []
      var lineIndices = Data()
      var triangleCount = 0
      for (band, data) in m.bands.enumerated() where m.bandCounts[band] > 0 && materials[band].transparency > 0 {
        elements.append(SCNGeometryElement(data: data, primitiveType: .triangles, primitiveCount: m.bandCounts[band], bytesPerIndex: 4))
        mats.append(materials[band])
        lineIndices.append(data)
        triangleCount += m.bandCounts[band]
      }
      guard !elements.isEmpty else {
        node.geometry = nil
        node.childNode(withName: "scan-triangles", recursively: false)?.removeFromParentNode()
        continue
      }
      let geometry = SCNGeometry(sources: [vertices, normals], elements: elements)
      geometry.materials = mats
      let isNew = node.geometry == nil
      node.geometry = geometry
      let edges = node.childNode(withName: "scan-triangles", recursively: false) ?? SCNNode()
      edges.name = "scan-triangles"
      let lines = SCNGeometry(sources: [vertices], elements: [
        SCNGeometryElement(data: lineIndices, primitiveType: .triangles,
                           primitiveCount: triangleCount, bytesPerIndex: 4)
      ])
      lines.materials = [triangleLine]
      edges.geometry = lines
      edges.renderingOrder = 1
      if edges.parent == nil { node.addChildNode(edges) }
      if isNew && !UIAccessibility.isReduceMotionEnabled {
        node.opacity = 0.15
        let reveal = SCNAction.fadeIn(duration: 0.65)
        reveal.timingMode = .easeOut
        node.runAction(reveal, forKey: "scan-reveal")
      }
    }
  }
}

/// A thread-safe on/off switch
final class FeedGate: @unchecked Sendable {
  private let lock = NSLock()
  private var on = true
  var isOn: Bool {
    get { lock.withLock { on } }
    set { lock.withLock { on = newValue } }
  }
}

/// Records LiDAR depth frames (with pose + intrinsics) up to 5× a second while moving:
/// zlib-compressed Float16 depth + UInt8 confidence + a 640×480 color JPEG of the same
/// frame. A full RGB-D sequence, so the Mac can build a fully colored mesh.
final class DepthRecorder: @unchecked Sendable {
  struct Record: Codable {
    /// Float16 meters, row-major, raw DEFLATE (zlib.decompress(data, -15) in Python)
    var file: String
    var confidence: String
    /// 640×480 JPEG of the camera image at this frame (sensor orientation, no EXIF rotation)
    var rgb: String?
    /// intrinsics for the rgb image (column-major 3×3)
    var rgbIntrinsics: [Float]?
    var camera: CameraState?
    var room: Int
    var timestamp: TimeInterval
    var width: Int
    var height: Int
    /// camera-to-world, column-major (same convention as photos)
    var transform: [Float]
    /// column-major 3x3, scaled to the depth map's resolution
    var intrinsics: [Float]
  }

  private let bundle: ScanBundle
  private let queue = DispatchQueue(label: "depth-writer", qos: .utility)
  private(set) var records: [Record] = []
  private var last: (time: TimeInterval, pose: simd_float4x4)?
  private var next = 0
  private var inFlight = 0
  private let ci = CIContext()

  init(bundle: ScanBundle) {
    self.bundle = bundle
  }

  /// Call every frame; keeps one when the camera has moved or turned enough (≤ 5/s)
  func consider(_ frame: ARFrame, room: Int, camera: CameraState?, interval: TimeInterval = 0.2) {
    guard let depth = frame.sceneDepth, inFlight < 3 else { return }
    guard case .normal = frame.camera.trackingState else { return }
    let pose = frame.camera.transform
    if let last {
      let moved = simd_distance(pose.columns.3, last.pose.columns.3)
      let turned = acos(min(1, max(-1, simd_dot(simd_make_float3(pose.columns.2), simd_make_float3(last.pose.columns.2)))))
      guard frame.timestamp - last.time > interval, moved > 0.04 || turned > 0.06 else { return }
    }
    last = (frame.timestamp, pose)
    let w = CVPixelBufferGetWidth(depth.depthMap), h = CVPixelBufferGetHeight(depth.depthMap)
    let scale = Float(w) / Float(frame.camera.imageResolution.width)
    var k = frame.camera.intrinsics
    k.columns.0.x *= scale; k.columns.1.y *= scale; k.columns.2.x *= scale; k.columns.2.y *= scale
    let name = String(format: "%05d", next)
    next += 1
    // color: downscale on the GPU now (so we don't hold ARKit's buffer), encode later
    let rgbScale = 640 / Float(frame.camera.imageResolution.width)
    var kc = frame.camera.intrinsics
    kc.columns.0.x *= rgbScale; kc.columns.1.y *= rgbScale; kc.columns.2.x *= rgbScale; kc.columns.2.y *= rgbScale
    let small = CIImage(cvPixelBuffer: frame.capturedImage).transformed(by: CGAffineTransform(scaleX: CGFloat(rgbScale), y: CGFloat(rgbScale)))
    let cg = ci.createCGImage(small, from: CGRect(x: 0, y: 0, width: 640, height: CGFloat((Float(frame.camera.imageResolution.height) * rgbScale).rounded())))
    let record = Record(
      file: "depth/\(name).f16z", confidence: "depth/\(name).conf", rgb: cg == nil ? nil : "depth/\(name).jpg",
      rgbIntrinsics: kc.columnMajor, camera: camera, room: room, timestamp: frame.timestamp,
      width: w, height: h, transform: pose.columnMajor, intrinsics: k.columnMajor)
    let depthData = Self.float16(depth.depthMap)
    let confData = depth.confidenceMap.map(Self.bytes)
    inFlight += 1
    queue.async {
      try? ((try? (depthData as NSData).compressed(using: .zlib)) as Data? ?? depthData).write(to: self.bundle.file(record.file))
      try? confData?.write(to: self.bundle.file(record.confidence))
      if let cg, let rgb = record.rgb, let dest = CGImageDestinationCreateWithURL(self.bundle.file(rgb) as CFURL, "public.jpeg" as CFString, 1, nil) {
        CGImageDestinationAddImage(dest, cg, [kCGImageDestinationLossyCompressionQuality: 0.8] as CFDictionary)
        CGImageDestinationFinalize(dest)
      }
      DispatchQueue.main.async {
        self.records.append(record)
        self.inFlight -= 1
      }
    }
  }

  func save() throws {
    let e = JSONEncoder()
    e.outputFormatting = [.prettyPrinted]
    try e.encode(records).write(to: bundle.file("depth/frames.json"))
  }

  func discard(room: Int) {
    for r in records where r.room == room {
      try? FileManager.default.removeItem(at: bundle.file(r.file))
      try? FileManager.default.removeItem(at: bundle.file(r.confidence))
      if let rgb = r.rgb { try? FileManager.default.removeItem(at: bundle.file(rgb)) }
    }
    records.removeAll { $0.room == room }
  }

  static func float16(_ buffer: CVPixelBuffer) -> Data {
    CVPixelBufferLockBaseAddress(buffer, .readOnly)
    defer { CVPixelBufferUnlockBaseAddress(buffer, .readOnly) }
    let w = CVPixelBufferGetWidth(buffer), h = CVPixelBufferGetHeight(buffer)
    let row = CVPixelBufferGetBytesPerRow(buffer)
    let base = CVPixelBufferGetBaseAddress(buffer)!
    var out = [Float16](repeating: 0, count: w * h)
    for y in 0..<h {
      let src = base.advanced(by: y * row).assumingMemoryBound(to: Float32.self)
      for x in 0..<w { out[y * w + x] = Float16(src[x]) }
    }
    return out.withUnsafeBufferPointer { Data(buffer: $0) }
  }

  static func bytes(_ buffer: CVPixelBuffer) -> Data {
    CVPixelBufferLockBaseAddress(buffer, .readOnly)
    defer { CVPixelBufferUnlockBaseAddress(buffer, .readOnly) }
    let w = CVPixelBufferGetWidth(buffer), h = CVPixelBufferGetHeight(buffer)
    let row = CVPixelBufferGetBytesPerRow(buffer)
    let base = CVPixelBufferGetBaseAddress(buffer)!
    var out = Data(capacity: w * h)
    for y in 0..<h { out.append(base.advanced(by: y * row).assumingMemoryBound(to: UInt8.self), count: w) }
    return out
  }

  /// 64×48 grid of each block's median depth (robust to LiDAR speckle; low-confidence
  /// pixels ignored), for the coverage engine's occlusion test
  static func grid(for frame: ARFrame, cells: (Int, Int) = (64, 48)) -> CoverageEngine.DepthGrid? {
    guard let sd = frame.sceneDepth else { return nil }
    let depth = sd.depthMap
    CVPixelBufferLockBaseAddress(depth, .readOnly)
    defer { CVPixelBufferUnlockBaseAddress(depth, .readOnly) }
    let conf = sd.confidenceMap
    if let conf { CVPixelBufferLockBaseAddress(conf, .readOnly) }
    defer { if let conf { CVPixelBufferUnlockBaseAddress(conf, .readOnly) } }
    let w = CVPixelBufferGetWidth(depth), h = CVPixelBufferGetHeight(depth)
    let row = CVPixelBufferGetBytesPerRow(depth)
    let base = CVPixelBufferGetBaseAddress(depth)!
    let confBase = conf.flatMap { CVPixelBufferGetBaseAddress($0) }
    let confRow = conf.map { CVPixelBufferGetBytesPerRow($0) } ?? 0
    let (gw, gh) = cells
    var buckets = [[Float]](repeating: [], count: gw * gh)
    for y in 0..<h {
      let src = base.advanced(by: y * row).assumingMemoryBound(to: Float32.self)
      let c = confBase?.advanced(by: y * confRow).assumingMemoryBound(to: UInt8.self)
      let gy = y * gh / h
      for x in 0..<w {
        let v = src[x]
        guard v > 0, (c?[x] ?? 2) >= 1 else { continue }
        buckets[gy * gw + x * gw / w].append(v)
      }
    }
    let values = buckets.map { b -> Float in
      guard !b.isEmpty else { return 0 } // unknown: the engine skips the test there
      let s = b.sorted()
      return s[s.count / 2]
    }
    let scale = Float(gw) / Float(frame.camera.imageResolution.width)
    let k = frame.camera.intrinsics
    return CoverageEngine.DepthGrid(
      width: gw, height: gh, values: values, worldToCamera: frame.camera.transform.inverse,
      fx: k.columns.0.x * scale, fy: k.columns.1.y * scale, cx: k.columns.2.x * scale, cy: k.columns.2.y * scale)
  }
}