Run a virtual iPhone on an Apple Silicon Mac.
Swift
feat(archive): add tree fingerprints, and fix the hardlink bug they found
Lakrf89366ed4f79
diff --git a/sources/VPhoneArchive/VPhoneArchiveExtractor.swift b/sources/VPhoneArchive/VPhoneArchiveExtractor.swiftindex fb556834..7b4a375d 100644--- a/sources/VPhoneArchive/VPhoneArchiveExtractor.swift+++ b/sources/VPhoneArchive/VPhoneArchiveExtractor.swift@@ -95,6 +95,27 @@ public enum VPhoneArchiveExtractor {}archive_entry_set_pathname(entry, targetPath)+ // A hardlink entry names its target as an archive-relative path,+ // and libarchive resolves that against the process's working+ // directory. Since the entry's own path has just been made+ // absolute, the target has to be too, or extraction fails with+ // "Hard-link target 'x' does not exist" — which is what happened.+ //+ // Symlinks are deliberately left alone: their target is data,+ // stored and restored verbatim, and rewriting one would change+ // what the link says.+ if let rawLink = archive_entry_hardlink(entry) {+ let linkPath = String(cString: rawLink)+ let linkTarget = resolvedDestination.appendingPathComponent(linkPath)+ let resolvedLink = linkTarget.standardized.path+ guard resolvedLink.hasPrefix(destinationPath + "/") else {+ throw VPhoneArchiveError.pathEscapesDestination(+ member: linkPath, destination: destinationPath+ )+ }+ archive_entry_set_hardlink(entry, resolvedLink)+ }+if options.noOverwriteDir, shouldSkipExistingDirectory(entry, at: target) {// Skip the header only. Everything under this directory that// does not yet exist is still written on its own entry, whichdiff --git a/sources/VPhoneArchive/VPhoneArchiveWriter.swift b/sources/VPhoneArchive/VPhoneArchiveWriter.swiftindex ab94fac2..a9c95370 100644--- a/sources/VPhoneArchive/VPhoneArchiveWriter.swift+++ b/sources/VPhoneArchive/VPhoneArchiveWriter.swift@@ -79,6 +79,21 @@ public enum VPhoneArchiveWriter {)}+ // Without a link resolver, every name of a hardlinked file is packed+ // as its own full copy, and unpacking gives back separate files. That+ // matters here: iosbinpack64 and the procursus bootstrap are full of+ // hardlinks, so flattening them silently doubles what lands on the+ // guest volume and breaks anything that expected the identity. GNU tar+ // does this by default, which is how the difference was found — see+ // `vphone-archive fingerprint`.+ //+ // With the tar strategy the resolver never defers: the first name is+ // returned as-is, and later ones come back as hardlink references with+ // size 0, which the data copy below already skips.+ let resolver = archive_entry_linkresolver_new()+ archive_entry_linkresolver_set_strategy(resolver, archive_format(writer))+ defer { archive_entry_linkresolver_free(resolver) }+let rootPath = resolvedRoot.pathvar written = 0var bytes: Int64 = 0@@ -104,13 +119,24 @@ public enum VPhoneArchiveWriter {if patterns.contains(where: { matches(relative, pattern: $0) }) { continue }archive_entry_set_pathname(entry, relative)- guard archive_write_header(writer, entry) == ARCHIVE_OK else {+ // Turns the second and later names of a hardlinked file into+ // references to the first. The archive-relative pathname has to be+ // set before this, or the resolver records absolute paths as the+ // link targets.+ var resolved: OpaquePointer? = entry+ var deferred: OpaquePointer?+ archive_entry_linkify(resolver, &resolved, &deferred)+ guard let resolved else { continue }++ guard archive_write_header(writer, resolved) == ARCHIVE_OK else {throw VPhoneArchiveError.writeFailed(path: relative, reason: archiveErrorString(writer))}- if archive_entry_size(entry) > 0 {+ // A hardlink reference comes back with size 0, so this also skips+ // re-storing contents we have already written once.+ if archive_entry_size(resolved) > 0 {bytes += try copyFile(at: absolute, into: writer, isCancelled: isCancelled)}diff --git a/sources/VPhoneArchive/VPhoneTreeFingerprint.swift b/sources/VPhoneArchive/VPhoneTreeFingerprint.swiftnew file mode 100644index 00000000..e7f3d955--- /dev/null+++ b/sources/VPhoneArchive/VPhoneTreeFingerprint.swift@@ -0,0 +1,232 @@+import CryptoKit+import Foundation++/// A complete description of a directory tree, for comparing two of them.+///+/// `diff -r` plus `stat` is not enough to decide that GNU tar and libarchive+/// produced the same result, and the four things it misses are exactly the+/// four most likely to differ: ACLs, extended attributes, which files are+/// hardlinked to which, and sparse-file occupancy. All four change how a guest+/// behaves and none of them shows up in a diff.+///+/// The output is deterministic and JSON, so two trees compare with a plain+/// equality check and any difference names itself.+public struct VPhoneTreeFingerprint: Codable, Equatable, Sendable {+ public struct Entry: Codable, Equatable, Sendable {+ /// Path relative to the tree root, so two trees in different places+ /// still compare.+ public let path: String+ /// file, dir, symlink, fifo, socket, blockdev, chardev+ public let type: String+ /// Permission bits including setuid, setgid and sticky, as octal.+ public let mode: String+ /// Numeric, never resolved to a name: the whole point of restoring+ /// ownership by number is that the name means something different on+ /// the host than it does in the guest.+ public let uid: Int+ public let gid: Int+ public let size: Int64+ /// Whole nanoseconds, because ARCHIVE_EXTRACT_TIME claims to restore+ /// them.+ public let mtimeNanoseconds: Int64+ /// Unresolved target of a symlink.+ public let linkTarget: String?+ /// Which files share an inode, normalised to a sequence number.+ ///+ /// Comparing inode numbers directly is meaningless across two+ /// extractions; what has to match is the *grouping*. Files with no+ /// other link have no id.+ public let hardlinkGroup: Int?+ /// Extended attribute names with a SHA-256 of each value. Covers the+ /// `com.apple.*` ones, which is where AppleDouble handling shows up.+ public let xattrs: [String: String]+ /// `acl_to_text` output, or nil when there is no ACL. libarchive and+ /// GNU tar disagree here more readily than anywhere else.+ public let acl: String?+ /// Blocks actually allocated. Two files can have the same logical size+ /// and different occupancy, which is what a lost sparse hole looks+ /// like.+ public let blocks: Int64+ /// Content digest, regular files only.+ public let contentSHA256: String?+ }++ public let root: String+ public let entries: [Entry]++ // MARK: - Capture++ public static func capture(_ root: URL, includeContentHashes: Bool = true) throws -> Self {+ let resolvedRoot = try VPhoneArchivePaths.resolved(root)+ let rootPath = resolvedRoot.path++ var paths: [String] = []++ // An explicit walk rather than FileManager's enumerator, which follows+ // its own rules about symlinks and packages. This has to see exactly+ // what is on disk, in a stable order.+ func walk(_ directory: String) throws {+ let names = try FileManager.default.contentsOfDirectory(atPath: directory)+ for name in names.sorted() {+ let full = directory + "/" + name+ paths.append(full)+ var info = stat()+ guard lstat(full, &info) == 0 else { continue }+ if (info.st_mode & S_IFMT) == S_IFDIR { try walk(full) }+ }+ }+ try walk(rootPath)++ // Hardlink groups, assigned in path order so the numbering is stable.+ var groupOf: [String: Int] = [:] // "dev:ino" -> group+ var linkCounts: [String: Int] = [:]+ for path in paths {+ var info = stat()+ guard lstat(path, &info) == 0, info.st_nlink > 1 else { continue }+ let key = "\(info.st_dev):\(info.st_ino)"+ linkCounts[key, default: 0] += 1+ }+ var nextGroup = 0+ for path in paths {+ var info = stat()+ guard lstat(path, &info) == 0, info.st_nlink > 1 else { continue }+ let key = "\(info.st_dev):\(info.st_ino)"+ guard linkCounts[key] ?? 0 > 1, groupOf[key] == nil else { continue }+ groupOf[key] = nextGroup+ nextGroup += 1+ }++ var entries: [Entry] = []+ entries.reserveCapacity(paths.count)+ for path in paths {+ var info = stat()+ guard lstat(path, &info) == 0 else { continue }+ let relative = VPhoneArchivePaths.relative(path, under: rootPath)+ let fileType = info.st_mode & S_IFMT+ let key = "\(info.st_dev):\(info.st_ino)"++ entries.append(Entry(+ path: relative,+ type: describe(fileType),+ mode: String(info.st_mode & 0o7777, radix: 8),+ uid: Int(info.st_uid),+ gid: Int(info.st_gid),+ size: fileType == S_IFREG ? info.st_size : 0,+ mtimeNanoseconds: Int64(info.st_mtimespec.tv_sec) * 1_000_000_000+ + Int64(info.st_mtimespec.tv_nsec),+ linkTarget: fileType == S_IFLNK+ ? try? FileManager.default.destinationOfSymbolicLink(atPath: path)+ : nil,+ hardlinkGroup: info.st_nlink > 1 ? groupOf[key] : nil,+ xattrs: extendedAttributes(of: path),+ acl: accessControlList(of: path),+ blocks: Int64(info.st_blocks),+ contentSHA256: includeContentHashes && fileType == S_IFREG+ ? digest(of: path)+ : nil+ ))+ }++ return VPhoneTreeFingerprint(root: rootPath, entries: entries)+ }++ /// Every way the two trees differ, described by path.+ ///+ /// The root itself is not compared: two extractions of the same archive+ /// into different directories are supposed to differ there.+ public func differences(from other: VPhoneTreeFingerprint) -> [String] {+ var found: [String] = []+ let mine = Dictionary(uniqueKeysWithValues: entries.map { ($0.path, $0) })+ let theirs = Dictionary(uniqueKeysWithValues: other.entries.map { ($0.path, $0) })++ for path in Set(mine.keys).subtracting(theirs.keys).sorted() {+ found.append("\(path): only in \(root)")+ }+ for path in Set(theirs.keys).subtracting(mine.keys).sorted() {+ found.append("\(path): only in \(other.root)")+ }+ for path in Set(mine.keys).intersection(theirs.keys).sorted() {+ let a = mine[path]!, b = theirs[path]!+ if a == b { continue }+ if a.type != b.type { found.append("\(path): type \(a.type) vs \(b.type)") }+ if a.mode != b.mode { found.append("\(path): mode \(a.mode) vs \(b.mode)") }+ if a.uid != b.uid || a.gid != b.gid {+ found.append("\(path): owner \(a.uid):\(a.gid) vs \(b.uid):\(b.gid)")+ }+ if a.size != b.size { found.append("\(path): size \(a.size) vs \(b.size)") }+ if a.mtimeNanoseconds != b.mtimeNanoseconds {+ found.append("\(path): mtime \(a.mtimeNanoseconds) vs \(b.mtimeNanoseconds)")+ }+ if a.linkTarget != b.linkTarget {+ found.append("\(path): symlink \(a.linkTarget ?? "-") vs \(b.linkTarget ?? "-")")+ }+ if a.hardlinkGroup != b.hardlinkGroup {+ found.append("\(path): hardlink group \(a.hardlinkGroup.map(String.init) ?? "-") vs \(b.hardlinkGroup.map(String.init) ?? "-")")+ }+ if a.xattrs != b.xattrs {+ found.append("\(path): xattrs \(a.xattrs.keys.sorted()) vs \(b.xattrs.keys.sorted())")+ }+ if a.acl != b.acl { found.append("\(path): ACL differs") }+ if a.blocks != b.blocks {+ found.append("\(path): occupancy \(a.blocks) vs \(b.blocks) blocks (sparseness)")+ }+ if a.contentSHA256 != b.contentSHA256 { found.append("\(path): contents differ") }+ }+ return found+ }++ // MARK: - Details++ private static func describe(_ fileType: mode_t) -> String {+ switch fileType {+ case S_IFREG: "file"+ case S_IFDIR: "dir"+ case S_IFLNK: "symlink"+ case S_IFIFO: "fifo"+ case S_IFSOCK: "socket"+ case S_IFBLK: "blockdev"+ case S_IFCHR: "chardev"+ default: "other"+ }+ }++ private static func extendedAttributes(of path: String) -> [String: String] {+ let size = listxattr(path, nil, 0, XATTR_NOFOLLOW)+ guard size > 0 else { return [:] }++ var names = [CChar](repeating: 0, count: size)+ guard listxattr(path, &names, size, XATTR_NOFOLLOW) == size else { return [:] }++ var result: [String: String] = [:]+ for chunk in names.split(separator: 0) {+ let name = String(decoding: chunk.map { UInt8(bitPattern: $0) }, as: UTF8.self)+ guard !name.isEmpty else { continue }+ let valueSize = getxattr(path, name, nil, 0, 0, XATTR_NOFOLLOW)+ guard valueSize >= 0 else { continue }+ var value = [UInt8](repeating: 0, count: max(valueSize, 1))+ guard getxattr(path, name, &value, valueSize, 0, XATTR_NOFOLLOW) == valueSize else { continue }+ result[name] = SHA256.hash(data: Data(value.prefix(valueSize)))+ .map { String(format: "%02x", $0) }.joined()+ }+ return result+ }++ private static func accessControlList(of path: String) -> String? {+ guard let acl = acl_get_link_np(path, ACL_TYPE_EXTENDED) else { return nil }+ defer { acl_free(UnsafeMutableRawPointer(acl)) }+ guard let text = acl_to_text(acl, nil) else { return nil }+ defer { acl_free(UnsafeMutableRawPointer(text)) }+ let value = String(cString: text)+ return value.isEmpty ? nil : value+ }++ private static func digest(of path: String) -> String? {+ guard let handle = FileHandle(forReadingAtPath: path) else { return nil }+ defer { try? handle.close() }+ var hasher = SHA256()+ while let chunk = try? handle.read(upToCount: 1 << 20), !chunk.isEmpty {+ hasher.update(data: chunk)+ }+ return hasher.finalize().map { String(format: "%02x", $0) }.joined()+ }+}diff --git a/sources/vphone-archive/main.swift b/sources/vphone-archive/main.swiftindex ebe6b3fc..584b5452 100644--- a/sources/vphone-archive/main.swift+++ b/sources/vphone-archive/main.swift@@ -22,7 +22,10 @@ struct VPhoneArchiveCLI: ParsableCommand {The compressor is detected when reading, so there is no --zstd to passand no way to pass the wrong one.""",- subcommands: [Extract.self, Create.self, Decompress.self, List.self, Cat.self],+ subcommands: [+ Extract.self, Create.self, Decompress.self,+ List.self, Cat.self, Fingerprint.self,+ ],defaultSubcommand: Extract.self)}@@ -216,4 +219,56 @@ struct Cat: ParsableCommand {}}+// MARK: - fingerprint++struct Fingerprint: ParsableCommand {+ static let configuration = CommandConfiguration(+ commandName: "fingerprint",+ abstract: "Describe a directory tree completely, or compare two of them",+ discussion: """+ For deciding whether GNU tar and vphone-archive produced the same+ result. `diff -r` and `stat` cannot answer that: they miss ACLs,+ extended attributes, which files are hardlinked to which, and+ sparse-file occupancy — the four things most likely to differ, and+ all four of which change how a guest behaves.++ With one path, writes JSON. With two, prints the differences and exits+ non-zero if there are any.+ """+ )++ @Argument(help: "Tree to describe", transform: URL.init(fileURLWithPath:))+ var tree: URL++ @Argument(help: "Second tree; given, the two are compared",+ transform: URL.init(fileURLWithPath:))+ var other: URL?++ @Flag(name: .customLong("no-content-hashes"),+ help: "Skip file digests — much faster, and enough to compare metadata")+ var noContentHashes = false++ func run() throws {+ let first = try VPhoneTreeFingerprint.capture(tree, includeContentHashes: !noContentHashes)++ guard let other else {+ let encoder = JSONEncoder()+ encoder.outputFormatting = [.prettyPrinted, .sortedKeys]+ print(String(decoding: try encoder.encode(first), as: UTF8.self))+ return+ }++ let second = try VPhoneTreeFingerprint.capture(+ other, includeContentHashes: !noContentHashes+ )+ let differences = first.differences(from: second)+ guard differences.isEmpty else {+ for line in differences { print(line) }+ print("\n\(differences.count) difference(s)")+ throw ExitCode(1)+ }+ print("identical (\(first.entries.count) entries)")+ }+}+VPhoneArchiveCLI.main()diff --git a/tests/VPhoneArchiveTests/RoundTripTests.swift b/tests/VPhoneArchiveTests/RoundTripTests.swiftindex 4d608fe7..f1514e4d 100644--- a/tests/VPhoneArchiveTests/RoundTripTests.swift+++ b/tests/VPhoneArchiveTests/RoundTripTests.swift@@ -125,6 +125,75 @@ struct RoundTripTests {}}+ @Test("hardlinks survive as hardlinks")+ func hardlinksArePreserved() throws {+ let source = try Self.makeTree()+ let archive = Self.scratch("hl").appendingPathExtension("tar")+ let destination = Self.scratch("hlout")+ try FileManager.default.createDirectory(at: destination, withIntermediateDirectories: true)+ defer {+ for url in [source, archive, destination] {+ try? FileManager.default.removeItem(at: url)+ }+ }++ try FileManager.default.linkItem(+ at: source.appendingPathComponent("top.txt"),+ to: source.appendingPathComponent("same.txt")+ )++ try VPhoneArchiveWriter.create(archive: archive, from: source)+ try VPhoneArchiveExtractor.extract(archive, into: destination, options: .intoHostDirectory)++ // Without archive_entry_linkresolver each name is packed as a separate+ // full copy, and unpacking gives back two unrelated files. That is not+ // cosmetic: iosbinpack64 and the procursus bootstrap are full of+ // hardlinks, so flattening them silently doubles what lands on the+ // guest volume. GNU tar preserves them, and `vphone-archive+ // fingerprint` is what caught the difference.+ var first = stat(), second = stat()+ #expect(lstat(destination.appendingPathComponent("top.txt").path, &first) == 0)+ #expect(lstat(destination.appendingPathComponent("same.txt").path, &second) == 0)+ #expect(first.st_ino == second.st_ino)+ #expect(first.st_nlink == 2)+ }++ @Test("a tree fingerprint notices what diff -r would miss")+ func fingerprintDetectsMetadataDifferences() throws {+ let a = Self.scratch("fpa")+ let b = Self.scratch("fpb")+ // The same mtime on both, because two files written a moment apart+ // genuinely differ and the fingerprint is right to say so — mtime is+ // one of the things ARCHIVE_EXTRACT_TIME is supposed to restore.+ let when = Date(timeIntervalSince1970: 1_700_000_000)+ for url in [a, b] {+ try FileManager.default.createDirectory(at: url, withIntermediateDirectories: true)+ try Data("same\n".utf8).write(to: url.appendingPathComponent("f.txt"))+ try FileManager.default.setAttributes(+ [.modificationDate: when], ofItemAtPath: url.appendingPathComponent("f.txt").path+ )+ try FileManager.default.setAttributes(+ [.modificationDate: when], ofItemAtPath: url.path+ )+ }+ defer {+ for url in [a, b] { try? FileManager.default.removeItem(at: url) }+ }++ #expect(try VPhoneTreeFingerprint.capture(a)+ .differences(from: VPhoneTreeFingerprint.capture(b)).isEmpty)++ // Identical contents, different mode — `diff -r` says nothing.+ try FileManager.default.setAttributes(+ [.posixPermissions: NSNumber(value: 0o600)],+ ofItemAtPath: b.appendingPathComponent("f.txt").path+ )+ let differences = try VPhoneTreeFingerprint.capture(a)+ .differences(from: VPhoneTreeFingerprint.capture(b))+ #expect(differences.count == 1)+ #expect(differences[0].contains("mode"))+ }+@Test("exclusions are applied")func exclusions() throws {let source = try Self.makeTree()