Skip to content

Run a virtual iPhone on an Apple Silicon Mac.

Swift

feat(archive): add tree fingerprints, and fix the hardlink bug they found

Lakrf89366ed4f79
diff --git a/sources/VPhoneArchive/VPhoneArchiveExtractor.swift b/sources/VPhoneArchive/VPhoneArchiveExtractor.swift
index fb556834..7b4a375d 100644
--- a/sources/VPhoneArchive/VPhoneArchiveExtractor.swift
+++ b/sources/VPhoneArchive/VPhoneArchiveExtractor.swift
@@ -95,6 +95,27 @@ public enum VPhoneArchiveExtractor {
}
archive_entry_set_pathname(entry, targetPath)
+ // A hardlink entry names its target as an archive-relative path,
+ // and libarchive resolves that against the process's working
+ // directory. Since the entry's own path has just been made
+ // absolute, the target has to be too, or extraction fails with
+ // "Hard-link target 'x' does not exist" — which is what happened.
+ //
+ // Symlinks are deliberately left alone: their target is data,
+ // stored and restored verbatim, and rewriting one would change
+ // what the link says.
+ if let rawLink = archive_entry_hardlink(entry) {
+ let linkPath = String(cString: rawLink)
+ let linkTarget = resolvedDestination.appendingPathComponent(linkPath)
+ let resolvedLink = linkTarget.standardized.path
+ guard resolvedLink.hasPrefix(destinationPath + "/") else {
+ throw VPhoneArchiveError.pathEscapesDestination(
+ member: linkPath, destination: destinationPath
+ )
+ }
+ archive_entry_set_hardlink(entry, resolvedLink)
+ }
+
if options.noOverwriteDir, shouldSkipExistingDirectory(entry, at: target) {
// Skip the header only. Everything under this directory that
// does not yet exist is still written on its own entry, which
diff --git a/sources/VPhoneArchive/VPhoneArchiveWriter.swift b/sources/VPhoneArchive/VPhoneArchiveWriter.swift
index ab94fac2..a9c95370 100644
--- a/sources/VPhoneArchive/VPhoneArchiveWriter.swift
+++ b/sources/VPhoneArchive/VPhoneArchiveWriter.swift
@@ -79,6 +79,21 @@ public enum VPhoneArchiveWriter {
)
}
+ // Without a link resolver, every name of a hardlinked file is packed
+ // as its own full copy, and unpacking gives back separate files. That
+ // matters here: iosbinpack64 and the procursus bootstrap are full of
+ // hardlinks, so flattening them silently doubles what lands on the
+ // guest volume and breaks anything that expected the identity. GNU tar
+ // does this by default, which is how the difference was found — see
+ // `vphone-archive fingerprint`.
+ //
+ // With the tar strategy the resolver never defers: the first name is
+ // returned as-is, and later ones come back as hardlink references with
+ // size 0, which the data copy below already skips.
+ let resolver = archive_entry_linkresolver_new()
+ archive_entry_linkresolver_set_strategy(resolver, archive_format(writer))
+ defer { archive_entry_linkresolver_free(resolver) }
+
let rootPath = resolvedRoot.path
var written = 0
var bytes: Int64 = 0
@@ -104,13 +119,24 @@ public enum VPhoneArchiveWriter {
if patterns.contains(where: { matches(relative, pattern: $0) }) { continue }
archive_entry_set_pathname(entry, relative)
- guard archive_write_header(writer, entry) == ARCHIVE_OK else {
+ // Turns the second and later names of a hardlinked file into
+ // references to the first. The archive-relative pathname has to be
+ // set before this, or the resolver records absolute paths as the
+ // link targets.
+ var resolved: OpaquePointer? = entry
+ var deferred: OpaquePointer?
+ archive_entry_linkify(resolver, &resolved, &deferred)
+ guard let resolved else { continue }
+
+ guard archive_write_header(writer, resolved) == ARCHIVE_OK else {
throw VPhoneArchiveError.writeFailed(
path: relative, reason: archiveErrorString(writer)
)
}
- if archive_entry_size(entry) > 0 {
+ // A hardlink reference comes back with size 0, so this also skips
+ // re-storing contents we have already written once.
+ if archive_entry_size(resolved) > 0 {
bytes += try copyFile(at: absolute, into: writer, isCancelled: isCancelled)
}
diff --git a/sources/VPhoneArchive/VPhoneTreeFingerprint.swift b/sources/VPhoneArchive/VPhoneTreeFingerprint.swift
new file mode 100644
index 00000000..e7f3d955
--- /dev/null
+++ b/sources/VPhoneArchive/VPhoneTreeFingerprint.swift
@@ -0,0 +1,232 @@
+import CryptoKit
+import Foundation
+
+/// A complete description of a directory tree, for comparing two of them.
+///
+/// `diff -r` plus `stat` is not enough to decide that GNU tar and libarchive
+/// produced the same result, and the four things it misses are exactly the
+/// four most likely to differ: ACLs, extended attributes, which files are
+/// hardlinked to which, and sparse-file occupancy. All four change how a guest
+/// behaves and none of them shows up in a diff.
+///
+/// The output is deterministic and JSON, so two trees compare with a plain
+/// equality check and any difference names itself.
+public struct VPhoneTreeFingerprint: Codable, Equatable, Sendable {
+ public struct Entry: Codable, Equatable, Sendable {
+ /// Path relative to the tree root, so two trees in different places
+ /// still compare.
+ public let path: String
+ /// file, dir, symlink, fifo, socket, blockdev, chardev
+ public let type: String
+ /// Permission bits including setuid, setgid and sticky, as octal.
+ public let mode: String
+ /// Numeric, never resolved to a name: the whole point of restoring
+ /// ownership by number is that the name means something different on
+ /// the host than it does in the guest.
+ public let uid: Int
+ public let gid: Int
+ public let size: Int64
+ /// Whole nanoseconds, because ARCHIVE_EXTRACT_TIME claims to restore
+ /// them.
+ public let mtimeNanoseconds: Int64
+ /// Unresolved target of a symlink.
+ public let linkTarget: String?
+ /// Which files share an inode, normalised to a sequence number.
+ ///
+ /// Comparing inode numbers directly is meaningless across two
+ /// extractions; what has to match is the *grouping*. Files with no
+ /// other link have no id.
+ public let hardlinkGroup: Int?
+ /// Extended attribute names with a SHA-256 of each value. Covers the
+ /// `com.apple.*` ones, which is where AppleDouble handling shows up.
+ public let xattrs: [String: String]
+ /// `acl_to_text` output, or nil when there is no ACL. libarchive and
+ /// GNU tar disagree here more readily than anywhere else.
+ public let acl: String?
+ /// Blocks actually allocated. Two files can have the same logical size
+ /// and different occupancy, which is what a lost sparse hole looks
+ /// like.
+ public let blocks: Int64
+ /// Content digest, regular files only.
+ public let contentSHA256: String?
+ }
+
+ public let root: String
+ public let entries: [Entry]
+
+ // MARK: - Capture
+
+ public static func capture(_ root: URL, includeContentHashes: Bool = true) throws -> Self {
+ let resolvedRoot = try VPhoneArchivePaths.resolved(root)
+ let rootPath = resolvedRoot.path
+
+ var paths: [String] = []
+
+ // An explicit walk rather than FileManager's enumerator, which follows
+ // its own rules about symlinks and packages. This has to see exactly
+ // what is on disk, in a stable order.
+ func walk(_ directory: String) throws {
+ let names = try FileManager.default.contentsOfDirectory(atPath: directory)
+ for name in names.sorted() {
+ let full = directory + "/" + name
+ paths.append(full)
+ var info = stat()
+ guard lstat(full, &info) == 0 else { continue }
+ if (info.st_mode & S_IFMT) == S_IFDIR { try walk(full) }
+ }
+ }
+ try walk(rootPath)
+
+ // Hardlink groups, assigned in path order so the numbering is stable.
+ var groupOf: [String: Int] = [:] // "dev:ino" -> group
+ var linkCounts: [String: Int] = [:]
+ for path in paths {
+ var info = stat()
+ guard lstat(path, &info) == 0, info.st_nlink > 1 else { continue }
+ let key = "\(info.st_dev):\(info.st_ino)"
+ linkCounts[key, default: 0] += 1
+ }
+ var nextGroup = 0
+ for path in paths {
+ var info = stat()
+ guard lstat(path, &info) == 0, info.st_nlink > 1 else { continue }
+ let key = "\(info.st_dev):\(info.st_ino)"
+ guard linkCounts[key] ?? 0 > 1, groupOf[key] == nil else { continue }
+ groupOf[key] = nextGroup
+ nextGroup += 1
+ }
+
+ var entries: [Entry] = []
+ entries.reserveCapacity(paths.count)
+ for path in paths {
+ var info = stat()
+ guard lstat(path, &info) == 0 else { continue }
+ let relative = VPhoneArchivePaths.relative(path, under: rootPath)
+ let fileType = info.st_mode & S_IFMT
+ let key = "\(info.st_dev):\(info.st_ino)"
+
+ entries.append(Entry(
+ path: relative,
+ type: describe(fileType),
+ mode: String(info.st_mode & 0o7777, radix: 8),
+ uid: Int(info.st_uid),
+ gid: Int(info.st_gid),
+ size: fileType == S_IFREG ? info.st_size : 0,
+ mtimeNanoseconds: Int64(info.st_mtimespec.tv_sec) * 1_000_000_000
+ + Int64(info.st_mtimespec.tv_nsec),
+ linkTarget: fileType == S_IFLNK
+ ? try? FileManager.default.destinationOfSymbolicLink(atPath: path)
+ : nil,
+ hardlinkGroup: info.st_nlink > 1 ? groupOf[key] : nil,
+ xattrs: extendedAttributes(of: path),
+ acl: accessControlList(of: path),
+ blocks: Int64(info.st_blocks),
+ contentSHA256: includeContentHashes && fileType == S_IFREG
+ ? digest(of: path)
+ : nil
+ ))
+ }
+
+ return VPhoneTreeFingerprint(root: rootPath, entries: entries)
+ }
+
+ /// Every way the two trees differ, described by path.
+ ///
+ /// The root itself is not compared: two extractions of the same archive
+ /// into different directories are supposed to differ there.
+ public func differences(from other: VPhoneTreeFingerprint) -> [String] {
+ var found: [String] = []
+ let mine = Dictionary(uniqueKeysWithValues: entries.map { ($0.path, $0) })
+ let theirs = Dictionary(uniqueKeysWithValues: other.entries.map { ($0.path, $0) })
+
+ for path in Set(mine.keys).subtracting(theirs.keys).sorted() {
+ found.append("\(path): only in \(root)")
+ }
+ for path in Set(theirs.keys).subtracting(mine.keys).sorted() {
+ found.append("\(path): only in \(other.root)")
+ }
+ for path in Set(mine.keys).intersection(theirs.keys).sorted() {
+ let a = mine[path]!, b = theirs[path]!
+ if a == b { continue }
+ if a.type != b.type { found.append("\(path): type \(a.type) vs \(b.type)") }
+ if a.mode != b.mode { found.append("\(path): mode \(a.mode) vs \(b.mode)") }
+ if a.uid != b.uid || a.gid != b.gid {
+ found.append("\(path): owner \(a.uid):\(a.gid) vs \(b.uid):\(b.gid)")
+ }
+ if a.size != b.size { found.append("\(path): size \(a.size) vs \(b.size)") }
+ if a.mtimeNanoseconds != b.mtimeNanoseconds {
+ found.append("\(path): mtime \(a.mtimeNanoseconds) vs \(b.mtimeNanoseconds)")
+ }
+ if a.linkTarget != b.linkTarget {
+ found.append("\(path): symlink \(a.linkTarget ?? "-") vs \(b.linkTarget ?? "-")")
+ }
+ if a.hardlinkGroup != b.hardlinkGroup {
+ found.append("\(path): hardlink group \(a.hardlinkGroup.map(String.init) ?? "-") vs \(b.hardlinkGroup.map(String.init) ?? "-")")
+ }
+ if a.xattrs != b.xattrs {
+ found.append("\(path): xattrs \(a.xattrs.keys.sorted()) vs \(b.xattrs.keys.sorted())")
+ }
+ if a.acl != b.acl { found.append("\(path): ACL differs") }
+ if a.blocks != b.blocks {
+ found.append("\(path): occupancy \(a.blocks) vs \(b.blocks) blocks (sparseness)")
+ }
+ if a.contentSHA256 != b.contentSHA256 { found.append("\(path): contents differ") }
+ }
+ return found
+ }
+
+ // MARK: - Details
+
+ private static func describe(_ fileType: mode_t) -> String {
+ switch fileType {
+ case S_IFREG: "file"
+ case S_IFDIR: "dir"
+ case S_IFLNK: "symlink"
+ case S_IFIFO: "fifo"
+ case S_IFSOCK: "socket"
+ case S_IFBLK: "blockdev"
+ case S_IFCHR: "chardev"
+ default: "other"
+ }
+ }
+
+ private static func extendedAttributes(of path: String) -> [String: String] {
+ let size = listxattr(path, nil, 0, XATTR_NOFOLLOW)
+ guard size > 0 else { return [:] }
+
+ var names = [CChar](repeating: 0, count: size)
+ guard listxattr(path, &names, size, XATTR_NOFOLLOW) == size else { return [:] }
+
+ var result: [String: String] = [:]
+ for chunk in names.split(separator: 0) {
+ let name = String(decoding: chunk.map { UInt8(bitPattern: $0) }, as: UTF8.self)
+ guard !name.isEmpty else { continue }
+ let valueSize = getxattr(path, name, nil, 0, 0, XATTR_NOFOLLOW)
+ guard valueSize >= 0 else { continue }
+ var value = [UInt8](repeating: 0, count: max(valueSize, 1))
+ guard getxattr(path, name, &value, valueSize, 0, XATTR_NOFOLLOW) == valueSize else { continue }
+ result[name] = SHA256.hash(data: Data(value.prefix(valueSize)))
+ .map { String(format: "%02x", $0) }.joined()
+ }
+ return result
+ }
+
+ private static func accessControlList(of path: String) -> String? {
+ guard let acl = acl_get_link_np(path, ACL_TYPE_EXTENDED) else { return nil }
+ defer { acl_free(UnsafeMutableRawPointer(acl)) }
+ guard let text = acl_to_text(acl, nil) else { return nil }
+ defer { acl_free(UnsafeMutableRawPointer(text)) }
+ let value = String(cString: text)
+ return value.isEmpty ? nil : value
+ }
+
+ private static func digest(of path: String) -> String? {
+ guard let handle = FileHandle(forReadingAtPath: path) else { return nil }
+ defer { try? handle.close() }
+ var hasher = SHA256()
+ while let chunk = try? handle.read(upToCount: 1 << 20), !chunk.isEmpty {
+ hasher.update(data: chunk)
+ }
+ return hasher.finalize().map { String(format: "%02x", $0) }.joined()
+ }
+}
diff --git a/sources/vphone-archive/main.swift b/sources/vphone-archive/main.swift
index ebe6b3fc..584b5452 100644
--- a/sources/vphone-archive/main.swift
+++ b/sources/vphone-archive/main.swift
@@ -22,7 +22,10 @@ struct VPhoneArchiveCLI: ParsableCommand {
The compressor is detected when reading, so there is no --zstd to pass
and no way to pass the wrong one.
""",
- subcommands: [Extract.self, Create.self, Decompress.self, List.self, Cat.self],
+ subcommands: [
+ Extract.self, Create.self, Decompress.self,
+ List.self, Cat.self, Fingerprint.self,
+ ],
defaultSubcommand: Extract.self
)
}
@@ -216,4 +219,56 @@ struct Cat: ParsableCommand {
}
}
+// MARK: - fingerprint
+
+struct Fingerprint: ParsableCommand {
+ static let configuration = CommandConfiguration(
+ commandName: "fingerprint",
+ abstract: "Describe a directory tree completely, or compare two of them",
+ discussion: """
+ For deciding whether GNU tar and vphone-archive produced the same
+ result. `diff -r` and `stat` cannot answer that: they miss ACLs,
+ extended attributes, which files are hardlinked to which, and
+ sparse-file occupancy — the four things most likely to differ, and
+ all four of which change how a guest behaves.
+
+ With one path, writes JSON. With two, prints the differences and exits
+ non-zero if there are any.
+ """
+ )
+
+ @Argument(help: "Tree to describe", transform: URL.init(fileURLWithPath:))
+ var tree: URL
+
+ @Argument(help: "Second tree; given, the two are compared",
+ transform: URL.init(fileURLWithPath:))
+ var other: URL?
+
+ @Flag(name: .customLong("no-content-hashes"),
+ help: "Skip file digests — much faster, and enough to compare metadata")
+ var noContentHashes = false
+
+ func run() throws {
+ let first = try VPhoneTreeFingerprint.capture(tree, includeContentHashes: !noContentHashes)
+
+ guard let other else {
+ let encoder = JSONEncoder()
+ encoder.outputFormatting = [.prettyPrinted, .sortedKeys]
+ print(String(decoding: try encoder.encode(first), as: UTF8.self))
+ return
+ }
+
+ let second = try VPhoneTreeFingerprint.capture(
+ other, includeContentHashes: !noContentHashes
+ )
+ let differences = first.differences(from: second)
+ guard differences.isEmpty else {
+ for line in differences { print(line) }
+ print("\n\(differences.count) difference(s)")
+ throw ExitCode(1)
+ }
+ print("identical (\(first.entries.count) entries)")
+ }
+}
+
VPhoneArchiveCLI.main()
diff --git a/tests/VPhoneArchiveTests/RoundTripTests.swift b/tests/VPhoneArchiveTests/RoundTripTests.swift
index 4d608fe7..f1514e4d 100644
--- a/tests/VPhoneArchiveTests/RoundTripTests.swift
+++ b/tests/VPhoneArchiveTests/RoundTripTests.swift
@@ -125,6 +125,75 @@ struct RoundTripTests {
}
}
+ @Test("hardlinks survive as hardlinks")
+ func hardlinksArePreserved() throws {
+ let source = try Self.makeTree()
+ let archive = Self.scratch("hl").appendingPathExtension("tar")
+ let destination = Self.scratch("hlout")
+ try FileManager.default.createDirectory(at: destination, withIntermediateDirectories: true)
+ defer {
+ for url in [source, archive, destination] {
+ try? FileManager.default.removeItem(at: url)
+ }
+ }
+
+ try FileManager.default.linkItem(
+ at: source.appendingPathComponent("top.txt"),
+ to: source.appendingPathComponent("same.txt")
+ )
+
+ try VPhoneArchiveWriter.create(archive: archive, from: source)
+ try VPhoneArchiveExtractor.extract(archive, into: destination, options: .intoHostDirectory)
+
+ // Without archive_entry_linkresolver each name is packed as a separate
+ // full copy, and unpacking gives back two unrelated files. That is not
+ // cosmetic: iosbinpack64 and the procursus bootstrap are full of
+ // hardlinks, so flattening them silently doubles what lands on the
+ // guest volume. GNU tar preserves them, and `vphone-archive
+ // fingerprint` is what caught the difference.
+ var first = stat(), second = stat()
+ #expect(lstat(destination.appendingPathComponent("top.txt").path, &first) == 0)
+ #expect(lstat(destination.appendingPathComponent("same.txt").path, &second) == 0)
+ #expect(first.st_ino == second.st_ino)
+ #expect(first.st_nlink == 2)
+ }
+
+ @Test("a tree fingerprint notices what diff -r would miss")
+ func fingerprintDetectsMetadataDifferences() throws {
+ let a = Self.scratch("fpa")
+ let b = Self.scratch("fpb")
+ // The same mtime on both, because two files written a moment apart
+ // genuinely differ and the fingerprint is right to say so — mtime is
+ // one of the things ARCHIVE_EXTRACT_TIME is supposed to restore.
+ let when = Date(timeIntervalSince1970: 1_700_000_000)
+ for url in [a, b] {
+ try FileManager.default.createDirectory(at: url, withIntermediateDirectories: true)
+ try Data("same\n".utf8).write(to: url.appendingPathComponent("f.txt"))
+ try FileManager.default.setAttributes(
+ [.modificationDate: when], ofItemAtPath: url.appendingPathComponent("f.txt").path
+ )
+ try FileManager.default.setAttributes(
+ [.modificationDate: when], ofItemAtPath: url.path
+ )
+ }
+ defer {
+ for url in [a, b] { try? FileManager.default.removeItem(at: url) }
+ }
+
+ #expect(try VPhoneTreeFingerprint.capture(a)
+ .differences(from: VPhoneTreeFingerprint.capture(b)).isEmpty)
+
+ // Identical contents, different mode — `diff -r` says nothing.
+ try FileManager.default.setAttributes(
+ [.posixPermissions: NSNumber(value: 0o600)],
+ ofItemAtPath: b.appendingPathComponent("f.txt").path
+ )
+ let differences = try VPhoneTreeFingerprint.capture(a)
+ .differences(from: VPhoneTreeFingerprint.capture(b))
+ #expect(differences.count == 1)
+ #expect(differences[0].contains("mode"))
+ }
+
@Test("exclusions are applied")
func exclusions() throws {
let source = try Self.makeTree()