Skip to content

Run a virtual iPhone on an Apple Silicon Mac.

Swift

Download IPSWs over several connections at once

Jackyfc6456f02c7a
diff --git a/Documents/Guides/create-and-run.md b/Documents/Guides/create-and-run.md
index 2f5b80a0..7c4e67c6 100644
--- a/Documents/Guides/create-and-run.md
+++ b/Documents/Guides/create-and-run.md
@@ -67,6 +67,8 @@ The online restore obtains its ticket in process. For an offline restore, see `v
`VPHONE_ROOT` relocates the VM library and both caches. `VPHONE_LIBRARY_ROOT` takes precedence for the library alone, and `--ipsw-cache <dir>` on `vm create` and `fw prepare` for the IPSW cache alone. Downloaded IPSWs stay cached until you delete them (in Launchpad, with File > Downloaded IPSWs…), so a second VM from the same URL downloads nothing; the prepared restore tree is removed after a successful `vm create` unless `--keep-artifacts` is set. VMs prepared by an earlier build may still hold a `.ipsw-cache/` directory; it is safe to delete, and `vm export` leaves it out.
+A remote IPSW downloads over four connections at once when its server answers HTTP range requests, as Apple's CDN does; each connection fetches 32 MB segments in turn and resumes a dropped segment where it stopped. `--download-connections <n>` on `fw prepare` sets the count (1 to 16); 1 downloads the IPSW as a single stream, which is also what a server that ignores ranges gets.
+
```sh
vphone-cli vm list
vphone-cli vm info myphone
diff --git a/VPhoneExecutable/VPhoneCommand/VPhoneCommand/Firmware/VPhoneFirmwareCommand.swift b/VPhoneExecutable/VPhoneCommand/VPhoneCommand/Firmware/VPhoneFirmwareCommand.swift
index 2fb85027..9b1a444c 100644
--- a/VPhoneExecutable/VPhoneCommand/VPhoneCommand/Firmware/VPhoneFirmwareCommand.swift
+++ b/VPhoneExecutable/VPhoneCommand/VPhoneCommand/Firmware/VPhoneFirmwareCommand.swift
@@ -1,6 +1,7 @@
import ArgumentParser
import FirmwarePatcher
import Foundation
+import VPhoneArchiveKit
import VPhoneCoreKit
import VPhonePatchKit
@@ -221,6 +222,8 @@ struct VPhoneFirmwarePrepareCommand: ParsableCommand {
var gpuDriverBundle: String?
@Option(help: "Directory for downloaded IPSWs, shared by every VM (default: ~/.vphone/ipsws or $VPHONE_ROOT/ipsws)")
var ipswCache: String?
+ @Option(help: "Connections each IPSW downloads over when its server answers range requests; 1 downloads it as one stream")
+ var downloadConnections = VPhoneIPSWCache.defaultDownloadConnections
@Option(help: "iPhone version to resolve to an IPSW") var iphoneVersion: String?
@Option(help: "iPhone build to resolve to an IPSW") var iphoneBuild: String?
@Option(help: ArgumentHelp(
@@ -243,6 +246,9 @@ struct VPhoneFirmwarePrepareCommand: ParsableCommand {
if let device, VPhoneGuestDevice.named(device) == nil {
throw ValidationError("vphone runs \(VPhoneGuestDevice.known.map(\.productType).joined(separator: ", ")) guests, not \(device).")
}
+ guard (1 ... 16).contains(downloadConnections) else {
+ throw ValidationError("--download-connections takes 1 to 16, not \(downloadConnections).")
+ }
let chosenDevice = device
let device = device ?? VPhoneFirmwareCatalog.device
let urls = if needsCatalog {
@@ -301,6 +307,7 @@ struct VPhoneFirmwarePrepareCommand: ParsableCommand {
ipswCacheDirectory: ipswCache.map {
URL(fileURLWithPath: ($0 as NSString).expandingTildeInPath, isDirectory: true)
} ?? VPhoneResources.ipswCacheDirectory(),
+ downloadConnections: downloadConnections,
device: chosenDevice,
bundle: bundle,
resources: resources,
diff --git a/VPhoneExecutable/VPhoneCommand/VPhoneCommand/Firmware/VPhoneFirmwarePreparer.swift b/VPhoneExecutable/VPhoneCommand/VPhoneCommand/Firmware/VPhoneFirmwarePreparer.swift
index 77002289..97183d27 100644
--- a/VPhoneExecutable/VPhoneCommand/VPhoneCommand/Firmware/VPhoneFirmwarePreparer.swift
+++ b/VPhoneExecutable/VPhoneCommand/VPhoneCommand/Firmware/VPhoneFirmwarePreparer.swift
@@ -34,6 +34,7 @@ enum VPhoneFirmwarePreparer {
cloudOSSource: String,
gpuDriverBundle: URL? = nil,
ipswCacheDirectory: URL = VPhoneResources.ipswCacheDirectory(),
+ downloadConnections: Int = VPhoneIPSWCache.defaultDownloadConnections,
device productType: String? = nil,
bundle: VPhoneBundle,
resources: VPhoneResources,
@@ -66,9 +67,13 @@ enum VPhoneFirmwarePreparer {
// inside the machine downloaded both again for each new one (#513).
// Local IPSWs are read in place and are never copied into the cache.
print("[*] Resolving iPhone IPSW...")
- let phone = try resolve(iPhoneSource, in: ipswCacheDirectory, label: "downloading iPhone IPSW")
+ let phone = try resolve(
+ iPhoneSource, in: ipswCacheDirectory, connections: downloadConnections, label: "downloading iPhone IPSW",
+ )
print("[*] Resolving cloudOS IPSW...")
- let cloud = try resolve(cloudOSSource, in: ipswCacheDirectory, label: "downloading cloudOS IPSW")
+ let cloud = try resolve(
+ cloudOSSource, in: ipswCacheDirectory, connections: downloadConnections, label: "downloading cloudOS IPSW",
+ )
try VPhoneIPSWCache.checkPair(iPhone: phone, cloudOS: cloud)
let device = VPhoneIPSWCache.guestDevice(for: phone, preferring: productType) ?? .default
if let productType, VPhoneGuestDevice.named(productType) != device {
@@ -155,10 +160,15 @@ enum VPhoneFirmwarePreparer {
/// Resolves one source, with a progress bar while it downloads. Launchpad
/// reads the bar as `progress` lines and shows the percentage on its
/// firmware step (#590).
- private static func resolve(_ source: String, in cacheDirectory: URL, label: String) throws -> VPhoneIPSWCache.Archive {
+ private static func resolve(
+ _ source: String,
+ in cacheDirectory: URL,
+ connections: Int,
+ label: String,
+ ) throws -> VPhoneIPSWCache.Archive {
let bar = VPhoneProgressBar(label: label)
let archive = try vphoneRunBlocking {
- try await VPhoneIPSWCache.resolve(source, in: cacheDirectory) { done, total in
+ try await VPhoneIPSWCache.resolve(source, in: cacheDirectory, connections: connections) { done, total in
bar.update(done: done, total: total)
}
}
diff --git a/VPhoneExecutable/VPhoneCommand/VPhoneCommand/Support/VPhoneProgressBar.swift b/VPhoneExecutable/VPhoneCommand/VPhoneCommand/Support/VPhoneProgressBar.swift
index e89c74f6..c0112ff5 100644
--- a/VPhoneExecutable/VPhoneCommand/VPhoneCommand/Support/VPhoneProgressBar.swift
+++ b/VPhoneExecutable/VPhoneCommand/VPhoneCommand/Support/VPhoneProgressBar.swift
@@ -11,9 +11,9 @@ import Foundation
/// stderr instead, at most a few per second.
///
/// Unchecked Sendable so a transfer's callback can drive it from another
-/// thread: every caller reports serially (URLSession calls one task's
-/// delegate one at a time) and calls `finish()` only after the transfer has
-/// returned.
+/// thread: every caller reports serially (`VPhoneIPSWCache` reports one at a
+/// time even across several connections) and calls `finish()` only after the
+/// transfer has returned.
final class VPhoneProgressBar: @unchecked Sendable {
private enum Mode {
case off, bar, lines
diff --git a/VPhoneKit/VPhoneArchiveKit/Transfer/VPhoneIPSWCache.swift b/VPhoneKit/VPhoneArchiveKit/Transfer/VPhoneIPSWCache.swift
index 0ab1465d..2aeb5f0d 100644
--- a/VPhoneKit/VPhoneArchiveKit/Transfer/VPhoneIPSWCache.swift
+++ b/VPhoneKit/VPhoneArchiveKit/Transfer/VPhoneIPSWCache.swift
@@ -1,5 +1,7 @@
import CryptoKit
+import Darwin
import Foundation
+import Synchronization
import VPhoneCoreKit
/// Resolves a local IPSW or downloads one into a reusable, validated cache.
@@ -22,6 +24,7 @@ public enum VPhoneIPSWCache {
case invalidManifest(URL)
case unexpectedHTTP(URL, Int)
case incompleteDownload(URL, expected: Int64, actual: Int64)
+ case changedDuringDownload(URL)
case swappedSources(iPhone: URL, cloudOS: URL)
case notIPhoneSource(URL, productTypes: [String])
case notCloudOSSource(URL)
@@ -35,6 +38,8 @@ public enum VPhoneIPSWCache {
case let .unexpectedHTTP(url, _): "Unable to download the IPSW from \(url). Try again later."
case let .incompleteDownload(url, expected, actual):
"The IPSW download from \(url) is incomplete (\(actual) of \(expected) bytes). Try again."
+ case let .changedDuringDownload(url):
+ "The IPSW at \(url) changed on the server while it downloaded. Try again."
case let .swappedSources(iPhone, cloudOS):
"The iPhone and cloudOS IPSWs are swapped: \(iPhone.lastPathComponent) is a cloudOS IPSW and \(cloudOS.lastPathComponent) is an iPhone IPSW. Swap the two sources, then try again."
case let .notIPhoneSource(file, productTypes):
@@ -49,11 +54,16 @@ public enum VPhoneIPSWCache {
/// `progress` is called with `(bytesDone, totalBytes)` while a remote
/// IPSW downloads, only when the server states the size. A local source
- /// or a cache hit never calls it.
+ /// or a cache hit never calls it. Reports come one at a time, even while
+ /// several connections download.
+ ///
+ /// `connections` is how many range requests download the IPSW at once when
+ /// its server answers them; 1 downloads it as one GET.
public static func resolve(
_ source: String,
in cacheDirectory: URL,
session: URLSession = URLSession(configuration: .ephemeral),
+ connections: Int = defaultDownloadConnections,
progress: (@Sendable (Int64, Int64) -> Void)? = nil,
) async throws -> Archive {
guard let url = URL(string: source), let scheme = url.scheme?.lowercased() else {
@@ -77,27 +87,12 @@ public enum VPhoneIPSWCache {
return valid
}
- var request = URLRequest(url: url)
- request.timeoutInterval = 3 * 60 * 60
let pending = cacheDirectory.appendingPathComponent(".\(cache.lastPathComponent).\(UUID().uuidString).partial")
defer { try? fm.removeItem(at: pending) }
guard fm.createFile(atPath: pending.path, contents: nil) else {
throw CocoaError(.fileWriteUnknown)
}
- let output = try FileHandle(forWritingTo: pending)
- defer { try? output.close() }
- let (response, size) = try await download(request, into: output, session: session, progress: progress)
- try output.close()
- guard let http = response as? HTTPURLResponse, http.statusCode == 200 else {
- throw Error.unexpectedHTTP(url, (response as? HTTPURLResponse)?.statusCode ?? 0)
- }
- if response.expectedContentLength > 0, size != response.expectedContentLength {
- throw Error.incompleteDownload(
- url,
- expected: response.expectedContentLength,
- actual: size,
- )
- }
+ try await download(url, into: pending, session: session, connections: connections, progress: progress)
let metadata = try inspect(pending)
do {
@@ -182,22 +177,318 @@ public enum VPhoneIPSWCache {
// MARK: - Download
- /// Writes the body into `output` chunk by chunk as URLSession delivers it,
- /// so the IPSW lands in the cache's own volume with no temporary copy.
- /// Iterating `bytes(for:)` one byte at a time was CPU-bound (issue #524),
- /// and `download(for:)` stages the file in the system temporary directory,
- /// which may be another volume. A non-200 response returns before any body.
- private static func download(
- _ request: URLRequest,
- into output: FileHandle,
+ /// Connections one remote IPSW downloads over when its server answers
+ /// range requests. Apple's CDN holds each connection well below a fast
+ /// link: four measured about 2.7 times the throughput of one.
+ public static let defaultDownloadConnections = 4
+
+ /// Bytes each range request asks for. Connections take segments in order,
+ /// so the partial file grows from the front, and a dropped connection
+ /// fetches again only what it had not yet written of one segment.
+ static let segmentSize: Int64 = 32 * 1024 * 1024
+
+ /// Range requests that make no progress this many times in a row fail the
+ /// download. Progress resets the count, so a long transfer survives any
+ /// number of dropped connections that each moved it forward.
+ static let segmentAttempts = 5
+
+ /// Writes the IPSW into `file`, which exists and is empty, in place on the
+ /// cache's own volume with no temporary copy. `download(for:)` would stage
+ /// it in the system temporary directory, which may be another volume, and
+ /// iterating `bytes(for:)` one byte at a time was CPU-bound (issue #524).
+ ///
+ /// With more than one connection and a server that answers range requests,
+ /// the file downloads as segments over that many connections at once.
+ /// Otherwise it is one GET, as a server that ignores ranges requires.
+ static func download(
+ _ url: URL,
+ into file: URL,
session: URLSession,
+ connections: Int,
+ segmentSize: Int64 = segmentSize,
progress: (@Sendable (Int64, Int64) -> Void)?,
- ) async throws -> (URLResponse, Int64) {
+ ) async throws {
+ let fd = open(file.path, O_WRONLY)
+ guard fd >= 0 else {
+ throw POSIXError(POSIXErrorCode(rawValue: errno) ?? .EIO)
+ }
+ defer { close(fd) }
+ let counter = ProgressCounter(progress)
+ if connections > 1, let resource = await probeRanges(url, session: session), resource.size > segmentSize {
+ try await downloadSegments(
+ resource,
+ into: fd,
+ configuration: session.configuration,
+ connections: connections,
+ segmentSize: segmentSize,
+ counter: counter,
+ )
+ } else {
+ try await downloadStream(url, into: fd, session: session, counter: counter)
+ }
+ }
+
+ /// One GET for the whole body. A non-200 response returns before any body.
+ private static func downloadStream(
+ _ url: URL,
+ into fd: Int32,
+ session: URLSession,
+ counter: ProgressCounter,
+ ) async throws {
+ var request = URLRequest(url: url)
+ request.timeoutInterval = 3 * 60 * 60
+ let outcome = await transfer(
+ request,
+ session: session,
+ into: fd,
+ at: 0,
+ accept: { $0.statusCode == 200 },
+ received: { count, response in counter.add(count, of: response.expectedContentLength) },
+ )
+ try Task.checkCancellation()
+ if let error = outcome.writeError {
+ throw error
+ }
+ guard let response = outcome.response else {
+ throw outcome.transportError ?? URLError(.badServerResponse)
+ }
+ guard outcome.accepted else {
+ throw Error.unexpectedHTTP(url, response.statusCode)
+ }
+ if let error = outcome.transportError {
+ throw error
+ }
+ if response.expectedContentLength > 0, outcome.written != response.expectedContentLength {
+ throw Error.incompleteDownload(url, expected: response.expectedContentLength, actual: outcome.written)
+ }
+ }
+
+ // MARK: Segments
+
+ struct RangedResource: Sendable {
+ let url: URL
+ let size: Int64
+ /// The ETag, else Last-Modified, sent as If-Range so a file replaced
+ /// on the server midway fails instead of mixing two files' bytes.
+ let validator: String?
+ }
+
+ /// Asks for the first byte and cancels at the headers. A server that
+ /// answers ranges says 206 and gives the full size in Content-Range; one
+ /// that ignores them says 200. Apple's CDN answers range requests without
+ /// advertising Accept-Ranges, so that header is not consulted. Any failure
+ /// returns nil, and the single GET that follows reports it.
+ static func probeRanges(_ url: URL, session: URLSession) async -> RangedResource? {
+ let outcome = await transfer(
+ rangeRequest(url, 0 ... 0, validator: nil),
+ session: session,
+ into: -1,
+ at: 0,
+ accept: { _ in false },
+ received: { _, _ in },
+ )
+ guard let response = outcome.response, response.statusCode == 206,
+ let range = contentRange(of: response), range.start == 0, range.total > 0
+ else {
+ return nil
+ }
+ let etag = response.value(forHTTPHeaderField: "ETag").flatMap { $0.hasPrefix("W/") ? nil : $0 }
+ return RangedResource(
+ url: url,
+ size: range.total,
+ validator: etag ?? response.value(forHTTPHeaderField: "Last-Modified"),
+ )
+ }
+
+ private static func downloadSegments(
+ _ resource: RangedResource,
+ into fd: Int32,
+ configuration: URLSessionConfiguration,
+ connections: Int,
+ segmentSize: Int64,
+ counter: ProgressCounter,
+ ) async throws {
+ let segments = SegmentQueue(size: resource.size, segmentSize: segmentSize)
+ try await withThrowingTaskGroup(of: Void.self) { group in
+ for _ in 0 ..< min(connections, segments.count) {
+ group.addTask {
+ // A session of its own: one session sends every request to
+ // a host over a single HTTP/2 connection, which would leave
+ // the workers sharing one connection's throughput.
+ let session = URLSession(configuration: configuration)
+ defer { session.invalidateAndCancel() }
+ while let segment = segments.next() {
+ try await downloadSegment(segment, of: resource, into: fd, session: session, counter: counter)
+ }
+ }
+ }
+ try await group.waitForAll()
+ }
+ var info = stat()
+ guard fstat(fd, &info) == 0 else {
+ throw POSIXError(POSIXErrorCode(rawValue: errno) ?? .EIO)
+ }
+ guard info.st_size == resource.size else {
+ throw Error.incompleteDownload(resource.url, expected: resource.size, actual: info.st_size)
+ }
+ }
+
+ /// Fetches one segment, resuming from the last byte written whenever a
+ /// connection drops or a server error comes back.
+ private static func downloadSegment(
+ _ segment: ClosedRange<Int64>,
+ of resource: RangedResource,
+ into fd: Int32,
+ session: URLSession,
+ counter: ProgressCounter,
+ ) async throws {
+ var offset = segment.lowerBound
+ var failures = 0
+ while offset <= segment.upperBound {
+ let start = offset
+ let outcome = await transfer(
+ rangeRequest(resource.url, start ... segment.upperBound, validator: resource.validator),
+ session: session,
+ into: fd,
+ at: start,
+ limit: segment.upperBound - start + 1,
+ accept: { response in
+ let range = contentRange(of: response)
+ return response.statusCode == 206 && range?.start == start && range?.total == resource.size
+ },
+ received: { count, _ in counter.add(count, of: resource.size) },
+ )
+ try Task.checkCancellation()
+ if let error = outcome.writeError {
+ throw error
+ }
+ offset += outcome.written
+ if let response = outcome.response, !outcome.accepted {
+ switch response.statusCode {
+ case 408, 429, 500...:
+ break
+ case 200, 206, 412:
+ // 200 is the whole body: If-Range no longer matches.
+ throw Error.changedDuringDownload(resource.url)
+ default:
+ throw Error.unexpectedHTTP(resource.url, response.statusCode)
+ }
+ }
+ guard offset <= segment.upperBound else { return }
+ failures = outcome.written > 0 ? 1 : failures + 1
+ guard failures < segmentAttempts else {
+ throw outcome.transportError ?? Error.incompleteDownload(
+ resource.url,
+ expected: segment.upperBound + 1,
+ actual: offset,
+ )
+ }
+ try await Task.sleep(for: .seconds(failures))
+ }
+ }
+
+ private static func rangeRequest(_ url: URL, _ range: ClosedRange<Int64>, validator: String?) -> URLRequest {
+ var request = URLRequest(url: url)
+ // An idle connection gives up after a minute and the segment resumes
+ // on a new one, instead of hanging the download.
+ request.timeoutInterval = 60
+ request.setValue("bytes=\(range.lowerBound)-\(range.upperBound)", forHTTPHeaderField: "Range")
+ // Byte offsets must index the file itself, not a compressed encoding.
+ request.setValue("identity", forHTTPHeaderField: "Accept-Encoding")
+ if let validator {
+ request.setValue(validator, forHTTPHeaderField: "If-Range")
+ }
+ return request
+ }
+
+ /// `bytes <start>-<end>/<total>`; nil for any other shape, including an
+ /// unknown total.
+ static func contentRange(of response: HTTPURLResponse) -> (start: Int64, end: Int64, total: Int64)? {
+ guard let value = response.value(forHTTPHeaderField: "Content-Range") else { return nil }
+ let parts = value.trimmingCharacters(in: .whitespaces).split(separator: " ", maxSplits: 1)
+ guard parts.count == 2, parts[0].lowercased() == "bytes" else { return nil }
+ let spanAndTotal = parts[1].split(separator: "/")
+ guard spanAndTotal.count == 2, let total = Int64(spanAndTotal[1]) else { return nil }
+ let span = spanAndTotal[0].split(separator: "-")
+ guard span.count == 2, let start = Int64(span[0]), let end = Int64(span[1]),
+ start <= end, end < total
+ else {
+ return nil
+ }
+ return (start, end, total)
+ }
+
+ /// Hands out the file's segments in order to every connection.
+ private final class SegmentQueue: Sendable {
+ let count: Int
+ private let size: Int64
+ private let segmentSize: Int64
+ private let nextIndex = Mutex(0)
+
+ init(size: Int64, segmentSize: Int64) {
+ self.size = size
+ self.segmentSize = segmentSize
+ count = Int((size + segmentSize - 1) / segmentSize)
+ }
+
+ func next() -> ClosedRange<Int64>? {
+ nextIndex.withLock { index -> ClosedRange<Int64>? in
+ guard index < count else { return nil }
+ let start = Int64(index) * segmentSize
+ index += 1
+ return start ... min(start + segmentSize, size) - 1
+ }
+ }
+ }
+
+ /// Adds up the bytes every connection writes and reports the sum one
+ /// report at a time, so a progress bar that is not thread-safe can take it.
+ private final class ProgressCounter: Sendable {
+ private let progress: (@Sendable (Int64, Int64) -> Void)?
+ private let done = Mutex<Int64>(0)
+
+ init(_ progress: (@Sendable (Int64, Int64) -> Void)?) {
+ self.progress = progress
+ }
+
+ func add(_ count: Int64, of total: Int64) {
+ guard let progress else { return }
+ done.withLock { done in
+ done += count
+ if total > 0 {
+ progress(done, total)
+ }
+ }
+ }
+ }
+
+ // MARK: Transfer
+
+ struct TransferOutcome {
+ var response: HTTPURLResponse?
+ /// Whether `accept` took the response; a refused one has no body.
+ var accepted = false
+ var written: Int64 = 0
+ var writeError: Swift.Error?
+ var transportError: Swift.Error?
+ }
+
+ /// One request whose body is written at `offset` onward as it arrives.
+ /// Never throws: the caller decides from the outcome whether to resume.
+ private static func transfer(
+ _ request: URLRequest,
+ session: URLSession,
+ into fd: Int32,
+ at offset: Int64,
+ limit: Int64 = .max,
+ accept: @escaping @Sendable (HTTPURLResponse) -> Bool,
+ received: @escaping @Sendable (Int64, HTTPURLResponse) -> Void,
+ ) async -> TransferOutcome {
let task = session.dataTask(with: request)
- let writer = DownloadWriter(output: output, progress: progress)
+ let writer = TransferWriter(fd: fd, offset: offset, limit: limit, accept: accept, received: received)
task.delegate = writer
- return try await withTaskCancellationHandler {
- try await withCheckedThrowingContinuation { continuation in
+ return await withTaskCancellationHandler {
+ await withCheckedContinuation { continuation in
writer.continuation = continuation
task.resume()
}
@@ -208,17 +499,27 @@ public enum VPhoneIPSWCache {
/// URLSession calls one task's delegate serially, and `continuation` is set
/// before the task resumes, so the mutable state is never shared.
- private final class DownloadWriter: NSObject, URLSessionDataDelegate, @unchecked Sendable {
- let output: FileHandle
- let progress: (@Sendable (Int64, Int64) -> Void)?
- var continuation: CheckedContinuation<(URLResponse, Int64), Swift.Error>?
- private var response: URLResponse?
- private var written: Int64 = 0
- private var writeError: Swift.Error?
-
- init(output: FileHandle, progress: (@Sendable (Int64, Int64) -> Void)?) {
- self.output = output
- self.progress = progress
+ private final class TransferWriter: NSObject, URLSessionDataDelegate, @unchecked Sendable {
+ let fd: Int32
+ let offset: Int64
+ let limit: Int64
+ let accept: @Sendable (HTTPURLResponse) -> Bool
+ let received: @Sendable (Int64, HTTPURLResponse) -> Void
+ var continuation: CheckedContinuation<TransferOutcome, Never>?
+ private var outcome = TransferOutcome()
+
+ init(
+ fd: Int32,
+ offset: Int64,
+ limit: Int64,
+ accept: @escaping @Sendable (HTTPURLResponse) -> Bool,
+ received: @escaping @Sendable (Int64, HTTPURLResponse) -> Void,
+ ) {
+ self.fd = fd
+ self.offset = offset
+ self.limit = limit
+ self.accept = accept
+ self.received = received
}
func urlSession(
@@ -227,38 +528,54 @@ public enum VPhoneIPSWCache {
didReceive response: URLResponse,
completionHandler: @escaping @Sendable (URLSession.ResponseDisposition) -> Void,
) {
- self.response = response
- let ok = (response as? HTTPURLResponse)?.statusCode == 200
- completionHandler(ok ? .allow : .cancel)
+ outcome.response = response as? HTTPURLResponse
+ outcome.accepted = outcome.response.map(accept) ?? false
+ completionHandler(outcome.accepted ? .allow : .cancel)
}
func urlSession(_: URLSession, dataTask: URLSessionDataTask, didReceive data: Data) {
- guard writeError == nil else { return }
+ guard outcome.writeError == nil, let response = outcome.response else { return }
do {
- try output.write(contentsOf: data)
- written += Int64(data.count)
- if let progress, let total = response?.expectedContentLength, total > 0 {
- progress(written, total)
+ // A segment's bytes may never reach the next segment's, which
+ // another connection is writing.
+ guard Int64(data.count) <= limit - outcome.written else {
+ throw Error.unexpectedHTTP(
+ dataTask.originalRequest?.url ?? response.url ?? URL(fileURLWithPath: "/"),
+ response.statusCode,
+ )
}
+ try write(data, at: offset + outcome.written)
+ outcome.written += Int64(data.count)
+ received(Int64(data.count), response)
} catch {
- writeError = error
+ outcome.writeError = error
dataTask.cancel()
}
}
func urlSession(_: URLSession, task _: URLSessionTask, didCompleteWithError error: Swift.Error?) {
- defer { continuation = nil }
- if let writeError {
- continuation?.resume(throwing: writeError)
- } else if let response, (response as? HTTPURLResponse)?.statusCode != 200 {
- // Cancelled on purpose in didReceive; the caller reports the status.
- continuation?.resume(returning: (response, 0))
- } else if let error {
- continuation?.resume(throwing: error)
- } else if let response {
- continuation?.resume(returning: (response, written))
- } else {
- continuation?.resume(throwing: URLError(.badServerResponse))
+ // A refused response was cancelled on purpose; the caller reads its status.
+ if outcome.accepted || outcome.response == nil {
+ outcome.transportError = error
+ }
+ continuation?.resume(returning: outcome)
+ continuation = nil
+ }
+
+ private func write(_ data: Data, at position: Int64) throws {
+ try data.withUnsafeBytes { buffer in
+ guard let base = buffer.baseAddress else { return }
+ var done = 0
+ while done < buffer.count {
+ let count = pwrite(fd, base + done, buffer.count - done, off_t(position) + off_t(done))
+ if count < 0 {
+ if errno == EINTR {
+ continue
+ }
+ throw POSIXError(POSIXErrorCode(rawValue: errno) ?? .EIO)
+ }
+ done += count
+ }
}
}
}
diff --git a/VPhoneKit/VPhoneArchiveKitTests/IPSWCacheTests.swift b/VPhoneKit/VPhoneArchiveKitTests/IPSWCacheTests.swift
index 9f1d19da..600378d8 100644
--- a/VPhoneKit/VPhoneArchiveKitTests/IPSWCacheTests.swift
+++ b/VPhoneKit/VPhoneArchiveKitTests/IPSWCacheTests.swift
@@ -42,6 +42,95 @@ private final class ProgressReports: @unchecked Sendable {
}
}
+/// A server that answers range requests, as Apple's CDN does, and can close a
+/// connection partway through a segment or replace the file between requests.
+private final class RangeServer: @unchecked Sendable {
+ private let lock = NSLock()
+ private var payload = Data()
+ private var etag = "\"v1\""
+ private var cutShortOnce: Set<Int64> = []
+ private var replaceAfterProbe = false
+ private var ranges: [String] = []
+
+ func reset(payload: Data, cutShortOnce: Set<Int64> = [], replaceAfterProbe: Bool = false) {
+ lock.withLock {
+ self.payload = payload
+ etag = "\"v1\""
+ self.cutShortOnce = cutShortOnce
+ self.replaceAfterProbe = replaceAfterProbe
+ ranges = []
+ }
+ }
+
+ var requestedRanges: [String] {
+ lock.withLock { ranges }
+ }
+
+ /// The response to `range` and `ifRange`, and whether to end it halfway.
+ func answer(range: String?, ifRange: String?) -> (status: Int, headers: [String: String], body: Data, cutShort: Bool) {
+ lock.withLock {
+ ranges.append(range ?? "")
+ let current = etag
+ if range == "bytes=0-0", replaceAfterProbe {
+ etag = "\"v2\""
+ }
+ guard let range, ifRange == nil || ifRange == current,
+ let bounds = Self.bounds(range, size: payload.count)
+ else {
+ return (200, ["Content-Length": "\(payload.count)", "ETag": current], payload, false)
+ }
+ let body = payload.subdata(in: bounds.lowerBound ..< bounds.upperBound + 1)
+ let headers = [
+ "Content-Length": "\(body.count)",
+ "Content-Range": "bytes \(bounds.lowerBound)-\(bounds.upperBound)/\(payload.count)",
+ "ETag": current,
+ ]
+ let cutShort = cutShortOnce.remove(Int64(bounds.lowerBound)) != nil
+ return (206, headers, body, cutShort)
+ }
+ }
+
+ private static func bounds(_ range: String, size: Int) -> ClosedRange<Int>? {
+ guard range.hasPrefix("bytes=") else { return nil }
+ let parts = range.dropFirst(6).split(separator: "-")
+ guard parts.count == 2, let start = Int(parts[0]), let end = Int(parts[1]), start <= end, start < size else {
+ return nil
+ }
+ return start ... min(end, size - 1)
+ }
+}
+
+private final class RangeStubProtocol: URLProtocol {
+ static let server = RangeServer()
+
+ override class func canInit(with _: URLRequest) -> Bool {
+ true
+ }
+
+ override class func canonicalRequest(for request: URLRequest) -> URLRequest {
+ request
+ }
+
+ override func startLoading() {
+ let answer = Self.server.answer(
+ range: request.value(forHTTPHeaderField: "Range"),
+ ifRange: request.value(forHTTPHeaderField: "If-Range"),
+ )
+ let response = HTTPURLResponse(
+ url: request.url!,
+ statusCode: answer.status,
+ httpVersion: "HTTP/1.1",
+ headerFields: answer.headers,
+ )!
+ client?.urlProtocol(self, didReceive: response, cacheStoragePolicy: .notAllowed)
+ // Cut short: the connection closes after half the stated length.
+ client?.urlProtocol(self, didLoad: answer.cutShort ? answer.body.prefix(answer.body.count / 2) : answer.body)
+ client?.urlProtocolDidFinishLoading(self)
+ }
+
+ override func stopLoading() {}
+}
+
@Suite("IPSW cache", .serialized)
struct IPSWCacheTests {
private func fixture(in root: URL) throws -> URL {
@@ -205,6 +294,145 @@ struct IPSWCacheTests {
#expect(FileManager.default.fileExists(atPath: live.path))
}
+ // MARK: - Concurrent download
+
+ private func rangeSession() -> URLSession {
+ let configuration = URLSessionConfiguration.ephemeral
+ configuration.protocolClasses = [RangeStubProtocol.self]
+ return URLSession(configuration: configuration)
+ }
+
+ private func randomPayload(_ count: Int) -> Data {
+ Data((0 ..< count).map { _ in UInt8.random(in: 0 ... 255) })
+ }
+
+ private func emptyFile(in root: URL) throws -> URL {
+ try FileManager.default.createDirectory(at: root, withIntermediateDirectories: true)
+ let file = root.appendingPathComponent("download.partial")
+ #expect(FileManager.default.createFile(atPath: file.path, contents: nil))
+ return file
+ }
+
+ /// Several connections write their segments into one file; the bytes must
+ /// land exactly where they belong, and progress must add up across them.
+ @Test func `segments over several connections reassemble the file`() async throws {
+ let root = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString)
+ defer { try? FileManager.default.removeItem(at: root) }
+ let payload = randomPayload(1_000_003)
+ RangeStubProtocol.server.reset(payload: payload)
+ let session = rangeSession()
+ defer { session.invalidateAndCancel() }
+ let file = try emptyFile(in: root)
+ let reports = ProgressReports()
+
+ try await VPhoneIPSWCache.download(
+ #require(URL(string: "https://example.invalid/input.ipsw")),
+ into: file,
+ session: session,
+ connections: 4,
+ segmentSize: 64 * 1024,
+ progress: { done, total in reports.append(done, total) },
+ )
+
+ #expect(try Data(contentsOf: file) == payload)
+ let ranges = RangeStubProtocol.server.requestedRanges
+ #expect(ranges.first == "bytes=0-0")
+ #expect(ranges.count == 1 + 16)
+ #expect(ranges.contains("bytes=983040-1000002"))
+ let values = reports.values
+ #expect(values.allSatisfy { $0.total == Int64(payload.count) })
+ #expect(values.last?.done == Int64(payload.count))
+ #expect(zip(values, values.dropFirst()).allSatisfy { $0.done < $1.done })
+ }
+
+ @Test func `a segment cut short resumes where it stopped`() async throws {
+ let root = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString)
+ defer { try? FileManager.default.removeItem(at: root) }
+ let payload = randomPayload(300_000)
+ RangeStubProtocol.server.reset(payload: payload, cutShortOnce: [100_000])
+ let session = rangeSession()
+ defer { session.invalidateAndCancel() }
+ let file = try emptyFile(in: root)
+
+ try await VPhoneIPSWCache.download(
+ #require(URL(string: "https://example.invalid/input.ipsw")),
+ into: file,
+ session: session,
+ connections: 3,
+ segmentSize: 100_000,
+ progress: nil,
+ )
+
+ #expect(try Data(contentsOf: file) == payload)
+ // Half of 100000-199999 arrived before the connection closed; only the rest is asked again.
+ #expect(RangeStubProtocol.server.requestedRanges.contains("bytes=150000-199999"))
+ }
+
+ /// If-Range carries the probe's ETag, so a file replaced on the server
+ /// fails the download instead of mixing two files' bytes.
+ @Test func `a file replaced during the download fails it`() async throws {
+ let root = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString)
+ defer { try? FileManager.default.removeItem(at: root) }
+ RangeStubProtocol.server.reset(payload: randomPayload(200_000), replaceAfterProbe: true)
+ let session = rangeSession()
+ defer { session.invalidateAndCancel() }
+ let file = try emptyFile(in: root)
+
+ await #expect {
+ try await VPhoneIPSWCache.download(
+ #require(URL(string: "https://example.invalid/input.ipsw")),
+ into: file,
+ session: session,
+ connections: 2,
+ segmentSize: 50000,
+ progress: nil,
+ )
+ } throws: { error in
+ if case .changedDuringDownload? = error as? VPhoneIPSWCache.Error {
+ true
+ } else {
+ false
+ }
+ }
+ }
+
+ /// One connection, or a server that ignores ranges, downloads as one GET.
+ @Test func `one connection downloads the file as a single request`() async throws {
+ let root = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString)
+ defer { try? FileManager.default.removeItem(at: root) }
+ let payload = randomPayload(200_000)
+ RangeStubProtocol.server.reset(payload: payload)
+ let session = rangeSession()
+ defer { session.invalidateAndCancel() }
+ let file = try emptyFile(in: root)
+
+ try await VPhoneIPSWCache.download(
+ #require(URL(string: "https://example.invalid/input.ipsw")),
+ into: file,
+ session: session,
+ connections: 1,
+ segmentSize: 50000,
+ progress: nil,
+ )
+
+ #expect(try Data(contentsOf: file) == payload)
+ #expect(RangeStubProtocol.server.requestedRanges == [""])
+ }
+
+ @Test func `content range parses only a complete byte range`() {
+ func parse(_ value: String) -> [Int64]? {
+ let url = URL(string: "https://example.invalid/")!
+ let response = HTTPURLResponse(url: url, statusCode: 206, httpVersion: nil, headerFields: ["Content-Range": value])!
+ return VPhoneIPSWCache.contentRange(of: response).map { [$0.start, $0.end, $0.total] }
+ }
+ #expect(parse("bytes 0-0/12343857610") == [0, 0, 12_343_857_610])
+ #expect(parse("bytes 100-199/1000") == [100, 199, 1000])
+ #expect(parse("bytes 0-0/*") == nil)
+ #expect(parse("bytes */1000") == nil)
+ #expect(parse("bytes 10-5/1000") == nil)
+ #expect(parse("bytes 0-1000/1000") == nil)
+ }
+
// MARK: - Pairing
private func ipsw(in root: URL, _ name: String, productTypes: [String], deviceClasses: [String]) throws -> VPhoneIPSWCache.Archive {