Skip to content

Free fork of NotProton: Windows Steam games in Mac Steam, on CodeWeavers' Wine and Apple's Game Porting Toolkit

fix(download): ask a busy host again before giving up on it (#2743)

Raymond Chin147ad6b38db7
diff --git a/app/Sources/BridgeplayApp/Support/PinnedDownload.swift b/app/Sources/BridgeplayApp/Support/PinnedDownload.swift
index 3de8855..4747b61 100644
--- a/app/Sources/BridgeplayApp/Support/PinnedDownload.swift
+++ b/app/Sources/BridgeplayApp/Support/PinnedDownload.swift
@@ -59,6 +59,7 @@ enum PinnedDownload {
step: String,
from source: String = "Valve",
session: URLSession = PinnedDownload.session,
+ backoff: Duration = .seconds(3),
report: @Sendable (Progress) -> Void = { _ in }
) async throws -> URL {
let archive = downloads.appending(path: file)
@@ -69,67 +70,78 @@ enum PinnedDownload {
var refusals: [String] = []
var kinds: Set<Refusal> = []
- for base in bases {
+ hosts: for base in bases {
let host = base.host() ?? base.absoluteString
- report(.fetching(host: host))
- let body: URL, response: URLResponse
- do {
- (body, response) = try await withoutActuallyEscaping(report) { report in
- // A dropped connection (or one the server closed during a pause) resumes
- // where it stopped rather than starting the 440 MB over.
- defer { Task { @MainActor in DownloadMeter.shared.clear() } }
- var resumeData: Data?
- for attempt in 1... {
- let relay = ProgressRelay(host: host, report: report)
- defer { relay.finish() }
- do {
- if let resumeData { return try await session.download(resumeFrom: resumeData, delegate: relay) }
- return try await session.download(from: base.appending(path: file), delegate: relay)
- } catch let error as URLError where attempt < 4 && error.downloadTaskResumeData != nil {
- resumeData = error.downloadTaskResumeData
- AppLog.note("\(file) from \(host) interrupted (\(error.localizedDescription)), resuming")
- // #279: a Wi-Fi drop spent every try within one second.
- await networkBack(within: .seconds(120))
- try await Task.sleep(for: .seconds(2 * attempt))
+ var busy = 0
+ while true {
+ report(.fetching(host: host))
+ let body: URL, response: URLResponse
+ do {
+ (body, response) = try await withoutActuallyEscaping(report) { report in
+ // A dropped connection (or one the server closed during a pause) resumes
+ // where it stopped rather than starting the 440 MB over.
+ defer { Task { @MainActor in DownloadMeter.shared.clear() } }
+ var resumeData: Data?
+ for attempt in 1... {
+ let relay = ProgressRelay(host: host, report: report)
+ defer { relay.finish() }
+ do {
+ if let resumeData { return try await session.download(resumeFrom: resumeData, delegate: relay) }
+ return try await session.download(from: base.appending(path: file), delegate: relay)
+ } catch let error as URLError where attempt < 4 && error.downloadTaskResumeData != nil {
+ resumeData = error.downloadTaskResumeData
+ AppLog.note("\(file) from \(host) interrupted (\(error.localizedDescription)), resuming")
+ // #279: a Wi-Fi drop spent every try within one second.
+ await networkBack(within: .seconds(120))
+ try await Task.sleep(for: .seconds(2 * attempt))
+ }
}
+ fatalError("unreachable")
}
- fatalError("unreachable")
+ } catch {
+ refusals.append("\(host): \(error.localizedDescription)")
+ kinds.insert(Self.connectionFailed(error) ? .unreachable : .failed)
+ continue hosts
}
- } catch {
- refusals.append("\(host): \(error.localizedDescription)")
- kinds.insert(Self.connectionFailed(error) ? .unreachable : .failed)
- continue
- }
- defer { try? FileManager.default.removeItem(at: body) }
+ defer { try? FileManager.default.removeItem(at: body) }
- // 206: resumed after a dropped connection.
- if let http = response as? HTTPURLResponse, ![200, 206].contains(http.statusCode) {
- refusals.append("\(host) answered \(http.statusCode)")
- kinds.insert(.answered)
- continue
- }
+ // 206: resumed after a dropped connection.
+ if let http = response as? HTTPURLResponse, ![200, 206].contains(http.statusCode) {
+ // #2727: github.com answers a passing 504 now and then, and the engine has
+ // only that one host. Ask it again before giving up on it.
+ if busy < 2, http.statusCode == 408 || http.statusCode == 429 || (500...599).contains(http.statusCode) {
+ busy += 1
+ AppLog.note("\(file) from \(host) answered \(http.statusCode), asking again")
+ try await Task.sleep(for: backoff * busy)
+ continue
+ }
+ refusals.append("\(host) answered \(http.statusCode)")
+ kinds.insert(.answered)
+ continue hosts
+ }
- let got = try Digest.sha256(of: body)
- guard got == sha256 else {
- refusals.append("\(host) served \(got.prefix(16)) rather than \(sha256.prefix(16))")
- kinds.insert(.mismatch)
- continue
- }
+ let got = try Digest.sha256(of: body)
+ guard got == sha256 else {
+ refusals.append("\(host) served \(got.prefix(16)) rather than \(sha256.prefix(16))")
+ kinds.insert(.mismatch)
+ continue hosts
+ }
- do {
- try FileManager.default.createDirectory(
- at: downloads, withIntermediateDirectories: true
- )
- try? FileManager.default.removeItem(at: archive)
- try FileManager.default.moveItem(at: body, to: archive)
- } catch {
- throw StepFailure(
- step: step,
- detail: String(localized: "\(downloads.path(percentEncoded: false)) could not be written. ")
- + error.localizedDescription
- )
+ do {
+ try FileManager.default.createDirectory(
+ at: downloads, withIntermediateDirectories: true
+ )
+ try? FileManager.default.removeItem(at: archive)
+ try FileManager.default.moveItem(at: body, to: archive)
+ } catch {
+ throw StepFailure(
+ step: step,
+ detail: String(localized: "\(downloads.path(percentEncoded: false)) could not be written. ")
+ + error.localizedDescription
+ )
+ }
+ return archive
}
- return archive
}
AppLog.note("\(file) refused by all \(bases.count) hosts: \(refusals.joined(separator: ". "))")
diff --git a/app/Tests/BridgeplayAppTests/PinnedDownloadTests.swift b/app/Tests/BridgeplayAppTests/PinnedDownloadTests.swift
index 61de8c1..97132ff 100644
--- a/app/Tests/BridgeplayAppTests/PinnedDownloadTests.swift
+++ b/app/Tests/BridgeplayAppTests/PinnedDownloadTests.swift
@@ -348,4 +348,87 @@ struct PinnedDownloadTests {
#expect(label.hasPrefix("Downloading from "))
#expect(label.contains(serving.lastPathComponent))
}
+
+ // Answers the status in the host name ("504.busy.invalid") to the first requests, then
+ // 200 with the body. How many times it says no is the first path component.
+ private final class Busy: URLProtocol, @unchecked Sendable {
+ static let body = Data("the real package".utf8)
+ private static let lock = NSLock()
+ nonisolated(unsafe) private static var asked: [String: Int] = [:]
+ static func count(_ key: String) -> Int { lock.lock(); defer { lock.unlock() }; return asked[key] ?? 0 }
+
+ override class func canInit(with request: URLRequest) -> Bool { request.url?.host()?.hasSuffix("busy.invalid") == true }
+ override class func canonicalRequest(for request: URLRequest) -> URLRequest { request }
+ override func startLoading() {
+ let url = request.url!
+ let key = url.pathComponents[1]
+ Self.lock.lock(); Self.asked[key, default: 0] += 1; let n = Self.asked[key]!; Self.lock.unlock()
+ let status = n <= Int(url.pathComponents[2])! ? Int(url.host()!.split(separator: ".")[0])! : 200
+ let response = HTTPURLResponse(url: url, statusCode: status, httpVersion: nil, headerFields: nil)!
+ client?.urlProtocol(self, didReceive: response, cacheStoragePolicy: .notAllowed)
+ client?.urlProtocol(self, didLoad: status == 200 ? Self.body : Data("busy".utf8))
+ client?.urlProtocolDidFinishLoading(self)
+ }
+ override func stopLoading() {}
+ }
+
+ private func busy(_ status: Int, refusals: Int) async throws -> (Result<URL, StepFailure>, asked: Int) {
+ let work = try scratchDirectory("pinned")
+ defer { try? FileManager.default.removeItem(at: work) }
+ let key = UUID().uuidString
+ let result: Result<URL, StepFailure>
+ do {
+ let got = try await PinnedDownload.obtain(
+ file: "package.tar", sha256: Digest.sha256(of: Busy.body),
+ bases: [URL(string: "https://\(status).busy.invalid/\(key)/\(refusals)/")!],
+ into: work.appending(path: "downloads"), step: "test",
+ session: PinnedDownload.session(deadline: 30, protocols: [Busy.self]), backoff: .zero)
+ result = .success(got)
+ } catch let failure as StepFailure {
+ result = .failure(failure)
+ }
+ return (result, Busy.count(key))
+ }
+
+ @Test("A host that answers a passing 5xx is asked again (#2727)", arguments: [502, 503, 504, 429])
+ func passingBusyIsAskedAgain(status: Int) async throws {
+ let (result, asked) = try await busy(status, refusals: 2)
+ #expect((try? result.get()) != nil)
+ #expect(asked == 3)
+ }
+
+ @Test("A host that stays busy is given up on after three asks")
+ func busyHostIsGivenUpOn() async throws {
+ let (result, asked) = try await busy(504, refusals: 99)
+ guard case .failure(let failure) = result else { Issue.record("downloaded from a busy host"); return }
+ #expect(failure.detail.contains("try again later"))
+ #expect(asked == 3)
+ }
+
+ @Test("A host given up on hands over to the next, which gets its own asks")
+ func busyHostFallsThroughToNextHost() async throws {
+ let work = try scratchDirectory("pinned")
+ defer { try? FileManager.default.removeItem(at: work) }
+ let (first, second) = (UUID().uuidString, UUID().uuidString)
+
+ let got = try await PinnedDownload.obtain(
+ file: "package.tar", sha256: Digest.sha256(of: Busy.body),
+ bases: [
+ URL(string: "https://504.busy.invalid/\(first)/99/")!,
+ URL(string: "https://503.busy.invalid/\(second)/2/")!,
+ ],
+ into: work.appending(path: "downloads"), step: "test",
+ session: PinnedDownload.session(deadline: 30, protocols: [Busy.self]), backoff: .zero)
+
+ #expect(try Data(contentsOf: got) == Busy.body)
+ #expect(Busy.count(first) == 3)
+ #expect(Busy.count(second) == 3)
+ }
+
+ @Test("A 404 is not asked again")
+ func notFoundIsNotRetried() async throws {
+ let (result, asked) = try await busy(404, refusals: 99)
+ #expect((try? result.get()) == nil)
+ #expect(asked == 1)
+ }
}