Free fork of NotProton: Windows Steam games in Mac Steam, on CodeWeavers' Wine and Apple's Game Porting Toolkit
Swift apple-silicond3dmetaldxmtdxvkgame-porting-toolkitgptkmac-gamingmacosmetalfxrosetta-2steamsteam-playwinewinecx
fix(download): ask a busy host again before giving up on it (#2743)
Raymond Chin147ad6b38db7
diff --git a/app/Sources/BridgeplayApp/Support/PinnedDownload.swift b/app/Sources/BridgeplayApp/Support/PinnedDownload.swiftindex 3de8855..4747b61 100644--- a/app/Sources/BridgeplayApp/Support/PinnedDownload.swift+++ b/app/Sources/BridgeplayApp/Support/PinnedDownload.swift@@ -59,6 +59,7 @@ enum PinnedDownload {step: String,from source: String = "Valve",session: URLSession = PinnedDownload.session,+ backoff: Duration = .seconds(3),report: @Sendable (Progress) -> Void = { _ in }) async throws -> URL {let archive = downloads.appending(path: file)@@ -69,67 +70,78 @@ enum PinnedDownload {var refusals: [String] = []var kinds: Set<Refusal> = []- for base in bases {+ hosts: for base in bases {let host = base.host() ?? base.absoluteString- report(.fetching(host: host))- let body: URL, response: URLResponse- do {- (body, response) = try await withoutActuallyEscaping(report) { report in- // A dropped connection (or one the server closed during a pause) resumes- // where it stopped rather than starting the 440 MB over.- defer { Task { @MainActor in DownloadMeter.shared.clear() } }- var resumeData: Data?- for attempt in 1... {- let relay = ProgressRelay(host: host, report: report)- defer { relay.finish() }- do {- if let resumeData { return try await session.download(resumeFrom: resumeData, delegate: relay) }- return try await session.download(from: base.appending(path: file), delegate: relay)- } catch let error as URLError where attempt < 4 && error.downloadTaskResumeData != nil {- resumeData = error.downloadTaskResumeData- AppLog.note("\(file) from \(host) interrupted (\(error.localizedDescription)), resuming")- // #279: a Wi-Fi drop spent every try within one second.- await networkBack(within: .seconds(120))- try await Task.sleep(for: .seconds(2 * attempt))+ var busy = 0+ while true {+ report(.fetching(host: host))+ let body: URL, response: URLResponse+ do {+ (body, response) = try await withoutActuallyEscaping(report) { report in+ // A dropped connection (or one the server closed during a pause) resumes+ // where it stopped rather than starting the 440 MB over.+ defer { Task { @MainActor in DownloadMeter.shared.clear() } }+ var resumeData: Data?+ for attempt in 1... {+ let relay = ProgressRelay(host: host, report: report)+ defer { relay.finish() }+ do {+ if let resumeData { return try await session.download(resumeFrom: resumeData, delegate: relay) }+ return try await session.download(from: base.appending(path: file), delegate: relay)+ } catch let error as URLError where attempt < 4 && error.downloadTaskResumeData != nil {+ resumeData = error.downloadTaskResumeData+ AppLog.note("\(file) from \(host) interrupted (\(error.localizedDescription)), resuming")+ // #279: a Wi-Fi drop spent every try within one second.+ await networkBack(within: .seconds(120))+ try await Task.sleep(for: .seconds(2 * attempt))+ }}+ fatalError("unreachable")}- fatalError("unreachable")+ } catch {+ refusals.append("\(host): \(error.localizedDescription)")+ kinds.insert(Self.connectionFailed(error) ? .unreachable : .failed)+ continue hosts}- } catch {- refusals.append("\(host): \(error.localizedDescription)")- kinds.insert(Self.connectionFailed(error) ? .unreachable : .failed)- continue- }- defer { try? FileManager.default.removeItem(at: body) }+ defer { try? FileManager.default.removeItem(at: body) }- // 206: resumed after a dropped connection.- if let http = response as? HTTPURLResponse, ![200, 206].contains(http.statusCode) {- refusals.append("\(host) answered \(http.statusCode)")- kinds.insert(.answered)- continue- }+ // 206: resumed after a dropped connection.+ if let http = response as? HTTPURLResponse, ![200, 206].contains(http.statusCode) {+ // #2727: github.com answers a passing 504 now and then, and the engine has+ // only that one host. Ask it again before giving up on it.+ if busy < 2, http.statusCode == 408 || http.statusCode == 429 || (500...599).contains(http.statusCode) {+ busy += 1+ AppLog.note("\(file) from \(host) answered \(http.statusCode), asking again")+ try await Task.sleep(for: backoff * busy)+ continue+ }+ refusals.append("\(host) answered \(http.statusCode)")+ kinds.insert(.answered)+ continue hosts+ }- let got = try Digest.sha256(of: body)- guard got == sha256 else {- refusals.append("\(host) served \(got.prefix(16)) rather than \(sha256.prefix(16))")- kinds.insert(.mismatch)- continue- }+ let got = try Digest.sha256(of: body)+ guard got == sha256 else {+ refusals.append("\(host) served \(got.prefix(16)) rather than \(sha256.prefix(16))")+ kinds.insert(.mismatch)+ continue hosts+ }- do {- try FileManager.default.createDirectory(- at: downloads, withIntermediateDirectories: true- )- try? FileManager.default.removeItem(at: archive)- try FileManager.default.moveItem(at: body, to: archive)- } catch {- throw StepFailure(- step: step,- detail: String(localized: "\(downloads.path(percentEncoded: false)) could not be written. ")- + error.localizedDescription- )+ do {+ try FileManager.default.createDirectory(+ at: downloads, withIntermediateDirectories: true+ )+ try? FileManager.default.removeItem(at: archive)+ try FileManager.default.moveItem(at: body, to: archive)+ } catch {+ throw StepFailure(+ step: step,+ detail: String(localized: "\(downloads.path(percentEncoded: false)) could not be written. ")+ + error.localizedDescription+ )+ }+ return archive}- return archive}AppLog.note("\(file) refused by all \(bases.count) hosts: \(refusals.joined(separator: ". "))")diff --git a/app/Tests/BridgeplayAppTests/PinnedDownloadTests.swift b/app/Tests/BridgeplayAppTests/PinnedDownloadTests.swiftindex 61de8c1..97132ff 100644--- a/app/Tests/BridgeplayAppTests/PinnedDownloadTests.swift+++ b/app/Tests/BridgeplayAppTests/PinnedDownloadTests.swift@@ -348,4 +348,87 @@ struct PinnedDownloadTests {#expect(label.hasPrefix("Downloading from "))#expect(label.contains(serving.lastPathComponent))}++ // Answers the status in the host name ("504.busy.invalid") to the first requests, then+ // 200 with the body. How many times it says no is the first path component.+ private final class Busy: URLProtocol, @unchecked Sendable {+ static let body = Data("the real package".utf8)+ private static let lock = NSLock()+ nonisolated(unsafe) private static var asked: [String: Int] = [:]+ static func count(_ key: String) -> Int { lock.lock(); defer { lock.unlock() }; return asked[key] ?? 0 }++ override class func canInit(with request: URLRequest) -> Bool { request.url?.host()?.hasSuffix("busy.invalid") == true }+ override class func canonicalRequest(for request: URLRequest) -> URLRequest { request }+ override func startLoading() {+ let url = request.url!+ let key = url.pathComponents[1]+ Self.lock.lock(); Self.asked[key, default: 0] += 1; let n = Self.asked[key]!; Self.lock.unlock()+ let status = n <= Int(url.pathComponents[2])! ? Int(url.host()!.split(separator: ".")[0])! : 200+ let response = HTTPURLResponse(url: url, statusCode: status, httpVersion: nil, headerFields: nil)!+ client?.urlProtocol(self, didReceive: response, cacheStoragePolicy: .notAllowed)+ client?.urlProtocol(self, didLoad: status == 200 ? Self.body : Data("busy".utf8))+ client?.urlProtocolDidFinishLoading(self)+ }+ override func stopLoading() {}+ }++ private func busy(_ status: Int, refusals: Int) async throws -> (Result<URL, StepFailure>, asked: Int) {+ let work = try scratchDirectory("pinned")+ defer { try? FileManager.default.removeItem(at: work) }+ let key = UUID().uuidString+ let result: Result<URL, StepFailure>+ do {+ let got = try await PinnedDownload.obtain(+ file: "package.tar", sha256: Digest.sha256(of: Busy.body),+ bases: [URL(string: "https://\(status).busy.invalid/\(key)/\(refusals)/")!],+ into: work.appending(path: "downloads"), step: "test",+ session: PinnedDownload.session(deadline: 30, protocols: [Busy.self]), backoff: .zero)+ result = .success(got)+ } catch let failure as StepFailure {+ result = .failure(failure)+ }+ return (result, Busy.count(key))+ }++ @Test("A host that answers a passing 5xx is asked again (#2727)", arguments: [502, 503, 504, 429])+ func passingBusyIsAskedAgain(status: Int) async throws {+ let (result, asked) = try await busy(status, refusals: 2)+ #expect((try? result.get()) != nil)+ #expect(asked == 3)+ }++ @Test("A host that stays busy is given up on after three asks")+ func busyHostIsGivenUpOn() async throws {+ let (result, asked) = try await busy(504, refusals: 99)+ guard case .failure(let failure) = result else { Issue.record("downloaded from a busy host"); return }+ #expect(failure.detail.contains("try again later"))+ #expect(asked == 3)+ }++ @Test("A host given up on hands over to the next, which gets its own asks")+ func busyHostFallsThroughToNextHost() async throws {+ let work = try scratchDirectory("pinned")+ defer { try? FileManager.default.removeItem(at: work) }+ let (first, second) = (UUID().uuidString, UUID().uuidString)++ let got = try await PinnedDownload.obtain(+ file: "package.tar", sha256: Digest.sha256(of: Busy.body),+ bases: [+ URL(string: "https://504.busy.invalid/\(first)/99/")!,+ URL(string: "https://503.busy.invalid/\(second)/2/")!,+ ],+ into: work.appending(path: "downloads"), step: "test",+ session: PinnedDownload.session(deadline: 30, protocols: [Busy.self]), backoff: .zero)++ #expect(try Data(contentsOf: got) == Busy.body)+ #expect(Busy.count(first) == 3)+ #expect(Busy.count(second) == 3)+ }++ @Test("A 404 is not asked again")+ func notFoundIsNotRetried() async throws {+ let (result, asked) = try await busy(404, refusals: 99)+ #expect((try? result.get()) == nil)+ #expect(asked == 1)+ }}