Skip to content

Run Windows games on Apple Silicon — free, open source, with an open compatibility database that tells you what actually works.

Save backups: one implementation for GOG and Epic

Gabe779da1b1de08
diff --git a/Sources/HighballApp/AppState.swift b/Sources/HighballApp/AppState.swift
index c4b2c00..4f7c632 100644
--- a/Sources/HighballApp/AppState.swift
+++ b/Sources/HighballApp/AppState.swift
@@ -3518,7 +3518,8 @@ final class AppState {
appendLog("\(title): Epic cloud saves left as they are; \(beforePlay ? "this Mac's saves won't be uploaded after this session, and " : "")Highball asks again next time")
return false
}
- let again = await Task.detached { saves.sync(app, install: install, prefix: prefix, direction: forced, onLine: log) }.value
+ // The player picked a side: the backup of what it replaces is kept, never pruned.
+ let again = await Task.detached { saves.sync(app, install: install, prefix: prefix, direction: forced, keepBackups: true, onLine: log) }.value
switch again {
case .synced:
appendLog("\(title): \(choice == .useCloud ? "Epic's cloud saves replaced this Mac's, after a backup of them" : "Epic's cloud now has this Mac's saves as its newest")")
diff --git a/Sources/HighballKit/EpicSaves.swift b/Sources/HighballKit/EpicSaves.swift
index e81959a..1d932b0 100644
--- a/Sources/HighballKit/EpicSaves.swift
+++ b/Sources/HighballKit/EpicSaves.swift
@@ -397,7 +397,9 @@ public struct EpicSaves: Sendable {
/// Syncs an Epic game's saves. `prefix` is the environment a Windows build is installed in,
/// which legendary reads to resolve the save folder; nil for a Mac build, whose folder it
/// resolves under the home folder. `install` keys the backups and the sync history.
- public func sync(_ app: String, install: URL, prefix: URL?, direction: Direction,
+ /// `keepBackups` makes the backup a kept one, never pruned: for the sync the player chose in
+ /// a conflict.
+ public func sync(_ app: String, install: URL, prefix: URL?, direction: Direction, keepBackups: Bool = false,
home: URL = FileManager.default.homeDirectoryForCurrentUser,
onLine: (@Sendable (String) -> Void)? = nil) -> Outcome {
let gameLock = Self.lock(for: app)
@@ -412,7 +414,9 @@ public struct EpicSaves: Sendable {
// A download that never finished: its backup goes back first.
if let pending = records()[key]?.restoring, let folder = savePath(app) {
do {
- try SaveBackups.restore(pending.isEmpty ? nil : URL(fileURLWithPath: pending), into: folder, filter: filter)
+ let backup = pending.isEmpty ? nil : URL(fileURLWithPath: pending)
+ try SaveBackups.restore(backup, into: folder, filter: filter)
+ if let backup { SaveBackups.unpin(backup) }
clearRestoring(key)
onLine?("Highball: an earlier download of these saves didn't finish, so this Mac's saves were put back from their backup")
} catch {
@@ -454,7 +458,7 @@ public struct EpicSaves: Sendable {
case .conflict(let first):
return .conflict(firstSync: first)
case .transfer(download: true):
- return download(app, folder: folder, filter: filter, backupDir: backupDir, key: key,
+ return download(app, folder: folder, filter: filter, backupDir: backupDir, key: key, kept: keepBackups,
before: before, environment: environment, onLine: onLine, noteSynced: noteSynced)
case .transfer(download: false):
let upload: Run
@@ -480,11 +484,15 @@ public struct EpicSaves: Sendable {
}
}
- private func download(_ app: String, folder: URL, filter: SaveFilter, backupDir: URL, key: String,
+ private func download(_ app: String, folder: URL, filter: SaveFilter, backupDir: URL, key: String, kept: Bool,
before: Check, environment: [String: String], onLine: (@Sendable (String) -> Void)?,
noteSynced: (String?) -> Void) -> Outcome {
let backup: URL?
- do { backup = try backups.backUpLocal(folder, into: backupDir, filter: filter) } catch {
+ do {
+ backup = try backups.backUpLocal(folder, into: backupDir, filter: filter, kept: kept)
+ // Pinned until the download is done or undone, so no backup made meanwhile prunes it.
+ if let backup { try SaveBackups.pin(backup) }
+ } catch {
return .failed("the saves couldn't be backed up first (\(error.localizedDescription)), so nothing was synced")
}
update(key) { record in
@@ -505,11 +513,13 @@ public struct EpicSaves: Sendable {
} catch { problem = error.localizedDescription }
guard let problem else {
+ if let backup { SaveBackups.unpin(backup) }
noteSynced(before.cloud)
return .synced
}
do {
try SaveBackups.restore(backup, into: folder, filter: filter)
+ if let backup { SaveBackups.unpin(backup) }
clearRestoring(key)
} catch {
return .failed("the download from Epic failed (\(problem)), and the saves couldn't be put back automatically; their backup is in \(((backup?.path ?? backupDir.path) as NSString).abbreviatingWithTildeInPath)")
diff --git a/Sources/HighballKit/GOGSaves.swift b/Sources/HighballKit/GOGSaves.swift
index ca9758b..fc7bea3 100644
--- a/Sources/HighballKit/GOGSaves.swift
+++ b/Sources/HighballKit/GOGSaves.swift
@@ -1,4 +1,3 @@
-import CryptoKit
import Foundation
import os
@@ -193,127 +192,44 @@ public struct GOGSaves: Sendable {
// MARK: Backups
- /// Backups taken before a sync could replace saves, stored at
- /// save-backups/<game id>-<location>-<install>/<date>-<mac|cloud>/. gogdl syncs both ways and
- /// deletes whatever the other side lacks: a download deletes local files missing from the
- /// cloud, and an upload deletes cloud files missing locally. The newest `keep` backups per
- /// install and location are kept.
- public var backupsDir: URL { store.dir.deletingLastPathComponent().appending(path: "save-backups", directoryHint: .isDirectory) }
- static let keep = 10
+ /// Backups taken before a sync could replace saves (SaveBackups), stored at
+ /// save-backups/<game id>-<location>-<install>/<date>-<mac|cloud>[-kept]/. gogdl syncs both
+ /// ways and deletes whatever the other side lacks: a download deletes local files missing
+ /// from the cloud, and an upload deletes cloud files missing locally.
+ public var backups: SaveBackups { SaveBackups(root: store.dir.deletingLastPathComponent().appending(path: "save-backups", directoryHint: .isDirectory)) }
+ public var backupsDir: URL { backups.root }
/// The backup folder for one install and location. The install is part of the name (sync
/// history is per install too), so a second copy of the game never rotates out the first
/// one's backups.
func backupsFolder(_ game: GOGStore.InstalledGame, _ location: Location) -> URL {
- // FNV-1a of the install path: short and stable across runs.
- let hash = game.folder.standardizedFileURL.path.utf8.reduce(UInt64(0xcbf2_9ce4_8422_2325)) { ($0 ^ UInt64($1)) &* 0x100_0000_01b3 }
- return backupsDir.appending(path: "\(game.id)-\(location.name)-\(String(hash, radix: 16).prefix(8))", directoryHint: .isDirectory)
- }
-
- private func newBackup(in dir: URL, label: String) -> URL {
- let formatter = ISO8601DateFormatter()
- formatter.formatOptions = [.withInternetDateTime, .withFractionalSeconds]
- let stamp = formatter.string(from: Date()).replacingOccurrences(of: ":", with: "")
- var name = "\(stamp)-\(label)", n = 1
- while FileManager.default.fileExists(atPath: dir.appending(path: name).path) { n += 1; name = "\(stamp)-\(label)-\(n)" }
- return dir.appending(path: name, directoryHint: .isDirectory)
- }
-
- /// Every file a sync touches, as gogdl walks a folder (create_directory_map): hidden files
- /// too, and into a linked folder, since its os.path.isdir follows links. A folder reached
- /// twice through links is walked once. Throws for a folder inside it that can't be read: its
- /// saves would go uncopied. A folder that isn't there holds nothing.
- static func files(in folder: URL) throws -> [URL] {
- guard FileManager.default.fileExists(atPath: folder.path) else { return [] }
- var files: [URL] = [], walked = Set<String>()
- func walk(_ dir: URL) throws {
- guard walked.insert(dir.resolvingSymlinksInPath().path).inserted else { return }
- for name in try FileManager.default.contentsOfDirectory(atPath: dir.path).sorted() {
- let item = dir.appending(path: name)
- var isFolder: ObjCBool = false
- if FileManager.default.fileExists(atPath: item.path, isDirectory: &isFolder), isFolder.boolValue { try walk(item) } else { files.append(item) }
- }
- }
- try walk(folder)
- return files
+ backups.folder("\(game.id)-\(location.name)", install: game.folder)
}
- /// Where a file from files(in:) sits under `folder`, as gogdl names it: by the path walked,
- /// not where a link leads.
- static func relativePath(of file: URL, in folder: URL) -> String {
- String(file.standardizedFileURL.path.dropFirst(folder.standardizedFileURL.path.count + 1))
- }
-
- /// A digest of a folder's files: each one's path, size and last change. When it matches a
- /// backup's, a new backup would be a duplicate.
- static func fingerprint(_ folder: URL, files chosen: [URL]? = nil) -> String {
- let keys: Set<URLResourceKey> = [.fileSizeKey, .contentModificationDateKey]
- let lines = (chosen ?? (try? files(in: folder)) ?? []).map { file -> String in
- let values = try? file.resolvingSymlinksInPath().resourceValues(forKeys: keys)
- return "\(relativePath(of: file, in: folder))\t\(values?.fileSize ?? 0)\t\(values?.contentModificationDate?.timeIntervalSince1970 ?? 0)"
- }
- return "mac " + digest(lines)
- }
+ typealias BackupKind = SaveBackups.Kind
+ static func kind(of backup: URL) -> BackupKind? { SaveBackups.kind(of: backup) }
+ static func isKept(_ backup: URL) -> Bool { SaveBackups.isKept(backup) }
+ static func files(in folder: URL) -> [URL] { SaveBackups.files(in: folder) }
+ /// files(in:), throwing for a folder inside it that can't be read: its saves would go uncopied.
+ static func walk(_ folder: URL) throws -> [URL] { try SaveBackups.walk(folder) }
+ static func relativePath(of file: URL, in folder: URL) -> String { SaveBackups.relativePath(file, in: folder) }
+ static func fingerprint(_ folder: URL) -> String { SaveBackups.fingerprint(folder) }
+ static func fingerprintFile(of backup: URL) -> URL { SaveBackups.fingerprintFile(of: backup) }
+ func backups(in dir: URL) -> [URL] { backups.backups(in: dir) }
/// A digest of a location's files in GOG's cloud: each one's path and hash.
static func fingerprint(cloud listing: [CloudFile], location: Location) -> String {
let own = listing.filter { $0.hash != deletedMark && $0.name.hasPrefix(location.name + "/") }
- return "cloud " + digest(own.map { "\($0.name)\t\($0.hash)" })
- }
-
- private static func digest(_ lines: [String]) -> String {
- SHA256.hash(data: Data(lines.sorted().joined(separator: "\n").utf8)).map { String(format: "%02x", $0) }.joined()
+ return "cloud " + SaveBackups.digest(own.map { "\($0.name)\t\($0.hash)" })
}
- /// What a backup holds: the Mac's saves or GOG's cloud's.
- enum BackupKind: String, Sendable { case mac, cloud }
-
- /// Backups are named <date>-<kind>[-kept]. Kept backups (made before an uninstall, or before
- /// the player picked a side in a conflict) are never pruned; the others are pruned per kind.
- static func kind(of backup: URL) -> BackupKind? {
- let name = backup.lastPathComponent
- return name.contains("-cloud") ? .cloud : name.contains("-mac") ? .mac : nil
- }
- static func isKept(_ backup: URL) -> Bool { backup.lastPathComponent.contains("-kept") }
-
- /// The newest backup of that kind when it holds exactly `fingerprint`. A kept backup asked
- /// for when the match isn't kept yet turns the match into a kept one instead of copying again.
- private func reuse(in dir: URL, kind: BackupKind, fingerprint: String, kept: Bool) -> URL? {
- guard let newest = backups(in: dir).first(where: { Self.kind(of: $0) == kind }),
- (try? String(contentsOf: Self.fingerprintFile(of: newest), encoding: .utf8)) == fingerprint else { return nil }
- guard kept, !Self.isKept(newest) else { return newest }
- let promoted = dir.appending(path: newest.lastPathComponent + "-kept", directoryHint: .isDirectory)
- guard (try? FileManager.default.moveItem(at: newest, to: promoted)) != nil else { return nil }
- try? FileManager.default.moveItem(at: Self.fingerprintFile(of: newest), to: Self.fingerprintFile(of: promoted))
- return promoted
- }
-
- /// Backs up the local saves before something could replace or delete them. Returns the
- /// newest Mac backup when nothing has changed since it, or nil when there is nothing to back
- /// up. `kept` backups are never pruned. `only` limits it to some of the folder's files, as
- /// files(in:) names them. Throws if the backup fails, leaving no partial copy behind; the
- /// caller must then stop.
+ /// Backs up the local saves before something could replace or delete them
+ /// (SaveBackups.backUpLocal). `kept` backups are never pruned. `only` limits it to some of
+ /// the folder's files, as files(in:) names them.
@discardableResult
public func backUpLocal(_ game: GOGStore.InstalledGame, location: Location, folder: URL, kept: Bool = false,
only: [URL]? = nil) throws -> URL? {
- let files = try only ?? Self.files(in: folder)
- guard !files.isEmpty else { return nil }
- let dir = backupsFolder(game, location)
- let fingerprint = Self.fingerprint(folder, files: files)
- if let same = reuse(in: dir, kind: .mac, fingerprint: fingerprint, kept: kept) { return same }
- let target = try finish(newBackup(in: dir, label: kept ? "mac-kept" : "mac")) { partial in
- // What a link points at, not the link: gogdl writes through it, so a download replaces
- // the file it points to. A link to nothing holds nothing to keep.
- for file in files {
- let source = file.resolvingSymlinksInPath()
- guard FileManager.default.fileExists(atPath: source.path) else { continue }
- let copy = partial.appending(path: Self.relativePath(of: file, in: folder))
- try FileManager.default.createDirectory(at: copy.deletingLastPathComponent(), withIntermediateDirectories: true)
- try FileManager.default.copyItem(at: source, to: copy)
- }
- }
- try? fingerprint.write(to: Self.fingerprintFile(of: target), atomically: true, encoding: .utf8)
- return target
+ try backups.backUpLocal(folder, into: backupsFolder(game, location), kept: kept, only: only)
}
/// Where the files a game wrote in its own folder are backed up (backUpGameFolder). Not a
@@ -323,7 +239,7 @@ public struct GOGSaves: Sendable {
/// The files under `folder` the game wrote, by the install's history, links counted by what
/// they point at.
static func files(in folder: URL, writtenPer history: GOGStore.InstallHistory) throws -> [URL] {
- try files(in: folder).filter { file in
+ try walk(folder).filter { file in
let changed = try? file.resolvingSymlinksInPath().resourceValues(forKeys: [.contentModificationDateKey]).contentModificationDate
return history.isGameWritten(changed ?? .distantPast)
}
@@ -348,17 +264,11 @@ public struct GOGSaves: Sendable {
/// How many files a backup holds and their size, for the log.
public static func contents(of backup: URL) -> (files: Int, bytes: Int64) {
- let all = (try? files(in: backup)) ?? []
+ let all = files(in: backup)
let bytes = all.reduce(Int64(0)) { $0 + Int64((try? $1.resourceValues(forKeys: [.fileSizeKey]).fileSize) ?? 0) }
return (all.count, bytes)
}
- /// Stored next to the backup rather than inside it, so restoring a backup by hand never puts
- /// this file in the game's folder, where gogdl would upload it with the saves.
- static func fingerprintFile(of backup: URL) -> URL {
- backup.deletingLastPathComponent().appending(path: backup.lastPathComponent + ".fingerprint")
- }
-
/// GOG's whole cloud listing for the game, from a gogdl run that can't move anything: an
/// empty folder and a location name no cloud file starts with, so gogdl finds no files on
/// either side and only logs the listing (saves.py `sync`). nil when gogdl logged none
@@ -387,26 +297,21 @@ public struct GOGSaves: Sendable {
let dir = backupsFolder(game, location)
if let listing = try listCloud(game) {
guard !Self.ownFiles(for: location, in: listing).isEmpty else { return nil }
- if let same = reuse(in: dir, kind: .cloud, fingerprint: Self.fingerprint(cloud: listing, location: location), kept: kept) {
+ if let same = backups.reuse(in: dir, kind: .cloud, fingerprint: Self.fingerprint(cloud: listing, location: location), kept: kept) {
return same
}
}
- let fingerprint = OSAllocatedUnfairLock<String?>(initialState: nil)
do {
- let target = try finish(newBackup(in: dir, label: kept ? "cloud-kept" : "cloud")) { partial in
+ return try backups.backUp(into: dir, kind: .cloud, kept: kept) { partial in
try FileManager.default.createDirectory(at: partial, withIntermediateDirectories: true)
let result = try run(game, location: location, folder: partial, timestamp: "0", direction: .forceDownload)
let missing = result.listing.map { Self.ownFiles(for: location, in: $0).filter { !FileManager.default.fileExists(atPath: partial.appending(path: $0).path) } } ?? []
guard result.status == 0, Self.unexplainedFailures(result, location: location, folder: partial, since: 0) == 0, missing.isEmpty else {
throw HighballError.failed("gogdl could not copy all of GOG's cloud saves (status \(result.status))")
}
- guard !(try Self.files(in: partial)).isEmpty else { throw NothingToBackUp() }
- fingerprint.withLock { $0 = result.listing.map { Self.fingerprint(cloud: $0, location: location) } }
- }
- if let value = fingerprint.withLock({ $0 }) {
- try? value.write(to: Self.fingerprintFile(of: target), atomically: true, encoding: .utf8)
+ guard !Self.files(in: partial).isEmpty else { throw NothingToBackUp() }
+ return result.listing.map { Self.fingerprint(cloud: $0, location: location) }
}
- return target
} catch is NothingToBackUp {
return nil
}
@@ -419,53 +324,6 @@ public struct GOGSaves: Sendable {
/// Any other error gogdl logs and continues past, such as a failed upload.
static func isFailure(_ line: String) -> Bool { isDownloadFailure(line) || line.contains(" ERROR: ") }
- /// Fills a backup under a ".partial" name and renames it only once complete, so a failed
- /// backup never sits among the real ones. Then prunes that kind's backups to the newest
- /// `keep`, never removing this one or a kept one.
- private func finish(_ target: URL, fill: (URL) throws -> Void) throws -> URL {
- let dir = target.deletingLastPathComponent()
- removeStalePartials(in: dir)
- let partial = dir.appending(path: target.lastPathComponent + ".partial", directoryHint: .isDirectory)
- do {
- try fill(partial)
- try FileManager.default.moveItem(at: partial, to: target)
- // A copied folder keeps the source's creation date, often years old. Date the backup
- // now so pruning keeps the newest ones.
- try FileManager.default.setAttributes([.creationDate: Date()], ofItemAtPath: target.path)
- } catch {
- try? FileManager.default.removeItem(at: partial)
- throw error
- }
- let kind = Self.kind(of: target)
- // Compare by name: the same folder can appear under both /var and /private/var.
- let rotating = backups(in: dir).filter {
- $0.lastPathComponent != target.lastPathComponent && Self.kind(of: $0) == kind && !Self.isKept($0)
- }
- for old in rotating.dropFirst(Self.keep - 1) {
- try? FileManager.default.removeItem(at: old)
- try? FileManager.default.removeItem(at: Self.fingerprintFile(of: old))
- }
- return target
- }
-
- /// A backup left half-filled by a crash or a forced quit: older than an hour, so no sync
- /// running now is still filling it.
- private func removeStalePartials(in dir: URL) {
- let all = (try? FileManager.default.contentsOfDirectory(at: dir, includingPropertiesForKeys: [.creationDateKey])) ?? []
- for partial in all where partial.pathExtension == "partial" {
- let made = (try? partial.resourceValues(forKeys: [.creationDateKey]).creationDate) ?? .distantPast
- if made < Date().addingTimeInterval(-3600) { try? FileManager.default.removeItem(at: partial) }
- }
- }
-
- /// A location's complete backups, newest first by creation date rather than by name, so a
- /// clock that was once wrong never gets today's backup pruned as the oldest.
- func backups(in dir: URL) -> [URL] {
- let all = (try? FileManager.default.contentsOfDirectory(at: dir, includingPropertiesForKeys: [.creationDateKey])) ?? []
- func made(_ url: URL) -> Date { (try? url.resourceValues(forKeys: [.creationDateKey]).creationDate) ?? .distantPast }
- return all.filter { !["partial", "fingerprint"].contains($0.pathExtension) }.sorted { made($0) > made($1) }
- }
-
// MARK: Locations sharing a prefix
/// A file in GOG's cloud storage, as gogdl lists it.
@@ -765,7 +623,7 @@ public struct GOGSaves: Sendable {
var count = 0
for entry in kept.entries {
let target = entry.path.isEmpty ? game.folder : game.folder.appending(path: entry.path)
- let files = try Self.files(in: entry.backup).map { ($0, Self.relativePath(of: $0, in: entry.backup)) }
+ let files = try Self.walk(entry.backup).map { ($0, Self.relativePath(of: $0, in: entry.backup)) }
let there = files.map { target.appending(path: $0.1) }.filter { FileManager.default.fileExists(atPath: $0.path) }
if !there.isEmpty {
try backUpLocal(game, location: Location(name: entry.name, template: ""), folder: target, kept: true, only: there)
diff --git a/Sources/HighballKit/SaveBackups.swift b/Sources/HighballKit/SaveBackups.swift
index 2848036..cb9c820 100644
--- a/Sources/HighballKit/SaveBackups.swift
+++ b/Sources/HighballKit/SaveBackups.swift
@@ -1,18 +1,29 @@
+import CryptoKit
+import Darwin
import Foundation
/// Backups of a game's saves, taken before a sync could replace them, stored at
-/// save-backups/<game>-<location>-<install>/<date>-<mac|cloud>/. The store tools replace saves
+/// save-backups/<game>-<location>-<install>/<date>-<kind>[-kept]/. The store tools replace saves
/// wholesale (gogdl deletes whatever the other side lacks, and legendary empties the save folder
/// before a download), so anything a sync could replace is backed up first, and a failed backup
-/// stops the sync. The newest `keep` backups per install and location are kept.
+/// stops the sync.
+///
+/// Backups come in two kinds, the Mac's saves and a cloud's, and each kind rotates on its own: the
+/// newest `keep` of each stay. Kept backups (made before an uninstall, or before the player picked
+/// a side in a conflict) and pinned ones (a download's, until it is done or undone) are never
+/// pruned. A backup identical to the newest of its kind is not made again.
public struct SaveBackups: Sendable {
public let root: URL
static let keep = 10
public init(root: URL) { self.root = root }
- /// The backup folder for one install and location. The install is part of the name, so a
- /// second copy of the game never rotates out the first one's backups.
+ /// What a backup holds: the Mac's saves or a cloud's.
+ public enum Kind: String, Sendable { case mac, cloud }
+
+ /// The backup folder for one install and location. The install is part of the name (sync
+ /// history is per install too), so a second copy of the game never rotates out the first
+ /// one's backups.
public func folder(_ name: String, install: URL) -> URL {
// FNV-1a of the install path: short and stable across runs.
let hash = install.standardizedFileURL.path.utf8.reduce(UInt64(0xcbf2_9ce4_8422_2325)) { ($0 ^ UInt64($1)) &* 0x100_0000_01b3 }
@@ -28,85 +39,133 @@ public struct SaveBackups: Sendable {
return dir.appending(path: name, directoryHint: .isDirectory)
}
- /// Every file a sync touches. The store tools include hidden files and symlinks, and replace
- /// or delete them too.
- public static func files(in folder: URL) -> [URL] {
- let keys: Set<URLResourceKey> = [.isRegularFileKey, .isSymbolicLinkKey]
- let walker = FileManager.default.enumerator(at: folder, includingPropertiesForKeys: Array(keys))
- var files: [URL] = []
- while let file = walker?.nextObject() as? URL {
- let values = try? file.resourceValues(forKeys: keys)
- if values?.isRegularFile == true || values?.isSymbolicLink == true { files.append(file) }
+ /// Backups are named <date>-<kind>[-kept].
+ public static func kind(of backup: URL) -> Kind? {
+ let name = backup.lastPathComponent
+ return name.contains("-cloud") ? .cloud : name.contains("-mac") ? .mac : nil
+ }
+ public static func isKept(_ backup: URL) -> Bool { backup.lastPathComponent.contains("-kept") }
+
+ // MARK: Files
+
+ /// Every file a sync touches, as the store tools walk a folder: hidden files too, and into a
+ /// linked folder (gogdl's os.path.isdir and legendary's os.walk follow links). A folder
+ /// reached twice through links is walked once. Throws for a folder inside it that can't be
+ /// read; one that isn't there holds nothing.
+ static func walk(_ folder: URL) throws -> [URL] {
+ guard FileManager.default.fileExists(atPath: folder.path) else { return [] }
+ var files: [URL] = [], walked = Set<String>()
+ func walk(_ dir: URL) throws {
+ guard walked.insert(dir.resolvingSymlinksInPath().path).inserted else { return }
+ for name in try FileManager.default.contentsOfDirectory(atPath: dir.path).sorted() {
+ let item = dir.appending(path: name)
+ var isFolder: ObjCBool = false
+ if FileManager.default.fileExists(atPath: item.path, isDirectory: &isFolder), isFolder.boolValue { try walk(item) } else { files.append(item) }
+ }
}
+ try walk(folder)
return files
}
+ public static func files(in folder: URL) -> [URL] { (try? walk(folder)) ?? [] }
+
/// The files in `folder` that `filter` picks, the ones a sync can replace.
public static func files(in folder: URL, matching filter: SaveFilter) -> [URL] {
guard !filter.isEverything else { return files(in: folder) }
return files(in: folder).filter { filter.matches(relativePath($0, in: folder)) }
}
- /// A file's path below `folder`, with "/" between components, as the filters see it.
+ /// A file's path below `folder`, with "/" between components, as the filters and the store
+ /// tools name it: by the path walked, not where a link leads.
static func relativePath(_ file: URL, in folder: URL) -> String {
- let base = folder.standardizedFileURL.resolvingSymlinksInPath().pathComponents
- let parts = file.standardizedFileURL.resolvingSymlinksInPath().pathComponents
- guard parts.starts(with: base) else { return file.lastPathComponent }
- return parts.dropFirst(base.count).joined(separator: "/")
+ String(file.standardizedFileURL.path.dropFirst(folder.standardizedFileURL.path.count + 1))
}
- /// A summary of a folder (file count, total bytes, latest change). When it matches the last
- /// backup's, a new backup would be a duplicate.
- static func fingerprint(_ folder: URL, filter: SaveFilter = .everything) -> String {
+ // MARK: Fingerprints
+
+ /// A digest of a folder's files (those `filter` picks): each one's path, size and last change.
+ /// When it matches a backup's, a new backup would be a duplicate.
+ public static func fingerprint(_ folder: URL, filter: SaveFilter = .everything) -> String {
+ fingerprint(folder, files: files(in: folder, matching: filter))
+ }
+
+ /// The same digest of some of `folder`'s files, as walk names them.
+ public static func fingerprint(_ folder: URL, files chosen: [URL]) -> String {
let keys: Set<URLResourceKey> = [.fileSizeKey, .contentModificationDateKey]
- let values = files(in: folder, matching: filter).map { try? $0.resourceValues(forKeys: keys) }
- let bytes = values.reduce(0) { $0 + ($1?.fileSize ?? 0) }
- let latest = values.compactMap { $0?.contentModificationDate?.timeIntervalSince1970 }.max() ?? 0
- return "\(values.count) \(bytes) \(latest)"
+ let lines = chosen.map { file -> String in
+ let values = try? file.resolvingSymlinksInPath().resourceValues(forKeys: keys)
+ return "\(relativePath(file, in: folder))\t\(values?.fileSize ?? 0)\t\(values?.contentModificationDate?.timeIntervalSince1970 ?? 0)"
+ }
+ return "mac " + digest(lines)
+ }
+
+ public static func digest(_ lines: [String]) -> String {
+ SHA256.hash(data: Data(lines.sorted().joined(separator: "\n").utf8)).map { String(format: "%02x", $0) }.joined()
}
/// Stored next to the backup rather than inside it, so restoring a backup by hand never puts
/// this file in the game's folder, where a sync would upload it with the saves.
- static func fingerprintFile(of backup: URL) -> URL {
+ public static func fingerprintFile(of backup: URL) -> URL {
backup.deletingLastPathComponent().appending(path: backup.lastPathComponent + ".fingerprint")
}
- /// Backs up the local saves before a download could replace them: the files `filter` picks,
- /// which are the ones a download replaces. Returns the newest existing backup when nothing
- /// has changed since it, or nil when there is nothing to back up. Throws if the backup fails,
- /// leaving no partial copy behind; the sync must then stop.
+ // MARK: Making backups
+
+ /// The newest backup of that kind when it holds exactly `fingerprint`. A kept backup asked
+ /// for when the match isn't kept yet turns the match into a kept one instead of copying again.
+ public func reuse(in dir: URL, kind: Kind, fingerprint: String, kept: Bool) -> URL? {
+ guard let newest = backups(in: dir).first(where: { Self.kind(of: $0) == kind }),
+ (try? String(contentsOf: Self.fingerprintFile(of: newest), encoding: .utf8)) == fingerprint else { return nil }
+ guard kept, !Self.isKept(newest) else { return newest }
+ let promoted = dir.appending(path: newest.lastPathComponent + "-kept", directoryHint: .isDirectory)
+ guard (try? FileManager.default.moveItem(at: newest, to: promoted)) != nil else { return nil }
+ try? FileManager.default.moveItem(at: Self.fingerprintFile(of: newest), to: Self.fingerprintFile(of: promoted))
+ try? FileManager.default.moveItem(at: Self.pinFile(of: newest), to: Self.pinFile(of: promoted))
+ return promoted
+ }
+
+ /// Backs up the local saves before something could replace or delete them: the files `filter`
+ /// picks, which are the ones a download replaces. Returns the newest Mac backup when nothing
+ /// has changed since it, or nil when there is nothing to back up. `kept` backups are never
+ /// pruned. `only` limits it to some of the folder's files, as walk names them (`filter` still
+ /// applies). Throws if the backup fails, leaving no partial copy behind; the caller must then
+ /// stop.
@discardableResult
- public func backUpLocal(_ folder: URL, into dir: URL, filter: SaveFilter = .everything) throws -> URL? {
- let picked = Self.files(in: folder, matching: filter)
+ public func backUpLocal(_ folder: URL, into dir: URL, filter: SaveFilter = .everything, kept: Bool = false,
+ only: [URL]? = nil) throws -> URL? {
+ // A folder that can't be read fails the copy: its saves would go uncopied.
+ let picked = try (only ?? Self.walk(folder)).filter { filter.isEverything || filter.matches(Self.relativePath($0, in: folder)) }
guard !picked.isEmpty else { return nil }
- let fingerprint = Self.fingerprint(folder, filter: filter)
- if let newest = backups(in: dir).first, (try? String(contentsOf: Self.fingerprintFile(of: newest), encoding: .utf8)) == fingerprint {
- return newest
- }
- let target = try backUp(into: dir, label: "mac") { partial in
+ let fingerprint = Self.fingerprint(folder, files: picked)
+ if let same = reuse(in: dir, kind: .mac, fingerprint: fingerprint, kept: kept) { return same }
+ return try backUp(into: dir, kind: .mac, kept: kept, fingerprint: fingerprint) { partial in
try FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true)
- if filter.isEverything {
- try FileManager.default.copyItem(at: folder, to: partial)
- } else {
- for file in picked {
- let destination = partial.appending(path: Self.relativePath(file, in: folder))
- try FileManager.default.createDirectory(at: destination.deletingLastPathComponent(), withIntermediateDirectories: true)
- try FileManager.default.copyItem(at: file, to: destination)
- }
+ // What a link points at, not the link: the tools write through it, so a download
+ // replaces the file it points to. A link to nothing holds nothing to keep.
+ for file in picked {
+ let source = file.resolvingSymlinksInPath()
+ guard FileManager.default.fileExists(atPath: source.path) else { continue }
+ let destination = partial.appending(path: Self.relativePath(file, in: folder))
+ try FileManager.default.createDirectory(at: destination.deletingLastPathComponent(), withIntermediateDirectories: true)
+ try FileManager.default.copyItem(at: source, to: destination)
}
+ return nil
}
- try? fingerprint.write(to: Self.fingerprintFile(of: target), atomically: true, encoding: .utf8)
- return target
}
- /// Makes a new backup, filled by `fill` under a ".partial" name and renamed only once
- /// complete, so a failed backup never sits among the real ones. Then prunes to the newest
- /// `keep`, never removing this one.
- public func backUp(into dir: URL, label: String, fill: (URL) throws -> Void) throws -> URL {
- let target = newBackup(in: dir, label: label)
+ /// Makes a new backup of `kind`, filled by `fill` under a ".partial" name and renamed only once
+ /// complete, so a failed backup never sits among the real ones. `fingerprint`, when known, is
+ /// stored beside it (`fill` may also return one it learned while filling). Then prunes that
+ /// kind's backups to the newest `keep`, never removing this one, a kept one or a pinned one.
+ public func backUp(into dir: URL, kind: Kind, kept: Bool, fingerprint: String? = nil,
+ fill: (URL) throws -> String?) throws -> URL {
+ try FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true)
+ removeStalePartials(in: dir)
+ let target = newBackup(in: dir, label: kept ? "\(kind.rawValue)-kept" : kind.rawValue)
let partial = dir.appending(path: target.lastPathComponent + ".partial", directoryHint: .isDirectory)
+ let learned: String?
do {
- try fill(partial)
+ learned = try fill(partial)
try FileManager.default.moveItem(at: partial, to: target)
// A copied folder keeps the source's creation date, often years old. Date the backup
// now so pruning keeps the newest ones.
@@ -115,21 +174,48 @@ public struct SaveBackups: Sendable {
try? FileManager.default.removeItem(at: partial)
throw error
}
+ if let value = fingerprint ?? learned {
+ try? value.write(to: Self.fingerprintFile(of: target), atomically: true, encoding: .utf8)
+ }
// Compare by name: the same folder can appear under both /var and /private/var.
- for old in backups(in: dir).filter({ $0.lastPathComponent != target.lastPathComponent }).dropFirst(Self.keep - 1) {
+ let rotating = backups(in: dir).filter {
+ $0.lastPathComponent != target.lastPathComponent && Self.kind(of: $0) == kind && !Self.isKept($0) && !Self.isPinned($0)
+ }
+ for old in rotating.dropFirst(Self.keep - 1) {
try? FileManager.default.removeItem(at: old)
try? FileManager.default.removeItem(at: Self.fingerprintFile(of: old))
}
return target
}
+ /// A backup left half-filled by a crash or a forced quit: older than an hour, so no sync
+ /// running now is still filling it.
+ private func removeStalePartials(in dir: URL) {
+ let all = (try? FileManager.default.contentsOfDirectory(at: dir, includingPropertiesForKeys: [.creationDateKey])) ?? []
+ for partial in all where partial.pathExtension == "partial" {
+ let made = (try? partial.resourceValues(forKeys: [.creationDateKey]).creationDate) ?? .distantPast
+ if made < Date().addingTimeInterval(-3600) { try? FileManager.default.removeItem(at: partial) }
+ }
+ }
+
/// A location's complete backups, newest first by creation date rather than by name, so a
/// clock that was once wrong never gets today's backup pruned as the oldest.
public func backups(in dir: URL) -> [URL] {
let all = (try? FileManager.default.contentsOfDirectory(at: dir, includingPropertiesForKeys: [.creationDateKey])) ?? []
func made(_ url: URL) -> Date { (try? url.resourceValues(forKeys: [.creationDateKey]).creationDate) ?? .distantPast }
- return all.filter { !["partial", "fingerprint"].contains($0.pathExtension) }.sorted { made($0) > made($1) }
+ return all.filter { !["partial", "fingerprint", "pinned"].contains($0.pathExtension) }.sorted { made($0) > made($1) }
+ }
+
+ // MARK: Pins and restoring
+
+ /// A pinned backup is never pruned: a download pins its backup until it is done or undone, so
+ /// the copy it may need to put back is still there however many backups are made meanwhile.
+ static func pinFile(of backup: URL) -> URL {
+ backup.deletingLastPathComponent().appending(path: backup.lastPathComponent + ".pinned")
}
+ public static func isPinned(_ backup: URL) -> Bool { FileManager.default.fileExists(atPath: pinFile(of: backup).path) }
+ public static func pin(_ backup: URL) throws { try Data().write(to: pinFile(of: backup)) }
+ public static func unpin(_ backup: URL) { try? FileManager.default.removeItem(at: pinFile(of: backup)) }
/// Puts a backup's files back into `folder`: first removes the files `filter` picks there
/// (what a failed download may have left half written), then copies the backup's files in,
diff --git a/Tests/HighballKitTests/EpicFakeLegendaryTests.swift b/Tests/HighballKitTests/EpicFakeLegendaryTests.swift
index 9846cd3..2d63379 100644
--- a/Tests/HighballKitTests/EpicFakeLegendaryTests.swift
+++ b/Tests/HighballKitTests/EpicFakeLegendaryTests.swift
@@ -205,6 +205,22 @@ final class EpicFakeLegendaryTests: XCTestCase {
XCTAssertNil(saves.records()[EpicSaves.key(app: app, install: install)]?.restoring)
}
+ /// Epic's backups are named like GOG's (<date>-mac), so they rotate per kind; the one made
+ /// for the player's pick in a conflict is kept, never pruned.
+ func testThePlayersPickKeepsItsBackup() throws {
+ try syncedOnce()
+ try cloudSave("from the PC", at: hourAgo)
+ try write("played here", to: folder.appending(path: "slot.sav"), at: Date())
+ XCTAssertEqual(sync(.download), .conflict(firstSync: false))
+ XCTAssertEqual(saves.sync(app, install: install, prefix: prefix, direction: .forceDownload, keepBackups: true, home: userHome), .synced)
+ XCTAssertEqual(read(folder.appending(path: "slot.sav")), "from the PC")
+ let backup = try XCTUnwrap(saves.backups.backups(in: saves.backups.folder("epic-\(app)", install: install)).first)
+ XCTAssertEqual(SaveBackups.kind(of: backup), .mac)
+ XCTAssertTrue(SaveBackups.isKept(backup))
+ XCTAssertEqual(read(backup.appending(path: "slot.sav")), "played here")
+ XCTAssertFalse(SaveBackups.isPinned(backup), "unpinned once the download is done")
+ }
+
/// Two syncs of one game never run at once: the second waits.
func testSyncsOfOneGameDoNotOverlap() throws {
try syncedOnce()
diff --git a/Tests/HighballKitTests/GOGSaveSafetyTests.swift b/Tests/HighballKitTests/GOGSaveSafetyTests.swift
index 134bd69..8acaf80 100644
--- a/Tests/HighballKitTests/GOGSaveSafetyTests.swift
+++ b/Tests/HighballKitTests/GOGSaveSafetyTests.swift
@@ -69,7 +69,7 @@ final class GOGSaveSafetyTests: XCTestCase {
let folder = root.appending(path: "Hidden")
try FileManager.default.createDirectory(at: folder, withIntermediateDirectories: true)
try Data("x".utf8).write(to: folder.appending(path: ".save"))
- XCTAssertEqual(try GOGSaves.files(in: folder).map(\.lastPathComponent), [".save"])
+ XCTAssertEqual(GOGSaves.files(in: folder).map(\.lastPathComponent), [".save"])
}
/// gogdl walks into a linked folder and writes through a linked file, so the copy holds what
@@ -245,7 +245,7 @@ final class GOGSaveSafetyTests: XCTestCase {
try write("after it", to: game.folder.appending(path: "SAVES/slot2.sav"), dated: setUp.addingTimeInterval(3000))
try write("cloud", to: game.folder.appending(path: "cloud/a.sav"), dated: setUp.addingTimeInterval(600))
let copy = try XCTUnwrap(try saves.backUpGameFolder(game, driveC: driveC, skipping: [game.folder.appending(path: "cloud")]))
- XCTAssertEqual(try GOGSaves.files(in: copy).map { GOGSaves.relativePath(of: $0, in: copy) }, ["SAVES/slot1.sav", "SAVES/slot2.sav"])
+ XCTAssertEqual(GOGSaves.files(in: copy).map { GOGSaves.relativePath(of: $0, in: copy) }, ["SAVES/slot1.sav", "SAVES/slot2.sav"])
XCTAssertTrue(GOGSaves.isKept(copy))
XCTAssertEqual(GOGSaves.contents(of: copy).files, 2)
XCTAssertEqual(GOGSaves.contents(of: copy).bytes, 25)
diff --git a/Tests/HighballKitTests/SaveBackupsTests.swift b/Tests/HighballKitTests/SaveBackupsTests.swift
new file mode 100644
index 0000000..79a483d
--- /dev/null
+++ b/Tests/HighballKitTests/SaveBackupsTests.swift
@@ -0,0 +1,28 @@
+import XCTest
+@testable import HighballKit
+
+/// The backups shared by GOG and Epic syncs (SaveBackups) beyond what GOGSaveSafetyTests covers.
+final class SaveBackupsTests: XCTestCase {
+ /// A pinned backup (a download's, until it is done or undone) outlives any number of newer
+ /// backups of its kind.
+ func testAPinnedBackupIsNeverPruned() throws {
+ let root = FileManager.default.temporaryDirectory.appending(path: "pinned-\(UUID().uuidString)")
+ defer { try? FileManager.default.removeItem(at: root) }
+ let backups = SaveBackups(root: root.appending(path: "backups"))
+ let folder = root.appending(path: "saves"), dir = backups.folder("game", install: root)
+ try FileManager.default.createDirectory(at: folder, withIntermediateDirectories: true)
+ try Data("0".utf8).write(to: folder.appending(path: "slot"))
+ let pinned = try XCTUnwrap(try backups.backUpLocal(folder, into: dir))
+ try SaveBackups.pin(pinned)
+ for i in 1...12 {
+ try Data("\(i)".utf8).write(to: folder.appending(path: "slot"))
+ try backups.backUpLocal(folder, into: dir)
+ }
+ XCTAssertTrue(FileManager.default.fileExists(atPath: pinned.path))
+ XCTAssertEqual(backups.backups(in: dir).count, 11, "ten rotating, plus the pinned one")
+ SaveBackups.unpin(pinned)
+ try Data("13".utf8).write(to: folder.appending(path: "slot"))
+ try backups.backUpLocal(folder, into: dir)
+ XCTAssertFalse(FileManager.default.fileExists(atPath: pinned.path), "pruned like the others once unpinned")
+ }
+}