Skip to content

Play your Steam library's Windows games on a jailbroken PS5.

C++

Restart downloads when their manifests changed

Leonardo Lobato9c260cda28d5
diff --git a/src/steam/backend/depot_installer.cpp b/src/steam/backend/depot_installer.cpp
index e46ef71..f44d9a2 100644
--- a/src/steam/backend/depot_installer.cpp
+++ b/src/steam/backend/depot_installer.cpp
@@ -430,16 +430,69 @@ InstallOutcome DepotInstaller::Run(const StoredSession& session, const InstallTa
if (std::fclose(list) != 0 || std::rename((list_path + ".tmp").c_str(), list_path.c_str()) != 0)
return fail(InstallError::kStorage, "cannot write file list");
}
+ std::string journal_header = "#v2 ";
+ for (std::size_t index = 0; index < access.grants.size(); ++index) {
+ if (index) journal_header += ',';
+ journal_header += std::to_string(access.grants[index].depot_id) + ':' +
+ std::to_string(access.grants[index].manifest_gid);
+ }
std::set<std::string> done;
- if (FILE* journal = std::fopen(journal_path.c_str(), "r")) {
- char line[8192];
- while (std::fgets(line, sizeof(line), journal)) {
- std::string entry(line);
+ bool has_header = false, nonempty_journal = false, reset_journal = false, first_line = true;
+ bool unterminated_journal = false;
+ if (FILE* input = std::fopen(journal_path.c_str(), "r")) {
+ const auto read_line = [&](std::string entry) {
while (!entry.empty() && (entry.back() == '\n' || entry.back() == '\r')) entry.pop_back();
- if (!entry.empty()) done.insert(entry);
+ const bool at_start = first_line;
+ first_line = false;
+ if (entry.empty()) return;
+ nonempty_journal = true;
+ if (entry[0] == '#') {
+ // Version headers must occur exactly once, on the first line. Other comments are ignored.
+ const bool version_prefix = entry.compare(0, 2, "#v") == 0 || entry.compare(0, 2, "#V") == 0;
+ const bool version_line = version_prefix && (entry.size() == 2 ||
+ (entry.size() > 2 && std::isdigit(static_cast<unsigned char>(entry[2]))));
+ if (version_line) {
+ if (!at_start || has_header || entry != journal_header) reset_journal = true;
+ has_header = true;
+ }
+ return;
+ }
+ done.insert(std::move(entry));
+ };
+ char buffer[8192];
+ std::string line;
+ bool oversized = false;
+ const auto max_line = std::max<std::size_t>(journal_header.size() + 2, sizeof(buffer));
+ while (std::fgets(buffer, sizeof(buffer), input)) {
+ const std::string part(buffer);
+ if (line.size() + part.size() > max_line) oversized = true;
+ if (!oversized) line += part;
+ if (!part.empty() && part.back() == '\n') {
+ if (oversized) { reset_journal = true; nonempty_journal = true; first_line = false; }
+ else read_line(std::move(line));
+ line.clear(); oversized = false;
+ }
}
- std::fclose(journal);
- }
+ if (oversized) { reset_journal = true; nonempty_journal = true; }
+ else if (!line.empty()) { read_line(std::move(line)); unterminated_journal = true; }
+ const bool read_ok = std::ferror(input) == 0;
+ const bool closed = std::fclose(input) == 0;
+ if (!read_ok || !closed) return fail(InstallError::kStorage, "cannot read journal");
+ } else if (Exists(journal_path)) return fail(InstallError::kStorage, "cannot read journal");
+ if (reset_journal || !nonempty_journal) {
+ done.clear();
+ const auto temporary = journal_path + ".tmp";
+ FILE* header = std::fopen(temporary.c_str(), "w");
+ if (!header) return fail(InstallError::kStorage, "cannot write journal");
+ const bool written = std::fprintf(header, "%s\n", journal_header.c_str()) >= 0;
+ const bool closed = std::fclose(header) == 0;
+ if (!written || !closed || std::rename(temporary.c_str(), journal_path.c_str()) != 0) {
+ std::remove(temporary.c_str());
+ return fail(InstallError::kStorage, "cannot write journal");
+ }
+ unterminated_journal = false;
+ if (reset_journal) trace("journal-reset");
+ } else if (!has_header) trace("journal-legacy");
std::uint64_t remaining_bytes = 0;
for (const auto& [key, placement] : placements) {
@@ -471,6 +524,11 @@ InstallOutcome DepotInstaller::Run(const StoredSession& session, const InstallTa
trace("write");
FILE* journal = std::fopen(journal_path.c_str(), "a");
if (!journal) return fail(InstallError::kStorage, "cannot write journal");
+ // A valid final line need not have a newline; separate it from newly appended chunk keys.
+ if (unterminated_journal && (std::fprintf(journal, "\n") < 0 || std::fflush(journal) != 0)) {
+ std::fclose(journal);
+ return fail(InstallError::kStorage, "cannot write journal");
+ }
download_start = last_stats = Clock::now();
ScopeExit end_stats{[&] { emit("end"); }};
auto report_progress = [&] {
diff --git a/src/steam/backend/steam_install_runner.cpp b/src/steam/backend/steam_install_runner.cpp
index 3448fc0..7b183b9 100644
--- a/src/steam/backend/steam_install_runner.cpp
+++ b/src/steam/backend/steam_install_runner.cpp
@@ -8,6 +8,13 @@
namespace playden::steam {
+void LogInstallTrace(const InstallLogFn& log, GameId app, const char* marker) {
+ if (!log) return;
+ if (std::string(marker) == "journal-reset") log(app, "journal", "reset", "");
+ else if (std::string(marker) == "journal-legacy") log(app, "journal", "legacy", "");
+ else log(app, "trace", marker, "");
+}
+
std::string FormatInstallStats(const char* phase, const InstallStats& stats) {
std::string detail;
const auto add = [&](const char* key, std::uint64_t value) {
diff --git a/src/steam/backend/steam_install_runner.h b/src/steam/backend/steam_install_runner.h
index 3b909e1..70357a6 100644
--- a/src/steam/backend/steam_install_runner.h
+++ b/src/steam/backend/steam_install_runner.h
@@ -8,6 +8,8 @@
namespace playden::steam {
std::string FormatInstallStats(const char* phase, const InstallStats& stats);
+using InstallLogFn = std::function<void(GameId app, const char* step, const char* category, const char* detail)>;
+void LogInstallTrace(const InstallLogFn& log, GameId app, const char* marker);
// Installs a Steam game: PICS app info for the depot selection (Windows, English, public
// branch), then the resumable depot installer.
@@ -21,7 +23,7 @@ class SteamInstallRunner final : public InstallRunner {
void SetTuning(std::function<DownloadTuning()> tuning) { tuning_ = std::move(tuning); }
// Optional sanitized log: the Steam app ID, step, error category and a short detail; never
// tokens or account identifiers. Called on the install worker.
- using LogFn = std::function<void(GameId app, const char* step, const char* category, const char* detail)>;
+ using LogFn = InstallLogFn;
void SetLog(LogFn log) {
log_ = log;
installer_.SetStats([this, log](const char* phase, const InstallStats& stats) {
@@ -30,7 +32,7 @@ class SteamInstallRunner final : public InstallRunner {
log(current_app_.load(), "stats", phase, detail.c_str());
}
});
- installer_.SetTrace([this, log](const char* step) { if (log) log(current_app_.load(), "trace", step, ""); });
+ installer_.SetTrace([this, log](const char* step) { LogInstallTrace(log, current_app_.load(), step); });
}
private:
diff --git a/tests/depot_installer_test.cpp b/tests/depot_installer_test.cpp
index 78de8e6..570d4fd 100644
--- a/tests/depot_installer_test.cpp
+++ b/tests/depot_installer_test.cpp
@@ -288,6 +288,7 @@ struct ParallelFixture {
return std::string("http=503");
};
fetcher.stats = [state] {
+ if (state->owner == std::thread::id{}) state->owner = std::this_thread::get_id();
CHECK(state->owner == std::this_thread::get_id());
CdnStats stats; stats.requests = state->requests; return stats;
};
@@ -388,7 +389,7 @@ void CancelStopsAllWorkers(const std::filesystem::path& root, bool during_backof
const auto fetches = fixture.entered;
usleep(20000);
CHECK(fixture.callbacks == calls && fixture.entered == fetches);
- CHECK(Read(root / DepotInstaller::kJournalDir / "journal").empty());
+ CHECK(Read(root / DepotInstaller::kJournalDir / "journal") == "#v2 441:11,442:12\n");
}
void WorkerFailureWinsAndJoins(const std::filesystem::path& root) {
@@ -427,7 +428,7 @@ void WriteFailureStopsWorkers(const std::filesystem::path& root) {
caller.join();
CHECK(outcome.error == InstallError::kStorage && outcome.detail == "cannot write file");
CHECK(fixture.active == 0 && fixture.last.chunks_done == 0);
- CHECK(Read(root / DepotInstaller::kJournalDir / "journal").empty());
+ CHECK(Read(root / DepotInstaller::kJournalDir / "journal") == "#v2 441:11,442:12\n");
}
void ParallelCorruptRetryAndResume(const std::filesystem::path& root) {
@@ -688,6 +689,131 @@ void ResumesWithoutRefetchingFinishedChunks(const std::filesystem::path& root) {
CHECK(Read(root / "game.exe") == "MZ-executable-second-chunk");
}
+void ChangedManifestRefetchesSameOffset(const std::filesystem::path& root) {
+ auto world = MakeWorld();
+ std::atomic<bool> cancel{false};
+ world.cancel = &cancel; world.cancel_after = 1;
+ auto hooks = world.Hooks();
+ std::vector<std::string> traces;
+ hooks.trace = [&](const char* step) { traces.emplace_back(step); };
+ DepotInstaller installer(hooks);
+ installer.SetTuning({1, 256});
+ CHECK(installer.Run({}, Target(root), cancel).error == InstallError::kCanceled);
+ CHECK(Read(root / "DATA/level.dat") == "level-patched");
+ world.manifests[442].gid = 13;
+ world.manifests[442].files[0] = world.AddFile("DATA/level.dat", {"updated-level"});
+ auto target = Target(root);
+ target.depots[1].manifest_gid = 13;
+ cancel.store(false); world.cancel_after = -1; world.chunk_calls = 0;
+ CHECK(installer.Run({}, target, cancel).error == InstallError::kNone);
+ CHECK(Read(root / "DATA/level.dat") == "updated-level");
+ CHECK(world.chunk_calls == 3);
+ CHECK(std::count(traces.begin(), traces.end(), "journal-reset") == 1);
+}
+
+void JournalHeaderModes(const std::filesystem::path& base) {
+ const std::string header = "#v2 441:11,442:12\n";
+ const std::string entry = "442\t0\tDATA/level.dat\n";
+ const std::vector<std::string> journals = {
+ header + "#comment ignored\n#verified chunks\n" + entry, entry, "", "#v2 441:11,442:99\n" + entry,
+ "#v2 broken\n" + entry, header + header + entry, "#v2 442:12,441:11\n" + entry,
+ "\n" + header + entry, "#v1 441:11,442:12\n" + entry, "#V2 441:11,442:12\n" + entry,
+ "#comment\n" + entry, header + entry.substr(0, entry.size() - 1),
+ };
+ for (std::size_t mode = 0; mode < journals.size(); ++mode) {
+ const auto root = base / std::to_string(mode);
+ auto world = MakeWorld();
+ std::atomic<bool> cancel{false}; world.cancel = &cancel; world.cancel_after = 1;
+ auto hooks = world.Hooks();
+ std::vector<std::string> traces;
+ hooks.trace = [&](const char* step) { traces.emplace_back(step); };
+ InstallStats last;
+ hooks.stats = [&](const char* phase, const InstallStats& stats) { if (std::string(phase) == "end") last = stats; };
+ DepotInstaller installer(hooks); installer.SetTuning({1, 256});
+ CHECK(installer.Run({}, Target(root), cancel).error == InstallError::kCanceled);
+ CHECK(Read(root / DepotInstaller::kJournalDir / "journal") == header + entry);
+ CHECK(std::count(traces.begin(), traces.end(), "journal-reset") == 0);
+ CHECK(std::count(traces.begin(), traces.end(), "journal-legacy") == 0);
+ std::ofstream(root / DepotInstaller::kJournalDir / "journal") << journals[mode];
+ cancel.store(false); world.cancel_after = -1; world.chunk_calls = 0; traces.clear();
+ CHECK(installer.Run({}, Target(root), cancel).error == InstallError::kNone);
+ const bool legacy = mode == 1 || mode == 10;
+ const bool reuse = mode == 0 || mode == 11 || legacy;
+ CHECK(world.chunk_calls == (reuse ? 2 : 3));
+ CHECK(last.chunks_skipped == (reuse ? 1u : 0u));
+ CHECK(std::count(traces.begin(), traces.end(), "journal-reset") == (mode >= 3 && mode <= 9 ? 1 : 0));
+ CHECK(std::count(traces.begin(), traces.end(), "journal-legacy") == (legacy ? 1 : 0));
+ CHECK(Read(root / "game.exe") == "MZ-executable-second-chunk");
+ CHECK(Read(root / "DATA/level.dat") == "level-patched");
+ }
+}
+
+void UnterminatedJournalGetsSeparator(const std::filesystem::path& root) {
+ auto world = MakeWorld();
+ std::filesystem::create_directories(root / DepotInstaller::kJournalDir);
+ std::filesystem::create_directories(root / "DATA");
+ std::ofstream(root / "DATA/level.dat") << "level-patched";
+ std::ofstream(root / DepotInstaller::kJournalDir / "journal") << "#v2 441:11,442:12\n442\t0\tDATA/level.dat";
+ std::atomic<bool> cancel{false}; world.cancel = &cancel; world.cancel_after = 1;
+ DepotInstaller installer(world.Hooks()); installer.SetTuning({1, 256});
+ CHECK(installer.Run({}, Target(root), cancel).error == InstallError::kCanceled);
+ CHECK(Read(root / DepotInstaller::kJournalDir / "journal") ==
+ "#v2 441:11,442:12\n442\t0\tDATA/level.dat\n441\t0\tgame.exe\n");
+ cancel.store(false); world.chunk_calls = 0; world.cancel_after = -1;
+ CHECK(installer.Run({}, Target(root), cancel).error == InstallError::kNone);
+ CHECK(world.chunk_calls == 1);
+}
+
+void LongMatchingJournalHeader(const std::filesystem::path& root) {
+ auto world = MakeWorld();
+ world.manifests[441].files = {world.AddFile("one", {"written"})};
+ auto target = Target(root);
+ const auto depot = target.depots.front();
+ target.depots.assign(1500, depot);
+ std::string header = "#v2 ";
+ for (int index = 0; index < 1500; ++index) header += (index ? ",441:11" : "441:11");
+ CHECK(header.size() > 8192);
+ std::filesystem::create_directories(root / DepotInstaller::kJournalDir);
+ std::ofstream(root / "one") << "written";
+ std::ofstream(root / DepotInstaller::kJournalDir / "journal") << header << "\n441\t0\tone\n";
+ auto hooks = world.Hooks();
+ InstallStats last;
+ hooks.stats = [&](const char* phase, const InstallStats& stats) { if (std::string(phase) == "end") last = stats; };
+ DepotInstaller installer(hooks);
+ std::atomic<bool> cancel{false};
+ const auto result = installer.Run({}, target, cancel);
+ CHECK(result.error == InstallError::kNone);
+ CHECK(world.chunk_calls == 0 && last.chunks_skipped == 1);
+ CHECK(Read(root / "one") == "written");
+}
+
+void HeaderWriteFailureCannotAppend(const std::filesystem::path& root) {
+ auto world = MakeWorld();
+ const auto folder = root / DepotInstaller::kJournalDir;
+ std::filesystem::create_directories(folder / "journal.tmp");
+ const std::string stale = "#v2 441:999,442:12\n442\t0\tDATA/level.dat\n";
+ std::ofstream(folder / "journal") << stale;
+ DepotInstaller installer(world.Hooks());
+ std::atomic<bool> cancel{false};
+ const auto outcome = installer.Run({}, Target(root), cancel);
+ CHECK(outcome.error == InstallError::kStorage && outcome.detail == "cannot write journal");
+ CHECK(world.chunk_calls == 0 && Read(folder / "journal") == stale);
+}
+
+void JournalTraceLogging() {
+ std::vector<std::string> lines;
+ const InstallLogFn log = [&](GameId app, const char* step, const char* category, const char* detail) {
+ lines.push_back(std::to_string(app) + " step=" + step + " result=" + category + " detail=" + detail);
+ };
+ LogInstallTrace(log, 440, "journal-reset");
+ LogInstallTrace(log, 440, "journal-legacy");
+ LogInstallTrace(log, 440, "write");
+ CHECK(lines == std::vector<std::string>({"440 step=journal result=reset detail=",
+ "440 step=journal result=legacy detail=",
+ "440 step=trace result=write detail="}));
+ LogInstallTrace({}, 440, "journal-reset");
+}
+
void RefusesWhenSpaceIsShort(const std::filesystem::path& root) {
auto world = MakeWorld();
world.free_bytes = 1024;
@@ -791,6 +917,12 @@ int main() {
RunStats(base / "stats");
InstallsMergedDepotsAndWritesRecord(base / "install");
ResumesWithoutRefetchingFinishedChunks(base / "resume");
+ ChangedManifestRefetchesSameOffset(base / "manifest-update");
+ JournalHeaderModes(base / "header-modes");
+ HeaderWriteFailureCannotAppend(base / "header-write-failure");
+ LongMatchingJournalHeader(base / "long-header");
+ UnterminatedJournalGetsSeparator(base / "unterminated-header");
+ JournalTraceLogging();
RefusesWhenSpaceIsShort(base / "space");
CorruptChunksFailAfterRetries(base / "corrupt");
BusyCdnIsRetriedAfterAWait(base / "busy");