Play your Steam library's Windows games on a jailbroken PS5.
C++
Show a game's exact size on the install sheet
Leonardo Lobatoe27e6340aa2a
diff --git a/src/homebrew/controller.cpp b/src/homebrew/controller.cppindex cc8f4e7..fa8b018 100644--- a/src/homebrew/controller.cpp+++ b/src/homebrew/controller.cpp@@ -2118,11 +2118,16 @@ void Controller::TickStoreDetails() {std::move(details.controller_support));}void Controller::TickAppDetails() {- if (!app_details_) return;+ if (!app_details_) {+ // Nothing measures sizes: Steam's figures stand.+ for (const auto& check : model_.TakeSizeChecks()) model_.UpdateMeasuredSize(check, false, false, {});+ return;+ }const auto& state = model_.state();const auto generation = state.account_generation;if (generation != requested_generation_) {app_details_->Invalidate(generation);+ model_.ResetSizeChecks();requested_generation_ = generation;requested_detail_.reset();launch_options_.clear();@@ -2138,6 +2143,7 @@ void Controller::TickAppDetails() {// a fetch starts and the host passes the new one on after this tick, so ask again next tick.requested_language_ = state.content_language;app_details_->Invalidate(generation);+ model_.ResetSizeChecks();requested_detail_.reset();reissue_after_language_ = true;return;@@ -2193,6 +2199,18 @@ void Controller::TickAppDetails() {}if (prepare_after_details_.erase(details.id) && runtime_) SubmitPrepare(details.id);}+ // The install sheet's exact sizes: the chosen version and language, from their manifests.+ const bool signed_in = model_.state().account().authentication == AuthState::SignedIn;+ for (const auto& check : model_.TakeSizeChecks()) {+ auto* details = Details(check.id.store);+ if (signed_in && details == app_details_) details->Measure({check.id, check.branch, check.language}, generation);+ else model_.UpdateMeasuredSize(check, false, !signed_in, {});+ }+ for (const auto& size : app_details_->PumpMeasured()) {+ if (size.account_generation != generation) continue;+ model_.UpdateMeasuredSize({size.request.id, size.request.branch, size.request.language}, size.measured, size.failed,+ {size.install_bytes, size.download_bytes});+ }}void Controller::RequestPrepare(GameId id) {if (!game_roots_.count(id)) return;diff --git a/src/homebrew/host.cpp b/src/homebrew/host.cppindex 6c05199..c770bff 100644--- a/src/homebrew/host.cpp+++ b/src/homebrew/host.cpp@@ -441,6 +441,9 @@ int main(int argc, char** argv) {std::unique_ptr<playden::OrganizationStore> organization;std::unique_ptr<playden::app::PersistenceService> persistence;std::unique_ptr<playden::steam::CmClient> details_cm;+ std::unique_ptr<playden::steam::CmClient> sizes_cm;+ std::unique_ptr<playden::steam::CdnClient> details_cdn;+ std::unique_ptr<playden::steam::DepotInstaller> details_measurer;std::unique_ptr<playden::steam::CmAppDetailsSource> details_source;std::unique_ptr<playden::LocalSessionRepository> details_sessions;std::unique_ptr<playden::steam::AppDetailsService> app_details;@@ -569,7 +572,17 @@ int main(int argc, char** argv) {details_cm = std::make_unique<playden::steam::CmClient>(PLAYDEN_HOMEBREW_CA_BUNDLE, *library_storage);details_cm->SetOwner("details");details_cm->SetSharedSession(steam_cm);+ // The install sheet's exact sizes: manifests through the details connection, never written anywhere.+ sizes_cm = std::make_unique<playden::steam::CmClient>(PLAYDEN_HOMEBREW_CA_BUNDLE, *library_storage);+ sizes_cm->SetOwner("sizes");+ sizes_cm->SetSharedSession(steam_cm);+ details_cdn = std::make_unique<playden::steam::CdnClient>(PLAYDEN_HOMEBREW_CA_BUNDLE);+ details_measurer = std::make_unique<playden::steam::DepotInstaller>(*sizes_cm, *details_cdn);details_source = std::make_unique<playden::steam::CmAppDetailsSource>(*details_cm);+ details_source->SetMeasurer(details_measurer.get(), sizes_cm.get());+ details_source->SetLog([log_directory](std::uint32_t app, const char* step, const char* result, const std::string& detail) {+ playden::app::AppendLogLine(log_directory, "install.log", "app=" + std::to_string(app) + " step=" + step + " result=" + result + " detail=" + detail);+ });details_source->SetLicensedDepots(&licensed_depots);details_source->SetContentLanguage([] { return selected_content_language(); });// Installed games' records, for their updates; the install service is made below and joined after thediff --git a/src/homebrew/ps5.cpp b/src/homebrew/ps5.cppindex db17e02..d96adbd 100644--- a/src/homebrew/ps5.cpp+++ b/src/homebrew/ps5.cpp@@ -376,7 +376,17 @@ int main() {playden::steam::CmClient details_cm(paths.trust_store, library_storage);details_cm.SetOwner("details");details_cm.SetSharedSession(steam_cm);+ // The install sheet's exact sizes: manifests through the details connection, never written anywhere.+ playden::steam::CmClient sizes_cm(paths.trust_store, library_storage);+ sizes_cm.SetOwner("sizes");+ sizes_cm.SetSharedSession(steam_cm);+ playden::steam::CdnClient details_cdn(paths.trust_store);+ playden::steam::DepotInstaller details_measurer(sizes_cm, details_cdn);playden::steam::CmAppDetailsSource details_source(details_cm);+ details_source.SetMeasurer(&details_measurer, &sizes_cm);+ details_source.SetLog([](std::uint32_t app, const char* step, const char* result, const std::string& detail) {+ playden::app::AppendLogLine(log_root, "install.log", "app=" + std::to_string(app) + " step=" + step + " result=" + result + " detail=" + detail);+ });details_source.SetLicensedDepots(&licensed_depots);details_source.SetContentLanguage([] { return selected_content_language(); });playden::LocalSessionRepository details_sessions(user_root, {}, &session_vault);diff --git a/src/homebrew/simulation.cpp b/src/homebrew/simulation.cppindex 0d1ed92..a2c08b2 100644--- a/src/homebrew/simulation.cpp+++ b/src/homebrew/simulation.cpp@@ -2357,6 +2357,12 @@ void Simulation::leave_steam_sign_in() {void Simulation::clear_account_specific_state() {++state_.account_generation;+ // Sizes follow the account's licenses; a check asked for the last account is never answered.+ size_checks_.clear();+ size_checks_running_.clear();+ size_checks_answered_.clear();+ size_checks_failed_.clear();+ measured_sizes_.clear();state_.account().challenge_url.clear();state_.account().challenge_expiry = 0;state_.account().last_sync = 0;@@ -2542,6 +2548,48 @@ const GameVersion* Simulation::install_sized_version(GameId id) const {return nullptr;}+SizeCheckRequest Simulation::install_size_check(GameId id) const {+ const auto* version = install_sized_version(id);+ // The language as an install asks for it (the sheet's choice, else Settings'): each part of the game falls+ // back from it on its own.+ return {id, version ? version->name : std::string("public"),+ install_language_.empty() ? state_.content_language : install_language_};+}++void Simulation::ResetSizeChecks() {+ size_checks_.clear();+ size_checks_running_.clear();+ size_checks_failed_.clear();+ rebuild_view();+}++const MeasuredInstallSize* Simulation::measured_install_size(GameId id) const {+ const auto check = install_size_check(id);+ const auto found = measured_sizes_.find({check.id, check.branch, check.language});+ return found == measured_sizes_.end() ? nullptr : &found->second;+}++bool Simulation::install_size_checking(GameId id) const {+ const auto check = install_size_check(id);+ return size_checks_running_.count({check.id, check.branch, check.language}) > 0;+}++InstallAssessment Simulation::install_assessment(GameId id) const {+ return assess_install(state_, id, install_target(), steam::kInstallSpaceMargin, install_sized_version(id),+ install_language(id), measured_install_size(id));+}++std::vector<SizeCheckRequest> Simulation::TakeSizeChecks() { return std::exchange(size_checks_, {}); }++void Simulation::UpdateMeasuredSize(const SizeCheckRequest& request, bool measured, bool failed, MeasuredInstallSize size) {+ const auto key = std::make_tuple(request.id, request.branch, request.language);+ size_checks_running_.erase(key);+ if (failed) size_checks_failed_.insert(key);+ else size_checks_answered_.insert(key);+ if (measured && size.install_bytes) measured_sizes_[key] = size;+ rebuild_view();+}+std::vector<std::string> Simulation::live_install_sheet_actions(GameId id) const {const auto* game = find_game(state_, id);std::vector<std::string> actions{"install-confirm"};@@ -2592,6 +2640,11 @@ bool Simulation::UpdateAppDetails(GameId id, std::optional<std::uint64_t> downlostd::optional<bool> windows_installable, std::vector<std::string> languages,std::uint32_t dlc_count, std::vector<GameVersion> versions, GameUpdate update) {if (!live()) return false;+ // New details may name other files (a newer build): the game's exact sizes are checked again.+ const auto of_game = [id](const auto& key) { return std::get<0>(key) == id; };+ std::erase_if(size_checks_answered_, of_game);+ std::erase_if(size_checks_failed_, of_game);+ std::erase_if(measured_sizes_, [id](const auto& entry) { return std::get<0>(entry.first) == id; });const auto game = std::find_if(state_.games.begin(), state_.games.end(),[id](const Game& candidate) { return candidate.id == id; });if (game == state_.games.end()) return false;@@ -4203,6 +4256,10 @@ void Simulation::rebuild_view() {!queue_return_focus_.empty();if (state_.navigation.route != Route::Downloads && !queue_details) queue_shown_ = false;if (state_.navigation.route != Route::Downloads && !queue_details) queue_return_focus_.clear();+ // A size check that failed is asked again when the install sheet opens next, not while it stays open.+ if (!rebuilding_overlay_ && std::none_of(state_.navigation.modals.begin(), state_.navigation.modals.end(),+ [](const ModalEntry& modal) { return modal.id == "install"; }))+ size_checks_failed_.clear();view_.surface = route_name(state_.navigation.route);view_.show_simulation_label = state_.simulation_mode;if (state_.editor.active) view_.surface = "keyboard";@@ -4289,17 +4346,26 @@ void Simulation::rebuild_view() {if (modal.id == "install" && live()) {// Shown when the game does not fit as is: say why, and where it would go.const auto id = state_.navigation.detail_id.value_or(state_.navigation.selected_game_id);- const auto assessment = assess_install(state_, id, install_target(), steam::kInstallSpaceMargin, install_sized_version(id),- install_language(id));+ // Steam's figures add up each part of the game; its exact size, once checked, replaces them.+ const auto check = install_size_check(id);+ const auto key = std::make_tuple(check.id, check.branch, check.language);+ if (!size_checks_running_.count(key) && !size_checks_answered_.count(key) && !size_checks_failed_.count(key)) {+ size_checks_.push_back(check);+ size_checks_running_.insert(key);+ }+ const auto assessment = install_assessment(id);+ const bool checking = install_size_checking(id);+ const bool exact = measured_install_size(id) != nullptr;const auto* volume = find_volume(state_, install_target());const auto* fallback = find_volume(state_, selected_volume_id_);view_.subtitle = "To " + (volume ? volume->label : std::string("unavailable storage")) +(install_volume_id_.empty() ? "" : " (this game only)") + ". ";view_.subtitle += assessment.free_bytes ? format_bytes(*assessment.free_bytes) + " free. " : "";view_.subtitle += assessment.installed_bytes- ? "Needs about " + format_bytes(*assessment.installed_bytes) + ". "+ ? (exact ? "Needs " : "Needs about ") + format_bytes(*assessment.installed_bytes) + ". ": "Its size is checked when the download starts. ";- if (assessment.readiness == InstallReadiness::Insufficient) view_.subtitle += "Not enough space: choose another drive or free up space.";+ if (assessment.readiness == InstallReadiness::Insufficient && checking) view_.subtitle += "Checking its exact size…";+ else if (assessment.readiness == InstallReadiness::Insufficient) view_.subtitle += "Not enough space: choose another drive or free up space.";else if (assessment.readiness == InstallReadiness::Unavailable) view_.subtitle += "This drive is disconnected.";else if (assessment.readiness == InstallReadiness::ReadOnly) view_.subtitle += "This drive is read-only.";else if (assessment.readiness == InstallReadiness::UnknownCapacity)@@ -5051,14 +5117,12 @@ void Simulation::rebuild_view() {else if (action_key == "install-confirm" || action_key == "install-estimate-retry") {const auto id = live() ? state_.navigation.detail_id.value_or(state_.navigation.selected_game_id): state_.navigation.selected_game_id;- const auto assessment = assess_install(state_, id, live() ? install_target() : selected_volume_id_,- live() ? steam::kInstallSpaceMargin : 0,- live() ? install_sized_version(id) : nullptr,- live() ? install_language(id) : std::string());+ const auto assessment = live() ? install_assessment(id) : assess_install(state_, id, selected_volume_id_);if (action_key == "install-estimate-retry") item.enabled = assessment.readiness == InstallReadiness::UnknownSize;else if (live()) {item.enabled = live_install_allowed(assessment.readiness, free_space_supported_);- item.subtitle = assessment.readiness == InstallReadiness::Insufficient ? "Not enough space"+ item.subtitle = assessment.readiness == InstallReadiness::Insufficient && install_size_checking(id) ? "Checking size"+ : assessment.readiness == InstallReadiness::Insufficient ? "Not enough space": assessment.readiness == InstallReadiness::Unavailable ? "Drive disconnected": assessment.readiness == InstallReadiness::ReadOnly ? "Drive is read-only": assessment.readiness == InstallReadiness::UnknownCapacity && free_space_supported_ ? "Free space unknown"@@ -7283,10 +7347,7 @@ void Simulation::activate() {} else if (action_name == "install-confirm") {const auto id = live() ? state_.navigation.detail_id.value_or(state_.navigation.selected_game_id): state_.navigation.selected_game_id;- const auto assessment = assess_install(state_, id, live() ? install_target() : selected_volume_id_,- live() ? steam::kInstallSpaceMargin : 0,- live() ? install_sized_version(id) : nullptr,- live() ? install_language(id) : std::string());+ const auto assessment = live() ? install_assessment(id) : assess_install(state_, id, selected_volume_id_);if (live()) {if (live_install_allowed(assessment.readiness, free_space_supported_)) {if (id.valid())diff --git a/src/homebrew/simulation.hpp b/src/homebrew/simulation.hppindex f644e6e..b482fd4 100644--- a/src/homebrew/simulation.hpp+++ b/src/homebrew/simulation.hpp@@ -15,6 +15,7 @@#include <map>#include <optional>#include <set>+#include <tuple>#include <string>#include <vector>@@ -476,6 +477,22 @@ struct GameVersion {std::vector<LanguageSize> languages;};+struct InstallAssessment; // storage_sim.hpp++// One install choice's size measured from its manifests: Steam's figures add up each depot, so depots holding the+// same files count them twice.+struct MeasuredInstallSize {+ std::uint64_t install_bytes = 0;+ std::uint64_t download_bytes = 0;+};++// A size check the install sheet asks for: a version (Steam branch) and a content language.+struct SizeCheckRequest {+ GameId id;+ std::string branch;+ std::string language;+};+// An installed game's update, from Steam app info compared with its install record.struct GameUpdate {bool newer = false; // a newer version of the game's content@@ -842,6 +859,13 @@ public:bool UpdateLibrary(LibraryUpdate update);// Live mode: requests for the real install service, and its state for display.[[nodiscard]] std::vector<InstallRequest> TakeInstallRequests();+ // Live mode: the exact sizes the install sheet asks for, and the answers. `measured` false: Steam's figures+ // stand (nothing to check, or the check failed; `failed` lets the sheet ask again when it opens next).+ [[nodiscard]] std::vector<SizeCheckRequest> TakeSizeChecks();+ void UpdateMeasuredSize(const SizeCheckRequest& request, bool measured, bool failed, MeasuredInstallSize size);+ // The details service dropped what it was asked (another account, or the language changed): checks+ // still waiting for an answer are asked again.+ void ResetSizeChecks();void UpdateInstalls(const std::vector<LiveInstallJob>& jobs, const std::vector<LiveInstalledGame>& installed,const std::vector<LiveCompletedDownload>& completed = {});// Live mode: Steam Cloud save requests for the cloud service, and its state for display.@@ -1148,6 +1172,12 @@ private:int queue_focus_index_ = 0; // Queue: the focused row's place, for when that row goes awaybool queue_shown_ = false; // Queue was on screen at the last rebuildstd::string queue_return_focus_; // Queue: the row a game page was opened from, focused again on Back+ // Install sheet size checks by (game, branch, language): asked and not answered, and the answers.+ std::vector<SizeCheckRequest> size_checks_;+ std::set<std::tuple<GameId, std::string, std::string>> size_checks_running_;+ std::set<std::tuple<GameId, std::string, std::string>> size_checks_answered_;+ std::set<std::tuple<GameId, std::string, std::string>> size_checks_failed_; // not asked again until the sheet reopens+ std::map<std::tuple<GameId, std::string, std::string>, MeasuredInstallSize> measured_sizes_;std::string settings_focus_section_;struct OrganizationJournalEntry {@@ -1351,6 +1381,12 @@ private:// The version chosen in the install sheet for `id`, when it is not the public one.[[nodiscard]] const GameVersion* install_version(GameId id) const;[[nodiscard]] const GameVersion* install_sized_version(GameId id) const;+ // The install sheet's choice for `id` as a size check names it, and its measured size once known.+ [[nodiscard]] SizeCheckRequest install_size_check(GameId id) const;+ [[nodiscard]] const MeasuredInstallSize* measured_install_size(GameId id) const;+ [[nodiscard]] bool install_size_checking(GameId id) const;+ // The live install sheet's fit check: the chosen version, language and drive, with the measured size.+ [[nodiscard]] InstallAssessment install_assessment(GameId id) const;[[nodiscard]] std::vector<std::string> live_install_sheet_actions(GameId id) const;// An installed game's update on its own drive: whether it can start, and the words the sheet shows.struct UpdateCheck {diff --git a/src/homebrew/storage_sim.cpp b/src/homebrew/storage_sim.cppindex b3966eb..619c210 100644--- a/src/homebrew/storage_sim.cpp+++ b/src/homebrew/storage_sim.cpp@@ -26,7 +26,8 @@ std::uint64_t add_saturated(std::uint64_t left, std::uint64_t right) noexcept {InstallAssessment assess_install(const State& state, GameId game_id,const std::string& volume_id, std::uint64_t safety_margin_bytes,- const GameVersion* version, const std::string& language) noexcept {+ const GameVersion* version, const std::string& language,+ const MeasuredInstallSize* measured) noexcept {InstallAssessment result;const auto game = find_item(state.games, [game_id](const Game& value) {return value.id == game_id;@@ -44,6 +45,10 @@ InstallAssessment assess_install(const State& state, GameId game_id,result.download_bytes = size.download_bytes ? std::optional<std::uint64_t>(size.download_bytes) : std::nullopt;result.installed_bytes = size.install_bytes;}+ if (measured != nullptr && measured->install_bytes) {+ result.installed_bytes = measured->install_bytes;+ if (measured->download_bytes) result.download_bytes = measured->download_bytes;+ }if (result.installed_bytes) result.required_bytes = add_saturated(*result.installed_bytes, safety_margin_bytes);}diff --git a/src/homebrew/storage_sim.hpp b/src/homebrew/storage_sim.hppindex 9ee6810..f39a724 100644--- a/src/homebrew/storage_sim.hpp+++ b/src/homebrew/storage_sim.hpp@@ -31,12 +31,14 @@ struct InstallAssessment {};// `version`: the version chosen for this install; its sizes, when known, replace the game's. `language`: the-// content language chosen for it; that language's sizes on `version`, when known, replace both.+// content language chosen for it; that language's sizes on `version`, when known, replace both. `measured`: that+// choice's size from its manifests, which replaces them all.[[nodiscard]] InstallAssessment assess_install(const State& state, GameId game_id,const std::string& volume_id,std::uint64_t safety_margin_bytes = 0,const GameVersion* version = nullptr,- const std::string& language = {}) noexcept;+ const std::string& language = {},+ const MeasuredInstallSize* measured = nullptr) noexcept;void reconcile_storage(State& state) noexcept;diff --git a/src/steam/app_details_service.cpp b/src/steam/app_details_service.cppindex dbd06c0..27f08e5 100644--- a/src/steam/app_details_service.cpp+++ b/src/steam/app_details_service.cpp@@ -1,11 +1,12 @@#include "steam/app_details_service.h"#include <algorithm>+#include <iterator>namespace playden::steam {AppDetailsService::AppDetailsService(AppDetailsSource& source, SessionRepository& sessions)- : source_(source), sessions_(sessions), worker_([this] { Run(); }) {}+ : source_(source), sessions_(sessions), worker_([this] { Run(); }), measure_worker_([this] { RunMeasures(); }) {}AppDetailsService::~AppDetailsService() { Shutdown(); }@@ -18,7 +19,11 @@ void AppDetailsService::Request(GameId id, std::uint64_t account_generation) {pending_.clear();requested_.clear();results_.clear();+ measure_pending_.clear();+ measure_requested_.clear();+ measured_.clear();cancel_.store(true, std::memory_order_release);+ measure_cancel_.store(true, std::memory_order_release);}if (requested_.count(id) || pending_.size() >= kMaxPending) return;requested_.insert(id);@@ -41,10 +46,43 @@ void AppDetailsService::Invalidate(std::uint64_t account_generation) {pending_.clear();requested_.clear();results_.clear();+ measure_pending_.clear();+ measure_requested_.clear();+ measured_.clear();cancel_.store(true, std::memory_order_release);+ measure_cancel_.store(true, std::memory_order_release);wake_.notify_all();}+void AppDetailsService::Measure(SizeRequest request, std::uint64_t account_generation) {+ if (!request.id.steam_app() || account_generation == 0) return;+ std::lock_guard<std::mutex> lock(mutex_);+ if (stopping_) return;+ if (account_generation != generation_) {+ // Another account: what was asked for the previous one goes, as for Request.+ generation_ = account_generation;+ pending_.clear();+ requested_.clear();+ results_.clear();+ measure_pending_.clear();+ measure_requested_.clear();+ measured_.clear();+ cancel_.store(true, std::memory_order_release);+ measure_cancel_.store(true, std::memory_order_release);+ }+ if (measure_requested_.count(request) || measure_pending_.size() >= kMaxPending) return;+ measure_requested_.insert(request);+ measure_pending_.push_back(std::move(request));+ wake_.notify_all();+}++std::vector<MeasuredSize> AppDetailsService::PumpMeasured() {+ std::lock_guard<std::mutex> lock(mutex_);+ std::vector<MeasuredSize> output(std::make_move_iterator(measured_.begin()), std::make_move_iterator(measured_.end()));+ measured_.clear();+ return output;+}+std::vector<AppDetails> AppDetailsService::Pump(std::size_t maximum) {std::lock_guard<std::mutex> lock(mutex_);std::vector<AppDetails> output;@@ -58,12 +96,14 @@ std::vector<AppDetails> AppDetailsService::Pump(std::size_t maximum) {void AppDetailsService::Shutdown() {{std::lock_guard<std::mutex> lock(mutex_);- if (stopping_ && !worker_.joinable()) return;+ if (stopping_ && !worker_.joinable() && !measure_worker_.joinable()) return;stopping_ = true;cancel_.store(true, std::memory_order_release);+ measure_cancel_.store(true, std::memory_order_release);}wake_.notify_all();if (worker_.joinable()) worker_.join();+ if (measure_worker_.joinable()) measure_worker_.join();}void AppDetailsService::Run() {@@ -105,4 +145,32 @@ void AppDetailsService::Run() {}}+// Size checks have their own worker: one that waits on manifests never holds up a game's page.+void AppDetailsService::RunMeasures() {+ for (;;) {+ SizeRequest measure;+ std::uint64_t generation = 0;+ {+ std::unique_lock<std::mutex> lock(mutex_);+ wake_.wait(lock, [this] { return stopping_ || !measure_pending_.empty(); });+ if (stopping_) return;+ measure = std::move(measure_pending_.front());+ measure_pending_.pop_front();+ generation = generation_;+ measure_cancel_.store(false, std::memory_order_release);+ }+ MeasuredSize result{measure};+ const auto session = sessions_.Load();+ if (session.ok() && session.value()) result = source_.Measure(*session.value(), measure, measure_cancel_);+ else result.failed = true;+ result.request = measure;+ result.account_generation = generation;+ std::lock_guard<std::mutex> lock(mutex_);+ if (stopping_) return;+ measure_requested_.erase(measure);+ if (generation != generation_ || measure_cancel_.load(std::memory_order_acquire)) continue;+ measured_.push_back(std::move(result));+ }+}+} // namespace playden::steamdiff --git a/src/steam/app_details_service.h b/src/steam/app_details_service.hindex 6fd33b1..8ce9925 100644--- a/src/steam/app_details_service.h+++ b/src/steam/app_details_service.h@@ -79,12 +79,43 @@ struct AppDetails {Update update;};+// One install choice whose exact size is asked for: a version (Steam branch, "public" by default) and a+// content language.+struct SizeRequest {+ GameId id;+ std::string branch;+ std::string language;+ bool operator==(const SizeRequest& other) const {+ return id == other.id && branch == other.branch && language == other.language;+ }+ bool operator<(const SizeRequest& other) const {+ if (!(id == other.id)) return id < other.id;+ return branch != other.branch ? branch < other.branch : language < other.language;+ }+};++// The size an install of that choice has once its depots are merged, from their manifests. Steam's figures add+// up each depot, so depots holding the same files count them twice.+struct MeasuredSize {+ SizeRequest request;+ std::uint64_t account_generation = 0;+ // False: Steam's figures are right as they are (nothing overlaps enough to matter), or the check failed.+ bool measured = false;+ bool failed = false; // the check could not finish (no session, the network); asking again may work+ std::uint64_t install_bytes = 0;+ std::uint64_t download_bytes = 0;+};+// Runs on the service worker; may block on the network.class AppDetailsSource {public:virtual ~AppDetailsSource() = default;virtual std::vector<AppDetails> Fetch(const StoredSession& session, const std::vector<GameId>& ids,const std::atomic<bool>& cancel) = 0;+ // Runs on its own worker, at the same time as Fetch. Default: Steam's figures stand.+ virtual MeasuredSize Measure(const StoredSession&, const SizeRequest& request, const std::atomic<bool>&) {+ return {request};+ }};// Fetches app details for games the player opens. Requests are deduplicated and cached for the@@ -102,6 +133,10 @@ class AppDetailsService {void Refresh(GameId id, std::uint64_t account_generation);void Invalidate(std::uint64_t account_generation);std::vector<AppDetails> Pump(std::size_t maximum = 8);+ // Asks for the exact size of one install choice, after the details already asked for. A choice already+ // queued or being checked is not asked twice.+ void Measure(SizeRequest request, std::uint64_t account_generation);+ std::vector<MeasuredSize> PumpMeasured();void Shutdown();static constexpr std::size_t kMaxPending = 64;@@ -109,6 +144,7 @@ class AppDetailsService {private:void Run();+ void RunMeasures();AppDetailsSource& source_;SessionRepository& sessions_;@@ -118,9 +154,14 @@ class AppDetailsService {std::deque<GameId> pending_;std::set<GameId> requested_; // queued, in flight or answered for this generationstd::deque<AppDetails> results_;+ std::deque<SizeRequest> measure_pending_;+ std::set<SizeRequest> measure_requested_; // queued or in flight+ std::deque<MeasuredSize> measured_;std::atomic<bool> cancel_{false};+ std::atomic<bool> measure_cancel_{false};bool stopping_ = false;std::thread worker_;+ std::thread measure_worker_; // size checks: the source's Measure runs here, beside Fetch};} // namespace playden::steamdiff --git a/src/steam/backend/cm_app_details.cpp b/src/steam/backend/cm_app_details.cppindex b9e18ae..02d1693 100644--- a/src/steam/backend/cm_app_details.cpp+++ b/src/steam/backend/cm_app_details.cpp@@ -3,6 +3,7 @@#include "steam/backend/app_info.h"#include <algorithm>+#include <chrono>namespace playden::steam {@@ -118,6 +119,27 @@ AppDetails DetailsFromAppInfo(std::uint32_t id, const std::string& buffer,return details;}+std::vector<AppInfo> CmAppDetailsSource::FetchDlcApps(CmClient& client, const StoredSession& session, std::uint32_t id,+ const std::string& buffer, const std::atomic<bool>& cancel) {+ std::vector<AppInfo> dlc_apps;+ const auto app = ParseAppInfo(id, buffer);+ if (!app) return dlc_apps;+ std::vector<std::uint32_t> dlc_ids;+ for (const auto dlc : app->dlc_app_ids)+ if (dlc != id && std::find(dlc_ids.begin(), dlc_ids.end(), dlc) == dlc_ids.end()) dlc_ids.push_back(dlc);+ for (std::size_t start = 0; start < dlc_ids.size() && !cancel.load(); start += kMaxAppInfoBatch) {+ const std::vector<std::uint32_t> batch(+ dlc_ids.begin() + static_cast<std::ptrdiff_t>(start),+ dlc_ids.begin() + static_cast<std::ptrdiff_t>(std::min(start + kMaxAppInfoBatch, dlc_ids.size())));+ const auto dlc_info = client.FetchAppInfo(session, batch, cancel);+ if (!dlc_info.ok()) continue;+ for (const auto dlc : batch)+ if (const auto found = dlc_info.buffers.find(dlc); found != dlc_info.buffers.end())+ if (auto parsed = ParseAppInfo(dlc, found->second)) dlc_apps.push_back(std::move(*parsed));+ }+ return dlc_apps;+}+std::vector<AppDetails> CmAppDetailsSource::Fetch(const StoredSession& session,const std::vector<GameId>& ids,const std::atomic<bool>& cancel) {@@ -127,9 +149,18 @@ std::vector<AppDetails> CmAppDetailsSource::Fetch(const StoredSession& session,if (const auto app = id.steam_app()) apps.push_back(*app);std::vector<std::uint32_t> app_ids(apps);if (app_ids.size() > kMaxAppInfoBatch) app_ids.resize(kMaxAppInfoBatch);+ const auto started = std::chrono::steady_clock::now();const auto fetched = client_.FetchAppInfo(session, app_ids, cancel);+ const auto ms = std::to_string(+ std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now() - started).count());std::vector<AppDetails> result;- if (!fetched.ok()) return result; // the service reports these as retryable+ if (!fetched.ok()) {+ if (log_) for (const auto id : app_ids) log_(id, "details", "failed", "batch=" + std::to_string(app_ids.size()) + " ms=" + ms);+ return result; // the service reports these as retryable+ }+ if (log_)+ for (const auto id : app_ids)+ log_(id, "details", fetched.buffers.count(id) ? "ok" : "missing", "batch=" + std::to_string(app_ids.size()) + " ms=" + ms);const auto language = language_ ? language_() : std::string(kDefaultContentLanguage);for (const auto id : apps) {const auto found = fetched.buffers.find(id);@@ -144,22 +175,7 @@ std::vector<AppDetails> CmAppDetailsSource::Fetch(const StoredSession& session,// An installed game's DLC apps list depots of their own: an update selects them too. A batch Steam// doesn't answer only leaves those DLC out of the comparison.std::vector<AppInfo> dlc_apps;- if (record && !record->depots.empty())- if (const auto app = ParseAppInfo(id, found->second)) {- std::vector<std::uint32_t> dlc_ids;- for (const auto dlc : app->dlc_app_ids)- if (dlc != id && std::find(dlc_ids.begin(), dlc_ids.end(), dlc) == dlc_ids.end()) dlc_ids.push_back(dlc);- for (std::size_t start = 0; start < dlc_ids.size() && !cancel.load(); start += kMaxAppInfoBatch) {- const std::vector<std::uint32_t> batch(- dlc_ids.begin() + static_cast<std::ptrdiff_t>(start),- dlc_ids.begin() + static_cast<std::ptrdiff_t>(std::min(start + kMaxAppInfoBatch, dlc_ids.size())));- const auto dlc_info = client_.FetchAppInfo(session, batch, cancel);- if (!dlc_info.ok()) continue;- for (const auto dlc : batch)- if (const auto buffer = dlc_info.buffers.find(dlc); buffer != dlc_info.buffers.end())- if (auto parsed = ParseAppInfo(dlc, buffer->second)) dlc_apps.push_back(std::move(*parsed));- }- }+ if (record && !record->depots.empty()) dlc_apps = FetchDlcApps(client_, session, id, found->second, cancel);if (cancel.load()) return {};result.push_back(DetailsFromAppInfo(id, found->second, licensed_ ? licensed_->Get() : std::nullopt, language, record,dlc_apps));@@ -167,4 +183,102 @@ std::vector<AppDetails> CmAppDetailsSource::Fetch(const StoredSession& session,return result;}+std::vector<DepotInfo> DepotsToMeasure(std::uint32_t id, const std::string& buffer,+ const std::optional<LicensedDepots::Grant>& licensed, const std::string& branch,+ std::string_view language, const std::vector<AppInfo>& dlc_apps) {+ const auto app = ParseAppInfo(id, buffer);+ if (!app) return {};+ DepotSelection selection;+ selection.language = std::string(language);+ if (!branch.empty()) selection.branch = branch;+ const std::set<std::uint32_t> owned{id};+ SelectionResult selected;+ bool any = false;+ if (licensed) {+ std::set<std::uint32_t> apps = owned;+ apps.insert(licensed->apps.begin(), licensed->apps.end());+ any = SelectDepots(*app, dlc_apps, apps, licensed->depots, selection, &selected);+ }+ if (!any && !SelectDepots(*app, dlc_apps, owned, std::nullopt, selection, &selected)) return {};+ const auto large = std::count_if(selected.depots.begin(), selected.depots.end(),+ [](const DepotInfo& depot) { return depot.size >= kMeasureDepotBytes; });+ if (large < 2) return {};+ return selected.depots;+}++MeasuredSize CmAppDetailsSource::Measure(const StoredSession& session, const SizeRequest& request,+ const std::atomic<bool>& cancel) {+ MeasuredSize result{request};+ const auto app = request.id.steam_app();+ if (!measurer_ || !measure_client_ || !app) return result;+ const auto started = std::chrono::steady_clock::now();+ const auto language = !request.language.empty() ? request.language+ : language_ ? language_() : std::string(kDefaultContentLanguage);+ const std::string branch = request.branch.empty() ? "public" : request.branch;+ std::size_t depot_count = 0;+ const auto log = [&](const char* outcome, const std::string& extra) {+ if (!log_) return;+ const auto ms = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now() - started).count();+ std::string detail = "branch=" + branch + " language=" + language + " depots=" + std::to_string(depot_count);+ if (result.measured)+ detail += " install=" + std::to_string(result.install_bytes) + " download=" + std::to_string(result.download_bytes);+ if (!extra.empty()) detail += " " + extra;+ log_(*app, "size", outcome, detail + " ms=" + std::to_string(ms));+ };+ const auto fetched = measure_client_->FetchAppInfo(session, {*app}, cancel);+ if (!fetched.ok()) {+ result.failed = true;+ log("failed", "why=appinfo");+ return result;+ }+ const auto buffer = fetched.buffers.find(*app);+ if (buffer == fetched.buffers.end()) {+ log("failed", "why=no-appinfo");+ return result;+ }+ // As an install selects: DLC that only its own appinfo lists counts too. Free DLC an install would claim first+ // can't be known yet; the download's own space check still has the last word.+ const auto dlc_apps = FetchDlcApps(*measure_client_, session, *app, buffer->second, cancel);+ if (cancel.load()) {+ result.failed = true;+ log("failed", "why=canceled");+ return result;+ }+ const auto depots = DepotsToMeasure(*app, buffer->second, licensed_ ? licensed_->Get() : std::nullopt,+ request.branch, language, dlc_apps);+ depot_count = depots.size();+ if (depots.empty()) {+ log("skipped", {});+ return result;+ }+ // What an account is granted can differ: its answers are its own.+ std::string key = std::to_string(session.steam_id) + " " + std::to_string(*app);+ for (const auto& depot : depots) key += " " + std::to_string(depot.id) + ":" + std::to_string(depot.manifest_gid.value_or(0));+ if (const auto known = measured_.find(key); known != measured_.end()) {+ result.measured = true;+ result.install_bytes = known->second.first;+ result.download_bytes = known->second.second;+ log("cached", {});+ return result;+ }+ // As an install asks: the branch its manifests come from, "public" by default.+ const auto measure = measurer_->Measure(session, *app, depots, branch, cancel);+ if (measure.error != InstallError::kNone || measure.install_bytes == 0) {+ // Refused or unmergeable: an install would say so itself; Steam's figures stand meanwhile.+ result.failed = measure.error == InstallError::kNetwork || measure.error == InstallError::kCanceled ||+ measure.error == InstallError::kNoSession;+ log("failed", std::string("why=") + InstallErrorName(measure.error) + (measure.detail.empty() ? "" : " (" + measure.detail + ")"));+ return result;+ }+ // Without a depot the account was refused, the size holds for these grants only: not kept.+ if (!measure.partial) measured_[key] = {measure.install_bytes, measure.download_bytes};+ result.measured = true;+ result.install_bytes = measure.install_bytes;+ result.download_bytes = measure.download_bytes;+ std::uint64_t summed = 0;+ for (const auto& depot : depots) summed += depot.size;+ log("fetched", "steam_sum=" + std::to_string(summed) + (measure.partial ? " partial=1" : ""));+ return result;+}+} // namespace playden::steamdiff --git a/src/steam/backend/cm_app_details.h b/src/steam/backend/cm_app_details.hindex 0ce713e..9ce88e7 100644--- a/src/steam/backend/cm_app_details.h+++ b/src/steam/backend/cm_app_details.h@@ -5,8 +5,11 @@#include "steam/backend/app_info.h"#include "steam/backend/cm_client.h"#include "steam/backend/content_language.h"+#include "steam/backend/depot_installer.h"#include <functional>+#include <map>+#include <utility>namespace playden::steam {@@ -28,16 +31,49 @@ class CmAppDetailsSource final : public AppDetailsSource {void SetInstalled(std::function<std::optional<installs::InstalledRecord>(std::uint32_t app)> installed) {installed_ = std::move(installed);}+ // What measures an install choice from its manifests (MeasuredSize), and the connection its size checks read+ // appinfo on: Measure runs on its own worker beside Fetch, so not the details one. Without them Steam's figures+ // stand.+ void SetMeasurer(DepotInstaller* measurer, CmClient* client) {+ measurer_ = measurer;+ measure_client_ = client;+ }+ // Optional: lines for the install log, never identifiers beyond app and depot IDs. Step "details": one per game a+ // fetch answers (result ok, missing or failed, with the time Steam took). Step "size": one per size check, result+ // fetched (the manifests were read), cached, skipped (Steam's sum stands: fewer than two large depots) or failed,+ // with the choice, the sizes and the time taken.+ void SetLog(std::function<void(std::uint32_t app, const char* step, const char* result, const std::string& detail)> log) {+ log_ = std::move(log);+ }std::vector<AppDetails> Fetch(const StoredSession& session, const std::vector<GameId>& ids,const std::atomic<bool>& cancel) override;+ MeasuredSize Measure(const StoredSession& session, const SizeRequest& request,+ const std::atomic<bool>& cancel) override;private:+ // The game's DLC apps' appinfo (a batch Steam doesn't answer is left out).+ std::vector<AppInfo> FetchDlcApps(CmClient& client, const StoredSession& session, std::uint32_t id,+ const std::string& buffer, const std::atomic<bool>& cancel);CmClient& client_;const LicensedDepots* licensed_ = nullptr;std::function<std::string()> language_;std::function<std::optional<installs::InstalledRecord>(std::uint32_t)> installed_;+ DepotInstaller* measurer_ = nullptr;+ CmClient* measure_client_ = nullptr;+ std::function<void(std::uint32_t, const char*, const char*, const std::string&)> log_;+ // Measured sizes (the size worker's only) by the depots and manifests measured: a manifest never changes, so neither does its size.+ std::map<std::string, std::pair<std::uint64_t, std::uint64_t>> measured_;};+// The depots an install of `branch` in `language` selects, as DetailsFromAppInfo selects them, when their+// merged size can differ enough from Steam's sum to be worth fetching their manifests: at least two of them+// hold kMeasureDepotBytes or more. Empty: Steam's figures stand.+inline constexpr std::uint64_t kMeasureDepotBytes = 64ull * 1000 * 1000;+// `dlc_apps`: the game's DLC apps' appinfo, for the depots only they list, as an install reads them.+std::vector<DepotInfo> DepotsToMeasure(std::uint32_t id, const std::string& buffer,+ const std::optional<LicensedDepots::Grant>& licensed, const std::string& branch,+ std::string_view language, const std::vector<AppInfo>& dlc_apps = {});+// Pure part, for tests: turns one parsed app buffer into details. With `licensed`, sizes count// only the depots those licenses grant, as an install selects them. With `installed` (the game's// install record), the record's branch and language select, and `update` compares the selectiondiff --git a/src/steam/backend/depot_installer.cpp b/src/steam/backend/depot_installer.cppindex c788636..dc878d1 100644--- a/src/steam/backend/depot_installer.cpp+++ b/src/steam/backend/depot_installer.cpp@@ -228,10 +228,7 @@ InstallError FromCm(CmError error) {return InstallError::kProtocol;}-struct Placement {- std::size_t depot = 0;- const ManifestFile* file = nullptr;-};+using Placement = ManifestPlacement;std::string JournalKey(std::uint32_t depot, const std::string& path, std::uint64_t offset) {return std::to_string(depot) + "\t" + std::to_string(offset) + "\t" + path;@@ -529,6 +526,57 @@ const char* InstallErrorName(InstallError error) {return "protocol";}+DepotMerge MergeDepotManifests(const std::vector<DepotManifest>& manifests,+ const std::function<void(const char* result, const std::string& detail)>& log) {+ DepotMerge merged;+ const auto entry = [&manifests](std::size_t depot, const ManifestFile& file) {+ std::string text = "depot=" + std::to_string(manifests[depot].depot_id) + " ";+ if (file.is_directory()) text += "folder";+ else if (file.is_symlink()) text += "link";+ else text += "file size=" + std::to_string(file.size);+ return text + " path=" + file.path;+ };+ auto& placements = merged.placements;+ for (std::size_t depot = 0; depot < manifests.size(); ++depot) {+ for (const auto& file : manifests[depot].files) {+ const auto key = Lower(file.path);+ const auto found = placements.find(key);+ if (found != placements.end()) {+ if (SameFile(*found->second.file, file)) continue;+ if (found->second.file->is_directory() != file.is_directory()) {+ const auto& folder = file.is_directory() ? file : *found->second.file;+ const auto& other = file.is_directory() ? *found->second.file : file;+ const auto detail = entry(found->second.depot, *found->second.file) + " | " + entry(depot, file);+ if (other.is_symlink()) {+ if (log) log("link-over-folder", detail);+ merged.conflict = file.path;+ return merged;+ }+ if (log) log("file-over-folder", detail);+ merged.files_over_folders.emplace_back(key, folder.path);+ if (file.is_directory()) continue;+ }+ }+ placements[key] = {depot, &file};+ }+ }+ for (const auto& [key, folder] : merged.files_over_folders) {+ const auto& winner = placements.at(key);+ const auto inside = placements.lower_bound(key + "/");+ if (inside != placements.end() && inside->first.compare(0, key.size() + 1, key + "/") == 0) {+ if (log) log("folder-not-empty", entry(winner.depot, *winner.file) + " | " + entry(inside->second.depot, *inside->second.file));+ merged.conflict = winner.file->path;+ return merged;+ }+ }+ for (const auto& [key, placement] : placements) {+ if (placement.file->is_directory() || placement.file->is_symlink()) continue;+ merged.install_bytes += placement.file->size;+ for (const auto& chunk : placement.file->chunks) merged.download_bytes += chunk.compressed_size;+ }+ return merged;+}+DepotInstaller::DepotInstaller(CmClient& cm, CdnClient& cdn) {hooks_.access = [&cm](const StoredSession& session, std::uint32_t app_id,const std::vector<DepotRequest>& depots, const std::atomic<bool>& cancel) {@@ -590,6 +638,54 @@ DepotInstaller::DepotInstaller(InstallerHooks hooks) : hooks_(std::move(hooks))if (!hooks_.free_bytes) hooks_.free_bytes = &DefaultFreeBytes;}+InstallMeasure DepotInstaller::Measure(const StoredSession& session, std::uint32_t app_id,+ const std::vector<DepotInfo>& depots, const std::string& branch,+ const std::atomic<bool>& cancel) {+ InstallMeasure measure;+ const auto fail = [&measure](InstallError error, std::string detail) {+ measure.error = error;+ measure.detail = std::move(detail);+ return measure;+ };+ if (app_id == 0 || depots.empty()) return fail(InstallError::kProtocol, "nothing to measure");+ std::vector<DepotRequest> requests;+ for (const auto& depot : depots) {+ if (!depot.manifest_gid || *depot.manifest_gid == 0) return fail(InstallError::kProtocol, "depot without manifest");+ requests.push_back({depot.id, *depot.manifest_gid, depot.depot_from_app.has_value(), depot.source_app, branch});+ }+ const auto access = hooks_.access(session, app_id, requests, cancel);+ if (!access.ok()) return fail(FromCm(access.error), access.detail.empty() ? "depot access failed" : access.detail);+ if (access.grants.empty()) return fail(InstallError::kRejected, "no depot granted");+ measure.partial = !access.skipped.empty();+ std::vector<ContentServer> servers;+ const auto listed = hooks_.servers(session.cell_id.value_or(0), cancel, &servers);+ if (listed == CdnError::kCanceled) return fail(InstallError::kCanceled, "canceled");+ if (listed != CdnError::kNone || servers.empty()) return fail(InstallError::kNetwork, "no content servers");+ // As Run: another server after a failure or a manifest that doesn't parse, with a pause between.+ std::vector<DepotManifest> manifests(access.grants.size());+ for (std::size_t index = 0; index < access.grants.size(); ++index) {+ InstallError fetched = InstallError::kNetwork;+ bool network_failed = false;+ const std::size_t attempts = std::min<std::size_t>(std::max<std::size_t>(servers.size() * 2, 4), 6);+ for (std::size_t attempt = 0; attempt < attempts; ++attempt) {+ fetched = hooks_.manifest(servers[attempt % servers.size()], access.grants[index], cancel, &manifests[index]);+ if (fetched != InstallError::kNetwork && fetched != InstallError::kCorrupt) break;+ network_failed = network_failed || fetched == InstallError::kNetwork;+ if (hooks_.pause && attempt + 1 < attempts) hooks_.pause(1000 << std::min<std::size_t>(attempt, 3), cancel);+ if (cancel.load()) return fail(InstallError::kCanceled, "canceled");+ }+ // As Run: broken only after a server failed to send it at all is a network problem, which a later try gets past.+ if (fetched == InstallError::kCorrupt && network_failed) fetched = InstallError::kNetwork;+ if (fetched != InstallError::kNone) return fail(fetched, "manifest unavailable");+ }+ const auto merged = MergeDepotManifests(manifests);+ if (!merged.conflict.empty()) return fail(InstallError::kCorrupt, "conflicting depots");+ measure.error = InstallError::kNone;+ measure.install_bytes = merged.install_bytes;+ measure.download_bytes = merged.download_bytes;+ return measure;+}+InstallOutcome DepotInstaller::Run(const StoredSession& session, const InstallTarget& target,const std::atomic<bool>& cancel,const std::function<void(const InstallProgress&)>& on_progress) {@@ -744,54 +840,16 @@ InstallOutcome DepotInstaller::Run(const StoredSession& session, const InstallTastats.manifests_ms = Milliseconds(Clock::now() - phase_start);phase_start = Clock::now();trace("merge");- // Later depots replace the same path from earlier ones; identical copies stay with the first. A file and a- // folder of the same name keep the file, as Steam does (Fallout 2's base depot has an empty master.dat- // folder that its language depot fills with the real master.dat), unless something lands inside the folder.- // A link is not a file here: it never displaces a folder. Every such clash is logged with both entries.- const auto entry = [&manifests](std::size_t depot, const ManifestFile& file) {- std::string text = "depot=" + std::to_string(manifests[depot].depot_id) + " ";- if (file.is_directory()) text += "folder";- else if (file.is_symlink()) text += "link";- else text += "file size=" + std::to_string(file.size);- return text + " path=" + file.path;- };+ // As MergeDepotManifests lays the files out; every clash of a file and a folder is logged with both entries.std::size_t merge_lines = 0;const auto log_merge = [&](const char* result, const std::string& detail) {if (hooks_.merge && ++merge_lines <= kMergeLogLimit) hooks_.merge(result, detail);};- std::map<std::string, Placement> placements;- struct FileOverFolder { std::string key, folder; };- std::vector<FileOverFolder> files_over_folders;- for (std::size_t depot = 0; depot < manifests.size(); ++depot) {- for (const auto& file : manifests[depot].files) {- const auto key = Lower(file.path);- const auto found = placements.find(key);- if (found != placements.end()) {- if (SameFile(*found->second.file, file)) continue;- if (found->second.file->is_directory() != file.is_directory()) {- const auto& folder = file.is_directory() ? file : *found->second.file;- const auto& other = file.is_directory() ? *found->second.file : file;- const auto detail = entry(found->second.depot, *found->second.file) + " | " + entry(depot, file);- if (other.is_symlink()) {- log_merge("link-over-folder", detail);- return fail(InstallError::kCorrupt, ("conflicting depots " + file.path).c_str());- }- log_merge("file-over-folder", detail);- files_over_folders.push_back({key, folder.path});- if (file.is_directory()) continue;- }- }- placements[key] = {depot, &file};- }- }- for (const auto& [key, folder] : files_over_folders) {+ auto merged = MergeDepotManifests(manifests, log_merge);+ if (!merged.conflict.empty()) return fail(InstallError::kCorrupt, ("conflicting depots " + merged.conflict).c_str());+ const auto& placements = merged.placements;+ for (const auto& [key, folder] : merged.files_over_folders) {const auto& winner = placements.at(key);- const auto inside = placements.lower_bound(key + "/");- if (inside != placements.end() && inside->first.compare(0, key.size() + 1, key + "/") == 0) {- log_merge("folder-not-empty", entry(winner.depot, *winner.file) + " | " +- entry(inside->second.depot, *inside->second.file));- return fail(InstallError::kCorrupt, ("conflicting depots " + winner.file->path).c_str());- }// An earlier install may have left the folder on disk; an empty one makes way for the file. An update// leaves the game alone until it puts its files in place.if (target.update) continue;diff --git a/src/steam/backend/depot_installer.h b/src/steam/backend/depot_installer.hindex 1c94afd..1823e33 100644--- a/src/steam/backend/depot_installer.h+++ b/src/steam/backend/depot_installer.h@@ -12,6 +12,7 @@#include <chrono>#include <cstdint>#include <functional>+#include <map>#include <optional>#include <string>#include <vector>@@ -118,6 +119,30 @@ struct InstallOutcome {std::vector<std::uint32_t> skipped_depots; // borrowed depots the account was refused, installed without};+// One file an install keeps from its depots: the depot (its index in the merged list) whose entry won.+struct ManifestPlacement {+ std::size_t depot = 0;+ const ManifestFile* file = nullptr;+};++// The depots' files as an install lays them out. Later depots replace the same path from earlier ones;+// identical copies stay with the first. A file and a folder of the same name keep the file, as Steam does+// (Fallout 2's base depot has an empty master.dat folder that its language depot fills with the real+// master.dat), unless something lands inside the folder. A link is not a file here: it never displaces a+// folder.+struct DepotMerge {+ std::map<std::string, ManifestPlacement> placements; // by lowercase path; points into the manifests+ // Each file that took the place of a folder of the same name: its key, then the folder's path.+ std::vector<std::pair<std::string, std::string>> files_over_folders;+ std::string conflict; // a path the depots can't share (a link over a folder, a folder with files over a file)+ std::uint64_t install_bytes = 0; // the merged files' bytes: what the install holds+ std::uint64_t download_bytes = 0; // their chunks' compressed bytes: what a new install downloads+};+// `log` hears how each clash of a file and a folder settled (the result, then both entries: depot, kind,+// size and path).+DepotMerge MergeDepotManifests(const std::vector<DepotManifest>& manifests,+ const std::function<void(const char* result, const std::string& detail)>& log = {});+// Seams for tests; the defaults use CmClient, CdnClient and the real filesystem.struct InstallerHooks {struct ChunkFetcher {@@ -160,6 +185,15 @@ struct InstallerHooks {std::function<std::chrono::steady_clock::time_point()> now;};+// What installing some depots holds once merged (DepotMerge's sizes), from their manifests.+struct InstallMeasure {+ InstallError error = InstallError::kProtocol;+ std::string detail; // short and identifier-free+ std::uint64_t install_bytes = 0;+ std::uint64_t download_bytes = 0;+ bool partial = false; // a depot the account was refused is left out+};+// Downloads and assembles an app's depots into install_root. Safe to run again after any// interruption: finished chunks recorded in the journal are not fetched twice. Runs// synchronously on the caller's worker thread.@@ -186,6 +220,12 @@ class DepotInstaller {// written before the first staged chunk.static constexpr const char* kUpdateDir = installs::kUpdateDirName;+ // The size an install of `depots` (Steam's listing order, from `branch`; empty: public) would have, from+ // the manifests Run would fetch: depot access, the server list and each manifest, merged. Nothing is+ // written. A depot the account is refused is left out, as an install leaves it.+ InstallMeasure Measure(const StoredSession& session, std::uint32_t app_id, const std::vector<DepotInfo>& depots,+ const std::string& branch, const std::atomic<bool>& cancel);+// Finishes an update whose files are all staged (phase replacing or deleting): renames them over the// game's, deletes the files the new version dropped and the folders that leaves empty (never anything// else), writes the new tree, file list and record and removes the work folder. Safe to run againdiff --git a/src/stores/details_source.h b/src/stores/details_source.hindex 18042c3..6c35e70 100644--- a/src/stores/details_source.h+++ b/src/stores/details_source.h@@ -22,6 +22,9 @@ class DetailsSource {virtual void Refresh(const GameId& id, std::uint64_t generation) { Request(id, generation); }virtual void Invalidate(std::uint64_t generation) = 0;virtual std::vector<GameDetails> Pump(std::size_t maximum) = 0;+ // The exact size of one install choice (steam::MeasuredSize); a store without it answers nothing.+ virtual void Measure(const steam::SizeRequest&, std::uint64_t) {}+ virtual std::vector<steam::MeasuredSize> PumpMeasured() { return {}; }virtual void Shutdown() = 0;};diff --git a/src/stores/steam.h b/src/stores/steam.hindex d4c747e..0ccc74e 100644--- a/src/stores/steam.h+++ b/src/stores/steam.h@@ -82,6 +82,10 @@ class SteamDetailsSource final : public DetailsSource {}void Invalidate(std::uint64_t generation) override { service_.Invalidate(generation); }std::vector<GameDetails> Pump(std::size_t maximum) override { return service_.Pump(maximum); }+ void Measure(const steam::SizeRequest& request, std::uint64_t generation) override {+ if (request.id.steam_app()) service_.Measure(request, generation);+ }+ std::vector<steam::MeasuredSize> PumpMeasured() override { return service_.PumpMeasured(); }void Shutdown() override { service_.Shutdown(); }private:diff --git a/tests/app_details_test.cpp b/tests/app_details_test.cppindex bb9edf9..5b2f0b6 100644--- a/tests/app_details_test.cpp+++ b/tests/app_details_test.cpp@@ -28,7 +28,24 @@ class Source final : public AppDetailsSource {public:std::mutex mutex;std::vector<std::vector<GameId>> calls;+ std::vector<SizeRequest> measures;std::atomic<bool> fail{false};+ std::atomic<bool> hold{false}; // a size check that waits (on manifests) until released or canceled+ MeasuredSize Measure(const StoredSession&, const SizeRequest& request, const std::atomic<bool>& cancel) override {+ {+ std::lock_guard<std::mutex> lock(mutex);+ measures.push_back(request);+ }+ while (hold && !cancel) std::this_thread::sleep_for(1ms);+ MeasuredSize result{request};+ if (fail) {+ result.failed = true;+ return result;+ }+ result.measured = true;+ result.install_bytes = *request.id.steam_app() * 7ull;+ return result;+ }std::vector<AppDetails> Fetch(const StoredSession&, const std::vector<GameId>& ids,const std::atomic<bool>&) override {{@@ -128,6 +145,106 @@ void FailuresAreRetryable() {assert(recovered[0].available && recovered[0].download_bytes == 90u);}+std::vector<MeasuredSize> AwaitMeasured(AppDetailsService& service, std::size_t count) {+ std::vector<MeasuredSize> all;+ const auto deadline = std::chrono::steady_clock::now() + 2s;+ while (all.size() < count && std::chrono::steady_clock::now() < deadline) {+ for (auto& size : service.PumpMeasured()) all.push_back(std::move(size));+ std::this_thread::sleep_for(1ms);+ }+ assert(all.size() >= count);+ return all;+}++// An install choice's exact size is asked once while it is queued or checked; answered, it can be asked again (the+// sheet decides when), and so can one that failed.+void SizeChecksAreAnsweredOncePerChoice() {+ Sessions sessions;+ Source source;+ AppDetailsService service(source, sessions);+ service.Measure({SteamId(400), "public", "english"}, 1);+ service.Measure({SteamId(400), "public", "english"}, 1);+ const auto first = AwaitMeasured(service, 1);+ assert(first.size() == 1 && first[0].measured && first[0].install_bytes == 2800u && first[0].account_generation == 1);+ assert(first[0].request == (SizeRequest{SteamId(400), "public", "english"}));+ {+ std::lock_guard<std::mutex> lock(source.mutex);+ assert(source.measures.size() == 1);+ }+ service.Measure({SteamId(400), "beta", "english"}, 1);+ const auto second = AwaitMeasured(service, 1);+ assert(second.size() == 1 && second[0].request.branch == "beta");+ service.Measure({SteamId(400), "public", "english"}, 1);+ assert(AwaitMeasured(service, 1)[0].request.branch == "public");+ source.fail = true;+ service.Measure({SteamId(401), "public", "english"}, 1);+ assert(AwaitMeasured(service, 1)[0].failed);+ source.fail = false;+ service.Measure({SteamId(401), "public", "english"}, 1);+ const auto retried = AwaitMeasured(service, 1);+ assert(retried[0].measured && retried[0].install_bytes == 2807u);+ // Another account asks again.+ service.Measure({SteamId(400), "public", "english"}, 2);+ assert(AwaitMeasured(service, 1)[0].account_generation == 2);+}++// A size check waiting on manifests doesn't hold up a game's details: they have their own worker.+void SlowSizeChecksDontBlockDetails() {+ Sessions sessions;+ Source source;+ source.hold = true;+ AppDetailsService service(source, sessions);+ service.Measure({SteamId(400), "public", "english"}, 1);+ const auto deadline = std::chrono::steady_clock::now() + 2s;+ for (;;) {+ {+ std::lock_guard<std::mutex> lock(source.mutex);+ if (!source.measures.empty()) break;+ }+ assert(std::chrono::steady_clock::now() < deadline);+ std::this_thread::sleep_for(1ms);+ }+ service.Request(SteamId(2005010), 1);+ const auto details = Await(service, 1);+ assert(details[0].id == SteamId(2005010) && details[0].available);+ assert(service.PumpMeasured().empty());+ source.hold = false;+ assert(AwaitMeasured(service, 1)[0].measured);+}++// Only a selection whose merged size can be far from Steam's sum is worth its manifests: two depots or more of+// kMeasureDepotBytes. Portal's 401 and 409 hold mostly the same files.+void LargeOverlappingDepotsAreMeasured() {+ const std::string portal = R"("appinfo"+{+ "appid" "400"+ "common" { "name" "Portal" "type" "Game" }+ "config" { "launch" { "0" { "executable" "portal.exe" "config" { "oslist" "windows" } } } }+ "depots"+ {+ "401" { "config" { "oslist" "windows" } "manifests" { "public" { "gid" "1" "size" "4184288203" "download" "2520000000" } } }+ "402" { "config" { "oslist" "windows" "language" "russian" } "manifests" { "public" { "gid" "2" "size" "160000000" "download" "90000000" } } }+ "409" { "manifests" { "public" { "gid" "9" "size" "3850000000" "download" "2360000000" } } }+ }+})";+ const auto english = DepotsToMeasure(400, portal, std::nullopt, "public", "english");+ assert(english.size() == 2 && english[0].id == 401 && english[1].id == 409 && english[1].manifest_gid == 9u);+ const auto russian = DepotsToMeasure(400, portal, std::nullopt, "", "russian");+ assert(russian.size() == 3);+ const std::string small = R"("appinfo"+{+ "appid" "440"+ "common" { "name" "Synthetic" "type" "Game" }+ "depots"+ {+ "441" { "config" { "oslist" "windows" } "manifests" { "public" { "gid" "1" "size" "4000000000" } } }+ "442" { "config" { "oslist" "windows" "language" "german" } "manifests" { "public" { "gid" "2" "size" "30000000" } } }+ }+})";+ assert(DepotsToMeasure(440, small, std::nullopt, "public", "german").empty());+ assert(DepotsToMeasure(440, "not vdf {", std::nullopt, "public", "english").empty());+}+void AppInfoBecomesWindowsEstimate() {const std::string buffer = R"("appinfo"{@@ -429,6 +546,9 @@ void InstalledGamesCompareWithTheirRecord() {}int main() {+ SizeChecksAreAnsweredOncePerChoice();+ SlowSizeChecksDontBlockDetails();+ LargeOverlappingDepotsAreMeasured();SizesCountOnlyLicensedDepots();RequestsAreDeduplicatedPerGeneration();NewGenerationDropsResultsAndRefetches();diff --git a/tests/depot_installer_test.cpp b/tests/depot_installer_test.cppindex 7ba17f7..400f31c 100644--- a/tests/depot_installer_test.cpp+++ b/tests/depot_installer_test.cpp@@ -1057,6 +1057,62 @@ void InstallsMergedDepotsAndWritesRecord(const std::filesystem::path& root) {CHECK(!std::filesystem::exists(root / (std::string(DepotInstaller::kFileTree) + ".tmp")));}+// Measure fetches the manifests an install would and merges them as it does, without writing anything: depots+// holding the same files count them once (Portal's 401 and 409 add up to twice the game).+void MeasureMatchesTheMergedInstall(const std::filesystem::path& root) {+ auto world = MakeWorld();+ DepotInstaller installer(world.Hooks());+ std::atomic<bool> cancel{false};+ const auto target = Target(root);+ const auto measured = installer.Measure({}, target.app_id, target.depots, {}, cancel);+ CHECK(measured.error == InstallError::kNone);+ CHECK(!std::filesystem::exists(root) && world.chunk_calls == 0);+ std::uint64_t summed = 0;+ for (const auto& [id, manifest] : world.manifests)+ for (const auto& file : manifest.files) summed += file.size;+ CHECK(measured.install_bytes == 26 + 13 && summed == 26 + 9 + 13); // the base's level.dat is replaced+ InstallProgress last;+ CHECK(installer.Run({}, target, cancel, [&](const InstallProgress& progress) { last = progress; }).error == InstallError::kNone);+ CHECK(measured.install_bytes == last.install_total && measured.download_bytes == last.download_total);+ // A depot the account is refused is left out, as an install leaves it; none granted is an error.+ auto hooks = world.Hooks();+ hooks.access = [](const StoredSession&, std::uint32_t, const std::vector<DepotRequest>& depots, const std::atomic<bool>&) {+ DepotAccessResult result;+ result.error = CmError::kNone;+ result.grants.push_back({depots[0].depot_id, depots[0].manifest_gid, kKey, 77});+ result.skipped.push_back(depots[1].depot_id);+ return result;+ };+ DepotInstaller refused(hooks);+ const auto base_only = refused.Measure({}, target.app_id, target.depots, {}, cancel);+ CHECK(base_only.error == InstallError::kNone && base_only.install_bytes == 26 + 9 && base_only.partial);+ CHECK(!measured.partial);+ hooks.access = [](const StoredSession&, std::uint32_t, const std::vector<DepotRequest>&, const std::atomic<bool>&) {+ DepotAccessResult result;+ result.error = CmError::kTimeout;+ return result;+ };+ DepotInstaller offline(hooks);+ CHECK(offline.Measure({}, target.app_id, target.depots, {}, cancel).error == InstallError::kNetwork);+ // A manifest that a server failed to send, then came back broken, is a network problem (a later try may work).+ hooks = world.Hooks();+ int manifest_calls = 0;+ hooks.manifest = [&manifest_calls](const ContentServer&, const DepotGrant&, const std::atomic<bool>&, DepotManifest*) {+ return ++manifest_calls == 1 ? InstallError::kNetwork : InstallError::kCorrupt;+ };+ DepotInstaller flaky(hooks);+ CHECK(flaky.Measure({}, target.app_id, target.depots, {}, cancel).error == InstallError::kNetwork);+ // A link over a folder can't be merged, for a measure as for an install.+ ManifestFile link;+ link.path = "data";+ link.link_target = "game.exe";+ world.manifests[442].files.push_back(link);+ DepotInstaller clash(world.Hooks());+ CHECK(clash.Measure({}, target.app_id, target.depots, {}, cancel).error == InstallError::kCorrupt);+ const auto merged = MergeDepotManifests({world.manifests[441], world.manifests[442]});+ CHECK(merged.conflict == "data");+}+// A file and an empty folder of the same name in different depots keep the file, in either order; a folder// with something inside it still conflicts.void FileReplacesEmptyFolderFromOtherDepot(const std::filesystem::path& base) {@@ -2616,6 +2672,7 @@ int main(int argc, char** argv) {StatsAndRemovalLogs(base / "removal");RunStats(base / "stats");InstallsMergedDepotsAndWritesRecord(base / "install");+ MeasureMatchesTheMergedInstall(base / "measure");LocalRecordNamesItsGame(base / "install-local");FileReplacesEmptyFolderFromOtherDepot(base / "file-over-folder");ResumesWithoutRefetchingFinishedChunks(base / "resume");diff --git a/tests/homebrew_library_model_test.cpp b/tests/homebrew_library_model_test.cppindex c6e38f9..4e955e8 100644--- a/tests/homebrew_library_model_test.cpp+++ b/tests/homebrew_library_model_test.cpp@@ -457,6 +457,8 @@ void live_install_confirmation_checks_space_and_unknowns() {activate(model, "details-install");check(model.view().surface == "install" && !find_row(model, "install-confirm")->enabled,"a game that does not fit opens the install sheet without a usable Install");+ // Its exact size has nothing to change (the controller answers so for a game without overlapping parts).+ for (const auto& size_check : model.TakeSizeChecks()) model.UpdateMeasuredSize(size_check, false, false, {});check(model.view().subtitle.find("Not enough space") != std::string::npos, "the sheet says why");check(model.TakeInstallRequests().empty(), "nothing is queued while it does not fit");@@ -699,6 +701,65 @@ void live_install_language_size() {check(!find_row(model, "install-confirm")->enabled, "and that size is the one checked");}+// Steam's figures add up each part of a game; the sheet asks for the exact size of the chosen version and language,+// and a game that only fits by it can install once it is known.+void live_install_exact_size() {+ using playden::homebrew::GameVersion;+ using playden::homebrew::SizeCheckRequest;+ auto model = make_live();+ model.UpdateAccount({AuthState::SignedIn, "123", "Synthetic", {}, {}, true});+ check(model.UpdateLibrary({{game(SteamId(400), "Portal")}, false, {}, false, false, {}}), "library loaded");+ model.UpdateVolumes({{"internal", "Console storage", true, 2'000'000'000, 1'000'000'000}}, "internal");+ GameVersion version{"public", "", 1760000000, 9000, 900'000'000, 1'500'000'000, {}};+ version.languages = {{"english", 900'000'000, 1'500'000'000}, {"russian", 1'000'000'000, 1'600'000'000}};+ model.UpdateAppDetails(SteamId(400), 900'000'000, 1'500'000'000, true, {"english", "russian"}, 0, {version});+ check(model.action("details:400"), "details open");+ activate(model, "details-install");+ auto checks = model.TakeSizeChecks();+ check(checks.size() == 1 && checks[0].id == SteamId(400) && checks[0].branch == "public" && checks[0].language == "english",+ "the sheet asks for its choice's exact size");+ check(model.view().subtitle.find("Needs about 1.5 GB") != std::string::npos &&+ model.view().subtitle.find("Checking its exact size") != std::string::npos &&+ !find_row(model, "install-confirm")->enabled && find_row(model, "install-confirm")->subtitle == "Checking size",+ "Steam's sum doesn't fit: the sheet waits for the exact size");+ check(model.TakeSizeChecks().empty(), "asked once");+ model.UpdateMeasuredSize(checks[0], true, false, {600'000'000, 400'000'000});+ check(model.view().subtitle.find("Needs 600 MB") != std::string::npos && find_row(model, "install-confirm")->enabled,+ "the exact size fits");+ // Another language is another check; one that fails keeps Steam's sum and isn't asked again while the sheet is open.+ activate(model, "install-language");+ activate(model, "install-language-russian");+ checks = model.TakeSizeChecks();+ check(checks.size() == 1 && checks[0].language == "russian", "Russian is checked too");+ model.UpdateMeasuredSize(checks[0], false, true, {});+ check(model.view().subtitle.find("Needs about 1.6 GB") != std::string::npos &&+ model.view().subtitle.find("Not enough space") != std::string::npos && !find_row(model, "install-confirm")->enabled,+ "a failed check keeps Steam's sum");+ check(model.TakeSizeChecks().empty(), "not asked again while the sheet is open");+ model.action("circle");+ activate(model, "details-install");+ activate(model, "install-language");+ activate(model, "install-language-russian");+ checks = model.TakeSizeChecks();+ check(checks.size() == 1 && checks[0].language == "russian", "asked again when the sheet opens next");+ // The service dropped it (the language changed in Settings): asked again.+ model.ResetSizeChecks();+ checks = model.TakeSizeChecks();+ check(checks.size() == 1 && checks[0].language == "russian", "a dropped check is asked again");+ // Nothing to measure: Steam's figures stand, and the sheet says so as before.+ model.UpdateMeasuredSize(checks[0], false, false, {});+ check(model.view().subtitle.find("Not enough space") != std::string::npos, "a choice with nothing to check keeps Steam's sum");+ check(model.TakeSizeChecks().empty(), "answered: not asked again");+ // New details (a newer build) forget the game's exact sizes: the sheet checks again.+ model.UpdateAppDetails(SteamId(400), 900'000'000, 1'500'000'000, true, {"english", "russian"}, 0, {version});+ checks = model.TakeSizeChecks();+ check(checks.size() == 1 && checks[0].language == "russian", "new details are checked again");+ activate(model, "install-language");+ activate(model, "install-language-english");+ model.TakeSizeChecks();+ check(model.view().subtitle.find("Needs about 1.5 GB") != std::string::npos, "the old exact size is gone");+}+// A branch named in lower case shows as the picker shows it.void live_install_version_title() {using playden::homebrew::GameVersion;@@ -3790,6 +3851,7 @@ int main() {live_install_language_row();live_install_version_row();live_install_language_size();+ live_install_exact_size();live_install_version_title();live_install_dlc_row();leftover_titles_require_confirmation_and_follow_the_queue();