Play your Steam library's Windows games on a jailbroken PS5.
C++
Download depot manifests larger than 16 MiB
Leonardo Lobatobb89f231126a
diff --git a/src/steam/backend/cdn_client.cpp b/src/steam/backend/cdn_client.cppindex 2f97198..4d9e34a 100644--- a/src/steam/backend/cdn_client.cpp+++ b/src/steam/backend/cdn_client.cpp@@ -74,10 +74,12 @@ CdnClient::CdnClient(Perform perform) : perform_(std::move(perform)) {}CdnClient::~CdnClient() = default;CdnError CdnClient::Get(const std::string& url, std::size_t max_bytes, std::uint32_t timeout_ms,- const std::atomic<bool>& cancel, std::vector<std::uint8_t>* out, bool plain_http) {+ const std::atomic<bool>& cancel, std::vector<std::uint8_t>* out, bool plain_http,+ bool depot_manifest) {HttpRequest request;request.url = url;request.allow_plain_http = plain_http;+ request.depot_manifest = depot_manifest;request.headers = {{"User-Agent", "Valve/Steam HTTP Client 1.0"}};request.timeout_ms = timeout_ms;request.max_response_bytes = max_bytes;@@ -166,7 +168,7 @@ CdnError CdnClient::Manifest(const ContentServer& server, std::uint32_t depot_idif (!ValidCdnHost(server.vhost) || !max_bytes) return CdnError::kProtocol;return Get(Scheme(server) + server.vhost + "/depot/" + std::to_string(depot_id) + "/manifest/" +std::to_string(gid) + "/5/" + std::to_string(request_code),- std::min(max_bytes, kMaxManifestDownloadBytes), 60'000, cancel, out, !server.https);+ std::min(max_bytes, kMaxManifestDownloadBytes), kMaxHttpTimeoutMs, cancel, out, !server.https, true);}CdnError CdnClient::Chunk(const ContentServer& server, std::uint32_t depot_id, const std::string& sha_hex,diff --git a/src/steam/backend/cdn_client.h b/src/steam/backend/cdn_client.hindex 5308541..a87372d 100644--- a/src/steam/backend/cdn_client.h+++ b/src/steam/backend/cdn_client.h@@ -47,12 +47,13 @@ class CdnClient {CdnError Servers(std::uint32_t cell_id, const std::atomic<bool>& cancel, std::vector<ContentServer>* out);CdnError Manifest(const ContentServer& server, std::uint32_t depot_id, std::uint64_t gid,std::uint64_t request_code, const std::atomic<bool>& cancel, std::vector<std::uint8_t>* out,- std::size_t max_bytes = kMaxHttpResponseBytes);+ std::size_t max_bytes = kMaxManifestDownloadBytes);CdnError Chunk(const ContentServer& server, std::uint32_t depot_id, const std::string& sha_hex,std::size_t max_bytes, const std::atomic<bool>& cancel, std::vector<std::uint8_t>* out);- // Bounded by the HTTP client's response cap.- static constexpr std::size_t kMaxManifestDownloadBytes = kMaxHttpResponseBytes;+ // A manifest has its own cap, above the HTTP client's general one: a large game's manifest is more than+ // 16 MiB. Its zip stays within what UnzipFirstEntry accepts (depot_manifest.h).+ static constexpr std::size_t kMaxManifestDownloadBytes = kMaxHttpManifestResponseBytes;static constexpr std::size_t kMaxServerListDownloadBytes = 512 * 1024;// The last failure as "http=<status>" or "transport=<category>", for sanitized logs.const CdnStats& stats() const { return stats_; }@@ -61,7 +62,8 @@ class CdnClient {private:CdnError Get(const std::string& url, std::size_t max_bytes, std::uint32_t timeout_ms,- const std::atomic<bool>& cancel, std::vector<std::uint8_t>* out, bool plain_http = false);+ const std::atomic<bool>& cancel, std::vector<std::uint8_t>* out, bool plain_http = false,+ bool depot_manifest = false);std::string ca_bundle_;Perform perform_;std::unique_ptr<HttpClient> http_;diff --git a/src/steam/backend/depot_installer.cpp b/src/steam/backend/depot_installer.cppindex 330aa1b..421851e 100644--- a/src/steam/backend/depot_installer.cpp+++ b/src/steam/backend/depot_installer.cpp@@ -600,6 +600,8 @@ DepotInstaller::DepotInstaller(CmClient& cm, CdnClient& cdn) {manifest_problem->clear(); // a manifest problem from an earlier run must not hide why this list failedreturn cdn.Servers(cell, cancel, out);};+ static_assert(CdnClient::kMaxManifestDownloadBytes <= kMaxManifestBytes && kMaxManifestBytes <= kMaxInflateRawBytes,+ "a manifest the CDN client accepts must unzip");hooks_.manifest = [&cdn, manifest_problem](const ContentServer& server, const DepotGrant& grant,const std::atomic<bool>& cancel, DepotManifest* out) {manifest_problem->clear();@@ -608,13 +610,20 @@ DepotInstaller::DepotInstaller(CmClient& cm, CdnClient& cdn) {cancel, &zip);if (fetched == CdnError::kCanceled) return InstallError::kCanceled;if (fetched != CdnError::kNone) return InstallError::kNetwork;+ const auto zip_bytes = zip.size();const auto corrupt = [&](std::string what) {- *manifest_problem = "depot=" + std::to_string(grant.depot_id) + " bytes=" + std::to_string(zip.size()) + " " + what;+ *manifest_problem = "depot=" + std::to_string(grant.depot_id) + " bytes=" + std::to_string(zip_bytes) + " " + what;return InstallError::kCorrupt;};+ // A large manifest is tens of MiB as a zip and inflated: each is freed once used.std::vector<std::uint8_t> raw;- if (!UnzipFirstEntry(zip, &InflateRaw, &raw)) return corrupt("unzip");- if (!ParseDepotManifest(raw, out)) return corrupt("parse raw=" + std::to_string(raw.size()));+ const bool unzipped = UnzipFirstEntry(zip, &InflateRaw, &raw, kMaxManifestBytes);+ std::vector<std::uint8_t>().swap(zip);+ if (!unzipped) return corrupt("unzip");+ const bool parsed = ParseDepotManifest(raw, out);+ const auto raw_bytes = raw.size();+ std::vector<std::uint8_t>().swap(raw);+ if (!parsed) return corrupt("parse raw=" + std::to_string(raw_bytes));if (out->filenames_encrypted && !DecryptManifestFilenames(out, grant.key, &SymmetricDecrypt))return corrupt("filenames");if (out->depot_id != grant.depot_id || out->gid != grant.manifest_gid)diff --git a/src/steam/backend/depot_manifest.cpp b/src/steam/backend/depot_manifest.cppindex d4cdd02..6895e8e 100644--- a/src/steam/backend/depot_manifest.cpp+++ b/src/steam/backend/depot_manifest.cpp@@ -419,8 +419,8 @@ bool ValidateDepotManifest(const DepotManifest& manifest, std::string* reason) {}bool UnzipFirstEntry(const std::vector<std::uint8_t>& zip, InflateRawFn inflate,- std::vector<std::uint8_t>* out) {- if (!out || zip.size() < 22 || zip.size() > kMaxManifestBytes + (1u << 20)) return false;+ std::vector<std::uint8_t>* out, std::size_t max_bytes) {+ if (!out || zip.size() < 22 || zip.size() > max_bytes + (1u << 20)) return false;const std::uint8_t* d = zip.data();const std::size_t n = zip.size();// End of central directory (PK\x05\x06), scanning back over a trailing comment.@@ -443,7 +443,7 @@ bool UnzipFirstEntry(const std::vector<std::uint8_t>& zip, InflateRawFn inflate,const std::size_t local = ReadLE32(d + cd + 42);if (flags & 1) return false; // encrypted entryif (csize == 0xFFFFFFFFu || usize == 0xFFFFFFFFu || local == 0xFFFFFFFFu) return false; // zip64- if (usize > kMaxManifestBytes) return false;+ if (usize > max_bytes) return false;if (local > n || n - local < 30 || ReadLE32(d + local) != 0x04034B50u) return false;const std::size_t start = local + 30 + ReadLE16(d + local + 26) + ReadLE16(d + local + 28);if (start > n || csize > n - start) return false;diff --git a/src/steam/backend/depot_manifest.h b/src/steam/backend/depot_manifest.hindex 7f5f070..5747b9b 100644--- a/src/steam/backend/depot_manifest.h+++ b/src/steam/backend/depot_manifest.h@@ -12,7 +12,11 @@// reference CDN.swift / ResumableDepotDownload.swift.namespace playden::steam {-inline constexpr std::size_t kMaxManifestBytes = 64 * 1024 * 1024;+// An inflated depot manifest. Manifests compress little (chunk hashes, encrypted file names), so a zip at the+// download cap (CdnClient::kMaxManifestDownloadBytes, 64 MiB) can inflate to about twice its size.+inline constexpr std::size_t kMaxManifestBytes = 128 * 1024 * 1024;+// What UnzipFirstEntry unpacks unless the caller says otherwise (a Cloud save).+inline constexpr std::size_t kMaxUnzipBytes = 64 * 1024 * 1024;inline constexpr std::size_t kMaxManifestFiles = 500'000;inline constexpr std::size_t kMaxManifestChunks = 4'000'000;@@ -76,8 +80,8 @@ bool ValidateDepotManifest(const DepotManifest& manifest, std::string* reason);using InflateRawFn = bool (*)(const std::uint8_t* data, std::size_t size, std::size_t expected,std::vector<std::uint8_t>* out);// Extracts the first zip entry (method 0 stored, or 8 deflate via `inflate`). Result is capped at-// kMaxManifestBytes.+// max_bytes (kMaxManifestBytes for a depot manifest).bool UnzipFirstEntry(const std::vector<std::uint8_t>& zip, InflateRawFn inflate,- std::vector<std::uint8_t>* out);+ std::vector<std::uint8_t>* out, std::size_t max_bytes = kMaxUnzipBytes);} // namespace playden::steamdiff --git a/src/steam/backend/download_server_source.cpp b/src/steam/backend/download_server_source.cppindex ab8bcf3..3b6c4db 100644--- a/src/steam/backend/download_server_source.cpp+++ b/src/steam/backend/download_server_source.cpp@@ -108,7 +108,8 @@ CdnDownloadServerSource::PayloadResult CdnDownloadServerSource::PrepareApp(std::vector<std::uint8_t> zip, raw;DepotManifest manifest;if (!metadata_budget_) return PayloadResult::kFailed;- const auto cap = static_cast<std::size_t>(std::min<std::uint64_t>(metadata_budget_, CdnClient::kMaxManifestDownloadBytes));+ // The speed test keeps the general response cap: it needs a few chunk hashes, not a large game's manifest.+ const auto cap = static_cast<std::size_t>(std::min<std::uint64_t>(metadata_budget_, kMaxHttpResponseBytes));const auto before = cdn_.stats().bytes;const auto outcome = cdn_.Manifest(servers_[attempt], grant.depot_id, grant.manifest_gid,grant.manifest_request_code, cancel, &zip, cap);@@ -116,7 +117,7 @@ CdnDownloadServerSource::PayloadResult CdnDownloadServerSource::PrepareApp(// fallback depots draw from this ledger, and the next request is capped before it starts.metadata_budget_ -= std::min(metadata_budget_, cdn_.stats().bytes - before);if (outcome != CdnError::kNone ||- !UnzipFirstEntry(zip, &InflateRaw, &raw) || !ParseDepotManifest(raw, &manifest) ||+ !UnzipFirstEntry(zip, &InflateRaw, &raw, kMaxManifestBytes) || !ParseDepotManifest(raw, &manifest) ||manifest.depot_id != grant.depot_id || manifest.gid != grant.manifest_gid) continue;// Names and content remain encrypted. Only bounded chunk hashes and compressed sizes are used.for (const auto& file : manifest.files) {diff --git a/src/steam/backend/gzip.cpp b/src/steam/backend/gzip.cppindex 43bb8d1..e4f0b6b 100644--- a/src/steam/backend/gzip.cpp+++ b/src/steam/backend/gzip.cpp@@ -113,7 +113,7 @@ bool Gunzip(const std::uint8_t* data, std::size_t size, std::size_t expected_sizbool InflateRaw(const std::uint8_t* data, std::size_t size, std::size_t expected_size,std::vector<std::uint8_t>* out) {- if (!data || !out || expected_size == 0 || expected_size > kMaxGunzipBytes * 16 ||+ if (!data || !out || expected_size == 0 || expected_size > kMaxInflateRawBytes ||size > static_cast<std::size_t>(INT_MAX) || expected_size > static_cast<std::size_t>(INT_MAX))return false;std::vector<std::uint8_t> result(expected_size);diff --git a/src/steam/backend/gzip.h b/src/steam/backend/gzip.hindex e114f96..413a182 100644--- a/src/steam/backend/gzip.h+++ b/src/steam/backend/gzip.h@@ -7,6 +7,8 @@namespace playden::steam {inline constexpr std::size_t kMaxGunzipBytes = 4 * 1024 * 1024;+// InflateRaw's own bound: an inflated depot manifest (kMaxManifestBytes); callers cap below it.+inline constexpr std::size_t kMaxInflateRawBytes = 128 * 1024 * 1024;// Decodes one RFC 1952 gzip member whose uncompressed size is declared in advance// (CMsgMulti.size_unzipped). Returns false on any malformed, truncated or oversized input.diff --git a/src/steam/backend/http_client.cpp b/src/steam/backend/http_client.cppindex 4f717a2..ca57df8 100644--- a/src/steam/backend/http_client.cpp+++ b/src/steam/backend/http_client.cpp@@ -141,7 +141,7 @@ bool ValidUrl(const std::string& url, bool allow_plain_http) {bool ValidRequest(const HttpRequest& request) {if (!ValidUrl(request.url, request.allow_plain_http) || request.timeout_ms == 0 ||request.timeout_ms > kMaxHttpTimeoutMs || request.max_response_bytes == 0 ||- request.max_response_bytes > kMaxHttpResponseBytes ||+ request.max_response_bytes > (request.depot_manifest ? kMaxHttpManifestResponseBytes : kMaxHttpResponseBytes) ||request.body.size() > kMaxHttpResponseBytes ||request.headers.size() > kMaxHttpResponseHeaderCount)return false;diff --git a/src/steam/backend/http_client.h b/src/steam/backend/http_client.hindex f67a4fc..05d939a 100644--- a/src/steam/backend/http_client.h+++ b/src/steam/backend/http_client.h@@ -34,6 +34,8 @@ const char* HttpErrorName(HttpError error);inline constexpr std::uint32_t kMaxHttpTimeoutMs = 120'000;inline constexpr std::size_t kMaxHttpResponseBytes = 16 * 1024 * 1024;+// Steam depot manifests only (HttpRequest::depot_manifest): a large game's manifest is more than 16 MiB.+inline constexpr std::size_t kMaxHttpManifestResponseBytes = 64 * 1024 * 1024;inline constexpr std::size_t kMaxHttpResponseHeaderBytes = 16 * 1024;inline constexpr std::size_t kMaxHttpResponseHeaderCount = 64;@@ -46,6 +48,8 @@ struct HttpRequest {std::size_t max_response_bytes = 1024 * 1024;// Also accept an http:// URL. Only for content checked by other means, such as Steam CDN chunks.bool allow_plain_http = false;+ // A Steam depot manifest: max_response_bytes may be up to kMaxHttpManifestResponseBytes, not kMaxHttpResponseBytes.+ bool depot_manifest = false;};struct HttpTiming {diff --git a/tests/cdn_client_test.cpp b/tests/cdn_client_test.cppindex 9d58721..4c9bdcf 100644--- a/tests/cdn_client_test.cpp+++ b/tests/cdn_client_test.cpp@@ -179,6 +179,44 @@ void PlainHttpServerUrls() {assert(!requests.back().allow_plain_http);}+// A depot manifest above the general 16 MiB response cap still downloads (Solo Leveling: ARISE OVERDRIVE, #451); one+// above the manifest cap does not, and chunks keep the cap their caller gives.+void LargeManifestsDownload() {+ std::vector<HttpRequest> requests;+ std::size_t size = 0;+ CdnClient client([&](const HttpRequest& request, const std::atomic<bool>&) {+ requests.push_back(request);+ HttpResponse response;+ if (size > request.max_response_bytes) { // as HttpClient's body limit does+ response.error = HttpError::kResponseTooLarge;+ return response;+ }+ response.status_code = 200;+ response.body.assign(size, 0x5a);+ return response;+ });+ std::atomic<bool> cancel{false};+ const ContentServer server{"h.example.invalid", "v.example.invalid", 1};+ std::vector<std::uint8_t> out;+ size = 16 * 1024 * 1024 + 1;+ assert(client.Manifest(server, 2373991, 11, 99, cancel, &out) == CdnError::kNone && out.size() == size);+ size = 64 * 1024 * 1024;+ assert(client.Manifest(server, 2373991, 11, 99, cancel, &out) == CdnError::kNone && out.size() == size);+ assert(requests.back().max_response_bytes == 64 * 1024 * 1024);+ out.clear();+ out.shrink_to_fit();+ size = 64 * 1024 * 1024 + 1;+ assert(client.Manifest(server, 2373991, 11, 99, cancel, &out) == CdnError::kNetwork && out.empty());+ assert(client.last_failure() == "transport=response_too_large");+ // A caller's smaller budget still holds.+ size = 1024;+ assert(client.Manifest(server, 2373991, 11, 99, cancel, &out, 1023) == CdnError::kNetwork);+ assert(requests.back().max_response_bytes == 1023);+ const std::string sha(40, 'c');+ assert(client.Chunk(server, 2373991, sha, 4096, cancel, &out) == CdnError::kNone);+ assert(requests.back().max_response_bytes == 4096);+}+} // namespaceint main() {@@ -191,4 +229,5 @@ int main() {EmptyServerListSaysWhy();ManifestAndChunkUrlsAndErrors();PlainHttpServerUrls();+ LargeManifestsDownload();}diff --git a/tests/http_client_test.cpp b/tests/http_client_test.cppindex 365eb6a..5c25d26 100644--- a/tests/http_client_test.cpp+++ b/tests/http_client_test.cpp@@ -217,6 +217,19 @@ int main(int argc, char** argv) {response = client.Perform(invalid_bounds, cancel);Check(response.error == HttpError::kInvalidRequest, "timeout should have a hard upper bound");+ // Only a depot manifest may ask for more than the general cap (#451), and only up to its own.+ HttpRequest large_cap = get;+ large_cap.max_response_bytes = playden::steam::kMaxHttpResponseBytes + 1;+ response = client.Perform(large_cap, cancel);+ Check(response.error == HttpError::kInvalidRequest, "a response cap above 16 MiB needs the manifest opt-in");+ large_cap.depot_manifest = true;+ large_cap.max_response_bytes = playden::steam::kMaxHttpManifestResponseBytes;+ response = client.Perform(large_cap, cancel);+ Check(response.ok() && response.status_code == 200, "a depot manifest may be up to 64 MiB");+ large_cap.max_response_bytes = playden::steam::kMaxHttpManifestResponseBytes + 1;+ response = client.Perform(large_cap, cancel);+ Check(response.error == HttpError::kInvalidRequest, "a depot manifest's cap has an upper bound");+HttpRequest invalid_get_body = get;invalid_get_body.body = {'x'};response = client.Perform(invalid_get_body, cancel);diff --git a/tests/steam_depot_manifest_test.cpp b/tests/steam_depot_manifest_test.cppindex e38e82c..f863e56 100644--- a/tests/steam_depot_manifest_test.cpp+++ b/tests/steam_depot_manifest_test.cpp@@ -174,6 +174,36 @@ void Unzip() {Bytes bad = zip;bad[0] = 'X'; // local header signatureCHECK(!UnzipFirstEntry(bad, InflateStub, &out));+ // A large game's manifest inflates past 64 MiB (#451): accepted up to kMaxManifestBytes when the caller+ // asks for it; other zips (Cloud saves) keep the 64 MiB bound.+ {+ const auto stored = [](std::size_t size) {+ Bytes z(30 + size + 46 + 22, 0);+ const auto put16 = [&](std::size_t at, std::uint32_t v) { z[at] = v & 0xff; z[at + 1] = (v >> 8) & 0xff; };+ const auto put32 = [&](std::size_t at, std::uint32_t v) { put16(at, v & 0xffff); put16(at + 2, v >> 16); };+ put32(0, 0x04034B50u);+ put32(18, static_cast<std::uint32_t>(size));+ put32(22, static_cast<std::uint32_t>(size));+ const std::size_t cd = 30 + size;+ put32(cd, 0x02014B50u);+ put32(cd + 20, static_cast<std::uint32_t>(size));+ put32(cd + 24, static_cast<std::uint32_t>(size));+ const std::size_t eocd = cd + 46;+ put32(eocd, 0x06054B50u);+ put16(eocd + 8, 1);+ put16(eocd + 10, 1);+ put32(eocd + 12, 46);+ put32(eocd + 16, static_cast<std::uint32_t>(cd));+ return z;+ };+ Bytes big = stored(kMaxUnzipBytes + 1);+ CHECK(!UnzipFirstEntry(big, InflateStub, &out));+ CHECK(UnzipFirstEntry(big, InflateStub, &out, kMaxManifestBytes) && out.size() == kMaxUnzipBytes + 1);+ out.clear();+ out.shrink_to_fit();+ big = stored(kMaxManifestBytes + 1);+ CHECK(!UnzipFirstEntry(big, InflateStub, &out, kMaxManifestBytes));+ }}void RejectBadInput() {diff --git a/tests/steam_gzip_test.cpp b/tests/steam_gzip_test.cppindex 6fbb0a9..00d6a2b 100644--- a/tests/steam_gzip_test.cpp+++ b/tests/steam_gzip_test.cpp@@ -1917,8 +1917,33 @@ void TestSizeLimits() {} // namespace+// A large game's depot manifest inflates past 64 MiB (#451); InflateRaw takes it, up to kMaxInflateRawBytes.+void TestLargeRawInflate() {+ const auto stored = [](std::size_t size) { // raw deflate in stored blocks of at most 65535 bytes+ std::vector<std::uint8_t> out;+ out.reserve(size + (size / 65535 + 1) * 5);+ for (std::size_t left = size; left > 0;) {+ const auto block = static_cast<std::uint16_t>(left < 65535 ? left : 65535);+ left -= block;+ out.push_back(left == 0 ? 1 : 0);+ out.push_back(block & 0xff);+ out.push_back(block >> 8);+ out.push_back(static_cast<std::uint8_t>(~block & 0xff));+ out.push_back(static_cast<std::uint8_t>((~block >> 8) & 0xff));+ out.insert(out.end(), block, 0x5a);+ }+ return out;+ };+ constexpr std::size_t kLarge = 64 * 1024 * 1024 + 1;+ std::vector<std::uint8_t> raw = stored(kLarge);+ std::vector<std::uint8_t> out;+ assert(playden::steam::InflateRaw(raw.data(), raw.size(), kLarge, &out) && out.size() == kLarge);+ assert(!playden::steam::InflateRaw(raw.data(), raw.size(), playden::steam::kMaxInflateRawBytes + 1, &out));+}+int main() {TestRoundTrips();+ TestLargeRawInflate();TestTruncation();TestCorruption();TestSizeLimits();