Skip to content

Play your Steam library's Windows games on a jailbroken PS5.

C++

Download depot manifests larger than 16 MiB

Leonardo Lobatobb89f231126a
diff --git a/src/steam/backend/cdn_client.cpp b/src/steam/backend/cdn_client.cpp
index 2f97198..4d9e34a 100644
--- a/src/steam/backend/cdn_client.cpp
+++ b/src/steam/backend/cdn_client.cpp
@@ -74,10 +74,12 @@ CdnClient::CdnClient(Perform perform) : perform_(std::move(perform)) {}
CdnClient::~CdnClient() = default;
CdnError CdnClient::Get(const std::string& url, std::size_t max_bytes, std::uint32_t timeout_ms,
- const std::atomic<bool>& cancel, std::vector<std::uint8_t>* out, bool plain_http) {
+ const std::atomic<bool>& cancel, std::vector<std::uint8_t>* out, bool plain_http,
+ bool depot_manifest) {
HttpRequest request;
request.url = url;
request.allow_plain_http = plain_http;
+ request.depot_manifest = depot_manifest;
request.headers = {{"User-Agent", "Valve/Steam HTTP Client 1.0"}};
request.timeout_ms = timeout_ms;
request.max_response_bytes = max_bytes;
@@ -166,7 +168,7 @@ CdnError CdnClient::Manifest(const ContentServer& server, std::uint32_t depot_id
if (!ValidCdnHost(server.vhost) || !max_bytes) return CdnError::kProtocol;
return Get(Scheme(server) + server.vhost + "/depot/" + std::to_string(depot_id) + "/manifest/" +
std::to_string(gid) + "/5/" + std::to_string(request_code),
- std::min(max_bytes, kMaxManifestDownloadBytes), 60'000, cancel, out, !server.https);
+ std::min(max_bytes, kMaxManifestDownloadBytes), kMaxHttpTimeoutMs, cancel, out, !server.https, true);
}
CdnError CdnClient::Chunk(const ContentServer& server, std::uint32_t depot_id, const std::string& sha_hex,
diff --git a/src/steam/backend/cdn_client.h b/src/steam/backend/cdn_client.h
index 5308541..a87372d 100644
--- a/src/steam/backend/cdn_client.h
+++ b/src/steam/backend/cdn_client.h
@@ -47,12 +47,13 @@ class CdnClient {
CdnError Servers(std::uint32_t cell_id, const std::atomic<bool>& cancel, std::vector<ContentServer>* out);
CdnError Manifest(const ContentServer& server, std::uint32_t depot_id, std::uint64_t gid,
std::uint64_t request_code, const std::atomic<bool>& cancel, std::vector<std::uint8_t>* out,
- std::size_t max_bytes = kMaxHttpResponseBytes);
+ std::size_t max_bytes = kMaxManifestDownloadBytes);
CdnError Chunk(const ContentServer& server, std::uint32_t depot_id, const std::string& sha_hex,
std::size_t max_bytes, const std::atomic<bool>& cancel, std::vector<std::uint8_t>* out);
- // Bounded by the HTTP client's response cap.
- static constexpr std::size_t kMaxManifestDownloadBytes = kMaxHttpResponseBytes;
+ // A manifest has its own cap, above the HTTP client's general one: a large game's manifest is more than
+ // 16 MiB. Its zip stays within what UnzipFirstEntry accepts (depot_manifest.h).
+ static constexpr std::size_t kMaxManifestDownloadBytes = kMaxHttpManifestResponseBytes;
static constexpr std::size_t kMaxServerListDownloadBytes = 512 * 1024;
// The last failure as "http=<status>" or "transport=<category>", for sanitized logs.
const CdnStats& stats() const { return stats_; }
@@ -61,7 +62,8 @@ class CdnClient {
private:
CdnError Get(const std::string& url, std::size_t max_bytes, std::uint32_t timeout_ms,
- const std::atomic<bool>& cancel, std::vector<std::uint8_t>* out, bool plain_http = false);
+ const std::atomic<bool>& cancel, std::vector<std::uint8_t>* out, bool plain_http = false,
+ bool depot_manifest = false);
std::string ca_bundle_;
Perform perform_;
std::unique_ptr<HttpClient> http_;
diff --git a/src/steam/backend/depot_installer.cpp b/src/steam/backend/depot_installer.cpp
index 330aa1b..421851e 100644
--- a/src/steam/backend/depot_installer.cpp
+++ b/src/steam/backend/depot_installer.cpp
@@ -600,6 +600,8 @@ DepotInstaller::DepotInstaller(CmClient& cm, CdnClient& cdn) {
manifest_problem->clear(); // a manifest problem from an earlier run must not hide why this list failed
return cdn.Servers(cell, cancel, out);
};
+ static_assert(CdnClient::kMaxManifestDownloadBytes <= kMaxManifestBytes && kMaxManifestBytes <= kMaxInflateRawBytes,
+ "a manifest the CDN client accepts must unzip");
hooks_.manifest = [&cdn, manifest_problem](const ContentServer& server, const DepotGrant& grant,
const std::atomic<bool>& cancel, DepotManifest* out) {
manifest_problem->clear();
@@ -608,13 +610,20 @@ DepotInstaller::DepotInstaller(CmClient& cm, CdnClient& cdn) {
cancel, &zip);
if (fetched == CdnError::kCanceled) return InstallError::kCanceled;
if (fetched != CdnError::kNone) return InstallError::kNetwork;
+ const auto zip_bytes = zip.size();
const auto corrupt = [&](std::string what) {
- *manifest_problem = "depot=" + std::to_string(grant.depot_id) + " bytes=" + std::to_string(zip.size()) + " " + what;
+ *manifest_problem = "depot=" + std::to_string(grant.depot_id) + " bytes=" + std::to_string(zip_bytes) + " " + what;
return InstallError::kCorrupt;
};
+ // A large manifest is tens of MiB as a zip and inflated: each is freed once used.
std::vector<std::uint8_t> raw;
- if (!UnzipFirstEntry(zip, &InflateRaw, &raw)) return corrupt("unzip");
- if (!ParseDepotManifest(raw, out)) return corrupt("parse raw=" + std::to_string(raw.size()));
+ const bool unzipped = UnzipFirstEntry(zip, &InflateRaw, &raw, kMaxManifestBytes);
+ std::vector<std::uint8_t>().swap(zip);
+ if (!unzipped) return corrupt("unzip");
+ const bool parsed = ParseDepotManifest(raw, out);
+ const auto raw_bytes = raw.size();
+ std::vector<std::uint8_t>().swap(raw);
+ if (!parsed) return corrupt("parse raw=" + std::to_string(raw_bytes));
if (out->filenames_encrypted && !DecryptManifestFilenames(out, grant.key, &SymmetricDecrypt))
return corrupt("filenames");
if (out->depot_id != grant.depot_id || out->gid != grant.manifest_gid)
diff --git a/src/steam/backend/depot_manifest.cpp b/src/steam/backend/depot_manifest.cpp
index d4cdd02..6895e8e 100644
--- a/src/steam/backend/depot_manifest.cpp
+++ b/src/steam/backend/depot_manifest.cpp
@@ -419,8 +419,8 @@ bool ValidateDepotManifest(const DepotManifest& manifest, std::string* reason) {
}
bool UnzipFirstEntry(const std::vector<std::uint8_t>& zip, InflateRawFn inflate,
- std::vector<std::uint8_t>* out) {
- if (!out || zip.size() < 22 || zip.size() > kMaxManifestBytes + (1u << 20)) return false;
+ std::vector<std::uint8_t>* out, std::size_t max_bytes) {
+ if (!out || zip.size() < 22 || zip.size() > max_bytes + (1u << 20)) return false;
const std::uint8_t* d = zip.data();
const std::size_t n = zip.size();
// End of central directory (PK\x05\x06), scanning back over a trailing comment.
@@ -443,7 +443,7 @@ bool UnzipFirstEntry(const std::vector<std::uint8_t>& zip, InflateRawFn inflate,
const std::size_t local = ReadLE32(d + cd + 42);
if (flags & 1) return false; // encrypted entry
if (csize == 0xFFFFFFFFu || usize == 0xFFFFFFFFu || local == 0xFFFFFFFFu) return false; // zip64
- if (usize > kMaxManifestBytes) return false;
+ if (usize > max_bytes) return false;
if (local > n || n - local < 30 || ReadLE32(d + local) != 0x04034B50u) return false;
const std::size_t start = local + 30 + ReadLE16(d + local + 26) + ReadLE16(d + local + 28);
if (start > n || csize > n - start) return false;
diff --git a/src/steam/backend/depot_manifest.h b/src/steam/backend/depot_manifest.h
index 7f5f070..5747b9b 100644
--- a/src/steam/backend/depot_manifest.h
+++ b/src/steam/backend/depot_manifest.h
@@ -12,7 +12,11 @@
// reference CDN.swift / ResumableDepotDownload.swift.
namespace playden::steam {
-inline constexpr std::size_t kMaxManifestBytes = 64 * 1024 * 1024;
+// An inflated depot manifest. Manifests compress little (chunk hashes, encrypted file names), so a zip at the
+// download cap (CdnClient::kMaxManifestDownloadBytes, 64 MiB) can inflate to about twice its size.
+inline constexpr std::size_t kMaxManifestBytes = 128 * 1024 * 1024;
+// What UnzipFirstEntry unpacks unless the caller says otherwise (a Cloud save).
+inline constexpr std::size_t kMaxUnzipBytes = 64 * 1024 * 1024;
inline constexpr std::size_t kMaxManifestFiles = 500'000;
inline constexpr std::size_t kMaxManifestChunks = 4'000'000;
@@ -76,8 +80,8 @@ bool ValidateDepotManifest(const DepotManifest& manifest, std::string* reason);
using InflateRawFn = bool (*)(const std::uint8_t* data, std::size_t size, std::size_t expected,
std::vector<std::uint8_t>* out);
// Extracts the first zip entry (method 0 stored, or 8 deflate via `inflate`). Result is capped at
-// kMaxManifestBytes.
+// max_bytes (kMaxManifestBytes for a depot manifest).
bool UnzipFirstEntry(const std::vector<std::uint8_t>& zip, InflateRawFn inflate,
- std::vector<std::uint8_t>* out);
+ std::vector<std::uint8_t>* out, std::size_t max_bytes = kMaxUnzipBytes);
} // namespace playden::steam
diff --git a/src/steam/backend/download_server_source.cpp b/src/steam/backend/download_server_source.cpp
index ab8bcf3..3b6c4db 100644
--- a/src/steam/backend/download_server_source.cpp
+++ b/src/steam/backend/download_server_source.cpp
@@ -108,7 +108,8 @@ CdnDownloadServerSource::PayloadResult CdnDownloadServerSource::PrepareApp(
std::vector<std::uint8_t> zip, raw;
DepotManifest manifest;
if (!metadata_budget_) return PayloadResult::kFailed;
- const auto cap = static_cast<std::size_t>(std::min<std::uint64_t>(metadata_budget_, CdnClient::kMaxManifestDownloadBytes));
+ // The speed test keeps the general response cap: it needs a few chunk hashes, not a large game's manifest.
+ const auto cap = static_cast<std::size_t>(std::min<std::uint64_t>(metadata_budget_, kMaxHttpResponseBytes));
const auto before = cdn_.stats().bytes;
const auto outcome = cdn_.Manifest(servers_[attempt], grant.depot_id, grant.manifest_gid,
grant.manifest_request_code, cancel, &zip, cap);
@@ -116,7 +117,7 @@ CdnDownloadServerSource::PayloadResult CdnDownloadServerSource::PrepareApp(
// fallback depots draw from this ledger, and the next request is capped before it starts.
metadata_budget_ -= std::min(metadata_budget_, cdn_.stats().bytes - before);
if (outcome != CdnError::kNone ||
- !UnzipFirstEntry(zip, &InflateRaw, &raw) || !ParseDepotManifest(raw, &manifest) ||
+ !UnzipFirstEntry(zip, &InflateRaw, &raw, kMaxManifestBytes) || !ParseDepotManifest(raw, &manifest) ||
manifest.depot_id != grant.depot_id || manifest.gid != grant.manifest_gid) continue;
// Names and content remain encrypted. Only bounded chunk hashes and compressed sizes are used.
for (const auto& file : manifest.files) {
diff --git a/src/steam/backend/gzip.cpp b/src/steam/backend/gzip.cpp
index 43bb8d1..e4f0b6b 100644
--- a/src/steam/backend/gzip.cpp
+++ b/src/steam/backend/gzip.cpp
@@ -113,7 +113,7 @@ bool Gunzip(const std::uint8_t* data, std::size_t size, std::size_t expected_siz
bool InflateRaw(const std::uint8_t* data, std::size_t size, std::size_t expected_size,
std::vector<std::uint8_t>* out) {
- if (!data || !out || expected_size == 0 || expected_size > kMaxGunzipBytes * 16 ||
+ if (!data || !out || expected_size == 0 || expected_size > kMaxInflateRawBytes ||
size > static_cast<std::size_t>(INT_MAX) || expected_size > static_cast<std::size_t>(INT_MAX))
return false;
std::vector<std::uint8_t> result(expected_size);
diff --git a/src/steam/backend/gzip.h b/src/steam/backend/gzip.h
index e114f96..413a182 100644
--- a/src/steam/backend/gzip.h
+++ b/src/steam/backend/gzip.h
@@ -7,6 +7,8 @@
namespace playden::steam {
inline constexpr std::size_t kMaxGunzipBytes = 4 * 1024 * 1024;
+// InflateRaw's own bound: an inflated depot manifest (kMaxManifestBytes); callers cap below it.
+inline constexpr std::size_t kMaxInflateRawBytes = 128 * 1024 * 1024;
// Decodes one RFC 1952 gzip member whose uncompressed size is declared in advance
// (CMsgMulti.size_unzipped). Returns false on any malformed, truncated or oversized input.
diff --git a/src/steam/backend/http_client.cpp b/src/steam/backend/http_client.cpp
index 4f717a2..ca57df8 100644
--- a/src/steam/backend/http_client.cpp
+++ b/src/steam/backend/http_client.cpp
@@ -141,7 +141,7 @@ bool ValidUrl(const std::string& url, bool allow_plain_http) {
bool ValidRequest(const HttpRequest& request) {
if (!ValidUrl(request.url, request.allow_plain_http) || request.timeout_ms == 0 ||
request.timeout_ms > kMaxHttpTimeoutMs || request.max_response_bytes == 0 ||
- request.max_response_bytes > kMaxHttpResponseBytes ||
+ request.max_response_bytes > (request.depot_manifest ? kMaxHttpManifestResponseBytes : kMaxHttpResponseBytes) ||
request.body.size() > kMaxHttpResponseBytes ||
request.headers.size() > kMaxHttpResponseHeaderCount)
return false;
diff --git a/src/steam/backend/http_client.h b/src/steam/backend/http_client.h
index f67a4fc..05d939a 100644
--- a/src/steam/backend/http_client.h
+++ b/src/steam/backend/http_client.h
@@ -34,6 +34,8 @@ const char* HttpErrorName(HttpError error);
inline constexpr std::uint32_t kMaxHttpTimeoutMs = 120'000;
inline constexpr std::size_t kMaxHttpResponseBytes = 16 * 1024 * 1024;
+// Steam depot manifests only (HttpRequest::depot_manifest): a large game's manifest is more than 16 MiB.
+inline constexpr std::size_t kMaxHttpManifestResponseBytes = 64 * 1024 * 1024;
inline constexpr std::size_t kMaxHttpResponseHeaderBytes = 16 * 1024;
inline constexpr std::size_t kMaxHttpResponseHeaderCount = 64;
@@ -46,6 +48,8 @@ struct HttpRequest {
std::size_t max_response_bytes = 1024 * 1024;
// Also accept an http:// URL. Only for content checked by other means, such as Steam CDN chunks.
bool allow_plain_http = false;
+ // A Steam depot manifest: max_response_bytes may be up to kMaxHttpManifestResponseBytes, not kMaxHttpResponseBytes.
+ bool depot_manifest = false;
};
struct HttpTiming {
diff --git a/tests/cdn_client_test.cpp b/tests/cdn_client_test.cpp
index 9d58721..4c9bdcf 100644
--- a/tests/cdn_client_test.cpp
+++ b/tests/cdn_client_test.cpp
@@ -179,6 +179,44 @@ void PlainHttpServerUrls() {
assert(!requests.back().allow_plain_http);
}
+// A depot manifest above the general 16 MiB response cap still downloads (Solo Leveling: ARISE OVERDRIVE, #451); one
+// above the manifest cap does not, and chunks keep the cap their caller gives.
+void LargeManifestsDownload() {
+ std::vector<HttpRequest> requests;
+ std::size_t size = 0;
+ CdnClient client([&](const HttpRequest& request, const std::atomic<bool>&) {
+ requests.push_back(request);
+ HttpResponse response;
+ if (size > request.max_response_bytes) { // as HttpClient's body limit does
+ response.error = HttpError::kResponseTooLarge;
+ return response;
+ }
+ response.status_code = 200;
+ response.body.assign(size, 0x5a);
+ return response;
+ });
+ std::atomic<bool> cancel{false};
+ const ContentServer server{"h.example.invalid", "v.example.invalid", 1};
+ std::vector<std::uint8_t> out;
+ size = 16 * 1024 * 1024 + 1;
+ assert(client.Manifest(server, 2373991, 11, 99, cancel, &out) == CdnError::kNone && out.size() == size);
+ size = 64 * 1024 * 1024;
+ assert(client.Manifest(server, 2373991, 11, 99, cancel, &out) == CdnError::kNone && out.size() == size);
+ assert(requests.back().max_response_bytes == 64 * 1024 * 1024);
+ out.clear();
+ out.shrink_to_fit();
+ size = 64 * 1024 * 1024 + 1;
+ assert(client.Manifest(server, 2373991, 11, 99, cancel, &out) == CdnError::kNetwork && out.empty());
+ assert(client.last_failure() == "transport=response_too_large");
+ // A caller's smaller budget still holds.
+ size = 1024;
+ assert(client.Manifest(server, 2373991, 11, 99, cancel, &out, 1023) == CdnError::kNetwork);
+ assert(requests.back().max_response_bytes == 1023);
+ const std::string sha(40, 'c');
+ assert(client.Chunk(server, 2373991, sha, 4096, cancel, &out) == CdnError::kNone);
+ assert(requests.back().max_response_bytes == 4096);
+}
+
} // namespace
int main() {
@@ -191,4 +229,5 @@ int main() {
EmptyServerListSaysWhy();
ManifestAndChunkUrlsAndErrors();
PlainHttpServerUrls();
+ LargeManifestsDownload();
}
diff --git a/tests/http_client_test.cpp b/tests/http_client_test.cpp
index 365eb6a..5c25d26 100644
--- a/tests/http_client_test.cpp
+++ b/tests/http_client_test.cpp
@@ -217,6 +217,19 @@ int main(int argc, char** argv) {
response = client.Perform(invalid_bounds, cancel);
Check(response.error == HttpError::kInvalidRequest, "timeout should have a hard upper bound");
+ // Only a depot manifest may ask for more than the general cap (#451), and only up to its own.
+ HttpRequest large_cap = get;
+ large_cap.max_response_bytes = playden::steam::kMaxHttpResponseBytes + 1;
+ response = client.Perform(large_cap, cancel);
+ Check(response.error == HttpError::kInvalidRequest, "a response cap above 16 MiB needs the manifest opt-in");
+ large_cap.depot_manifest = true;
+ large_cap.max_response_bytes = playden::steam::kMaxHttpManifestResponseBytes;
+ response = client.Perform(large_cap, cancel);
+ Check(response.ok() && response.status_code == 200, "a depot manifest may be up to 64 MiB");
+ large_cap.max_response_bytes = playden::steam::kMaxHttpManifestResponseBytes + 1;
+ response = client.Perform(large_cap, cancel);
+ Check(response.error == HttpError::kInvalidRequest, "a depot manifest's cap has an upper bound");
+
HttpRequest invalid_get_body = get;
invalid_get_body.body = {'x'};
response = client.Perform(invalid_get_body, cancel);
diff --git a/tests/steam_depot_manifest_test.cpp b/tests/steam_depot_manifest_test.cpp
index e38e82c..f863e56 100644
--- a/tests/steam_depot_manifest_test.cpp
+++ b/tests/steam_depot_manifest_test.cpp
@@ -174,6 +174,36 @@ void Unzip() {
Bytes bad = zip;
bad[0] = 'X'; // local header signature
CHECK(!UnzipFirstEntry(bad, InflateStub, &out));
+ // A large game's manifest inflates past 64 MiB (#451): accepted up to kMaxManifestBytes when the caller
+ // asks for it; other zips (Cloud saves) keep the 64 MiB bound.
+ {
+ const auto stored = [](std::size_t size) {
+ Bytes z(30 + size + 46 + 22, 0);
+ const auto put16 = [&](std::size_t at, std::uint32_t v) { z[at] = v & 0xff; z[at + 1] = (v >> 8) & 0xff; };
+ const auto put32 = [&](std::size_t at, std::uint32_t v) { put16(at, v & 0xffff); put16(at + 2, v >> 16); };
+ put32(0, 0x04034B50u);
+ put32(18, static_cast<std::uint32_t>(size));
+ put32(22, static_cast<std::uint32_t>(size));
+ const std::size_t cd = 30 + size;
+ put32(cd, 0x02014B50u);
+ put32(cd + 20, static_cast<std::uint32_t>(size));
+ put32(cd + 24, static_cast<std::uint32_t>(size));
+ const std::size_t eocd = cd + 46;
+ put32(eocd, 0x06054B50u);
+ put16(eocd + 8, 1);
+ put16(eocd + 10, 1);
+ put32(eocd + 12, 46);
+ put32(eocd + 16, static_cast<std::uint32_t>(cd));
+ return z;
+ };
+ Bytes big = stored(kMaxUnzipBytes + 1);
+ CHECK(!UnzipFirstEntry(big, InflateStub, &out));
+ CHECK(UnzipFirstEntry(big, InflateStub, &out, kMaxManifestBytes) && out.size() == kMaxUnzipBytes + 1);
+ out.clear();
+ out.shrink_to_fit();
+ big = stored(kMaxManifestBytes + 1);
+ CHECK(!UnzipFirstEntry(big, InflateStub, &out, kMaxManifestBytes));
+ }
}
void RejectBadInput() {
diff --git a/tests/steam_gzip_test.cpp b/tests/steam_gzip_test.cpp
index 6fbb0a9..00d6a2b 100644
--- a/tests/steam_gzip_test.cpp
+++ b/tests/steam_gzip_test.cpp
@@ -1917,8 +1917,33 @@ void TestSizeLimits() {
} // namespace
+// A large game's depot manifest inflates past 64 MiB (#451); InflateRaw takes it, up to kMaxInflateRawBytes.
+void TestLargeRawInflate() {
+ const auto stored = [](std::size_t size) { // raw deflate in stored blocks of at most 65535 bytes
+ std::vector<std::uint8_t> out;
+ out.reserve(size + (size / 65535 + 1) * 5);
+ for (std::size_t left = size; left > 0;) {
+ const auto block = static_cast<std::uint16_t>(left < 65535 ? left : 65535);
+ left -= block;
+ out.push_back(left == 0 ? 1 : 0);
+ out.push_back(block & 0xff);
+ out.push_back(block >> 8);
+ out.push_back(static_cast<std::uint8_t>(~block & 0xff));
+ out.push_back(static_cast<std::uint8_t>((~block >> 8) & 0xff));
+ out.insert(out.end(), block, 0x5a);
+ }
+ return out;
+ };
+ constexpr std::size_t kLarge = 64 * 1024 * 1024 + 1;
+ std::vector<std::uint8_t> raw = stored(kLarge);
+ std::vector<std::uint8_t> out;
+ assert(playden::steam::InflateRaw(raw.data(), raw.size(), kLarge, &out) && out.size() == kLarge);
+ assert(!playden::steam::InflateRaw(raw.data(), raw.size(), playden::steam::kMaxInflateRawBytes + 1, &out));
+}
+
int main() {
TestRoundTrips();
+ TestLargeRawInflate();
TestTruncation();
TestCorruption();
TestSizeLimits();