Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

Fix zstd (#547)

Bill Yangd36f3b0f1c0d
diff --git a/server/src/services/storage/r2StorageService.ts b/server/src/services/storage/r2StorageService.ts
index 0f99481bc..87a588196 100644
--- a/server/src/services/storage/r2StorageService.ts
+++ b/server/src/services/storage/r2StorageService.ts
@@ -1,4 +1,11 @@
-import { S3Client, PutObjectCommand, GetObjectCommand, DeleteObjectCommand } from "@aws-sdk/client-s3";
+import {
+ S3Client,
+ PutObjectCommand,
+ GetObjectCommand,
+ DeleteObjectCommand,
+ GetObjectCommandOutput,
+} from "@aws-sdk/client-s3";
+import { NodeHttpHandler } from "@smithy/node-http-handler";
import { Readable } from "stream";
import { gunzipSync } from "zlib";
import { compress as zstdCompress, decompress as zstdDecompress } from "@mongodb-js/zstd";
@@ -14,6 +21,25 @@ class R2StorageService {
constructor() {
// Only initialize R2 in cloud environment
if (IS_CLOUD && process.env.R2_ACCESS_KEY_ID && process.env.R2_SECRET_ACCESS_KEY) {
+ // Create a custom HTTP handler that strips checksum headers
+ const httpHandler = new NodeHttpHandler();
+ const originalHandle = httpHandler.handle.bind(httpHandler);
+
+ httpHandler.handle = async (request: any, options?: any) => {
+ const response = await originalHandle(request, options);
+
+ // Wrap the response to strip checksum headers
+ return {
+ ...response,
+ response: {
+ ...response.response,
+ headers: Object.fromEntries(
+ Object.entries(response.response.headers).filter(([key]) => !key.toLowerCase().includes("checksum")),
+ ),
+ },
+ };
+ };
+
this.client = new S3Client({
region: "auto",
endpoint: `https://${process.env.R2_ACCOUNT_ID}.r2.cloudflarestorage.com`,
@@ -23,7 +49,10 @@ class R2StorageService {
},
// Required for R2 compatibility
forcePathStyle: true,
+ // Use our custom HTTP handler
+ requestHandler: httpHandler,
});
+
this.bucketName = process.env.R2_BUCKET_NAME || "rybbit";
this.enabled = true;
this.logger.info({ bucket: this.bucketName }, "R2Storage initialized");
@@ -58,8 +87,8 @@ class R2StorageService {
Bucket: this.bucketName,
Key: key,
Body: compressed,
- ContentType: "application/json",
- ContentEncoding: "zstd",
+ ContentType: "application/octet-stream", // Binary data, not JSON
+ // Don't set ContentEncoding - that triggers automatic decompression
Metadata: {
siteId: siteId.toString(),
sessionId: sessionId,
@@ -107,18 +136,45 @@ class R2StorageService {
const buffer = Buffer.concat(chunks);
- // Detect compression based on file extension
- let decompressed: Buffer;
- if (key.endsWith(".zst")) {
- decompressed = await zstdDecompress(buffer);
- } else if (key.endsWith(".gz")) {
- decompressed = gunzipSync(buffer);
- } else {
- // Assume zstd for unknown extensions
- decompressed = await zstdDecompress(buffer);
+ // Check if data is already JSON (uncompressed)
+ // This happens when ContentEncoding was set to "zstd" and R2 auto-decompressed it
+ const bufferStr = buffer.toString("utf8", 0, Math.min(buffer.length, 100));
+ const isLikelyJSON = bufferStr.trimStart().startsWith("[") || bufferStr.trimStart().startsWith("{");
+
+ if (isLikelyJSON) {
+ try {
+ // Try parsing as JSON first
+ return JSON.parse(buffer.toString());
+ } catch (e) {
+ // Not valid JSON, proceed with decompression
+ }
}
- return JSON.parse(decompressed.toString());
+ // Try to decompress based on file extension
+ let decompressed: Buffer;
+
+ try {
+ if (key.endsWith(".zst")) {
+ decompressed = await zstdDecompress(buffer);
+ } else if (key.endsWith(".gz")) {
+ decompressed = gunzipSync(buffer);
+ } else {
+ // Assume zstd for unknown extensions
+ decompressed = await zstdDecompress(buffer);
+ }
+ return JSON.parse(decompressed.toString());
+ } catch (decompressionError: any) {
+ // If decompression fails and we haven't tried JSON yet, try it now
+ if (!isLikelyJSON) {
+ try {
+ return JSON.parse(buffer.toString());
+ } catch (jsonError) {
+ // Data is truly corrupted, throw the original error
+ throw decompressionError;
+ }
+ }
+ throw decompressionError;
+ }
} catch (error) {
console.error("[R2Storage] Failed to retrieve batch:", error);
throw error;