Skip to content

SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.

compression: compress JSON uploads, and match file types by extension (#12339)

Pat Whited650abe0e76a
diff --git a/seaweed-volume/src/server/handlers.rs b/seaweed-volume/src/server/handlers.rs
index d2d02d35b0..91c2d2a7ca 100644
--- a/seaweed-volume/src/server/handlers.rs
+++ b/seaweed-volume/src/server/handlers.rs
@@ -3101,15 +3101,12 @@ pub async fn post_handler(
// Auto-compress compressible file types (matches Go's IsCompressableFileType).
// Only compress if not already gzipped and compression saves >10%.
- // Go uses filepath.Base(pu.FileName) for extension detection (not the URL path).
let (final_data, final_is_gzipped) = if !is_gzipped && !is_chunk_manifest {
- let ext = {
- let dot_pos = filename.rfind('.');
- dot_pos
- .map(|p| filename[p..].to_lowercase())
- .unwrap_or_default()
- };
- if is_compressible_file_type(&ext, &mime_type) {
+ let ext = filename
+ .rfind('.')
+ .map(|p| &filename[p..])
+ .unwrap_or_default();
+ if is_compressible_file_type(ext, &mime_type) {
if let Some(compressed) = try_gzip_data(&body_data) {
if compressed.len() * 10 < body_data.len() * 9 {
(compressed, true)
@@ -4728,6 +4725,15 @@ fn extract_jwt(headers: &HeaderMap, uri: &axum::http::Uri) -> Option<String> {
/// Check if a file type should be compressed based on extension and MIME type.
/// Returns true only when we are sure the type is compressible.
fn is_compressible_file_type(ext: &str, mtype: &str) -> bool {
+ let ext = ext.to_ascii_lowercase();
+ let ext = ext.as_str();
+ let mtype = mtype
+ .split(';')
+ .next()
+ .unwrap_or_default()
+ .trim()
+ .to_ascii_lowercase();
+ let mtype = mtype.as_str();
// text/*
if mtype.starts_with("text/") {
return true;
@@ -4760,6 +4766,9 @@ fn is_compressible_file_type(ext: &str, mtype: &str) -> bool {
if mtype.ends_with("script") {
return true;
}
+ if mtype.ends_with("json") || mtype.ends_with("json-seq") {
+ return true;
+ }
if mtype.ends_with("vnd.rar") {
return false;
}
@@ -5245,6 +5254,18 @@ mod tests {
assert!(is_compressible_file_type("", "application/javascript"));
assert!(!is_compressible_file_type("", "application/zstd"));
assert!(!is_compressible_file_type("", "application/vnd.rar"));
+ assert!(is_compressible_file_type("", "application/json"));
+ assert!(is_compressible_file_type(
+ "",
+ "application/json; charset=utf-8"
+ ));
+ assert!(is_compressible_file_type("", "Application/JSON"));
+ assert!(is_compressible_file_type("", "application/ld+json"));
+ assert!(is_compressible_file_type("", "application/x-ndjson"));
+ assert!(is_compressible_file_type("", "application/json-seq"));
+ assert!(is_compressible_file_type("", "text/plain; charset=utf-8"));
+ assert!(is_compressible_file_type(".JSON", ""));
+ assert!(!is_compressible_file_type(".gz", "application/json"));
// Audio
assert!(is_compressible_file_type(".wav", "audio/wav"));
diff --git a/weed/operation/needle_parse_test.go b/weed/operation/needle_parse_test.go
index 339d4507e2..fb1d530a78 100644
--- a/weed/operation/needle_parse_test.go
+++ b/weed/operation/needle_parse_test.go
@@ -5,6 +5,7 @@ import (
"context"
"fmt"
"io"
+ "math/rand"
"net/http"
"testing"
@@ -118,6 +119,33 @@ func TestCreateNeedleFromRequest(t *testing.T) {
}
+func TestUploadKeepsMarginalGzipUncompressed(t *testing.T) {
+ data := make([]byte, 4300)
+ rand.New(rand.NewSource(1)).Read(data[:4000])
+ gzipped, _ := util.GzipData(data)
+ if len(gzipped) >= len(data) || len(gzipped)*10 < len(data)*9 {
+ t.Fatalf("gzip size %d of %d is not a marginal saving", len(gzipped), len(data))
+ }
+
+ mockClient := &MockClient{}
+ var stored *needle.Needle
+ mockClient.needleHandling = func(n *needle.Needle, originalSize int, err error) {
+ assert.Equal(t, nil, err, "upload: %v", err)
+ stored = n
+ }
+ uploadOption := &UploadOption{
+ UploadUrl: "http://localhost:8080/389,0f084d17353afda0",
+ Filename: "doc.pdf",
+ MimeType: "application/pdf",
+ }
+ newUploader(mockClient).Upload(context.Background(), bytes.NewReader(data), uploadOption)
+ if stored == nil {
+ t.Fatal("no needle reached the volume server")
+ }
+ assert.False(t, stored.IsCompressed(), "gzip saved less than 10%, data should stay uncompressed")
+ assert.Equal(t, data, stored.Data)
+}
+
var textContent = `Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
diff --git a/weed/operation/upload_content.go b/weed/operation/upload_content.go
index a44dc38e8e..4d829e33e7 100644
--- a/weed/operation/upload_content.go
+++ b/weed/operation/upload_content.go
@@ -390,7 +390,7 @@ func (uploader *Uploader) doUploadData(ctx context.Context, data []byte, option
option.MimeType = ""
}
}
- if shouldBeCompressed, iAmSure := util.IsCompressableFileType(filepath.Base(option.Filename), option.MimeType); iAmSure && shouldBeCompressed {
+ if shouldBeCompressed, iAmSure := util.IsCompressableFileType(filepath.Ext(option.Filename), option.MimeType); iAmSure && shouldBeCompressed {
shouldGzipNow = true
} else if !iAmSure && option.MimeType == "" && len(data) > 16*1024 {
var compressed []byte
@@ -412,7 +412,7 @@ func (uploader *Uploader) doUploadData(ctx context.Context, data []byte, option
compressed, compressErr := util.GzipData(data)
// fmt.Printf("data is compressed from %d ==> %d\n", len(data), len(compressed))
if compressErr == nil {
- if len(compressed) < len(data) {
+ if len(compressed)*10 < len(data)*9 {
data = compressed
contentIsGzipped = true
}
diff --git a/weed/storage/needle/needle_parse_upload.go b/weed/storage/needle/needle_parse_upload.go
index 0707acef14..3a668de9b6 100644
--- a/weed/storage/needle/needle_parse_upload.go
+++ b/weed/storage/needle/needle_parse_upload.go
@@ -96,7 +96,7 @@ func ParseUpload(r *http.Request, sizeLimit int64, bytesBuffer *bytes.Buffer) (p
}
} else if r.FormValue("type") != "replicate" {
// replica writes must keep the source needle's compression state, not re-derive it
- ext := filepath.Base(pu.FileName)
+ ext := filepath.Ext(pu.FileName)
mimeType := pu.MimeType
if mimeType == "" {
mimeType = http.DetectContentType(pu.Data)
diff --git a/weed/util/compression.go b/weed/util/compression.go
index d62ba90883..aac6b1e012 100644
--- a/weed/util/compression.go
+++ b/weed/util/compression.go
@@ -99,6 +99,12 @@ func IsZstdContent(data []byte) bool {
/*
* Default not to compressed since compression can be done on client side.
*/func IsCompressableFileType(ext, mtype string) (shouldBeCompressed, iAmSure bool) {
+ ext = strings.ToLower(ext)
+ // Match on the media type alone: "application/json; charset=utf-8" is JSON.
+ if i := strings.IndexByte(mtype, ';'); i >= 0 {
+ mtype = strings.TrimSpace(mtype[:i])
+ }
+ mtype = strings.ToLower(mtype)
// text
if strings.HasPrefix(mtype, "text/") {
@@ -137,6 +143,10 @@ func IsZstdContent(data []byte) bool {
if strings.HasSuffix(mtype, "script") {
return true, true
}
+ // application/json, application/ld+json, application/x-ndjson, application/json-seq, ...
+ if strings.HasSuffix(mtype, "json") || strings.HasSuffix(mtype, "json-seq") {
+ return true, true
+ }
if strings.HasSuffix(mtype, "vnd.rar") {
return false, true
}
diff --git a/weed/util/compression_test.go b/weed/util/compression_test.go
new file mode 100644
index 0000000000..e9486be3e1
--- /dev/null
+++ b/weed/util/compression_test.go
@@ -0,0 +1,31 @@
+package util
+
+import "testing"
+
+func TestIsCompressableFileType(t *testing.T) {
+ tests := []struct {
+ ext, mtype string
+ compress, isSure bool
+ }{
+ {".json", "", true, true},
+ {".JSON", "", true, true},
+ {"", "application/json", true, true},
+ {"", "application/json; charset=utf-8", true, true},
+ {"", "application/ld+json", true, true},
+ {"", "application/x-ndjson", true, true},
+ {"", "application/json-seq", true, true},
+ {"", "Application/JSON", true, true},
+ {"", "text/plain; charset=utf-8", true, true},
+ {"", "application/xml", true, true},
+ {".gz", "application/json", false, true},
+ {"", "application/gzip", false, false},
+ {"", "image/png", false, true},
+ {"", "", false, false},
+ }
+ for _, tt := range tests {
+ compress, isSure := IsCompressableFileType(tt.ext, tt.mtype)
+ if compress != tt.compress || isSure != tt.isSure {
+ t.Errorf("IsCompressableFileType(%q, %q) = (%v, %v), want (%v, %v)", tt.ext, tt.mtype, compress, isSure, tt.compress, tt.isSure)
+ }
+ }
+}