SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
s3tables: accept Puffin statistics files in a table's metadata/ (#11892)
Javier de la Torre593cfee85b8f
diff --git a/weed/s3api/s3tables/iceberg_layout.go b/weed/s3api/s3tables/iceberg_layout.goindex 00c2030285..1e21f8c41f 100644--- a/weed/s3api/s3tables/iceberg_layout.go+++ b/weed/s3api/s3tables/iceberg_layout.go@@ -15,6 +15,7 @@ import (// - metadata/v*.metadata.json (table metadata)// - metadata/snap-*.avro (snapshot manifest lists)// - metadata/*.avro (manifest files)+// - metadata/*.stats, metadata/*.puffin (statistics files)// - data/*.parquet, *.orc, *.avro, *.puffin (data files)const uuidPattern = `[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}`@@ -68,6 +69,9 @@ var (// whose naming we can't anticipate across engines and versions.regexp.MustCompile(`^[A-Za-z0-9._-]+\.avro$`),regexp.MustCompile(`^[A-Za-z0-9._-]+\.metadata\.json$`),+ // Table and partition statistics files listed in the table metadata+ // are Puffin files; engines name them *.stats or *.puffin.+ regexp.MustCompile(`^[A-Za-z0-9._-]+\.puffin$`),}// Patterns for valid data filesdiff --git a/weed/s3api/s3tables/iceberg_layout_test.go b/weed/s3api/s3tables/iceberg_layout_test.goindex 855361297f..88d97ee8a6 100644--- a/weed/s3api/s3tables/iceberg_layout_test.go+++ b/weed/s3api/s3tables/iceberg_layout_test.go@@ -45,6 +45,10 @@ func TestIcebergLayoutValidator_AcceptsRealWorldManifestNames(t *testing.T) {"trino/iceberg stats file","metadata/table.stats",},+ {+ "puffin statistics file",+ "metadata/20260928-1-0199a3c2-7d1e-7c55-9a0e-3b1f2c4d5e6f.puffin",+ },}v := NewIcebergLayoutValidator()@@ -70,6 +74,8 @@ func TestIcebergLayoutValidator_RejectsClearlyBadMetadataNames(t *testing.T) {{"subdirectory under metadata is not allowed", "metadata/sub/file.avro"},{"top-level dir other than metadata or data", "garbage/file.avro"},{"catch-all anchors must reject trailing extension", "metadata/file.avro.txt"},+ {"puffin with a trailing extension", "metadata/stats.puffin.txt"},+ {"puffin in a subdirectory under metadata", "metadata/sub/stats.puffin"},}v := NewIcebergLayoutValidator()@@ -137,3 +143,31 @@ func TestValidateFilePathAcceptsLanceLayout(t *testing.T) {}}}++// Statistics files are referenced from the table metadata and kept by orphan+// cleanup; the upload door has to admit them in the table's metadata/, and+// only there.+func TestTableBucketFileValidator_PuffinStatisticsOnlyInTableMetadata(t *testing.T) {+ v := NewTableBucketFileValidator()+ accepted := []string{+ TablesPath + "/cat/ns/t/metadata/stats-1.puffin",+ TablesPath + "/cat/ns/t/metadata/table.stats",+ TablesPath + "/cat/ns/t/data/0199a3c2-deletes.puffin",+ }+ for _, p := range accepted {+ if err := v.ValidateTableBucketUpload(p); err != nil {+ t.Errorf("ValidateTableBucketUpload(%q) = %v, want nil", p, err)+ }+ }+ rejected := []string{+ TablesPath + "/cat/ns/t/stats-1.puffin",+ TablesPath + "/cat/ns/t/other/stats-1.puffin",+ TablesPath + "/cat/ns/t/metadata/sub/stats-1.puffin",+ TablesPath + "/cat/ns/t/metadata/notes.txt",+ }+ for _, p := range rejected {+ if err := v.ValidateTableBucketUpload(p); err == nil {+ t.Errorf("ValidateTableBucketUpload(%q) = nil, want an error", p)+ }+ }+}