SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
filer: skip UpdateEntry when inline content is unchanged (#11580)
Chris Lueafe79ebff4d
diff --git a/weed/filer/filer_conf_test.go b/weed/filer/filer_conf_test.goindex f17719be29..b71da89b68 100644--- a/weed/filer/filer_conf_test.go+++ b/weed/filer/filer_conf_test.go@@ -262,8 +262,9 @@ func TestClearReadOnly(t *testing.T) {type fakeFilerConfClient struct {filer_pb.SeaweedFilerClient- mu sync.Mutex- entries map[string]*filer_pb.Entry // key: dir+"/"+name+ mu sync.Mutex+ entries map[string]*filer_pb.Entry // key: dir+"/"+name+ updateCalls int}func newFakeFilerConfClient() *fakeFilerConfClient {@@ -310,6 +311,7 @@ func (c *fakeFilerConfClient) CreateEntry(_ context.Context, in *filer_pb.Createfunc (c *fakeFilerConfClient) UpdateEntry(_ context.Context, in *filer_pb.UpdateEntryRequest, _ ...grpc.CallOption) (*filer_pb.UpdateEntryResponse, error) {c.mu.Lock()defer c.mu.Unlock()+ c.updateCalls++key := c.key(in.Directory, in.Entry.Name)if !conditionHoldsLocked(in.Condition, c.entries[key]) {return nil, errors.New("precondition failed")diff --git a/weed/filer/read_write.go b/weed/filer/read_write.goindex 7f5032c164..7fa572fd95 100644--- a/weed/filer/read_write.go+++ b/weed/filer/read_write.go@@ -75,6 +75,9 @@ func SaveInsideFiler(ctx context.Context, client filer_pb.SeaweedFilerClient, di})} else if err == nil {entry := resp.Entry+ if bytes.Equal(entry.Content, content) && bytes.Equal(entry.GetAttributes().GetMd5(), contentMd5[:]) {+ return nil+ }entry.Content = contententry.Attributes.Mtime = time.Now().Unix()entry.Attributes.FileSize = uint64(len(content))diff --git a/weed/filer/read_write_test.go b/weed/filer/read_write_test.gonew file mode 100644index 0000000000..59cec8c6cf--- /dev/null+++ b/weed/filer/read_write_test.go@@ -0,0 +1,51 @@+package filer++import (+ "context"+ "testing"++ "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"+ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+)++// Rewriting identical inline content must not issue UpdateEntry: each one is a+// metadata event the local meta log persists to /topics/.system/log, and a+// client that rewrites the same config on a timer (e.g. the operator's 5-minute+// resync) keeps the volumes growing on an otherwise idle cluster.+func TestSaveInsideFilerSkipsIdenticalContent(t *testing.T) {+ ctx := context.Background()+ client := newFakeFilerConfClient()++ require.NoError(t, SaveInsideFiler(ctx, client, DirectoryEtcSeaweedFS, "a.json", []byte("v1")))+ require.Equal(t, 0, client.updateCalls)++ require.NoError(t, SaveInsideFiler(ctx, client, DirectoryEtcSeaweedFS, "a.json", []byte("v1")))+ assert.Equal(t, 0, client.updateCalls)++ require.NoError(t, SaveInsideFiler(ctx, client, DirectoryEtcSeaweedFS, "a.json", []byte("v2")))+ assert.Equal(t, 1, client.updateCalls)++ content, err := ReadInsideFiler(ctx, client, DirectoryEtcSeaweedFS, "a.json")+ require.NoError(t, err)+ assert.Equal(t, []byte("v2"), content)+}++// A legacy entry holding identical bytes but no Md5 still gets one write to+// stamp the hash that IF_ETAG_MATCH conditional writes key off; later+// identical writes skip.+func TestSaveInsideFilerStampsMd5OnLegacyEntry(t *testing.T) {+ ctx := context.Background()+ client := newFakeFilerConfClient()+ client.entries[client.key(DirectoryEtcSeaweedFS, "b.json")] = &filer_pb.Entry{+ Name: "b.json",+ Content: []byte("v1"),+ Attributes: &filer_pb.FuseAttributes{},+ }++ require.NoError(t, SaveInsideFiler(ctx, client, DirectoryEtcSeaweedFS, "b.json", []byte("v1")))+ assert.Equal(t, 1, client.updateCalls)++ require.NoError(t, SaveInsideFiler(ctx, client, DirectoryEtcSeaweedFS, "b.json", []byte("v1")))+ assert.Equal(t, 1, client.updateCalls)+}