SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
filer: make a retried ObjectTransaction replay its recorded outcome (#11788)
Chris Lu6da432b8bff1
diff --git a/weed/admin/dash/admin_server.go b/weed/admin/dash/admin_server.goindex d4485bfcf6..c77ab3c391 100644--- a/weed/admin/dash/admin_server.go+++ b/weed/admin/dash/admin_server.go@@ -14,6 +14,7 @@ import ("sync""time"+ "github.com/google/uuid""github.com/seaweedfs/seaweedfs/weed/admin/maintenance"adminplugin "github.com/seaweedfs/seaweedfs/weed/admin/plugin""github.com/seaweedfs/seaweedfs/weed/cluster"@@ -1155,6 +1156,7 @@ func (s *AdminServer) SetBucketLifecycle(bucketName string, rules []BucketLifecybucketPath := filerConfig.BucketsPath + "/" + bucketNameresp, err := client.ObjectTransaction(context.Background(), &filer_pb.ObjectTransactionRequest{LockKey: bucketPath,+ RequestId: uuid.NewString(),RouteKey: s3_constants.ObjectWriteRouteKeyPrefix + bucketPath,Mutations: []*filer_pb.ObjectMutation{bucketLifecycleMutation(filerConfig.BucketsPath, bucketName, lifecycleXML)},})diff --git a/weed/admin/dash/bucket_policy.go b/weed/admin/dash/bucket_policy.goindex 31a14f6142..ebc1b397a3 100644--- a/weed/admin/dash/bucket_policy.go+++ b/weed/admin/dash/bucket_policy.go@@ -6,6 +6,7 @@ import ("errors""fmt"+ "github.com/google/uuid""github.com/seaweedfs/seaweedfs/weed/pb/filer_pb""github.com/seaweedfs/seaweedfs/weed/s3api""github.com/seaweedfs/seaweedfs/weed/s3api/policy_engine"@@ -126,6 +127,7 @@ func (s *AdminServer) writeBucketPolicy(bucketName string, policyJSON []byte) erbucketPath := filerConfig.BucketsPath + "/" + bucketNameresp, err := client.ObjectTransaction(context.Background(), &filer_pb.ObjectTransactionRequest{LockKey: bucketPath,+ RequestId: uuid.NewString(),RouteKey: s3_constants.ObjectWriteRouteKeyPrefix + bucketPath,Mutations: []*filer_pb.ObjectMutation{bucketPolicyMutation(filerConfig.BucketsPath, bucketName, policyJSON)},})diff --git a/weed/iam/integration/oidc_provider_store.go b/weed/iam/integration/oidc_provider_store.goindex 0eaea1f4c6..76ac002d04 100644--- a/weed/iam/integration/oidc_provider_store.go+++ b/weed/iam/integration/oidc_provider_store.go@@ -14,6 +14,7 @@ import ("sync""time"+ "github.com/google/uuid""github.com/seaweedfs/seaweedfs/weed/glog""github.com/seaweedfs/seaweedfs/weed/pb""github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"@@ -563,8 +564,9 @@ func (f *FilerOIDCProviderStore) UpdateProvider(ctx context.Context, filerAddresfunc (f *FilerOIDCProviderStore) deleteProviderEntryIfUnchanged(ctx context.Context, client filer_pb.SeaweedFilerClient, entry *filer_pb.Entry) (bool, error) {fullPath := f.basePath + "/" + entry.Nameresp, err := client.ObjectTransaction(ctx, &filer_pb.ObjectTransactionRequest{- LockKey: fullPath,- RouteKey: s3_constants.ObjectWriteRouteKeyPrefix + fullPath,+ LockKey: fullPath,+ RequestId: uuid.NewString(),+ RouteKey: s3_constants.ObjectWriteRouteKeyPrefix + fullPath,Condition: &filer_pb.WriteCondition{Clauses: []*filer_pb.WriteCondition_Clause{{Kind: filer_pb.WriteCondition_IF_ENTRY_EQUAL, ExpectedEntry: entry,}}},diff --git a/weed/iam/integration/role_store.go b/weed/iam/integration/role_store.goindex 27ff79f4dc..f418b46dcb 100644--- a/weed/iam/integration/role_store.go+++ b/weed/iam/integration/role_store.go@@ -11,6 +11,7 @@ import ("sync""time"+ "github.com/google/uuid""github.com/karlseguin/ccache/v2""github.com/seaweedfs/seaweedfs/weed/glog""github.com/seaweedfs/seaweedfs/weed/iam/policy"@@ -380,8 +381,9 @@ func (f *FilerRoleStore) UpdateRole(ctx context.Context, filerAddress string, rofunc (f *FilerRoleStore) deleteRoleEntryIfUnchanged(ctx context.Context, client filer_pb.SeaweedFilerClient, entry *filer_pb.Entry) (bool, error) {fullPath := f.basePath + "/" + entry.Nameresp, err := client.ObjectTransaction(ctx, &filer_pb.ObjectTransactionRequest{- LockKey: fullPath,- RouteKey: s3_constants.ObjectWriteRouteKeyPrefix + fullPath,+ LockKey: fullPath,+ RequestId: uuid.NewString(),+ RouteKey: s3_constants.ObjectWriteRouteKeyPrefix + fullPath,Condition: &filer_pb.WriteCondition{Clauses: []*filer_pb.WriteCondition_Clause{{Kind: filer_pb.WriteCondition_IF_ENTRY_EQUAL, ExpectedEntry: entry,}}},diff --git a/weed/pb/filer.proto b/weed/pb/filer.protoindex bfa1de2912..0abe60c556 100644--- a/weed/pb/filer.proto+++ b/weed/pb/filer.proto@@ -392,6 +392,7 @@ message ObjectTransactionRequest {string condition_key = 6; // if set, evaluate the condition against this entry instead of lock_key (still locking lock_key)string route_key = 7; // ring key identifying the owner filer; a non-owner forwards the whole transaction to itbool is_moved = 8; // set on a forwarded transaction so the receiver applies it locally instead of forwarding again+ string request_id = 9; // idempotency token: the owner records the outcome and replays it on a retried request instead of re-applyingstring condition_alt_key = 10; // if the condition's entry is absent, evaluate it against this entry instead (e.g. null slot, else latest version)}diff --git a/weed/pb/filer_pb/filer.pb.go b/weed/pb/filer_pb/filer.pb.goindex 1d82746e26..0c9190f81f 100644--- a/weed/pb/filer_pb/filer.pb.go+++ b/weed/pb/filer_pb/filer.pb.go@@ -1,7 +1,7 @@// Code generated by protoc-gen-go. DO NOT EDIT.// versions:// protoc-gen-go v1.36.6-// protoc v3.21.12+// protoc v7.35.0// source: filer.protopackage filer_pb@@ -1814,6 +1814,7 @@ type ObjectTransactionRequest struct {ConditionKey string `protobuf:"bytes,6,opt,name=condition_key,json=conditionKey,proto3" json:"condition_key,omitempty"` // if set, evaluate the condition against this entry instead of lock_key (still locking lock_key)RouteKey string `protobuf:"bytes,7,opt,name=route_key,json=routeKey,proto3" json:"route_key,omitempty"` // ring key identifying the owner filer; a non-owner forwards the whole transaction to itIsMoved bool `protobuf:"varint,8,opt,name=is_moved,json=isMoved,proto3" json:"is_moved,omitempty"` // set on a forwarded transaction so the receiver applies it locally instead of forwarding again+ RequestId string `protobuf:"bytes,9,opt,name=request_id,json=requestId,proto3" json:"request_id,omitempty"` // idempotency token: the owner records the outcome and replays it on a retried request instead of re-applyingConditionAltKey string `protobuf:"bytes,10,opt,name=condition_alt_key,json=conditionAltKey,proto3" json:"condition_alt_key,omitempty"` // if the condition's entry is absent, evaluate it against this entry instead (e.g. null slot, else latest version)unknownFields protoimpl.UnknownFieldssizeCache protoimpl.SizeCache@@ -1905,6 +1906,13 @@ func (x *ObjectTransactionRequest) GetIsMoved() bool {return false}+func (x *ObjectTransactionRequest) GetRequestId() string {+ if x != nil {+ return x.RequestId+ }+ return ""+}+func (x *ObjectTransactionRequest) GetConditionAltKey() string {if x != nil {return x.ConditionAltKey@@ -7441,7 +7449,7 @@ const file_filer_proto_rawDesc = "" +"\fexclude_name\x18\t \x01(\tR\vexcludeName\x1a?\n" +"\x11CopyExtendedEntry\x12\x10\n" +"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +- "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\x81\x03\n" ++ "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\xa0\x03\n" +"\x18ObjectTransactionRequest\x12\x19\n" +"\block_key\x18\x01 \x01(\tR\alockKey\x126\n" +"\tcondition\x18\x02 \x01(\v2\x18.filer_pb.WriteConditionR\tcondition\x126\n" +@@ -7452,7 +7460,9 @@ const file_filer_proto_rawDesc = "" +"signatures\x12#\n" +"\rcondition_key\x18\x06 \x01(\tR\fconditionKey\x12\x1b\n" +"\troute_key\x18\a \x01(\tR\brouteKey\x12\x19\n" +- "\bis_moved\x18\b \x01(\bR\aisMoved\x12*\n" ++ "\bis_moved\x18\b \x01(\bR\aisMoved\x12\x1d\n" ++ "\n" ++ "request_id\x18\t \x01(\tR\trequestId\x12*\n" +"\x11condition_alt_key\x18\n" +" \x01(\tR\x0fconditionAltKey\"f\n" +"\x19ObjectTransactionResponse\x12\x14\n" +diff --git a/weed/pb/filer_pb/filer_vtproto.pb.go b/weed/pb/filer_pb/filer_vtproto.pb.goindex e11dd00454..cac2d01f9a 100644--- a/weed/pb/filer_pb/filer_vtproto.pb.go+++ b/weed/pb/filer_pb/filer_vtproto.pb.go@@ -1574,6 +1574,13 @@ func (m *ObjectTransactionRequest) MarshalToSizedBufferVT(dAtA []byte) (int, erri--dAtA[i] = 0x52}+ if len(m.RequestId) > 0 {+ i -= len(m.RequestId)+ copy(dAtA[i:], m.RequestId)+ i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.RequestId)))+ i--+ dAtA[i] = 0x4a+ }if m.IsMoved {i--if m.IsMoved {@@ -7195,6 +7202,10 @@ func (m *ObjectTransactionRequest) SizeVT() (n int) {if m.IsMoved {n += 2}+ l = len(m.RequestId)+ if l > 0 {+ n += 1 + l + protohelpers.SizeOfVarint(uint64(l))+ }l = len(m.ConditionAltKey)if l > 0 {n += 1 + l + protohelpers.SizeOfVarint(uint64(l))@@ -13852,6 +13863,38 @@ func (m *ObjectTransactionRequest) UnmarshalVT(dAtA []byte) error {}}m.IsMoved = bool(v != 0)+ case 9:+ if wireType != 2 {+ return fmt.Errorf("proto: wrong wireType = %d for field RequestId", wireType)+ }+ var stringLen uint64+ for shift := uint(0); ; shift += 7 {+ if shift >= 64 {+ return protohelpers.ErrIntOverflow+ }+ if iNdEx >= l {+ return io.ErrUnexpectedEOF+ }+ b := dAtA[iNdEx]+ iNdEx+++ stringLen |= uint64(b&0x7F) << shift+ if b < 0x80 {+ break+ }+ }+ intStringLen := int(stringLen)+ if intStringLen < 0 {+ return protohelpers.ErrInvalidLength+ }+ postIndex := iNdEx + intStringLen+ if postIndex < 0 {+ return protohelpers.ErrInvalidLength+ }+ if postIndex > l {+ return io.ErrUnexpectedEOF+ }+ m.RequestId = string(dAtA[iNdEx:postIndex])+ iNdEx = postIndexcase 10:if wireType != 2 {return fmt.Errorf("proto: wrong wireType = %d for field ConditionAltKey", wireType)diff --git a/weed/s3api/s3api_object_routed_write.go b/weed/s3api/s3api_object_routed_write.goindex 7a323f1f55..5e315aa873 100644--- a/weed/s3api/s3api_object_routed_write.go+++ b/weed/s3api/s3api_object_routed_write.go@@ -9,6 +9,7 @@ import ("strings""time"+ "github.com/google/uuid""github.com/seaweedfs/seaweedfs/weed/pb""github.com/seaweedfs/seaweedfs/weed/pb/filer_pb""github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"@@ -157,6 +158,11 @@ func singleStrongETag(v string) (string, bool) {// the write. The reached filer forwards to the real owner by route_key, keeping its// per-path lock authoritative. Mirrors getObjectEntryRoutedByKey on the read path.func (s3a *S3ApiServer) objectTxnOnFiler(owner pb.ServerAddress, req *filer_pb.ObjectTransactionRequest) (*filer_pb.ObjectTransactionResponse, error) {+ // One id per logical transaction: failover retries reuse it so the owner+ // replays the recorded outcome instead of re-applying the mutations.+ if req.RequestId == "" {+ req.RequestId = uuid.NewString()+ }var resp *filer_pb.ObjectTransactionResponsetxn := func(client filer_pb.SeaweedFilerClient) error {var e errordiff --git a/weed/server/filer_grpc_server.go b/weed/server/filer_grpc_server.goindex a5dfdd7fcb..c338e051c5 100644--- a/weed/server/filer_grpc_server.go+++ b/weed/server/filer_grpc_server.go@@ -344,6 +344,7 @@ func (fs *FilerServer) ObjectTransaction(ctx context.Context, req *filer_pb.ObjeConditionKey: req.ConditionKey,RouteKey: req.RouteKey,IsMoved: true,+ RequestId: req.RequestId,ConditionAltKey: req.ConditionAltKey,}var resp *filer_pb.ObjectTransactionResponse@@ -367,6 +368,28 @@ func (fs *FilerServer) ObjectTransaction(ctx context.Context, req *filer_pb.ObjepathLock := fs.entryLockTable.AcquireLock("ObjectTransaction", lockPath, util.ExclusiveLock)defer fs.entryLockTable.ReleaseLock(lockPath, pathLock)+ // Under the same lock the first application committed with, a failover+ // replay of the same request replays its recorded outcome instead of+ // re-applying mutations over whatever superseded it.+ if resp, found := fs.recentObjectTxn(req.RequestId, lockPath); found {+ glog.V(2).InfofCtx(ctx, "ObjectTransaction %s: replaying recorded outcome for request %s", req.LockKey, req.RequestId)+ return resp, nil+ }++ resp := &filer_pb.ObjectTransactionResponse{}+ // Outcomes are replayed rather than re-run: a mutation error may follow a+ // partial apply, and a retry that re-runs could write the stale mutation+ // over whatever superseded it, while replaying the recorded failure at+ // least matches what the first attempt reported. The one exception is the+ // condition lookup below: it fails before any mutation runs, so the retry+ // must re-evaluate rather than replay an empty outcome.+ record := true+ defer func() {+ if record {+ fs.rememberObjectTxn(req.RequestId, lockPath, resp)+ }+ }()+if conditionIsSet(req.Condition) {// The condition is evaluated against condition_key when set (e.g. a// version entry whose WORM guards gate the delete), while the lock stays@@ -393,28 +416,102 @@ func (fs *FilerServer) ObjectTransaction(ctx context.Context, req *filer_pb.Objecurrent, conditionPath, findErr = fs.latestVersionedConditionEntry(ctx, util.FullPath(req.ConditionAltKey))}if findErr != nil && findErr != filer_pb.ErrNotFound {- return &filer_pb.ObjectTransactionResponse{}, fmt.Errorf("ObjectTransaction condition %s: %w", conditionPath, findErr)+ record = false // a lookup error decides nothing; a retry must re-run, not replay+ return nil, fmt.Errorf("ObjectTransaction condition %s: %w", conditionPath, findErr)}if findErr == filer_pb.ErrNotFound {current = nil}if !writeConditionSatisfied(req.Condition, current) {glog.V(3).InfofCtx(ctx, "ObjectTransaction %s: precondition failed", conditionPath)- return &filer_pb.ObjectTransactionResponse{- Error: "precondition failed",- ErrorCode: filer_pb.FilerError_PRECONDITION_FAILED,- }, nil+ resp.Error = "precondition failed"+ resp.ErrorCode = filer_pb.FilerError_PRECONDITION_FAILED+ return resp, nil}}for i, m := range req.Mutations {if err := fs.applyObjectMutation(ctx, m, req.IsFromOtherCluster, req.Signatures); err != nil {glog.V(2).InfofCtx(ctx, "ObjectTransaction %s mutation %d (%v): %v", lockPath, i, m.Type, err)- return &filer_pb.ObjectTransactionResponse{Error: fmt.Sprintf("mutation %d: %v", i, err)}, nil+ resp.Error = fmt.Sprintf("mutation %d: %v", i, err)+ return resp, nil}}- return &filer_pb.ObjectTransactionResponse{}, nil+ return resp, nil+}++const (+ recentObjectTxnTTL = 2 * time.Minute+ recentObjectTxnSweepCap = 10000+ recentObjectTxnSweepPeriod = 30 * time.Second+)++type recentObjectTxn struct {+ resp *filer_pb.ObjectTransactionResponse+ expiresAt time.Time+}++// recentObjectTxnKey scopes an idempotency token to the transaction's lock+// path, so a token reused on a different object cannot replay this outcome.+func recentObjectTxnKey(requestID string, lockPath util.FullPath) string {+ return requestID + "\x00" + string(lockPath)+}++// recentObjectTxn returns the recorded outcome of an already-applied request,+// so a failover replay of the same ObjectTransaction does not re-apply it.+func (fs *FilerServer) recentObjectTxn(requestID string, lockPath util.FullPath) (*filer_pb.ObjectTransactionResponse, bool) {+ if requestID == "" {+ return nil, false+ }+ fs.recentObjectTxnsMu.Lock()+ defer fs.recentObjectTxnsMu.Unlock()++ key := recentObjectTxnKey(requestID, lockPath)+ txn, found := fs.recentObjectTxns[key]+ if !found {+ return nil, false+ }+ if time.Now().After(txn.expiresAt) {+ delete(fs.recentObjectTxns, key)+ return nil, false+ }+ return txn.resp, true+}++func (fs *FilerServer) rememberObjectTxn(requestID string, lockPath util.FullPath, resp *filer_pb.ObjectTransactionResponse) {+ if requestID == "" || resp == nil {+ return+ }+ fs.recentObjectTxnsMu.Lock()+ defer fs.recentObjectTxnsMu.Unlock()+ if fs.recentObjectTxns == nil {+ fs.recentObjectTxns = make(map[string]recentObjectTxn)+ }+ // A live record is never evicted early: a replay that misses re-applies the+ // transaction. The sweep is amortized so a hot cache does not scan on every+ // write; between sweeps the map may exceed the cap, still bounded by the+ // insert rate over the TTL.+ now := time.Now()+ if len(fs.recentObjectTxns) >= recentObjectTxnSweepCap {+ if now.After(fs.recentObjectTxnsSweepAt) {+ fs.recentObjectTxnsSweepAt = now.Add(recentObjectTxnSweepPeriod)+ for id, txn := range fs.recentObjectTxns {+ if now.After(txn.expiresAt) {+ delete(fs.recentObjectTxns, id)+ }+ }+ }+ if len(fs.recentObjectTxns) >= recentObjectTxnSweepCap {+ // Full of unexpired records: drop the new one rather than evict+ // an outcome a replay may still need.+ return+ }+ }+ fs.recentObjectTxns[recentObjectTxnKey(requestID, lockPath)] = recentObjectTxn{+ resp: resp,+ expiresAt: now.Add(recentObjectTxnTTL),+ }}// latestVersionedConditionEntry resolves the object a conditional write shoulddiff --git a/weed/server/filer_grpc_server_object_txn_test.go b/weed/server/filer_grpc_server_object_txn_test.goindex 51b1442fb9..c73badd7a3 100644--- a/weed/server/filer_grpc_server_object_txn_test.go+++ b/weed/server/filer_grpc_server_object_txn_test.go@@ -2,6 +2,7 @@ package weed_serverimport ("context"+ "fmt""net""os""strconv"@@ -855,6 +856,139 @@ func TestObjectTransactionForwardsToOwner(t *testing.T) {}}+// A failover replay carries the same request_id. Under the path lock the owner+// replays the recorded outcome instead of re-applying the mutations over a+// newer entry.+func TestObjectTransactionRequestIdDedupesReplay(t *testing.T) {+ now := time.Unix(1700000000, 0)+ fs, store := newTxnTestServer(t, nil)++ put := func(etag string) *filer_pb.ObjectMutation {+ return &filer_pb.ObjectMutation{+ Type: filer_pb.ObjectMutation_PUT,+ Directory: "/buckets/bkt",+ Entry: &filer_pb.Entry{+ Name: "obj",+ Attributes: &filer_pb.FuseAttributes{Mtime: now.Unix(), FileMode: 0644, Inode: 1},+ Extended: map[string][]byte{"etag": []byte(etag)},+ },+ }+ }+ txn := func(requestID, etag string) *filer_pb.ObjectTransactionRequest {+ return &filer_pb.ObjectTransactionRequest{+ LockKey: "/buckets/bkt/obj",+ RequestId: requestID,+ Mutations: []*filer_pb.ObjectMutation{put(etag)},+ }+ }++ respA, err := fs.ObjectTransaction(context.Background(), txn("req-a", "A"))+ if err != nil || respA.Error != "" {+ t.Fatalf("first put: err=%v resp=%q", err, respA.Error)+ }++ // a newer writer lands the same path+ respB, err := fs.ObjectTransaction(context.Background(), txn("req-b", "B"))+ if err != nil || respB.Error != "" {+ t.Fatalf("second put: err=%v resp=%q", err, respB.Error)+ }++ // the replayed request replays A's outcome; the stored entry stays B+ respReplay, err := fs.ObjectTransaction(context.Background(), txn("req-a", "A"))+ if err != nil || respReplay.Error != "" {+ t.Fatalf("replayed put: err=%v resp=%q", err, respReplay.Error)+ }+ entry := store.entries["/buckets/bkt/obj"]+ if entry == nil {+ t.Fatal("entry missing after replay")+ }+ if got := string(entry.Extended["etag"]); got != "B" {+ t.Fatalf("replayed transaction re-applied the stale entry: etag = %q, want B", got)+ }+}++// A recorded application-level failure replays to the retried request even+// after the world changes enough for the condition to pass, and its mutation+// still does not run.+func TestObjectTransactionRequestIdReplaysRecordedError(t *testing.T) {+ now := time.Unix(1700000000, 0)+ fs, store := newTxnTestServer(t, nil)++ req := &filer_pb.ObjectTransactionRequest{+ LockKey: "/buckets/bkt/obj",+ RequestId: "req-fail",+ Condition: &filer_pb.WriteCondition{+ Clauses: []*filer_pb.WriteCondition_Clause{{Kind: filer_pb.WriteCondition_IF_EXISTS}},+ },+ Mutations: []*filer_pb.ObjectMutation{{+ Type: filer_pb.ObjectMutation_PUT,+ Directory: "/buckets/bkt",+ Entry: &filer_pb.Entry{+ Name: "obj",+ Attributes: &filer_pb.FuseAttributes{Mtime: now.Unix(), FileMode: 0644, Inode: 1},+ Extended: map[string][]byte{"etag": []byte("stale")},+ },+ }},+ }++ resp, err := fs.ObjectTransaction(context.Background(), req)+ if err != nil || resp.ErrorCode != filer_pb.FilerError_PRECONDITION_FAILED {+ t.Fatalf("precondition: err=%v code=%v", err, resp.GetErrorCode())+ }++ // another writer lands the object the precondition wanted+ other := &filer_pb.ObjectTransactionRequest{+ LockKey: "/buckets/bkt/obj",+ RequestId: "req-other",+ Mutations: []*filer_pb.ObjectMutation{{+ Type: filer_pb.ObjectMutation_PUT,+ Directory: "/buckets/bkt",+ Entry: &filer_pb.Entry{+ Name: "obj",+ Attributes: &filer_pb.FuseAttributes{Mtime: now.Unix(), FileMode: 0644, Inode: 2},+ Extended: map[string][]byte{"etag": []byte("live")},+ },+ }},+ }+ respOther, err := fs.ObjectTransaction(context.Background(), other)+ if err != nil || respOther.Error != "" {+ t.Fatalf("other put: err=%v resp=%q", err, respOther.Error)+ }++ // the retry replays the recorded failure instead of re-evaluating and+ // overwriting the live entry+ resp2, err := fs.ObjectTransaction(context.Background(), req)+ if err != nil || resp2.ErrorCode != filer_pb.FilerError_PRECONDITION_FAILED {+ t.Fatalf("replayed precondition: err=%v code=%v", err, resp2.GetErrorCode())+ }+ if got := string(store.entries["/buckets/bkt/obj"].Extended["etag"]); got != "live" {+ t.Fatalf("replayed transaction overwrote the live entry: etag = %q", got)+ }+}++// At capacity the cache drops new recordings rather than evicting a live+// outcome a replay may still need.+func TestObjectTransactionReplayCacheDropsNewAtCap(t *testing.T) {+ fs, _ := newTxnTestServer(t, nil)+ path := util.FullPath("/buckets/bkt/obj")+ resp := &filer_pb.ObjectTransactionResponse{}++ for i := 0; i < recentObjectTxnSweepCap; i++ {+ fs.rememberObjectTxn(fmt.Sprintf("req-%d", i), path, resp)+ }+ fs.rememberObjectTxn("req-overflow", path, resp)++ if _, found := fs.recentObjectTxn("req-0", path); !found {+ t.Fatal("a live record was evicted to admit a new one")+ }+ if _, found := fs.recentObjectTxn("req-overflow", path); found {+ t.Fatal("overflow record was recorded instead of dropped")+ }+ if len(fs.recentObjectTxns) > recentObjectTxnSweepCap {+ t.Fatalf("cache exceeded cap: %d", len(fs.recentObjectTxns))+ }+}+// A suspended-versioning conditional write falls back to the latest version// under the transaction lock: a real latest satisfies existence checks, and a// delete marker reads as absent.diff --git a/weed/server/filer_server.go b/weed/server/filer_server.goindex 58748422c4..41c7256172 100644--- a/weed/server/filer_server.go+++ b/weed/server/filer_server.go@@ -142,6 +142,13 @@ type FilerServer struct {recentCopyRequestsMu sync.MutexrecentCopyRequests map[string]recentCopyRequest+ // outcomes of recently applied object transactions keyed by request_id, so+ // a failover replay of the same transaction replays its outcome instead of+ // re-applying its mutations over a newer entry+ recentObjectTxnsMu sync.Mutex+ recentObjectTxns map[string]recentObjectTxn+ recentObjectTxnsSweepAt time.Time+// credential manager for IAM operationsCredentialManager *credential.CredentialManager@@ -236,6 +243,7 @@ func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption)subscribers: make(map[int32]*metadataSubscriber),inFlightDataLimitCond: sync.NewCond(new(sync.Mutex)),recentCopyRequests: make(map[string]recentCopyRequest),+ recentObjectTxns: make(map[string]recentObjectTxn),CredentialManager: option.CredentialManager,entryLockTable: util.NewLockTable[util.FullPath](),posixLocks: posixlock.NewManager(),