SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
filer: conditional UpdateEntry with a chunk-set write condition (#10382)
Chris Lucdb60069a643
diff --git a/other/java/client/src/main/proto/filer.proto b/other/java/client/src/main/proto/filer.protoindex db62bff65d..d350b117af 100644--- a/other/java/client/src/main/proto/filer.proto+++ b/other/java/client/src/main/proto/filer.proto@@ -265,6 +265,7 @@ message WriteCondition {IF_MODIFIED_SINCE = 6; // fail if present and mtime <= unix_timeIF_EXTENDED_NOT_EQUAL = 7; // fail if present and extended[ext_key] == ext_valueIF_EXTENDED_TIME_ELAPSED = 8; // fail if present and extended[ext_key] (unix seconds) is in the future+ IF_CHUNKS_EQUAL = 9; // fail unless the stored chunk fid multiset equals fids (absent entry = no chunks)}// Clause is one primitive comparison. IF_ETAG_MATCH holds when the current// entry's ETag equals any value in etags; IF_ETAG_NOT_MATCH holds when it@@ -278,6 +279,11 @@ message WriteCondition {// clock). The caller composes these and, for governance-bypass, simply omits// the retention clause when the bypass is authorized — the filer makes no// authorization decision.+ //+ // IF_CHUNKS_EQUAL guards a chunk-preserving read-modify-write: the stored+ // chunk fid set must still equal what the caller read, so a stale write+ // cannot resurrect needles that a concurrent update already diffed away+ // and queued for deletion. An empty fids list expects no chunks.message Clause {Kind kind = 1;repeated string etags = 2; // ETag set for IF_ETAG_* kinds@@ -287,6 +293,7 @@ message WriteCondition {string ext_value = 6; // blocking value for IF_EXTENDED_NOT_EQUALstring gate_key = 7; // IF_EXTENDED_TIME_ELAPSED: only enforce when extended[gate_key] == gate_valuestring gate_value = 8; // gate value (e.g. retention mode COMPLIANCE for governance bypass)+ repeated string fids = 9; // chunk fid strings for IF_CHUNKS_EQUAL}repeated Clause clauses = 1; // all must hold (logical AND)}@@ -447,6 +454,10 @@ message UpdateEntryRequest {bool is_from_other_cluster = 3;repeated int32 signatures = 4;map<string, bytes> expected_extended = 5;+ // Optional precondition evaluated against the current entry atomically with+ // the write, under the filer's per-path lock. The caller must route the+ // key's writes to this entry's owner filer for the check to be authoritative.+ WriteCondition condition = 6;}message UpdateEntryResponse {SubscribeMetadataResponse metadata_event = 1;diff --git a/weed/pb/filer.proto b/weed/pb/filer.protoindex db62bff65d..d350b117af 100644--- a/weed/pb/filer.proto+++ b/weed/pb/filer.proto@@ -265,6 +265,7 @@ message WriteCondition {IF_MODIFIED_SINCE = 6; // fail if present and mtime <= unix_timeIF_EXTENDED_NOT_EQUAL = 7; // fail if present and extended[ext_key] == ext_valueIF_EXTENDED_TIME_ELAPSED = 8; // fail if present and extended[ext_key] (unix seconds) is in the future+ IF_CHUNKS_EQUAL = 9; // fail unless the stored chunk fid multiset equals fids (absent entry = no chunks)}// Clause is one primitive comparison. IF_ETAG_MATCH holds when the current// entry's ETag equals any value in etags; IF_ETAG_NOT_MATCH holds when it@@ -278,6 +279,11 @@ message WriteCondition {// clock). The caller composes these and, for governance-bypass, simply omits// the retention clause when the bypass is authorized — the filer makes no// authorization decision.+ //+ // IF_CHUNKS_EQUAL guards a chunk-preserving read-modify-write: the stored+ // chunk fid set must still equal what the caller read, so a stale write+ // cannot resurrect needles that a concurrent update already diffed away+ // and queued for deletion. An empty fids list expects no chunks.message Clause {Kind kind = 1;repeated string etags = 2; // ETag set for IF_ETAG_* kinds@@ -287,6 +293,7 @@ message WriteCondition {string ext_value = 6; // blocking value for IF_EXTENDED_NOT_EQUALstring gate_key = 7; // IF_EXTENDED_TIME_ELAPSED: only enforce when extended[gate_key] == gate_valuestring gate_value = 8; // gate value (e.g. retention mode COMPLIANCE for governance bypass)+ repeated string fids = 9; // chunk fid strings for IF_CHUNKS_EQUAL}repeated Clause clauses = 1; // all must hold (logical AND)}@@ -447,6 +454,10 @@ message UpdateEntryRequest {bool is_from_other_cluster = 3;repeated int32 signatures = 4;map<string, bytes> expected_extended = 5;+ // Optional precondition evaluated against the current entry atomically with+ // the write, under the filer's per-path lock. The caller must route the+ // key's writes to this entry's owner filer for the check to be authoritative.+ WriteCondition condition = 6;}message UpdateEntryResponse {SubscribeMetadataResponse metadata_event = 1;diff --git a/weed/pb/filer_pb/filer.pb.go b/weed/pb/filer_pb/filer.pb.goindex 0013fb0ff3..059dde3f3f 100644--- a/weed/pb/filer_pb/filer.pb.go+++ b/weed/pb/filer_pb/filer.pb.go@@ -206,6 +206,7 @@ const (WriteCondition_IF_MODIFIED_SINCE WriteCondition_Kind = 6 // fail if present and mtime <= unix_timeWriteCondition_IF_EXTENDED_NOT_EQUAL WriteCondition_Kind = 7 // fail if present and extended[ext_key] == ext_valueWriteCondition_IF_EXTENDED_TIME_ELAPSED WriteCondition_Kind = 8 // fail if present and extended[ext_key] (unix seconds) is in the future+ WriteCondition_IF_CHUNKS_EQUAL WriteCondition_Kind = 9 // fail unless the stored chunk fid multiset equals fids (absent entry = no chunks))// Enum value maps for WriteCondition_Kind.@@ -220,6 +221,7 @@ var (6: "IF_MODIFIED_SINCE",7: "IF_EXTENDED_NOT_EQUAL",8: "IF_EXTENDED_TIME_ELAPSED",+ 9: "IF_CHUNKS_EQUAL",}WriteCondition_Kind_value = map[string]int32{"NONE": 0,@@ -231,6 +233,7 @@ var ("IF_MODIFIED_SINCE": 6,"IF_EXTENDED_NOT_EQUAL": 7,"IF_EXTENDED_TIME_ELAPSED": 8,+ "IF_CHUNKS_EQUAL": 9,})@@ -2310,8 +2313,12 @@ type UpdateEntryRequest struct {IsFromOtherCluster bool `protobuf:"varint,3,opt,name=is_from_other_cluster,json=isFromOtherCluster,proto3" json:"is_from_other_cluster,omitempty"`Signatures []int32 `protobuf:"varint,4,rep,packed,name=signatures,proto3" json:"signatures,omitempty"`ExpectedExtended map[string][]byte `protobuf:"bytes,5,rep,name=expected_extended,json=expectedExtended,proto3" json:"expected_extended,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`- unknownFields protoimpl.UnknownFields- sizeCache protoimpl.SizeCache+ // Optional precondition evaluated against the current entry atomically with+ // the write, under the filer's per-path lock. The caller must route the+ // key's writes to this entry's owner filer for the check to be authoritative.+ Condition *WriteCondition `protobuf:"bytes,6,opt,name=condition,proto3" json:"condition,omitempty"`+ unknownFields protoimpl.UnknownFields+ sizeCache protoimpl.SizeCache}func (x *UpdateEntryRequest) Reset() {@@ -2379,6 +2386,13 @@ func (x *UpdateEntryRequest) GetExpectedExtended() map[string][]byte {return nil}+func (x *UpdateEntryRequest) GetCondition() *WriteCondition {+ if x != nil {+ return x.Condition+ }+ return nil+}+type UpdateEntryResponse struct {state protoimpl.MessageState `protogen:"open.v1"`MetadataEvent *SubscribeMetadataResponse `protobuf:"bytes,1,opt,name=metadata_event,json=metadataEvent,proto3" json:"metadata_event,omitempty"`@@ -6491,6 +6505,11 @@ func (x *MountInfo) GetDataCenter() string {// clock). The caller composes these and, for governance-bypass, simply omits// the retention clause when the bypass is authorized — the filer makes no// authorization decision.+//+// IF_CHUNKS_EQUAL guards a chunk-preserving read-modify-write: the stored+// chunk fid set must still equal what the caller read, so a stale write+// cannot resurrect needles that a concurrent update already diffed away+// and queued for deletion. An empty fids list expects no chunks.type WriteCondition_Clause struct {state protoimpl.MessageState `protogen:"open.v1"`Kind WriteCondition_Kind `protobuf:"varint,1,opt,name=kind,proto3,enum=filer_pb.WriteCondition_Kind" json:"kind,omitempty"`@@ -6501,6 +6520,7 @@ type WriteCondition_Clause struct {ExtValue string `protobuf:"bytes,6,opt,name=ext_value,json=extValue,proto3" json:"ext_value,omitempty"` // blocking value for IF_EXTENDED_NOT_EQUALGateKey string `protobuf:"bytes,7,opt,name=gate_key,json=gateKey,proto3" json:"gate_key,omitempty"` // IF_EXTENDED_TIME_ELAPSED: only enforce when extended[gate_key] == gate_valueGateValue string `protobuf:"bytes,8,opt,name=gate_value,json=gateValue,proto3" json:"gate_value,omitempty"` // gate value (e.g. retention mode COMPLIANCE for governance bypass)+ Fids []string `protobuf:"bytes,9,rep,name=fids,proto3" json:"fids,omitempty"` // chunk fid strings for IF_CHUNKS_EQUALunknownFields protoimpl.UnknownFieldssizeCache protoimpl.SizeCache}@@ -6591,6 +6611,13 @@ func (x *WriteCondition_Clause) GetGateValue() string {return ""}+func (x *WriteCondition_Clause) GetFids() []string {+ if x != nil {+ return x.Fids+ }+ return nil+}+// if found, send the exact address// if not found, send the full list of existing brokerstype LocateBrokerResponse_Resource struct {@@ -6925,9 +6952,9 @@ const file_filer_proto_rawDesc = "" +"signatures\x18\x05 \x03(\x05R\n" +"signatures\x12=\n" +"\x1bskip_check_parent_directory\x18\x06 \x01(\bR\x18skipCheckParentDirectory\x126\n" +- "\tcondition\x18\a \x01(\v2\x18.filer_pb.WriteConditionR\tcondition\"\x93\x04\n" ++ "\tcondition\x18\a \x01(\v2\x18.filer_pb.WriteConditionR\tcondition\"\xbc\x04\n" +"\x0eWriteCondition\x129\n" +- "\aclauses\x18\x01 \x03(\v2\x1f.filer_pb.WriteCondition.ClauseR\aclauses\x1a\xfd\x01\n" ++ "\aclauses\x18\x01 \x03(\v2\x1f.filer_pb.WriteCondition.ClauseR\aclauses\x1a\x91\x02\n" +"\x06Clause\x121\n" +"\x04kind\x18\x01 \x01(\x0e2\x1d.filer_pb.WriteCondition.KindR\x04kind\x12\x14\n" +"\x05etags\x18\x02 \x03(\tR\x05etags\x12\x1b\n" +@@ -6938,7 +6965,8 @@ const file_filer_proto_rawDesc = "" +"\text_value\x18\x06 \x01(\tR\bextValue\x12\x19\n" +"\bgate_key\x18\a \x01(\tR\agateKey\x12\x1d\n" +"\n" +- "gate_value\x18\b \x01(\tR\tgateValue\"\xc5\x01\n" ++ "gate_value\x18\b \x01(\tR\tgateValue\x12\x12\n" ++ "\x04fids\x18\t \x03(\tR\x04fids\"\xda\x01\n" +"\x04Kind\x12\b\n" +"\x04NONE\x10\x00\x12\x11\n" +"\rIF_NOT_EXISTS\x10\x01\x12\r\n" +@@ -6948,7 +6976,8 @@ const file_filer_proto_rawDesc = "" +"\x13IF_UNMODIFIED_SINCE\x10\x05\x12\x15\n" +"\x11IF_MODIFIED_SINCE\x10\x06\x12\x19\n" +"\x15IF_EXTENDED_NOT_EQUAL\x10\a\x12\x1c\n" +- "\x18IF_EXTENDED_TIME_ELAPSED\x10\b\"\xa2\x05\n" ++ "\x18IF_EXTENDED_TIME_ELAPSED\x10\b\x12\x13\n" ++ "\x0fIF_CHUNKS_EQUAL\x10\t\"\xa2\x05\n" +"\x0eObjectMutation\x121\n" +"\x04type\x18\x01 \x01(\x0e2\x1d.filer_pb.ObjectMutation.TypeR\x04type\x12\x1c\n" +"\tdirectory\x18\x02 \x01(\tR\tdirectory\x12\x12\n" +@@ -7034,7 +7063,7 @@ const file_filer_proto_rawDesc = "" +"\x05error\x18\x01 \x01(\tR\x05error\x12J\n" +"\x0emetadata_event\x18\x02 \x01(\v2#.filer_pb.SubscribeMetadataResponseR\rmetadataEvent\x123\n" +"\n" +- "error_code\x18\x03 \x01(\x0e2\x14.filer_pb.FilerErrorR\terrorCode\"\xd2\x02\n" ++ "error_code\x18\x03 \x01(\x0e2\x14.filer_pb.FilerErrorR\terrorCode\"\x8a\x03\n" +"\x12UpdateEntryRequest\x12\x1c\n" +"\tdirectory\x18\x01 \x01(\tR\tdirectory\x12%\n" +"\x05entry\x18\x02 \x01(\v2\x0f.filer_pb.EntryR\x05entry\x121\n" +@@ -7042,7 +7071,8 @@ const file_filer_proto_rawDesc = "" +"\n" +"signatures\x18\x04 \x03(\x05R\n" +"signatures\x12_\n" +- "\x11expected_extended\x18\x05 \x03(\v22.filer_pb.UpdateEntryRequest.ExpectedExtendedEntryR\x10expectedExtended\x1aC\n" ++ "\x11expected_extended\x18\x05 \x03(\v22.filer_pb.UpdateEntryRequest.ExpectedExtendedEntryR\x10expectedExtended\x126\n" ++ "\tcondition\x18\x06 \x01(\v2\x18.filer_pb.WriteConditionR\tcondition\x1aC\n" +"\x15ExpectedExtendedEntry\x12\x10\n" +"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +"\x05value\x18\x02 \x01(\fR\x05value:\x028\x01\"a\n" +@@ -7620,110 +7650,111 @@ var file_filer_proto_depIdxs = []int32{1, // 31: filer_pb.CreateEntryResponse.error_code:type_name -> filer_pb.FilerError10, // 32: filer_pb.UpdateEntryRequest.entry:type_name -> filer_pb.Entry100, // 33: filer_pb.UpdateEntryRequest.expected_extended:type_name -> filer_pb.UpdateEntryRequest.ExpectedExtendedEntry- 59, // 34: filer_pb.UpdateEntryResponse.metadata_event:type_name -> filer_pb.SubscribeMetadataResponse- 13, // 35: filer_pb.AppendToEntryRequest.chunks:type_name -> filer_pb.FileChunk- 59, // 36: filer_pb.DeleteEntryResponse.metadata_event:type_name -> filer_pb.SubscribeMetadataResponse- 12, // 37: filer_pb.StreamRenameEntryResponse.event_notification:type_name -> filer_pb.EventNotification- 45, // 38: filer_pb.AssignVolumeResponse.location:type_name -> filer_pb.Location- 45, // 39: filer_pb.AssignVolumeResponse.replicas:type_name -> filer_pb.Location- 45, // 40: filer_pb.Locations.locations:type_name -> filer_pb.Location- 101, // 41: filer_pb.LookupVolumeResponse.locations_map:type_name -> filer_pb.LookupVolumeResponse.LocationsMapEntry- 47, // 42: filer_pb.CollectionListResponse.collections:type_name -> filer_pb.Collection- 12, // 43: filer_pb.SubscribeMetadataResponse.event_notification:type_name -> filer_pb.EventNotification- 59, // 44: filer_pb.SubscribeMetadataResponse.events:type_name -> filer_pb.SubscribeMetadataResponse- 63, // 45: filer_pb.SubscribeMetadataResponse.log_file_refs:type_name -> filer_pb.LogFileChunkRef- 62, // 46: filer_pb.ListMetadataSubscribersResponse.subscribers:type_name -> filer_pb.MetadataSubscriber- 13, // 47: filer_pb.LogFileChunkRef.chunks:type_name -> filer_pb.FileChunk- 10, // 48: filer_pb.TraverseBfsMetadataResponse.entry:type_name -> filer_pb.Entry- 102, // 49: filer_pb.LocateBrokerResponse.resources:type_name -> filer_pb.LocateBrokerResponse.Resource- 103, // 50: filer_pb.FilerConf.locations:type_name -> filer_pb.FilerConf.PathConf- 10, // 51: filer_pb.CacheRemoteObjectToLocalClusterResponse.entry:type_name -> filer_pb.Entry- 59, // 52: filer_pb.CacheRemoteObjectToLocalClusterResponse.metadata_event:type_name -> filer_pb.SubscribeMetadataResponse- 84, // 53: filer_pb.TransferLocksRequest.locks:type_name -> filer_pb.Lock- 17, // 54: filer_pb.StreamMutateEntryRequest.create_request:type_name -> filer_pb.CreateEntryRequest- 29, // 55: filer_pb.StreamMutateEntryRequest.update_request:type_name -> filer_pb.UpdateEntryRequest- 35, // 56: filer_pb.StreamMutateEntryRequest.delete_request:type_name -> filer_pb.DeleteEntryRequest- 39, // 57: filer_pb.StreamMutateEntryRequest.rename_request:type_name -> filer_pb.StreamRenameEntryRequest- 28, // 58: filer_pb.StreamMutateEntryResponse.create_response:type_name -> filer_pb.CreateEntryResponse- 30, // 59: filer_pb.StreamMutateEntryResponse.update_response:type_name -> filer_pb.UpdateEntryResponse- 36, // 60: filer_pb.StreamMutateEntryResponse.delete_response:type_name -> filer_pb.DeleteEntryResponse- 40, // 61: filer_pb.StreamMutateEntryResponse.rename_response:type_name -> filer_pb.StreamRenameEntryResponse- 95, // 62: filer_pb.MountListResponse.mounts:type_name -> filer_pb.MountInfo- 3, // 63: filer_pb.WriteCondition.Clause.kind:type_name -> filer_pb.WriteCondition.Kind- 44, // 64: filer_pb.LookupVolumeResponse.LocationsMapEntry.value:type_name -> filer_pb.Locations- 5, // 65: filer_pb.SeaweedFiler.LookupDirectoryEntry:input_type -> filer_pb.LookupDirectoryEntryRequest- 7, // 66: filer_pb.SeaweedFiler.ListEntries:input_type -> filer_pb.ListEntriesRequest- 17, // 67: filer_pb.SeaweedFiler.CreateEntry:input_type -> filer_pb.CreateEntryRequest- 29, // 68: filer_pb.SeaweedFiler.UpdateEntry:input_type -> filer_pb.UpdateEntryRequest- 31, // 69: filer_pb.SeaweedFiler.TouchAccessTime:input_type -> filer_pb.TouchAccessTimeRequest- 33, // 70: filer_pb.SeaweedFiler.AppendToEntry:input_type -> filer_pb.AppendToEntryRequest- 35, // 71: filer_pb.SeaweedFiler.DeleteEntry:input_type -> filer_pb.DeleteEntryRequest- 21, // 72: filer_pb.SeaweedFiler.ObjectTransaction:input_type -> filer_pb.ObjectTransactionRequest- 26, // 73: filer_pb.SeaweedFiler.ObjectTransactionBatch:input_type -> filer_pb.ObjectTransactionBatchRequest- 24, // 74: filer_pb.SeaweedFiler.PosixLock:input_type -> filer_pb.PosixLockRequest- 37, // 75: filer_pb.SeaweedFiler.AtomicRenameEntry:input_type -> filer_pb.AtomicRenameEntryRequest- 39, // 76: filer_pb.SeaweedFiler.StreamRenameEntry:input_type -> filer_pb.StreamRenameEntryRequest- 89, // 77: filer_pb.SeaweedFiler.StreamMutateEntry:input_type -> filer_pb.StreamMutateEntryRequest- 41, // 78: filer_pb.SeaweedFiler.AssignVolume:input_type -> filer_pb.AssignVolumeRequest- 43, // 79: filer_pb.SeaweedFiler.LookupVolume:input_type -> filer_pb.LookupVolumeRequest- 48, // 80: filer_pb.SeaweedFiler.CollectionList:input_type -> filer_pb.CollectionListRequest- 50, // 81: filer_pb.SeaweedFiler.DeleteCollection:input_type -> filer_pb.DeleteCollectionRequest- 52, // 82: filer_pb.SeaweedFiler.Statistics:input_type -> filer_pb.StatisticsRequest- 54, // 83: filer_pb.SeaweedFiler.Ping:input_type -> filer_pb.PingRequest- 56, // 84: filer_pb.SeaweedFiler.GetFilerConfiguration:input_type -> filer_pb.GetFilerConfigurationRequest- 64, // 85: filer_pb.SeaweedFiler.TraverseBfsMetadata:input_type -> filer_pb.TraverseBfsMetadataRequest- 58, // 86: filer_pb.SeaweedFiler.SubscribeMetadata:input_type -> filer_pb.SubscribeMetadataRequest- 58, // 87: filer_pb.SeaweedFiler.SubscribeLocalMetadata:input_type -> filer_pb.SubscribeMetadataRequest- 60, // 88: filer_pb.SeaweedFiler.ListMetadataSubscribers:input_type -> filer_pb.ListMetadataSubscribersRequest- 71, // 89: filer_pb.SeaweedFiler.KvGet:input_type -> filer_pb.KvGetRequest- 73, // 90: filer_pb.SeaweedFiler.KvPut:input_type -> filer_pb.KvPutRequest- 76, // 91: filer_pb.SeaweedFiler.CacheRemoteObjectToLocalCluster:input_type -> filer_pb.CacheRemoteObjectToLocalClusterRequest- 78, // 92: filer_pb.SeaweedFiler.DistributedLock:input_type -> filer_pb.LockRequest- 80, // 93: filer_pb.SeaweedFiler.DistributedUnlock:input_type -> filer_pb.UnlockRequest- 82, // 94: filer_pb.SeaweedFiler.FindLockOwner:input_type -> filer_pb.FindLockOwnerRequest- 85, // 95: filer_pb.SeaweedFiler.TransferLocks:input_type -> filer_pb.TransferLocksRequest- 87, // 96: filer_pb.SeaweedFiler.ReplicateLock:input_type -> filer_pb.ReplicateLockRequest- 91, // 97: filer_pb.SeaweedFiler.MountRegister:input_type -> filer_pb.MountRegisterRequest- 93, // 98: filer_pb.SeaweedFiler.MountList:input_type -> filer_pb.MountListRequest- 6, // 99: filer_pb.SeaweedFiler.LookupDirectoryEntry:output_type -> filer_pb.LookupDirectoryEntryResponse- 8, // 100: filer_pb.SeaweedFiler.ListEntries:output_type -> filer_pb.ListEntriesResponse- 28, // 101: filer_pb.SeaweedFiler.CreateEntry:output_type -> filer_pb.CreateEntryResponse- 30, // 102: filer_pb.SeaweedFiler.UpdateEntry:output_type -> filer_pb.UpdateEntryResponse- 32, // 103: filer_pb.SeaweedFiler.TouchAccessTime:output_type -> filer_pb.TouchAccessTimeResponse- 34, // 104: filer_pb.SeaweedFiler.AppendToEntry:output_type -> filer_pb.AppendToEntryResponse- 36, // 105: filer_pb.SeaweedFiler.DeleteEntry:output_type -> filer_pb.DeleteEntryResponse- 22, // 106: filer_pb.SeaweedFiler.ObjectTransaction:output_type -> filer_pb.ObjectTransactionResponse- 27, // 107: filer_pb.SeaweedFiler.ObjectTransactionBatch:output_type -> filer_pb.ObjectTransactionBatchResponse- 25, // 108: filer_pb.SeaweedFiler.PosixLock:output_type -> filer_pb.PosixLockResponse- 38, // 109: filer_pb.SeaweedFiler.AtomicRenameEntry:output_type -> filer_pb.AtomicRenameEntryResponse- 40, // 110: filer_pb.SeaweedFiler.StreamRenameEntry:output_type -> filer_pb.StreamRenameEntryResponse- 90, // 111: filer_pb.SeaweedFiler.StreamMutateEntry:output_type -> filer_pb.StreamMutateEntryResponse- 42, // 112: filer_pb.SeaweedFiler.AssignVolume:output_type -> filer_pb.AssignVolumeResponse- 46, // 113: filer_pb.SeaweedFiler.LookupVolume:output_type -> filer_pb.LookupVolumeResponse- 49, // 114: filer_pb.SeaweedFiler.CollectionList:output_type -> filer_pb.CollectionListResponse- 51, // 115: filer_pb.SeaweedFiler.DeleteCollection:output_type -> filer_pb.DeleteCollectionResponse- 53, // 116: filer_pb.SeaweedFiler.Statistics:output_type -> filer_pb.StatisticsResponse- 55, // 117: filer_pb.SeaweedFiler.Ping:output_type -> filer_pb.PingResponse- 57, // 118: filer_pb.SeaweedFiler.GetFilerConfiguration:output_type -> filer_pb.GetFilerConfigurationResponse- 65, // 119: filer_pb.SeaweedFiler.TraverseBfsMetadata:output_type -> filer_pb.TraverseBfsMetadataResponse- 59, // 120: filer_pb.SeaweedFiler.SubscribeMetadata:output_type -> filer_pb.SubscribeMetadataResponse- 59, // 121: filer_pb.SeaweedFiler.SubscribeLocalMetadata:output_type -> filer_pb.SubscribeMetadataResponse- 61, // 122: filer_pb.SeaweedFiler.ListMetadataSubscribers:output_type -> filer_pb.ListMetadataSubscribersResponse- 72, // 123: filer_pb.SeaweedFiler.KvGet:output_type -> filer_pb.KvGetResponse- 74, // 124: filer_pb.SeaweedFiler.KvPut:output_type -> filer_pb.KvPutResponse- 77, // 125: filer_pb.SeaweedFiler.CacheRemoteObjectToLocalCluster:output_type -> filer_pb.CacheRemoteObjectToLocalClusterResponse- 79, // 126: filer_pb.SeaweedFiler.DistributedLock:output_type -> filer_pb.LockResponse- 81, // 127: filer_pb.SeaweedFiler.DistributedUnlock:output_type -> filer_pb.UnlockResponse- 83, // 128: filer_pb.SeaweedFiler.FindLockOwner:output_type -> filer_pb.FindLockOwnerResponse- 86, // 129: filer_pb.SeaweedFiler.TransferLocks:output_type -> filer_pb.TransferLocksResponse- 88, // 130: filer_pb.SeaweedFiler.ReplicateLock:output_type -> filer_pb.ReplicateLockResponse- 92, // 131: filer_pb.SeaweedFiler.MountRegister:output_type -> filer_pb.MountRegisterResponse- 94, // 132: filer_pb.SeaweedFiler.MountList:output_type -> filer_pb.MountListResponse- 99, // [99:133] is the sub-list for method output_type- 65, // [65:99] is the sub-list for method input_type- 65, // [65:65] is the sub-list for extension type_name- 65, // [65:65] is the sub-list for extension extendee- 0, // [0:65] is the sub-list for field type_name+ 18, // 34: filer_pb.UpdateEntryRequest.condition:type_name -> filer_pb.WriteCondition+ 59, // 35: filer_pb.UpdateEntryResponse.metadata_event:type_name -> filer_pb.SubscribeMetadataResponse+ 13, // 36: filer_pb.AppendToEntryRequest.chunks:type_name -> filer_pb.FileChunk+ 59, // 37: filer_pb.DeleteEntryResponse.metadata_event:type_name -> filer_pb.SubscribeMetadataResponse+ 12, // 38: filer_pb.StreamRenameEntryResponse.event_notification:type_name -> filer_pb.EventNotification+ 45, // 39: filer_pb.AssignVolumeResponse.location:type_name -> filer_pb.Location+ 45, // 40: filer_pb.AssignVolumeResponse.replicas:type_name -> filer_pb.Location+ 45, // 41: filer_pb.Locations.locations:type_name -> filer_pb.Location+ 101, // 42: filer_pb.LookupVolumeResponse.locations_map:type_name -> filer_pb.LookupVolumeResponse.LocationsMapEntry+ 47, // 43: filer_pb.CollectionListResponse.collections:type_name -> filer_pb.Collection+ 12, // 44: filer_pb.SubscribeMetadataResponse.event_notification:type_name -> filer_pb.EventNotification+ 59, // 45: filer_pb.SubscribeMetadataResponse.events:type_name -> filer_pb.SubscribeMetadataResponse+ 63, // 46: filer_pb.SubscribeMetadataResponse.log_file_refs:type_name -> filer_pb.LogFileChunkRef+ 62, // 47: filer_pb.ListMetadataSubscribersResponse.subscribers:type_name -> filer_pb.MetadataSubscriber+ 13, // 48: filer_pb.LogFileChunkRef.chunks:type_name -> filer_pb.FileChunk+ 10, // 49: filer_pb.TraverseBfsMetadataResponse.entry:type_name -> filer_pb.Entry+ 102, // 50: filer_pb.LocateBrokerResponse.resources:type_name -> filer_pb.LocateBrokerResponse.Resource+ 103, // 51: filer_pb.FilerConf.locations:type_name -> filer_pb.FilerConf.PathConf+ 10, // 52: filer_pb.CacheRemoteObjectToLocalClusterResponse.entry:type_name -> filer_pb.Entry+ 59, // 53: filer_pb.CacheRemoteObjectToLocalClusterResponse.metadata_event:type_name -> filer_pb.SubscribeMetadataResponse+ 84, // 54: filer_pb.TransferLocksRequest.locks:type_name -> filer_pb.Lock+ 17, // 55: filer_pb.StreamMutateEntryRequest.create_request:type_name -> filer_pb.CreateEntryRequest+ 29, // 56: filer_pb.StreamMutateEntryRequest.update_request:type_name -> filer_pb.UpdateEntryRequest+ 35, // 57: filer_pb.StreamMutateEntryRequest.delete_request:type_name -> filer_pb.DeleteEntryRequest+ 39, // 58: filer_pb.StreamMutateEntryRequest.rename_request:type_name -> filer_pb.StreamRenameEntryRequest+ 28, // 59: filer_pb.StreamMutateEntryResponse.create_response:type_name -> filer_pb.CreateEntryResponse+ 30, // 60: filer_pb.StreamMutateEntryResponse.update_response:type_name -> filer_pb.UpdateEntryResponse+ 36, // 61: filer_pb.StreamMutateEntryResponse.delete_response:type_name -> filer_pb.DeleteEntryResponse+ 40, // 62: filer_pb.StreamMutateEntryResponse.rename_response:type_name -> filer_pb.StreamRenameEntryResponse+ 95, // 63: filer_pb.MountListResponse.mounts:type_name -> filer_pb.MountInfo+ 3, // 64: filer_pb.WriteCondition.Clause.kind:type_name -> filer_pb.WriteCondition.Kind+ 44, // 65: filer_pb.LookupVolumeResponse.LocationsMapEntry.value:type_name -> filer_pb.Locations+ 5, // 66: filer_pb.SeaweedFiler.LookupDirectoryEntry:input_type -> filer_pb.LookupDirectoryEntryRequest+ 7, // 67: filer_pb.SeaweedFiler.ListEntries:input_type -> filer_pb.ListEntriesRequest+ 17, // 68: filer_pb.SeaweedFiler.CreateEntry:input_type -> filer_pb.CreateEntryRequest+ 29, // 69: filer_pb.SeaweedFiler.UpdateEntry:input_type -> filer_pb.UpdateEntryRequest+ 31, // 70: filer_pb.SeaweedFiler.TouchAccessTime:input_type -> filer_pb.TouchAccessTimeRequest+ 33, // 71: filer_pb.SeaweedFiler.AppendToEntry:input_type -> filer_pb.AppendToEntryRequest+ 35, // 72: filer_pb.SeaweedFiler.DeleteEntry:input_type -> filer_pb.DeleteEntryRequest+ 21, // 73: filer_pb.SeaweedFiler.ObjectTransaction:input_type -> filer_pb.ObjectTransactionRequest+ 26, // 74: filer_pb.SeaweedFiler.ObjectTransactionBatch:input_type -> filer_pb.ObjectTransactionBatchRequest+ 24, // 75: filer_pb.SeaweedFiler.PosixLock:input_type -> filer_pb.PosixLockRequest+ 37, // 76: filer_pb.SeaweedFiler.AtomicRenameEntry:input_type -> filer_pb.AtomicRenameEntryRequest+ 39, // 77: filer_pb.SeaweedFiler.StreamRenameEntry:input_type -> filer_pb.StreamRenameEntryRequest+ 89, // 78: filer_pb.SeaweedFiler.StreamMutateEntry:input_type -> filer_pb.StreamMutateEntryRequest+ 41, // 79: filer_pb.SeaweedFiler.AssignVolume:input_type -> filer_pb.AssignVolumeRequest+ 43, // 80: filer_pb.SeaweedFiler.LookupVolume:input_type -> filer_pb.LookupVolumeRequest+ 48, // 81: filer_pb.SeaweedFiler.CollectionList:input_type -> filer_pb.CollectionListRequest+ 50, // 82: filer_pb.SeaweedFiler.DeleteCollection:input_type -> filer_pb.DeleteCollectionRequest+ 52, // 83: filer_pb.SeaweedFiler.Statistics:input_type -> filer_pb.StatisticsRequest+ 54, // 84: filer_pb.SeaweedFiler.Ping:input_type -> filer_pb.PingRequest+ 56, // 85: filer_pb.SeaweedFiler.GetFilerConfiguration:input_type -> filer_pb.GetFilerConfigurationRequest+ 64, // 86: filer_pb.SeaweedFiler.TraverseBfsMetadata:input_type -> filer_pb.TraverseBfsMetadataRequest+ 58, // 87: filer_pb.SeaweedFiler.SubscribeMetadata:input_type -> filer_pb.SubscribeMetadataRequest+ 58, // 88: filer_pb.SeaweedFiler.SubscribeLocalMetadata:input_type -> filer_pb.SubscribeMetadataRequest+ 60, // 89: filer_pb.SeaweedFiler.ListMetadataSubscribers:input_type -> filer_pb.ListMetadataSubscribersRequest+ 71, // 90: filer_pb.SeaweedFiler.KvGet:input_type -> filer_pb.KvGetRequest+ 73, // 91: filer_pb.SeaweedFiler.KvPut:input_type -> filer_pb.KvPutRequest+ 76, // 92: filer_pb.SeaweedFiler.CacheRemoteObjectToLocalCluster:input_type -> filer_pb.CacheRemoteObjectToLocalClusterRequest+ 78, // 93: filer_pb.SeaweedFiler.DistributedLock:input_type -> filer_pb.LockRequest+ 80, // 94: filer_pb.SeaweedFiler.DistributedUnlock:input_type -> filer_pb.UnlockRequest+ 82, // 95: filer_pb.SeaweedFiler.FindLockOwner:input_type -> filer_pb.FindLockOwnerRequest+ 85, // 96: filer_pb.SeaweedFiler.TransferLocks:input_type -> filer_pb.TransferLocksRequest+ 87, // 97: filer_pb.SeaweedFiler.ReplicateLock:input_type -> filer_pb.ReplicateLockRequest+ 91, // 98: filer_pb.SeaweedFiler.MountRegister:input_type -> filer_pb.MountRegisterRequest+ 93, // 99: filer_pb.SeaweedFiler.MountList:input_type -> filer_pb.MountListRequest+ 6, // 100: filer_pb.SeaweedFiler.LookupDirectoryEntry:output_type -> filer_pb.LookupDirectoryEntryResponse+ 8, // 101: filer_pb.SeaweedFiler.ListEntries:output_type -> filer_pb.ListEntriesResponse+ 28, // 102: filer_pb.SeaweedFiler.CreateEntry:output_type -> filer_pb.CreateEntryResponse+ 30, // 103: filer_pb.SeaweedFiler.UpdateEntry:output_type -> filer_pb.UpdateEntryResponse+ 32, // 104: filer_pb.SeaweedFiler.TouchAccessTime:output_type -> filer_pb.TouchAccessTimeResponse+ 34, // 105: filer_pb.SeaweedFiler.AppendToEntry:output_type -> filer_pb.AppendToEntryResponse+ 36, // 106: filer_pb.SeaweedFiler.DeleteEntry:output_type -> filer_pb.DeleteEntryResponse+ 22, // 107: filer_pb.SeaweedFiler.ObjectTransaction:output_type -> filer_pb.ObjectTransactionResponse+ 27, // 108: filer_pb.SeaweedFiler.ObjectTransactionBatch:output_type -> filer_pb.ObjectTransactionBatchResponse+ 25, // 109: filer_pb.SeaweedFiler.PosixLock:output_type -> filer_pb.PosixLockResponse+ 38, // 110: filer_pb.SeaweedFiler.AtomicRenameEntry:output_type -> filer_pb.AtomicRenameEntryResponse+ 40, // 111: filer_pb.SeaweedFiler.StreamRenameEntry:output_type -> filer_pb.StreamRenameEntryResponse+ 90, // 112: filer_pb.SeaweedFiler.StreamMutateEntry:output_type -> filer_pb.StreamMutateEntryResponse+ 42, // 113: filer_pb.SeaweedFiler.AssignVolume:output_type -> filer_pb.AssignVolumeResponse+ 46, // 114: filer_pb.SeaweedFiler.LookupVolume:output_type -> filer_pb.LookupVolumeResponse+ 49, // 115: filer_pb.SeaweedFiler.CollectionList:output_type -> filer_pb.CollectionListResponse+ 51, // 116: filer_pb.SeaweedFiler.DeleteCollection:output_type -> filer_pb.DeleteCollectionResponse+ 53, // 117: filer_pb.SeaweedFiler.Statistics:output_type -> filer_pb.StatisticsResponse+ 55, // 118: filer_pb.SeaweedFiler.Ping:output_type -> filer_pb.PingResponse+ 57, // 119: filer_pb.SeaweedFiler.GetFilerConfiguration:output_type -> filer_pb.GetFilerConfigurationResponse+ 65, // 120: filer_pb.SeaweedFiler.TraverseBfsMetadata:output_type -> filer_pb.TraverseBfsMetadataResponse+ 59, // 121: filer_pb.SeaweedFiler.SubscribeMetadata:output_type -> filer_pb.SubscribeMetadataResponse+ 59, // 122: filer_pb.SeaweedFiler.SubscribeLocalMetadata:output_type -> filer_pb.SubscribeMetadataResponse+ 61, // 123: filer_pb.SeaweedFiler.ListMetadataSubscribers:output_type -> filer_pb.ListMetadataSubscribersResponse+ 72, // 124: filer_pb.SeaweedFiler.KvGet:output_type -> filer_pb.KvGetResponse+ 74, // 125: filer_pb.SeaweedFiler.KvPut:output_type -> filer_pb.KvPutResponse+ 77, // 126: filer_pb.SeaweedFiler.CacheRemoteObjectToLocalCluster:output_type -> filer_pb.CacheRemoteObjectToLocalClusterResponse+ 79, // 127: filer_pb.SeaweedFiler.DistributedLock:output_type -> filer_pb.LockResponse+ 81, // 128: filer_pb.SeaweedFiler.DistributedUnlock:output_type -> filer_pb.UnlockResponse+ 83, // 129: filer_pb.SeaweedFiler.FindLockOwner:output_type -> filer_pb.FindLockOwnerResponse+ 86, // 130: filer_pb.SeaweedFiler.TransferLocks:output_type -> filer_pb.TransferLocksResponse+ 88, // 131: filer_pb.SeaweedFiler.ReplicateLock:output_type -> filer_pb.ReplicateLockResponse+ 92, // 132: filer_pb.SeaweedFiler.MountRegister:output_type -> filer_pb.MountRegisterResponse+ 94, // 133: filer_pb.SeaweedFiler.MountList:output_type -> filer_pb.MountListResponse+ 100, // [100:134] is the sub-list for method output_type+ 66, // [66:100] is the sub-list for method input_type+ 66, // [66:66] is the sub-list for extension type_name+ 66, // [66:66] is the sub-list for extension extendee+ 0, // [0:66] is the sub-list for field type_name}func init() { file_filer_proto_init() }diff --git a/weed/pb/filer_pb/filer_vtproto.pb.go b/weed/pb/filer_pb/filer_vtproto.pb.goindex 39d59d9ed8..9bdbaba8e7 100644--- a/weed/pb/filer_pb/filer_vtproto.pb.go+++ b/weed/pb/filer_pb/filer_vtproto.pb.go@@ -1105,6 +1105,15 @@ func (m *WriteCondition_Clause) MarshalToSizedBufferVT(dAtA []byte) (int, error)i -= len(m.unknownFields)copy(dAtA[i:], m.unknownFields)}+ if len(m.Fids) > 0 {+ for iNdEx := len(m.Fids) - 1; iNdEx >= 0; iNdEx-- {+ i -= len(m.Fids[iNdEx])+ copy(dAtA[i:], m.Fids[iNdEx])+ i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Fids[iNdEx])))+ i--+ dAtA[i] = 0x4a+ }+ }if len(m.GateValue) > 0 {i -= len(m.GateValue)copy(dAtA[i:], m.GateValue)@@ -2038,6 +2047,16 @@ func (m *UpdateEntryRequest) MarshalToSizedBufferVT(dAtA []byte) (int, error) {i -= len(m.unknownFields)copy(dAtA[i:], m.unknownFields)}+ if m.Condition != nil {+ size, err := m.Condition.MarshalToSizedBufferVT(dAtA[:i])+ if err != nil {+ return 0, err+ }+ i -= size+ i = protohelpers.EncodeVarint(dAtA, i, uint64(size))+ i--+ dAtA[i] = 0x32+ }if len(m.ExpectedExtended) > 0 {for k := range m.ExpectedExtended {v := m.ExpectedExtended[k]@@ -6615,6 +6634,12 @@ func (m *WriteCondition_Clause) SizeVT() (n int) {if l > 0 {n += 1 + l + protohelpers.SizeOfVarint(uint64(l))}+ if len(m.Fids) > 0 {+ for _, s := range m.Fids {+ l = len(s)+ n += 1 + l + protohelpers.SizeOfVarint(uint64(l))+ }+ }n += len(m.unknownFields)return n}@@ -6979,6 +7004,10 @@ func (m *UpdateEntryRequest) SizeVT() (n int) {n += mapEntrySize + 1 + protohelpers.SizeOfVarint(uint64(mapEntrySize))}}+ if m.Condition != nil {+ l = m.Condition.SizeVT()+ n += 1 + l + protohelpers.SizeOfVarint(uint64(l))+ }n += len(m.unknownFields)return n}@@ -11801,6 +11830,38 @@ func (m *WriteCondition_Clause) UnmarshalVT(dAtA []byte) error {}m.GateValue = string(dAtA[iNdEx:postIndex])iNdEx = postIndex+ case 9:+ if wireType != 2 {+ return fmt.Errorf("proto: wrong wireType = %d for field Fids", wireType)+ }+ var stringLen uint64+ for shift := uint(0); ; shift += 7 {+ if shift >= 64 {+ return protohelpers.ErrIntOverflow+ }+ if iNdEx >= l {+ return io.ErrUnexpectedEOF+ }+ b := dAtA[iNdEx]+ iNdEx+++ stringLen |= uint64(b&0x7F) << shift+ if b < 0x80 {+ break+ }+ }+ intStringLen := int(stringLen)+ if intStringLen < 0 {+ return protohelpers.ErrInvalidLength+ }+ postIndex := iNdEx + intStringLen+ if postIndex < 0 {+ return protohelpers.ErrInvalidLength+ }+ if postIndex > l {+ return io.ErrUnexpectedEOF+ }+ m.Fids = append(m.Fids, string(dAtA[iNdEx:postIndex]))+ iNdEx = postIndexdefault:iNdEx = preIndexskippy, err := protohelpers.Skip(dAtA[iNdEx:])@@ -14420,6 +14481,42 @@ func (m *UpdateEntryRequest) UnmarshalVT(dAtA []byte) error {}m.ExpectedExtended[mapkey] = mapvalueiNdEx = postIndex+ case 6:+ if wireType != 2 {+ return fmt.Errorf("proto: wrong wireType = %d for field Condition", wireType)+ }+ var msglen int+ for shift := uint(0); ; shift += 7 {+ if shift >= 64 {+ return protohelpers.ErrIntOverflow+ }+ if iNdEx >= l {+ return io.ErrUnexpectedEOF+ }+ b := dAtA[iNdEx]+ iNdEx+++ msglen |= int(b&0x7F) << shift+ if b < 0x80 {+ break+ }+ }+ if msglen < 0 {+ return protohelpers.ErrInvalidLength+ }+ postIndex := iNdEx + msglen+ if postIndex < 0 {+ return protohelpers.ErrInvalidLength+ }+ if postIndex > l {+ return io.ErrUnexpectedEOF+ }+ if m.Condition == nil {+ m.Condition = &WriteCondition{}+ }+ if err := m.Condition.UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil {+ return err+ }+ iNdEx = postIndexdefault:iNdEx = preIndexskippy, err := protohelpers.Skip(dAtA[iNdEx:])diff --git a/weed/server/filer_grpc_server.go b/weed/server/filer_grpc_server.goindex 985607d00e..f4d2025f96 100644--- a/weed/server/filer_grpc_server.go+++ b/weed/server/filer_grpc_server.go@@ -594,13 +594,26 @@ func (fs *FilerServer) UpdateEntry(ctx context.Context, req *filer_pb.UpdateEntr}fullpath := util.Join(req.Directory, req.Entry.Name)- entry, err := fs.filer.FindEntry(ctx, util.FullPath(fullpath))++ // Serialize concurrent mutations to the same path on this filer so the+ // read (preconditions, garbage diff) and the write are atomic. Callers+ // route a key's writes to this owner filer, making this local lock+ // sufficient.+ lockPath := util.FullPath(fullpath)+ pathLock := fs.entryLockTable.AcquireLock("UpdateEntry", lockPath, util.ExclusiveLock)+ defer fs.entryLockTable.ReleaseLock(lockPath, pathLock)++ entry, err := fs.filer.FindEntry(ctx, lockPath)if err != nil {return &filer_pb.UpdateEntryResponse{}, fmt.Errorf("not found %s: %v", fullpath, err)}if err := validateUpdateEntryPreconditions(entry, req.ExpectedExtended); err != nil {return &filer_pb.UpdateEntryResponse{}, err}+ if conditionIsSet(req.Condition) && !writeConditionSatisfied(req.Condition, entry) {+ glog.V(3).InfofCtx(ctx, "UpdateEntry %s: precondition failed: %v", fullpath, req.Condition)+ return &filer_pb.UpdateEntryResponse{}, status.Errorf(codes.FailedPrecondition, "precondition failed: %s", fullpath)+ }chunks, garbage, err2 := fs.cleanupChunks(ctx, fullpath, entry, req.Entry)if err2 != nil {@@ -706,6 +719,12 @@ func (fs *FilerServer) AppendToEntry(ctx context.Context, req *filer_pb.AppendTolock := lockClient.NewShortLivedLock(string(fullpath), string(fs.option.Host))defer lock.StopShortLivedLock()+ // The cluster lock serializes appenders across filers; the path lock makes+ // this read-modify-write atomic against conditional updates and deletes on+ // the owner filer.+ pathLock := fs.entryLockTable.AcquireLock("AppendToEntry", fullpath, util.ExclusiveLock)+ defer fs.entryLockTable.ReleaseLock(fullpath, pathLock)+var offset int64 = 0entry, err := fs.filer.FindEntry(ctx, fullpath)if err == filer_pb.ErrNotFound {@@ -749,8 +768,16 @@ func (fs *FilerServer) DeleteEntry(ctx context.Context, req *filer_pb.DeleteEntrglog.V(4).InfofCtx(ctx, "DeleteEntry %v", req)+ // A delete queues the entry's chunks for deletion, so it must not+ // interleave with a conditional update's check-then-write on the same+ // path: the update would pass its precondition and then resurrect fids+ // that are already on the deletion queue.+ fullpath := util.JoinPath(req.Directory, req.Name)+ pathLock := fs.entryLockTable.AcquireLock("DeleteEntry", fullpath, util.ExclusiveLock)+ defer fs.entryLockTable.ReleaseLock(fullpath, pathLock)+ctx, eventSink := filer.WithMetadataEventSink(ctx)- err = fs.filer.DeleteEntryMetaAndData(ctx, util.JoinPath(req.Directory, req.Name), req.IsRecursive, req.IgnoreRecursiveError, req.IsDeleteData, req.IsFromOtherCluster, req.Signatures, req.IfNotModifiedAfter)+ err = fs.filer.DeleteEntryMetaAndData(ctx, fullpath, req.IsRecursive, req.IgnoreRecursiveError, req.IsDeleteData, req.IsFromOtherCluster, req.Signatures, req.IfNotModifiedAfter)resp = &filer_pb.DeleteEntryResponse{}if err != nil && err != filer_pb.ErrNotFound {resp.Error = err.Error()diff --git a/weed/server/filer_grpc_server_chunk_condition_test.go b/weed/server/filer_grpc_server_chunk_condition_test.gonew file mode 100644index 0000000000..6ebdcba652--- /dev/null+++ b/weed/server/filer_grpc_server_chunk_condition_test.go@@ -0,0 +1,187 @@+package weed_server++import (+ "context"+ "testing"+ "time"++ "github.com/seaweedfs/seaweedfs/weed/filer"+ "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"+ "github.com/seaweedfs/seaweedfs/weed/util"+ "google.golang.org/grpc/codes"+ "google.golang.org/grpc/status"+)++func chunkFid(fid string) *filer_pb.FileChunk {+ return &filer_pb.FileChunk{FileId: fid, Size: 100}+}++func ifChunksEqual(fids ...string) *filer_pb.WriteCondition {+ return one(&filer_pb.WriteCondition_Clause{Kind: filer_pb.WriteCondition_IF_CHUNKS_EQUAL, Fids: fids})+}++func TestChunkFidsEqualClause(t *testing.T) {+ withChunks := func(fids ...string) *filer.Entry {+ e := &filer.Entry{FullPath: "/test/obj"}+ for _, fid := range fids {+ e.Chunks = append(e.Chunks, chunkFid(fid))+ }+ return e+ }++ cases := []struct {+ name string+ cur *filer.Entry+ fids []string+ want bool+ }{+ {"absent-no-chunks", nil, nil, true},+ {"absent-expected", nil, []string{"3,01"}, false},+ {"empty-expects-none", withChunks(), nil, true},+ {"exact", withChunks("3,01", "3,02"), []string{"3,01", "3,02"}, true},+ {"reordered", withChunks("3,01", "3,02"), []string{"3,02", "3,01"}, true},+ {"duplicates", withChunks("3,01", "3,01", "3,02"), []string{"3,02", "3,01", "3,01"}, true},+ // The strand: a concurrent update emptied the stored chunk list, so the+ // stale writer's expectation no longer holds.+ {"stored-emptied", withChunks(), []string{"3,01"}, false},+ // The reverse strand: the writer read no chunks, but a concurrent update+ // has since added one; an empty fids list still guards it.+ {"stored-filled", withChunks("3,01"), nil, false},+ {"chunk-added", withChunks("3,01", "3,02"), []string{"3,01"}, false},+ {"chunk-removed", withChunks("3,01"), []string{"3,01", "3,02"}, false},+ {"chunk-replaced", withChunks("3,09"), []string{"3,01"}, false},+ {"duplicate-count", withChunks("3,01"), []string{"3,01", "3,01"}, false},+ }+ for _, tc := range cases {+ if got := writeConditionSatisfied(ifChunksEqual(tc.fids...), tc.cur); got != tc.want {+ t.Errorf("%s: got %v want %v", tc.name, got, tc.want)+ }+ }+}++// TestUpdateEntryChunkConditionPreventsStrand drives the UpdateEntry handler+// through the interleaving that strands an entry on a dead needle:+//+// 1. entry has chunks=[F]; needle F is live.+// 2. R (a chunk-preserving read-modify-write) snapshots chunks=[F].+// 3. D (eviction) commits chunks=[] first; F is diffed away and queued for+// deletion. The store is seeded at chunks=[] to model D's commit.+// 4. R commits its stale snapshot last.+//+// Without a condition R's write wins and resurrects the reference to F; with+// IF_CHUNKS_EQUAL it fails with FailedPrecondition and the entry stays empty.+func TestUpdateEntryChunkConditionPreventsStrand(t *testing.T) {+ const fidF = "3,01637037d6"++ newServer := func(storedChunks ...*filer_pb.FileChunk) (*FilerServer, *renameTestStore) {+ store := newRenameTestStore()+ store.entries["/test/obj"] = &filer.Entry{+ FullPath: "/test/obj",+ Attr: filer.Attr{Inode: 1, Mtime: time.Unix(1700000000, 0), Mode: 0644},+ Chunks: storedChunks,+ }+ f := newRenameTestFiler(store)+ return &FilerServer{filer: f, option: &FilerOption{}, entryLockTable: util.NewLockTable[util.FullPath]()}, store+ }++ // R's stale request: it read chunks=[F] and writes back chunks=[F].+ staleReq := func(cond *filer_pb.WriteCondition) *filer_pb.UpdateEntryRequest {+ return &filer_pb.UpdateEntryRequest{+ Directory: "/test",+ Entry: &filer_pb.Entry{+ Name: "obj",+ Attributes: &filer_pb.FuseAttributes{Mtime: 1700000005, FileMode: 0644, Inode: 1},+ Chunks: []*filer_pb.FileChunk{chunkFid(fidF)},+ },+ Condition: cond,+ }+ }++ t.Run("unconditional-resurrects", func(t *testing.T) {+ fs, store := newServer()+ if _, err := fs.UpdateEntry(context.Background(), staleReq(nil)); err != nil {+ t.Fatalf("UpdateEntry: %v", err)+ }+ if got := len(store.entries["/test/obj"].GetChunks()); got != 1 {+ t.Fatalf("last-write-wins should have resurrected chunks=[F], got %d chunks", got)+ }+ })++ t.Run("condition-rejects-stale", func(t *testing.T) {+ fs, store := newServer()+ _, err := fs.UpdateEntry(context.Background(), staleReq(ifChunksEqual(fidF)))+ if status.Code(err) != codes.FailedPrecondition {+ t.Fatalf("stale write must fail with FailedPrecondition, got %v", err)+ }+ if got := len(store.entries["/test/obj"].GetChunks()); got != 0 {+ t.Fatalf("entry must stay chunks=[], got %d chunks", got)+ }+ })++ t.Run("condition-passes-fresh", func(t *testing.T) {+ fs, store := newServer(chunkFid(fidF))+ if _, err := fs.UpdateEntry(context.Background(), staleReq(ifChunksEqual(fidF))); err != nil {+ t.Fatalf("fresh read-modify-write must succeed: %v", err)+ }+ chunks := store.entries["/test/obj"].GetChunks()+ if len(chunks) != 1 || chunks[0].GetFileIdString() != fidF {+ t.Fatalf("chunks=[F] must be preserved, got %v", chunks)+ }+ })++ // The reverse strand: R read the entry before a concurrent writer added F,+ // so its stale write would wipe the chunk and queue F for deletion. An+ // IF_CHUNKS_EQUAL clause with no fids guards the emptiness it observed.+ t.Run("empty-expectation-rejects", func(t *testing.T) {+ fs, store := newServer(chunkFid(fidF))+ req := &filer_pb.UpdateEntryRequest{+ Directory: "/test",+ Entry: &filer_pb.Entry{+ Name: "obj",+ Attributes: &filer_pb.FuseAttributes{Mtime: 1700000005, FileMode: 0644, Inode: 1},+ },+ Condition: ifChunksEqual(),+ }+ _, err := fs.UpdateEntry(context.Background(), req)+ if status.Code(err) != codes.FailedPrecondition {+ t.Fatalf("stale wipe must fail with FailedPrecondition, got %v", err)+ }+ if got := len(store.entries["/test/obj"].GetChunks()); got != 1 {+ t.Fatalf("chunks=[F] must be preserved, got %d chunks", got)+ }+ })+}++// DeleteEntry queues chunk deletions, so it must serialize on the same path+// lock the conditional writers hold; otherwise it can interleave with a+// passed precondition and the stale write resurrects the queued fids.+func TestDeleteEntryWaitsForPathLock(t *testing.T) {+ store := newRenameTestStore()+ store.entries["/test/obj"] = &filer.Entry{+ FullPath: "/test/obj",+ Attr: filer.Attr{Inode: 1, Mtime: time.Unix(1700000000, 0), Mode: 0644},+ }+ f := newRenameTestFiler(store)+ fs := &FilerServer{filer: f, option: &FilerOption{}, entryLockTable: util.NewLockTable[util.FullPath]()}++ lockPath := util.FullPath("/test/obj")+ hold := fs.entryLockTable.AcquireLock("test", lockPath, util.ExclusiveLock)++ done := make(chan struct{})+ go func() {+ fs.DeleteEntry(context.Background(), &filer_pb.DeleteEntryRequest{Directory: "/test", Name: "obj"})+ close(done)+ }()++ select {+ case <-done:+ t.Fatal("DeleteEntry completed while the path lock was held")+ case <-time.After(100 * time.Millisecond):+ }++ fs.entryLockTable.ReleaseLock(lockPath, hold)+ <-done+ if _, ok := store.entries["/test/obj"]; ok {+ t.Fatal("entry not deleted after the lock was released")+ }+}diff --git a/weed/server/filer_grpc_server_condition.go b/weed/server/filer_grpc_server_condition.goindex e832bc3d2f..82406c367e 100644--- a/weed/server/filer_grpc_server_condition.go+++ b/weed/server/filer_grpc_server_condition.go@@ -80,6 +80,8 @@ func clauseSatisfied(c *filer_pb.WriteCondition_Clause, current *filer.Entry) boreturn false}return deadline <= time.Now().Unix()+ case filer_pb.WriteCondition_IF_CHUNKS_EQUAL:+ return chunkFidsEqual(current, c.Fids)default:// An unrecognized clause kind (e.g. from a newer client) must not be// treated as satisfied, which would silently bypass the guard. Fail@@ -88,6 +90,30 @@ func clauseSatisfied(c *filer_pb.WriteCondition_Clause, current *filer.Entry) bo}}+// chunkFidsEqual compares the stored chunk fids (absent entry = none) against+// expected as multisets; chunk order carries no meaning for needle liveness.+func chunkFidsEqual(current *filer.Entry, expected []string) bool {+ var chunks []*filer_pb.FileChunk+ if current != nil {+ chunks = current.GetChunks()+ }+ if len(chunks) != len(expected) {+ return false+ }+ counts := make(map[string]int, len(expected))+ for _, fid := range expected {+ counts[fid]+++ }+ for _, chunk := range chunks {+ fid := chunk.GetFileIdString()+ if counts[fid] == 0 {+ return false+ }+ counts[fid]--+ }+ return true+}+// etagInSet reports whether stored matches any candidate. A strong comparison// (allowWeak false) treats a weak ETag as never equal; a weak comparison// ignores the W/ marker on both sides.diff --git a/weed/server/filer_grpc_server_condition_test.go b/weed/server/filer_grpc_server_condition_test.goindex 3dcb0f7922..2285a4baef 100644--- a/weed/server/filer_grpc_server_condition_test.go+++ b/weed/server/filer_grpc_server_condition_test.go@@ -10,6 +10,8 @@ import ("github.com/seaweedfs/seaweedfs/weed/pb/filer_pb""github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants""github.com/seaweedfs/seaweedfs/weed/util"+ "google.golang.org/grpc/codes"+ "google.golang.org/grpc/status")func entryWithETag(etag string, mtime time.Time) *filer.Entry {@@ -284,3 +286,46 @@ func TestCreateEntryReusesProvidedExisting(t *testing.T) {t.Fatalf("providing existing should save one path lookup: existing=%d nil=%d", withExisting, withNil)}}++// The UpdateEntry handler enforces the precondition under the per-path lock: a+// matching If-Match applies the update, a non-matching one fails with+// FailedPrecondition and leaves the stored entry untouched.+func TestUpdateEntryConditionEnforced(t *testing.T) {+ newServer := func() (*FilerServer, *renameTestStore) {+ store := newRenameTestStore()+ store.entries["/test/obj"] = &filer.Entry{+ FullPath: "/test/obj",+ Attr: filer.Attr{Inode: 1, Mtime: time.Unix(1700000000, 0), Mode: 0644},+ Extended: map[string][]byte{s3_constants.ExtETagKey: []byte("abc")},+ }+ f := newRenameTestFiler(store)+ return &FilerServer{filer: f, option: &FilerOption{}, entryLockTable: util.NewLockTable[util.FullPath]()}, store+ }++ req := func(etag string) *filer_pb.UpdateEntryRequest {+ return &filer_pb.UpdateEntryRequest{+ Directory: "/test",+ Entry: &filer_pb.Entry{+ Name: "obj",+ Attributes: &filer_pb.FuseAttributes{Mtime: 1700000005, FileMode: 0644, Inode: 1},+ },+ Condition: one(&filer_pb.WriteCondition_Clause{Kind: filer_pb.WriteCondition_IF_ETAG_MATCH, Etags: []string{etag}}),+ }+ }++ fs, store := newServer()+ if _, err := fs.UpdateEntry(context.Background(), req(`"zzz"`)); status.Code(err) != codes.FailedPrecondition {+ t.Fatalf("mismatched etag: want FailedPrecondition, got %v", err)+ }+ if got := store.entries["/test/obj"].Attr.Mtime.Unix(); got != 1700000000 {+ t.Fatalf("rejected update must not be applied, mtime %d", got)+ }++ fs, store = newServer()+ if _, err := fs.UpdateEntry(context.Background(), req(`"abc"`)); err != nil {+ t.Fatalf("matching etag should update: %v", err)+ }+ if got := store.entries["/test/obj"].Attr.Mtime.Unix(); got != 1700000005 {+ t.Fatalf("update not applied, mtime %d", got)+ }+}