SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
filer: add extended-attribute guard clauses for object-lock (#9648)
Chris Lue2203b2a0bee
diff --git a/other/java/client/src/main/proto/filer.proto b/other/java/client/src/main/proto/filer.protoindex c89b1f1f6e..da7e47e959 100644--- a/other/java/client/src/main/proto/filer.proto+++ b/other/java/client/src/main/proto/filer.proto@@ -242,22 +242,35 @@ message CreateEntryRequest {// compound conditions (e.g. If-Match + If-Unmodified-Since together).message WriteCondition {enum Kind {- NONE = 0; // unconditional- IF_NOT_EXISTS = 1; // fail if the entry exists (If-None-Match: *)- IF_EXISTS = 2; // fail if the entry is absent (If-Match: *)- IF_ETAG_MATCH = 3; // fail if absent or etag matches none of the set (If-Match)- IF_ETAG_NOT_MATCH = 4; // fail if present and etag matches any of the set (If-None-Match)- IF_UNMODIFIED_SINCE = 5; // fail if present and mtime > unix_time- IF_MODIFIED_SINCE = 6; // fail if present and mtime <= unix_time+ NONE = 0; // unconditional+ IF_NOT_EXISTS = 1; // fail if the entry exists (If-None-Match: *)+ IF_EXISTS = 2; // fail if the entry is absent (If-Match: *)+ IF_ETAG_MATCH = 3; // fail if absent or etag matches none of the set (If-Match)+ IF_ETAG_NOT_MATCH = 4; // fail if present and etag matches any of the set (If-None-Match)+ IF_UNMODIFIED_SINCE = 5; // fail if present and mtime > unix_time+ IF_MODIFIED_SINCE = 6; // fail if present and mtime <= unix_time+ IF_EXTENDED_NOT_EQUAL = 7; // fail if present and extended[ext_key] == ext_value+ IF_EXTENDED_TIME_ELAPSED = 8; // fail if present and extended[ext_key] (unix seconds) is in the future}// Clause is one primitive comparison. IF_ETAG_MATCH holds when the current// entry's ETag equals any value in etags; IF_ETAG_NOT_MATCH holds when it// equals none. allow_weak permits weak-comparison (ignoring the W/ prefix).+ //+ // The IF_EXTENDED_* kinds are generic guards on an extended attribute, used+ // to enforce object-lock without teaching the filer S3 semantics:+ // IF_EXTENDED_NOT_EQUAL expresses a legal hold (block while a key equals a+ // value), and IF_EXTENDED_TIME_ELAPSED expresses retention (block while a+ // stored unix-second deadline is in the future, compared to the filer's+ // clock). The caller composes these and, for governance-bypass, simply omits+ // the retention clause when the bypass is authorized — the filer makes no+ // authorization decision.message Clause {Kind kind = 1;repeated string etags = 2; // ETag set for IF_ETAG_* kindsint64 unix_time = 3; // bound (unix seconds) for IF_*_SINCE kindsbool allow_weak = 4; // compare ETags ignoring the weak (W/) marker+ string ext_key = 5; // extended attribute name for IF_EXTENDED_* kinds+ string ext_value = 6; // blocking value for IF_EXTENDED_NOT_EQUAL}repeated Clause clauses = 1; // all must hold (logical AND)}diff --git a/weed/pb/filer.proto b/weed/pb/filer.protoindex c89b1f1f6e..da7e47e959 100644--- a/weed/pb/filer.proto+++ b/weed/pb/filer.proto@@ -242,22 +242,35 @@ message CreateEntryRequest {// compound conditions (e.g. If-Match + If-Unmodified-Since together).message WriteCondition {enum Kind {- NONE = 0; // unconditional- IF_NOT_EXISTS = 1; // fail if the entry exists (If-None-Match: *)- IF_EXISTS = 2; // fail if the entry is absent (If-Match: *)- IF_ETAG_MATCH = 3; // fail if absent or etag matches none of the set (If-Match)- IF_ETAG_NOT_MATCH = 4; // fail if present and etag matches any of the set (If-None-Match)- IF_UNMODIFIED_SINCE = 5; // fail if present and mtime > unix_time- IF_MODIFIED_SINCE = 6; // fail if present and mtime <= unix_time+ NONE = 0; // unconditional+ IF_NOT_EXISTS = 1; // fail if the entry exists (If-None-Match: *)+ IF_EXISTS = 2; // fail if the entry is absent (If-Match: *)+ IF_ETAG_MATCH = 3; // fail if absent or etag matches none of the set (If-Match)+ IF_ETAG_NOT_MATCH = 4; // fail if present and etag matches any of the set (If-None-Match)+ IF_UNMODIFIED_SINCE = 5; // fail if present and mtime > unix_time+ IF_MODIFIED_SINCE = 6; // fail if present and mtime <= unix_time+ IF_EXTENDED_NOT_EQUAL = 7; // fail if present and extended[ext_key] == ext_value+ IF_EXTENDED_TIME_ELAPSED = 8; // fail if present and extended[ext_key] (unix seconds) is in the future}// Clause is one primitive comparison. IF_ETAG_MATCH holds when the current// entry's ETag equals any value in etags; IF_ETAG_NOT_MATCH holds when it// equals none. allow_weak permits weak-comparison (ignoring the W/ prefix).+ //+ // The IF_EXTENDED_* kinds are generic guards on an extended attribute, used+ // to enforce object-lock without teaching the filer S3 semantics:+ // IF_EXTENDED_NOT_EQUAL expresses a legal hold (block while a key equals a+ // value), and IF_EXTENDED_TIME_ELAPSED expresses retention (block while a+ // stored unix-second deadline is in the future, compared to the filer's+ // clock). The caller composes these and, for governance-bypass, simply omits+ // the retention clause when the bypass is authorized — the filer makes no+ // authorization decision.message Clause {Kind kind = 1;repeated string etags = 2; // ETag set for IF_ETAG_* kindsint64 unix_time = 3; // bound (unix seconds) for IF_*_SINCE kindsbool allow_weak = 4; // compare ETags ignoring the weak (W/) marker+ string ext_key = 5; // extended attribute name for IF_EXTENDED_* kinds+ string ext_value = 6; // blocking value for IF_EXTENDED_NOT_EQUAL}repeated Clause clauses = 1; // all must hold (logical AND)}diff --git a/weed/pb/filer_pb/filer.pb.go b/weed/pb/filer_pb/filer.pb.goindex f8f3687386..21b94d8b0e 100644--- a/weed/pb/filer_pb/filer.pb.go+++ b/weed/pb/filer_pb/filer.pb.go@@ -139,13 +139,15 @@ func (FilerError) EnumDescriptor() ([]byte, []int) {type WriteCondition_Kind int32const (- WriteCondition_NONE WriteCondition_Kind = 0 // unconditional- WriteCondition_IF_NOT_EXISTS WriteCondition_Kind = 1 // fail if the entry exists (If-None-Match: *)- WriteCondition_IF_EXISTS WriteCondition_Kind = 2 // fail if the entry is absent (If-Match: *)- WriteCondition_IF_ETAG_MATCH WriteCondition_Kind = 3 // fail if absent or etag matches none of the set (If-Match)- WriteCondition_IF_ETAG_NOT_MATCH WriteCondition_Kind = 4 // fail if present and etag matches any of the set (If-None-Match)- WriteCondition_IF_UNMODIFIED_SINCE WriteCondition_Kind = 5 // fail if present and mtime > unix_time- WriteCondition_IF_MODIFIED_SINCE WriteCondition_Kind = 6 // fail if present and mtime <= unix_time+ WriteCondition_NONE WriteCondition_Kind = 0 // unconditional+ WriteCondition_IF_NOT_EXISTS WriteCondition_Kind = 1 // fail if the entry exists (If-None-Match: *)+ WriteCondition_IF_EXISTS WriteCondition_Kind = 2 // fail if the entry is absent (If-Match: *)+ WriteCondition_IF_ETAG_MATCH WriteCondition_Kind = 3 // fail if absent or etag matches none of the set (If-Match)+ WriteCondition_IF_ETAG_NOT_MATCH WriteCondition_Kind = 4 // fail if present and etag matches any of the set (If-None-Match)+ WriteCondition_IF_UNMODIFIED_SINCE WriteCondition_Kind = 5 // fail if present and mtime > unix_time+ WriteCondition_IF_MODIFIED_SINCE WriteCondition_Kind = 6 // fail if present and mtime <= unix_time+ WriteCondition_IF_EXTENDED_NOT_EQUAL WriteCondition_Kind = 7 // fail if present and extended[ext_key] == ext_value+ WriteCondition_IF_EXTENDED_TIME_ELAPSED WriteCondition_Kind = 8 // fail if present and extended[ext_key] (unix seconds) is in the future)// Enum value maps for WriteCondition_Kind.@@ -158,15 +160,19 @@ var (4: "IF_ETAG_NOT_MATCH",5: "IF_UNMODIFIED_SINCE",6: "IF_MODIFIED_SINCE",+ 7: "IF_EXTENDED_NOT_EQUAL",+ 8: "IF_EXTENDED_TIME_ELAPSED",}WriteCondition_Kind_value = map[string]int32{- "NONE": 0,- "IF_NOT_EXISTS": 1,- "IF_EXISTS": 2,- "IF_ETAG_MATCH": 3,- "IF_ETAG_NOT_MATCH": 4,- "IF_UNMODIFIED_SINCE": 5,- "IF_MODIFIED_SINCE": 6,+ "NONE": 0,+ "IF_NOT_EXISTS": 1,+ "IF_EXISTS": 2,+ "IF_ETAG_MATCH": 3,+ "IF_ETAG_NOT_MATCH": 4,+ "IF_UNMODIFIED_SINCE": 5,+ "IF_MODIFIED_SINCE": 6,+ "IF_EXTENDED_NOT_EQUAL": 7,+ "IF_EXTENDED_TIME_ELAPSED": 8,})@@ -5771,12 +5777,23 @@ func (x *MountInfo) GetDataCenter() string {// Clause is one primitive comparison. IF_ETAG_MATCH holds when the current// entry's ETag equals any value in etags; IF_ETAG_NOT_MATCH holds when it// equals none. allow_weak permits weak-comparison (ignoring the W/ prefix).+//+// The IF_EXTENDED_* kinds are generic guards on an extended attribute, used+// to enforce object-lock without teaching the filer S3 semantics:+// IF_EXTENDED_NOT_EQUAL expresses a legal hold (block while a key equals a+// value), and IF_EXTENDED_TIME_ELAPSED expresses retention (block while a+// stored unix-second deadline is in the future, compared to the filer's+// clock). The caller composes these and, for governance-bypass, simply omits+// the retention clause when the bypass is authorized — the filer makes no+// authorization decision.type WriteCondition_Clause struct {state protoimpl.MessageState `protogen:"open.v1"`Kind WriteCondition_Kind `protobuf:"varint,1,opt,name=kind,proto3,enum=filer_pb.WriteCondition_Kind" json:"kind,omitempty"`Etags []string `protobuf:"bytes,2,rep,name=etags,proto3" json:"etags,omitempty"` // ETag set for IF_ETAG_* kindsUnixTime int64 `protobuf:"varint,3,opt,name=unix_time,json=unixTime,proto3" json:"unix_time,omitempty"` // bound (unix seconds) for IF_*_SINCE kindsAllowWeak bool `protobuf:"varint,4,opt,name=allow_weak,json=allowWeak,proto3" json:"allow_weak,omitempty"` // compare ETags ignoring the weak (W/) marker+ ExtKey string `protobuf:"bytes,5,opt,name=ext_key,json=extKey,proto3" json:"ext_key,omitempty"` // extended attribute name for IF_EXTENDED_* kinds+ ExtValue string `protobuf:"bytes,6,opt,name=ext_value,json=extValue,proto3" json:"ext_value,omitempty"` // blocking value for IF_EXTENDED_NOT_EQUALunknownFields protoimpl.UnknownFieldssizeCache protoimpl.SizeCache}@@ -5839,6 +5856,20 @@ func (x *WriteCondition_Clause) GetAllowWeak() bool {return false}+func (x *WriteCondition_Clause) GetExtKey() string {+ if x != nil {+ return x.ExtKey+ }+ return ""+}++func (x *WriteCondition_Clause) GetExtValue() string {+ if x != nil {+ return x.ExtValue+ }+ return ""+}+// if found, send the exact address// if not found, send the full list of existing brokerstype LocateBrokerResponse_Resource struct {@@ -6171,15 +6202,17 @@ const file_filer_proto_rawDesc = "" +"signatures\x18\x05 \x03(\x05R\n" +"signatures\x12=\n" +"\x1bskip_check_parent_directory\x18\x06 \x01(\bR\x18skipCheckParentDirectory\x126\n" +- "\tcondition\x18\a \x01(\v2\x18.filer_pb.WriteConditionR\tcondition\"\xea\x02\n" ++ "\tcondition\x18\a \x01(\v2\x18.filer_pb.WriteConditionR\tcondition\"\xd9\x03\n" +"\x0eWriteCondition\x129\n" +- "\aclauses\x18\x01 \x03(\v2\x1f.filer_pb.WriteCondition.ClauseR\aclauses\x1a\x8d\x01\n" ++ "\aclauses\x18\x01 \x03(\v2\x1f.filer_pb.WriteCondition.ClauseR\aclauses\x1a\xc3\x01\n" +"\x06Clause\x121\n" +"\x04kind\x18\x01 \x01(\x0e2\x1d.filer_pb.WriteCondition.KindR\x04kind\x12\x14\n" +"\x05etags\x18\x02 \x03(\tR\x05etags\x12\x1b\n" +"\tunix_time\x18\x03 \x01(\x03R\bunixTime\x12\x1d\n" +"\n" +- "allow_weak\x18\x04 \x01(\bR\tallowWeak\"\x8c\x01\n" ++ "allow_weak\x18\x04 \x01(\bR\tallowWeak\x12\x17\n" ++ "\aext_key\x18\x05 \x01(\tR\x06extKey\x12\x1b\n" ++ "\text_value\x18\x06 \x01(\tR\bextValue\"\xc5\x01\n" +"\x04Kind\x12\b\n" +"\x04NONE\x10\x00\x12\x11\n" +"\rIF_NOT_EXISTS\x10\x01\x12\r\n" +@@ -6187,7 +6220,9 @@ const file_filer_proto_rawDesc = "" +"\rIF_ETAG_MATCH\x10\x03\x12\x15\n" +"\x11IF_ETAG_NOT_MATCH\x10\x04\x12\x17\n" +"\x13IF_UNMODIFIED_SINCE\x10\x05\x12\x15\n" +- "\x11IF_MODIFIED_SINCE\x10\x06\"\x96\x04\n" ++ "\x11IF_MODIFIED_SINCE\x10\x06\x12\x19\n" ++ "\x15IF_EXTENDED_NOT_EQUAL\x10\a\x12\x1c\n" ++ "\x18IF_EXTENDED_TIME_ELAPSED\x10\b\"\x96\x04\n" +"\x0eObjectMutation\x121\n" +"\x04type\x18\x01 \x01(\x0e2\x1d.filer_pb.ObjectMutation.TypeR\x04type\x12\x1c\n" +"\tdirectory\x18\x02 \x01(\tR\tdirectory\x12\x12\n" +diff --git a/weed/server/filer_grpc_server_condition.go b/weed/server/filer_grpc_server_condition.goindex 62ded896b8..809dfd4e12 100644--- a/weed/server/filer_grpc_server_condition.go+++ b/weed/server/filer_grpc_server_condition.go@@ -1,7 +1,9 @@package weed_serverimport (+ "strconv""strings"+ "time""github.com/seaweedfs/seaweedfs/weed/filer""github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"@@ -27,10 +29,14 @@ func writeConditionSatisfied(cond *filer_pb.WriteCondition, current *filer.Entry// clauseSatisfied evaluates one primitive against the current entry. For the// ETag kinds, etags is a set: IF_ETAG_MATCH holds when the current ETag equals-// any member, IF_ETAG_NOT_MATCH when it equals none.+// any member, IF_ETAG_NOT_MATCH when it equals none. The IF_EXTENDED_* kinds are+// generic guards on an extended attribute used to enforce object-lock (legal+// hold and retention) without S3 knowledge in the filer.func clauseSatisfied(c *filer_pb.WriteCondition_Clause, current *filer.Entry) bool {exists := current != nilswitch c.Kind {+ case filer_pb.WriteCondition_NONE:+ return truecase filer_pb.WriteCondition_IF_NOT_EXISTS:return !existscase filer_pb.WriteCondition_IF_EXISTS:@@ -43,8 +49,32 @@ func clauseSatisfied(c *filer_pb.WriteCondition_Clause, current *filer.Entry) boreturn !exists || current.Attr.Mtime.Unix() <= c.UnixTimecase filer_pb.WriteCondition_IF_MODIFIED_SINCE:return !exists || current.Attr.Mtime.Unix() > c.UnixTime+ case filer_pb.WriteCondition_IF_EXTENDED_NOT_EQUAL:+ if !exists {+ return true+ }+ v, ok := current.Extended[c.ExtKey]+ return !ok || string(v) != c.ExtValue+ case filer_pb.WriteCondition_IF_EXTENDED_TIME_ELAPSED:+ if !exists {+ return true+ }+ v, ok := current.Extended[c.ExtKey]+ if !ok {+ return true+ }+ deadline, err := strconv.ParseInt(strings.TrimSpace(string(v)), 10, 64)+ if err != nil {+ // An unparseable retention deadline is treated as still in force, so+ // a malformed attribute fails safe (write blocked) rather than open.+ return false+ }+ return deadline <= time.Now().Unix()default:- return true+ // An unrecognized clause kind (e.g. from a newer client) must not be+ // treated as satisfied, which would silently bypass the guard. Fail+ // closed so the write is blocked rather than slipping through.+ return false}}diff --git a/weed/server/filer_grpc_server_condition_test.go b/weed/server/filer_grpc_server_condition_test.goindex 0a2af9e5e9..9ee4ea6d73 100644--- a/weed/server/filer_grpc_server_condition_test.go+++ b/weed/server/filer_grpc_server_condition_test.go@@ -2,6 +2,7 @@ package weed_serverimport ("context"+ "strconv""testing""time"@@ -105,6 +106,80 @@ func TestWriteConditionClauses(t *testing.T) {}}+// The generic IF_EXTENDED_* guards express object-lock without S3 knowledge in+// the filer: a legal hold (IF_EXTENDED_NOT_EQUAL) and a retention deadline+// (IF_EXTENDED_TIME_ELAPSED).+func TestWriteConditionObjectLockGuards(t *testing.T) {+ now := time.Now()+ withExt := func(ext map[string]string) *filer.Entry {+ e := &filer.Entry{FullPath: "/test/obj", Attr: filer.Attr{Mtime: now}, Extended: map[string][]byte{}}+ for k, v := range ext {+ e.Extended[k] = []byte(v)+ }+ return e+ }+ legalHold := &filer_pb.WriteCondition{Clauses: []*filer_pb.WriteCondition_Clause{+ {Kind: filer_pb.WriteCondition_IF_EXTENDED_NOT_EQUAL, ExtKey: "lock-hold", ExtValue: "ON"},+ }}+ retention := &filer_pb.WriteCondition{Clauses: []*filer_pb.WriteCondition_Clause{+ {Kind: filer_pb.WriteCondition_IF_EXTENDED_TIME_ELAPSED, ExtKey: "retain-until"},+ }}+ future := strconv.FormatInt(now.Add(time.Hour).Unix(), 10)+ past := strconv.FormatInt(now.Add(-time.Hour).Unix(), 10)++ cases := []struct {+ name string+ cond *filer_pb.WriteCondition+ cur *filer.Entry+ want bool+ }{+ {"hold-on-blocks", legalHold, withExt(map[string]string{"lock-hold": "ON"}), false},+ {"hold-off-allows", legalHold, withExt(map[string]string{"lock-hold": "OFF"}), true},+ {"hold-absent-allows", legalHold, withExt(nil), true},+ {"hold-on-new-object", legalHold, nil, true}, // nothing to protect yet+ {"retain-future-blocks", retention, withExt(map[string]string{"retain-until": future}), false},+ {"retain-past-allows", retention, withExt(map[string]string{"retain-until": past}), true},+ {"retain-absent-allows", retention, withExt(nil), true},+ {"retain-malformed-blocks", retention, withExt(map[string]string{"retain-until": "soon"}), false},+ // Composed WORM guard: legal hold AND retention, both clear -> allowed.+ {"worm-both-clear", &filer_pb.WriteCondition{Clauses: []*filer_pb.WriteCondition_Clause{+ {Kind: filer_pb.WriteCondition_IF_EXTENDED_NOT_EQUAL, ExtKey: "lock-hold", ExtValue: "ON"},+ {Kind: filer_pb.WriteCondition_IF_EXTENDED_TIME_ELAPSED, ExtKey: "retain-until"},+ }}, withExt(map[string]string{"lock-hold": "OFF", "retain-until": past}), true},+ // Either guard tripping blocks the whole WORM condition.+ {"worm-hold-trips", &filer_pb.WriteCondition{Clauses: []*filer_pb.WriteCondition_Clause{+ {Kind: filer_pb.WriteCondition_IF_EXTENDED_NOT_EQUAL, ExtKey: "lock-hold", ExtValue: "ON"},+ {Kind: filer_pb.WriteCondition_IF_EXTENDED_TIME_ELAPSED, ExtKey: "retain-until"},+ }}, withExt(map[string]string{"lock-hold": "ON", "retain-until": past}), false},+ }+ for _, tc := range cases {+ if got := writeConditionSatisfied(tc.cond, tc.cur); got != tc.want {+ t.Errorf("%s: got %v want %v", tc.name, got, tc.want)+ }+ }+}++// An unrecognized clause kind (e.g. from a newer client) fails closed, so a+// guard can't be silently bypassed by an older filer. NONE stays a no-op.+func TestWriteConditionUnknownKindFailsClosed(t *testing.T) {+ present := entryWithETag("abc", time.Now())+ unknown := &filer_pb.WriteCondition{Clauses: []*filer_pb.WriteCondition_Clause{+ {Kind: filer_pb.WriteCondition_Kind(9999)},+ }}+ if writeConditionSatisfied(unknown, present) {+ t.Error("unknown clause kind must not be satisfied (fail closed) for an existing entry")+ }+ if writeConditionSatisfied(unknown, nil) {+ t.Error("unknown clause kind must not be satisfied (fail closed) for an absent entry")+ }+ none := &filer_pb.WriteCondition{Clauses: []*filer_pb.WriteCondition_Clause{+ {Kind: filer_pb.WriteCondition_NONE},+ }}+ if !writeConditionSatisfied(none, present) {+ t.Error("a NONE clause must be satisfied (no-op)")+ }+}+// storedEntryETag prefers the stored Seaweed ETag attribute and falls back to// the Md5-derived ETag, matching the S3 gateway.func TestStoredEntryETag(t *testing.T) {