SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
filer: reuse the caller's fetched entry in CreateEntry (#9645)
Chris Lu83195fc111be
diff --git a/weed/filer/filer.go b/weed/filer/filer.goindex 420aa1761b..c4421ed378 100644--- a/weed/filer/filer.go+++ b/weed/filer/filer.go@@ -209,7 +209,11 @@ func (f *Filer) RollbackTransaction(ctx context.Context) error {return f.Store.RollbackTransaction(ctx)}-func (f *Filer) CreateEntry(ctx context.Context, entry *Entry, o_excl bool, isFromOtherCluster bool, signatures []int32, skipCreateParentDir bool, maxFilenameLength uint32) error {+// CreateEntry creates or replaces an entry. When existing is non-nil the caller+// has already fetched the current entry at this path under a path lock, and it+// is reused instead of looking the store up again; pass nil to have CreateEntry+// look it up itself.+func (f *Filer) CreateEntry(ctx context.Context, entry *Entry, existing *Entry, o_excl bool, isFromOtherCluster bool, signatures []int32, skipCreateParentDir bool, maxFilenameLength uint32) error {if string(entry.FullPath) == "/" {return nil@@ -227,7 +231,10 @@ func (f *Filer) CreateEntry(ctx context.Context, entry *Entry, o_excl bool, isFrentry.Attr.Atime = entryInitialAtime(entry.Attr)}- oldEntry, _ := f.FindEntry(ctx, entry.FullPath)+ oldEntry := existing+ if oldEntry == nil {+ oldEntry, _ = f.FindEntry(ctx, entry.FullPath)+ }/*if !hasWritePermission(lastDirectoryEntry, entry) {diff --git a/weed/filer/filer_inode_test.go b/weed/filer/filer_inode_test.goindex e965ab017b..e19ae680a2 100644--- a/weed/filer/filer_inode_test.go+++ b/weed/filer/filer_inode_test.go@@ -28,7 +28,7 @@ func TestCreateEntryAssignsInodeWhenMissing(t *testing.T) {},}- err := f.CreateEntry(context.Background(), entry, false, false, nil, false, f.MaxFilenameLength)+ err := f.CreateEntry(context.Background(), entry, nil, false, false, nil, false, f.MaxFilenameLength)require.NoError(t, err)stored, findErr := store.FindEntry(context.Background(), entry.FullPath)@@ -48,7 +48,7 @@ func TestCreateEntryAssignsInodesToAutoCreatedParents(t *testing.T) {},}- err := f.CreateEntry(context.Background(), entry, false, false, nil, false, f.MaxFilenameLength)+ err := f.CreateEntry(context.Background(), entry, nil, false, false, nil, false, f.MaxFilenameLength)require.NoError(t, err)for _, path := range []string{"/a", "/a/b", "/a/b/c.txt"} {diff --git a/weed/filer/filer_lazy_remote.go b/weed/filer/filer_lazy_remote.goindex a7025d3045..77599ca8c3 100644--- a/weed/filer/filer_lazy_remote.go+++ b/weed/filer/filer_lazy_remote.go@@ -96,7 +96,7 @@ func (f *Filer) maybeLazyFetchFromRemote(ctx context.Context, p util.FullPath) (persistBaseCtx, cancelPersist := context.WithTimeout(context.Background(), 30*time.Second)defer cancelPersist()persistCtx := context.WithValue(persistBaseCtx, lazyFetchContextKey{}, true)- saveErr := f.CreateEntry(persistCtx, entry, false, false, nil, true, f.MaxFilenameLength)+ saveErr := f.CreateEntry(persistCtx, entry, nil, false, false, nil, true, f.MaxFilenameLength)if saveErr != nil {glog.Warningf("maybeLazyFetchFromRemote: failed to persist filer entry for %s: %v", p, saveErr)f.lazyFetchGroup.Forget(key)diff --git a/weed/filer/filer_lazy_remote_listing.go b/weed/filer/filer_lazy_remote_listing.goindex c6d8271574..59b2d2ea3f 100644--- a/weed/filer/filer_lazy_remote_listing.go+++ b/weed/filer/filer_lazy_remote_listing.go@@ -152,7 +152,7 @@ func (f *Filer) maybeLazyListFromRemote(ctx context.Context, p util.FullPath) {entry.Attr.FileSize = uint64(remoteEntry.RemoteSize)}}- if saveErr := f.CreateEntry(persistCtx, entry, false, false, nil, true, f.MaxFilenameLength); saveErr != nil {+ if saveErr := f.CreateEntry(persistCtx, entry, nil, false, false, nil, true, f.MaxFilenameLength); saveErr != nil {glog.Warningf("maybeLazyListFromRemote: persist %s: %v", childPath, saveErr)}}@@ -193,7 +193,7 @@ func (f *Filer) updateDirectoryListingSyncedAt(ctx context.Context, p util.FullPdirEntry.Extended = make(map[string][]byte)}dirEntry.Extended[xattrRemoteListingSyncedAt] = []byte(fmt.Sprintf("%d", syncTime.Unix()))- if saveErr := f.CreateEntry(ctx, dirEntry, false, false, nil, true, f.MaxFilenameLength); saveErr != nil {+ if saveErr := f.CreateEntry(ctx, dirEntry, nil, false, false, nil, true, f.MaxFilenameLength); saveErr != nil {glog.Warningf("maybeLazyListFromRemote: create dir synced_at for %s: %v", p, saveErr)}returndiff --git a/weed/filer/filer_notify_append.go b/weed/filer/filer_notify_append.goindex 036c825cfe..432153dfb4 100644--- a/weed/filer/filer_notify_append.go+++ b/weed/filer/filer_notify_append.go@@ -43,7 +43,7 @@ func (f *Filer) appendToFile(targetFile string, data []byte) error {entry.Chunks = append(entry.GetChunks(), uploadResult.ToPbFileChunk(assignResult.Fid, offset, time.Now().UnixNano()))// update the entry- err = f.CreateEntry(context.Background(), entry, false, false, nil, false, f.MaxFilenameLength)+ err = f.CreateEntry(context.Background(), entry, nil, false, false, nil, false, f.MaxFilenameLength)return err}diff --git a/weed/filer/leveldb/leveldb_store_test.go b/weed/filer/leveldb/leveldb_store_test.goindex a491d969b0..b5384fc778 100644--- a/weed/filer/leveldb/leveldb_store_test.go+++ b/weed/filer/leveldb/leveldb_store_test.go@@ -32,7 +32,7 @@ func TestCreateAndFind(t *testing.T) {},}- if err := testFiler.CreateEntry(ctx, entry1, false, false, nil, false, testFiler.MaxFilenameLength); err != nil {+ if err := testFiler.CreateEntry(ctx, entry1, nil, false, false, nil, false, testFiler.MaxFilenameLength); err != nil {t.Errorf("create entry %v: %v", entry1.FullPath, err)return}diff --git a/weed/filer/leveldb2/leveldb2_store_test.go b/weed/filer/leveldb2/leveldb2_store_test.goindex 6f6667422f..5c6a3a4d71 100644--- a/weed/filer/leveldb2/leveldb2_store_test.go+++ b/weed/filer/leveldb2/leveldb2_store_test.go@@ -29,7 +29,7 @@ func TestCreateAndFind(t *testing.T) {},}- if err := testFiler.CreateEntry(ctx, entry1, false, false, nil, false, testFiler.MaxFilenameLength); err != nil {+ if err := testFiler.CreateEntry(ctx, entry1, nil, false, false, nil, false, testFiler.MaxFilenameLength); err != nil {t.Errorf("create entry %v: %v", entry1.FullPath, err)return}diff --git a/weed/filer/leveldb3/leveldb3_store_test.go b/weed/filer/leveldb3/leveldb3_store_test.goindex aea6edd041..8a3cd2d82e 100644--- a/weed/filer/leveldb3/leveldb3_store_test.go+++ b/weed/filer/leveldb3/leveldb3_store_test.go@@ -29,7 +29,7 @@ func TestCreateAndFind(t *testing.T) {},}- if err := testFiler.CreateEntry(ctx, entry1, false, false, nil, false, testFiler.MaxFilenameLength); err != nil {+ if err := testFiler.CreateEntry(ctx, entry1, nil, false, false, nil, false, testFiler.MaxFilenameLength); err != nil {t.Errorf("create entry %v: %v", entry1.FullPath, err)return}diff --git a/weed/filer/rocksdb/rocksdb_store_test.go b/weed/filer/rocksdb/rocksdb_store_test.goindex 35181d6016..00c772d4d6 100644--- a/weed/filer/rocksdb/rocksdb_store_test.go+++ b/weed/filer/rocksdb/rocksdb_store_test.go@@ -35,7 +35,7 @@ func TestCreateAndFind(t *testing.T) {},}- if err := testFiler.CreateEntry(ctx, entry1, false, false, nil, false, testFiler.MaxFilenameLength); err != nil {+ if err := testFiler.CreateEntry(ctx, entry1, nil, false, false, nil, false, testFiler.MaxFilenameLength); err != nil {t.Errorf("create entry %v: %v", entry1.FullPath, err)return}@@ -149,7 +149,7 @@ func TestListDirectoryWithPrefix(t *testing.T) {Gid: 1,},}- if err := testFiler.CreateEntry(ctx, entry, false, false, nil, false, testFiler.MaxFilenameLength); err != nil {+ if err := testFiler.CreateEntry(ctx, entry, nil, false, false, nil, false, testFiler.MaxFilenameLength); err != nil {t.Fatalf("Failed to create entry %s: %v", fullpath, err)}}diff --git a/weed/server/filer_grpc_server.go b/weed/server/filer_grpc_server.goindex 4aadf677a3..ef67e2cb3c 100644--- a/weed/server/filer_grpc_server.go+++ b/weed/server/filer_grpc_server.go@@ -195,6 +195,9 @@ func (fs *FilerServer) CreateEntry(ctx context.Context, req *filer_pb.CreateEntr// Evaluate the optional precondition against the current entry while the// path lock is held, so the check and the write are atomic on this filer.+ // The fetched entry is then handed to CreateEntry below so it does not look+ // the same path up again under the lock.+ var existing *filer.Entryif conditionIsSet(req.Condition) {current, findErr := fs.filer.FindEntry(ctx, fullpath)if findErr != nil && findErr != filer_pb.ErrNotFound {@@ -210,10 +213,11 @@ func (fs *FilerServer) CreateEntry(ctx context.Context, req *filer_pb.CreateEntrErrorCode: filer_pb.FilerError_PRECONDITION_FAILED,}, nil}+ existing = current}ctx, eventSink := filer.WithMetadataEventSink(ctx)- createErr := fs.filer.CreateEntry(ctx, newEntry, req.OExcl, req.IsFromOtherCluster, req.Signatures, req.SkipCheckParentDirectory, so.MaxFileNameLength)+ createErr := fs.filer.CreateEntry(ctx, newEntry, existing, req.OExcl, req.IsFromOtherCluster, req.Signatures, req.SkipCheckParentDirectory, so.MaxFileNameLength)if createErr == nil {fs.filer.DeleteChunksNotRecursive(garbage)@@ -327,7 +331,7 @@ func (fs *FilerServer) applyObjectMutation(ctx context.Context, m *filer_pb.Objereturn fmt.Errorf("PUT requires an entry")}newEntry := filer.FromPbEntry(m.Directory, m.Entry)- return fs.filer.CreateEntry(ctx, newEntry, false, fromOtherCluster, signatures, false, fs.filer.MaxFilenameLength)+ return fs.filer.CreateEntry(ctx, newEntry, nil, false, fromOtherCluster, signatures, false, fs.filer.MaxFilenameLength)case filer_pb.ObjectMutation_DELETE:fullpath := util.NewFullPath(m.Directory, m.Name)@@ -592,7 +596,7 @@ func (fs *FilerServer) AppendToEntry(ctx context.Context, req *filer_pb.AppendToglog.V(0).InfofCtx(ctx, "MaybeManifestize: %v", err)}- err = fs.filer.CreateEntry(context.Background(), entry, false, false, nil, false, fs.filer.MaxFilenameLength)+ err = fs.filer.CreateEntry(context.Background(), entry, nil, false, false, nil, false, fs.filer.MaxFilenameLength)return &filer_pb.AppendToEntryResponse{}, err}diff --git a/weed/server/filer_grpc_server_condition_test.go b/weed/server/filer_grpc_server_condition_test.goindex 9ee4ea6d73..ce9007a55c 100644--- a/weed/server/filer_grpc_server_condition_test.go+++ b/weed/server/filer_grpc_server_condition_test.go@@ -234,3 +234,42 @@ func TestCreateEntryConditionEnforced(t *testing.T) {t.Fatalf("matching etag should overwrite, got error %q", resp.Error)}}++// CreateEntry reuses a provided existing entry instead of reading the store+// again; passing nil makes it look the path up itself.+func TestCreateEntryReusesProvidedExisting(t *testing.T) {+ existing := &filer.Entry{+ FullPath: "/test/obj",+ Attr: filer.Attr{Inode: 1, Mtime: time.Unix(1700000000, 0), Crtime: time.Unix(1700000000, 0)},+ Extended: map[string][]byte{s3_constants.ExtETagKey: []byte("abc")},+ }+ newEntry := func() *filer.Entry {+ return &filer.Entry{+ FullPath: "/test/obj",+ Attr: filer.Attr{Inode: 1, Mtime: time.Unix(1700000001, 0)},+ }+ }++ // Provided existing vs nil: the only difference is CreateEntry's own lookup,+ // so providing it must save exactly one path read (other store-layer reads+ // during the overwrite are the same in both runs).+ store := newRenameTestStore()+ store.entries["/test/obj"] = existing.ShallowClone()+ f := newRenameTestFiler(store)+ if err := f.CreateEntry(context.Background(), newEntry(), existing, false, false, nil, true, f.MaxFilenameLength); err != nil {+ t.Fatalf("create with existing: %v", err)+ }+ withExisting := store.findEntryCallCount("/test/obj")++ store2 := newRenameTestStore()+ store2.entries["/test/obj"] = existing.ShallowClone()+ f2 := newRenameTestFiler(store2)+ if err := f2.CreateEntry(context.Background(), newEntry(), nil, false, false, nil, true, f2.MaxFilenameLength); err != nil {+ t.Fatalf("create with nil: %v", err)+ }+ withNil := store2.findEntryCallCount("/test/obj")++ if withNil != withExisting+1 {+ t.Fatalf("providing existing should save one path lookup: existing=%d nil=%d", withExisting, withNil)+ }+}diff --git a/weed/server/filer_grpc_server_rename.go b/weed/server/filer_grpc_server_rename.goindex 3c8aae6b1d..3b60e9e75e 100644--- a/weed/server/filer_grpc_server_rename.go+++ b/weed/server/filer_grpc_server_rename.go@@ -245,7 +245,7 @@ func (fs *FilerServer) moveSelfEntry(ctx context.Context, stream filer_pb.Seaweereturn fmt.Errorf("insert entry %s: %v", newEntry.FullPath, createErr)}} else {- if createErr := fs.filer.CreateEntry(filer.WithSuppressedMetadataEvents(ctx), newEntry, false, false, signatures, false, fs.filer.MaxFilenameLength); createErr != nil {+ if createErr := fs.filer.CreateEntry(filer.WithSuppressedMetadataEvents(ctx), newEntry, nil, false, false, signatures, false, fs.filer.MaxFilenameLength); createErr != nil {return createErr}}diff --git a/weed/server/filer_server_handlers_copy.go b/weed/server/filer_server_handlers_copy.goindex 2577c41b27..041c19d89e 100644--- a/weed/server/filer_server_handlers_copy.go+++ b/weed/server/filer_server_handlers_copy.go@@ -169,7 +169,7 @@ func (fs *FilerServer) copy(ctx context.Context, w http.ResponseWriter, r *http.// Pass o_excl = !overwrite so the default copy refuses to replace an// existing destination, while overwrite=true updates the pre-created target.- if createErr := fs.filer.CreateEntry(ctx, newEntry, !overwrite, false, nil, false, fs.filer.MaxFilenameLength); createErr != nil {+ if createErr := fs.filer.CreateEntry(ctx, newEntry, nil, !overwrite, false, nil, false, fs.filer.MaxFilenameLength); createErr != nil {err = fmt.Errorf("failed to create copied entry from '%s' to '%s': %w", src, dst, createErr)writeJsonError(w, r, http.StatusInternalServerError, err)returndiff --git a/weed/server/filer_server_handlers_tagging.go b/weed/server/filer_server_handlers_tagging.goindex 8d69619594..06cd1295e9 100644--- a/weed/server/filer_server_handlers_tagging.go+++ b/weed/server/filer_server_handlers_tagging.go@@ -42,7 +42,7 @@ func (fs *FilerServer) PutTaggingHandler(w http.ResponseWriter, r *http.Request)}}- if dbErr := fs.filer.CreateEntry(ctx, existingEntry, false, false, nil, false, fs.filer.MaxFilenameLength); dbErr != nil {+ if dbErr := fs.filer.CreateEntry(ctx, existingEntry, nil, false, false, nil, false, fs.filer.MaxFilenameLength); dbErr != nil {glog.V(0).InfofCtx(ctx, "failing to update %s tagging : %v", path, dbErr)writeJsonError(w, r, http.StatusInternalServerError, dbErr)return@@ -108,7 +108,7 @@ func (fs *FilerServer) DeleteTaggingHandler(w http.ResponseWriter, r *http.Requereturn}- if dbErr := fs.filer.CreateEntry(ctx, existingEntry, false, false, nil, false, fs.filer.MaxFilenameLength); dbErr != nil {+ if dbErr := fs.filer.CreateEntry(ctx, existingEntry, nil, false, false, nil, false, fs.filer.MaxFilenameLength); dbErr != nil {glog.V(0).InfofCtx(ctx, "failing to delete %s tagging : %v", path, dbErr)writeJsonError(w, r, http.StatusInternalServerError, dbErr)returndiff --git a/weed/server/filer_server_handlers_write_autochunk.go b/weed/server/filer_server_handlers_write_autochunk.goindex 94af39b8db..c7959682a0 100644--- a/weed/server/filer_server_handlers_write_autochunk.go+++ b/weed/server/filer_server_handlers_write_autochunk.go@@ -339,7 +339,7 @@ func (fs *FilerServer) saveMetaData(ctx context.Context, r *http.Request, fileNa}}- dbErr := fs.filer.CreateEntry(context.WithoutCancel(ctx), entry, false, false, nil, skipCheckParentDirEntry(r), so.MaxFileNameLength)+ dbErr := fs.filer.CreateEntry(context.WithoutCancel(ctx), entry, nil, false, false, nil, skipCheckParentDirEntry(r), so.MaxFileNameLength)if dbErr != nil {replyerr = dbErrfilerResult.Error = dbErr.Error()@@ -437,7 +437,7 @@ func (fs *FilerServer) mkdir(ctx context.Context, w http.ResponseWriter, r *httpName: util.FullPath(path).Name(),}- if dbErr := fs.filer.CreateEntry(context.WithoutCancel(ctx), entry, false, false, nil, false, so.MaxFileNameLength); dbErr != nil {+ if dbErr := fs.filer.CreateEntry(context.WithoutCancel(ctx), entry, nil, false, false, nil, false, so.MaxFileNameLength); dbErr != nil {replyerr = dbErrfilerResult.Error = dbErr.Error()glog.V(0).InfofCtx(ctx, "failing to create dir %s on filer server : %v", path, dbErr)diff --git a/weed/server/filer_server_tus_session.go b/weed/server/filer_server_tus_session.goindex 90aba9c996..2b804b3d83 100644--- a/weed/server/filer_server_tus_session.go+++ b/weed/server/filer_server_tus_session.go@@ -132,7 +132,7 @@ func (fs *FilerServer) createTusSession(ctx context.Context, uploadID, targetPatUid: OS_UID,Gid: OS_GID,},- }, false, false, nil, false, fs.filer.MaxFilenameLength); err != nil {+ }, nil, false, false, nil, false, fs.filer.MaxFilenameLength); err != nil {return nil, fmt.Errorf("create session directory: %w", err)}@@ -167,7 +167,7 @@ func (fs *FilerServer) saveTusSession(ctx context.Context, session *TusSession)Content: sessionData,}- if err := fs.filer.CreateEntry(ctx, entry, false, false, nil, false, fs.filer.MaxFilenameLength); err != nil {+ if err := fs.filer.CreateEntry(ctx, entry, nil, false, false, nil, false, fs.filer.MaxFilenameLength); err != nil {return fmt.Errorf("save session info entry: %w", err)}@@ -265,7 +265,7 @@ func (fs *FilerServer) saveTusChunk(ctx context.Context, uploadID string, chunkUid: OS_UID,Gid: OS_GID,},- }, false, false, nil, false, fs.filer.MaxFilenameLength); err != nil {+ }, nil, false, false, nil, false, fs.filer.MaxFilenameLength); err != nil {return fmt.Errorf("save chunk info: %w", err)}@@ -371,7 +371,7 @@ func (fs *FilerServer) completeTusUpload(ctx context.Context, session *TusSessio}// Ensure parent directory exists- if err := fs.filer.CreateEntry(ctx, entry, false, false, nil, false, fs.filer.MaxFilenameLength); err != nil {+ if err := fs.filer.CreateEntry(ctx, entry, nil, false, false, nil, false, fs.filer.MaxFilenameLength); err != nil {return fmt.Errorf("create final file entry: %w", err)}