SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
[volume] preserve volume data mtime across tier moves (#9947)
Chris Luaabd44fbb535
diff --git a/seaweed-volume/src/server/grpc_server.rs b/seaweed-volume/src/server/grpc_server.rsindex e3e4f95147..2f20e58bba 100644--- a/seaweed-volume/src/server/grpc_server.rs+++ b/seaweed-volume/src/server/grpc_server.rs@@ -1247,7 +1247,7 @@ impl VolumeServer for VolumeGrpcService {.await.map_err(|e| Status::internal(e))?;if dat_modified_ts_ns > 0 {- set_file_mtime(&dat_path, dat_modified_ts_ns);+ let _ = set_file_mtime(&dat_path, dat_modified_ts_ns);}}@@ -1272,7 +1272,7 @@ impl VolumeServer for VolumeGrpcService {.await.map_err(|e| Status::internal(e))?;if idx_modified_ts_ns > 0 {- set_file_mtime(&idx_path, idx_modified_ts_ns);+ let _ = set_file_mtime(&idx_path, idx_modified_ts_ns);}// Copy .vif file (ignore if not found on source)@@ -1296,7 +1296,7 @@ impl VolumeServer for VolumeGrpcService {.await.map_err(|e| Status::internal(e))?;if vif_modified_ts_ns > 0 {- set_file_mtime(&vif_path, vif_modified_ts_ns);+ let _ = set_file_mtime(&vif_path, vif_modified_ts_ns);}// Remove the .note file@@ -3087,6 +3087,15 @@ impl VolumeServer for VolumeGrpcService {dat_path};+ // Store the source .dat mtime, not the upload time, so a reload computes+ // TTL from real data age (matches Go VolumeTierMoveDatToRemote).+ let dat_modified_secs = std::fs::metadata(&dat_path)+ .and_then(|m| m.modified())+ .map_err(|e| Status::internal(format!("stat data file {}: {}", dat_path, e)))?+ .duration_since(std::time::UNIX_EPOCH)+ .map(|d| d.as_secs())+ .unwrap_or(0);+// Look up the S3 tier backendlet backend = {let registry = self.state.s3_tier_registry.read().unwrap();@@ -3139,18 +3148,13 @@ impl VolumeServer for VolumeGrpcService {{let mut store = state.store.write().unwrap();if let Some((_, vol)) = store.find_volume_mut(vid) {- let now_unix = std::time::SystemTime::now()- .duration_since(std::time::UNIX_EPOCH)- .unwrap_or_default()- .as_secs();-vol.volume_info.files.push(volume_server_pb::RemoteFile {backend_type: backend_type.clone(),backend_id: backend_id.clone(),key,offset: 0,file_size: size,- modified_time: now_unix,+ modified_time: dat_modified_secs,extension: ".dat".to_string(),});vol.refresh_remote_write_mode();@@ -3201,7 +3205,7 @@ impl VolumeServer for VolumeGrpcService {let vid = VolumeId(req.volume_id);// Validate volume and get remote storage info- let (dat_path, storage_name, storage_key) = {+ let (dat_path, storage_name, storage_key, remote_modified_secs) = {let store = self.state.store.read().unwrap();let (_, vol) = store.find_volume(vid)@@ -3231,7 +3235,14 @@ impl VolumeServer for VolumeGrpcService {)));}- (dat_path, storage_name, storage_key)+ let remote_modified_secs = vol+ .volume_info+ .files+ .first()+ .map(|f| f.modified_time)+ .unwrap_or(0);++ (dat_path, storage_name, storage_key, remote_modified_secs)};// Look up the S3 tier backend@@ -3279,6 +3290,15 @@ impl VolumeServer for VolumeGrpcService {))})?;+ // Restore the .dat mtime so a reload computes TTL from real data age,+ // not download time (matches Go VolumeTierMoveDatFromRemote).+ if remote_modified_secs > 0 {+ let modified_ts_ns = (remote_modified_secs as i64).saturating_mul(1_000_000_000);+ if let Err(e) = set_file_mtime(&dat_path, modified_ts_ns) {+ tracing::warn!("volume {} restore data file {} modified time: {}", vid, dat_path, e);+ }+ }+if !keep_remote {// Delete remote filebackend.delete_file(&storage_key).await.map_err(|e| {@@ -4150,17 +4170,16 @@ async fn ping_filer_target(use super::grpc_client::parse_grpc_address;/// Set the modification time of a file from nanoseconds since Unix epoch.-fn set_file_mtime(path: &str, modified_ts_ns: i64) {+fn set_file_mtime(path: &str, modified_ts_ns: i64) -> std::io::Result<()> {use std::time::{Duration, SystemTime};let ts = if modified_ts_ns >= 0 {SystemTime::UNIX_EPOCH + Duration::from_nanos(modified_ts_ns as u64)} else {SystemTime::UNIX_EPOCH};- if let Ok(file) = std::fs::File::open(path) {- let ft = std::fs::FileTimes::new().set_accessed(ts).set_modified(ts);- let _ = file.set_times(ft);- }+ let file = std::fs::File::open(path)?;+ let ft = std::fs::FileTimes::new().set_accessed(ts).set_modified(ts);+ file.set_times(ft)}/// Copy a file from a remote volume server via CopyFile streaming RPC.diff --git a/weed/server/volume_grpc_tier_download.go b/weed/server/volume_grpc_tier_download.goindex e7a93e7bd6..afde9a44f1 100644--- a/weed/server/volume_grpc_tier_download.go+++ b/weed/server/volume_grpc_tier_download.go@@ -2,8 +2,10 @@ package weed_serverimport ("fmt"+ "os""time"+ "github.com/seaweedfs/seaweedfs/weed/glog""github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb""github.com/seaweedfs/seaweedfs/weed/storage/backend""github.com/seaweedfs/seaweedfs/weed/storage/needle"@@ -28,6 +30,7 @@ func (vs *VolumeServer) VolumeTierMoveDatFromRemote(req *volume_server_pb.Volumeif storageName == "" || storageKey == "" {return fmt.Errorf("volume %d is already on local disk", req.VolumeId)}+ remoteFileModifiedTime := v.GetVolumeInfo().GetFiles()[0].GetModifiedTime()// check whether the local .dat already exists_, ok := v.DataBackend.(*backend.DiskFile)@@ -62,6 +65,14 @@ func (vs *VolumeServer) VolumeTierMoveDatFromRemote(req *volume_server_pb.Volumeif err != nil {return fmt.Errorf("backend %s copy file %s: %v", storageName, v.FileName(".dat"), err)}+ if remoteFileModifiedTime > 0 {+ modifiedTime := time.Unix(int64(remoteFileModifiedTime), 0)+ // best-effort: a cosmetic mtime failure should not abort an otherwise+ // complete tier-down, matching the EC copy path+ if err := os.Chtimes(v.FileName(".dat"), modifiedTime, modifiedTime); err != nil {+ glog.Warningf("volume %d restore data file %s modified time: %v", v.Id, v.FileName(".dat"), err)+ }+ }if req.KeepRemoteDatFile {return nildiff --git a/weed/server/volume_grpc_tier_test.go b/weed/server/volume_grpc_tier_test.gonew file mode 100644index 0000000000..1329b0bf4b--- /dev/null+++ b/weed/server/volume_grpc_tier_test.go@@ -0,0 +1,220 @@+package weed_server++import (+ "fmt"+ "io"+ "os"+ "path/filepath"+ "testing"+ "time"++ "google.golang.org/grpc"++ "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"+ "github.com/seaweedfs/seaweedfs/weed/stats"+ "github.com/seaweedfs/seaweedfs/weed/storage"+ "github.com/seaweedfs/seaweedfs/weed/storage/backend"+ "github.com/seaweedfs/seaweedfs/weed/storage/needle"+ "github.com/seaweedfs/seaweedfs/weed/storage/types"+ "github.com/seaweedfs/seaweedfs/weed/util"+)++const tierTimestampTestBackendName = "tier_timestamp_test.default"++type discardServerStream[T any] struct {+ grpc.ServerStream+}++func (s *discardServerStream[T]) Send(*T) error {+ return nil+}++type tierTimestampTestBackend struct {+ root string+}++func (b *tierTimestampTestBackend) ToProperties() map[string]string {+ return map[string]string{"root": b.root}+}++func (b *tierTimestampTestBackend) NewStorageFile(key string, volumeInfo *volume_server_pb.VolumeInfo) backend.BackendStorageFile {+ return &tierTimestampTestBackendFile{+ path: filepath.Join(b.root, key),+ volumeInfo: volumeInfo,+ }+}++func (b *tierTimestampTestBackend) CopyFile(file *os.File, fn func(progressed int64, percentage float32) error) (key string, size int64, err error) {+ key = "remote.dat"+ fileInfo, err := file.Stat()+ if err != nil {+ return "", 0, err+ }++ output, err := os.Create(filepath.Join(b.root, key))+ if err != nil {+ return "", 0, err+ }+ defer output.Close()++ size, err = io.Copy(output, io.NewSectionReader(file, 0, fileInfo.Size()))+ if err == nil && fn != nil {+ err = fn(size, 100)+ }+ return key, size, err+}++func (b *tierTimestampTestBackend) DownloadFile(fileName string, key string, fn func(progressed int64, percentage float32) error) (size int64, err error) {+ input, err := os.Open(filepath.Join(b.root, key))+ if err != nil {+ return 0, err+ }+ defer input.Close()++ output, err := os.Create(fileName)+ if err != nil {+ return 0, err+ }+ defer output.Close()++ size, err = io.Copy(output, input)+ if err == nil && fn != nil {+ err = fn(size, 100)+ }+ return size, err+}++func (b *tierTimestampTestBackend) DeleteFile(key string) error {+ return os.Remove(filepath.Join(b.root, key))+}++type tierTimestampTestBackendFile struct {+ path string+ volumeInfo *volume_server_pb.VolumeInfo+}++func (f *tierTimestampTestBackendFile) ReadAt(p []byte, off int64) (int, error) {+ file, err := os.Open(f.path)+ if err != nil {+ return 0, err+ }+ defer file.Close()+ return file.ReadAt(p, off)+}++func (f *tierTimestampTestBackendFile) WriteAt(p []byte, off int64) (int, error) {+ return 0, fmt.Errorf("remote test file is read-only")+}++func (f *tierTimestampTestBackendFile) Truncate(off int64) error {+ return fmt.Errorf("remote test file is read-only")+}++func (f *tierTimestampTestBackendFile) Close() error {+ return nil+}++func (f *tierTimestampTestBackendFile) GetStat() (datSize int64, modTime time.Time, err error) {+ files := f.volumeInfo.GetFiles()+ if len(files) == 0 {+ return 0, time.Time{}, fmt.Errorf("remote file info not found")+ }+ return int64(files[0].GetFileSize()), time.Unix(int64(files[0].GetModifiedTime()), 0), nil+}++func (f *tierTimestampTestBackendFile) Name() string {+ return f.path+}++func (f *tierTimestampTestBackendFile) Sync() error {+ return nil+}++func TestVolumeTierMoveDatPreservesModifiedTime(t *testing.T) {+ dataDir := t.TempDir()+ remoteDir := t.TempDir()+ testBackend := &tierTimestampTestBackend{root: remoteDir}+ backend.BackendStorages[tierTimestampTestBackendName] = testBackend+ t.Cleanup(func() {+ delete(backend.BackendStorages, tierTimestampTestBackendName)+ })++ store := storage.NewStore(+ nil,+ "localhost",+ 8080,+ 18080,+ "http://localhost:8080",+ "store-id",+ []string{dataDir},+ []int32{10},+ []util.MinFreeSpace{{}},+ "",+ storage.NeedleMapInMemory,+ []types.DiskType{types.HardDriveType},+ nil,+ 0,+ stats.DefaultDiskIOProbeConfig(),+ )+ t.Cleanup(store.Close)++ const volumeId = needle.VolumeId(1)+ if err := store.AddVolume(volumeId, "", storage.NeedleMapInMemory, "000", "", 0, needle.Version3, 0, types.HardDriveType, 0); err != nil {+ t.Fatalf("add volume: %v", err)+ }++ volume := store.GetVolume(volumeId)+ dataFileName := volume.FileName(".dat")+ sourceModifiedTime := time.Unix(1_700_000_000, 0)+ if err := os.Chtimes(dataFileName, sourceModifiedTime, sourceModifiedTime); err != nil {+ t.Fatalf("set source modified time: %v", err)+ }++ // Re-open the data backend so the DiskFile caches the on-disk mtime, the way+ // a volume freshly loaded from disk does.+ volume.DataBackend.Close()+ reopened, err := os.OpenFile(dataFileName, os.O_RDWR, 0644)+ if err != nil {+ t.Fatalf("reopen data file: %v", err)+ }+ volume.DataBackend = backend.NewDiskFile(reopened)++ volumeServer := &VolumeServer{store: store}+ if err := volumeServer.VolumeTierMoveDatToRemote(+ &volume_server_pb.VolumeTierMoveDatToRemoteRequest{+ VolumeId: uint32(volumeId),+ DestinationBackendName: tierTimestampTestBackendName,+ },+ &discardServerStream[volume_server_pb.VolumeTierMoveDatToRemoteResponse]{},+ ); err != nil {+ t.Fatalf("move data to remote: %v", err)+ }++ remoteFiles := volume.GetVolumeInfo().GetFiles()+ if len(remoteFiles) != 1 {+ t.Fatalf("remote file count = %d, want 1", len(remoteFiles))+ }+ if got := remoteFiles[0].GetModifiedTime(); got != uint64(sourceModifiedTime.Unix()) {+ t.Fatalf("remote modified time = %d, want %d", got, sourceModifiedTime.Unix())+ }+ if _, err := os.Stat(dataFileName); !os.IsNotExist(err) {+ t.Fatalf("local data file still exists after upload: %v", err)+ }++ if err := volumeServer.VolumeTierMoveDatFromRemote(+ &volume_server_pb.VolumeTierMoveDatFromRemoteRequest{+ VolumeId: uint32(volumeId),+ },+ &discardServerStream[volume_server_pb.VolumeTierMoveDatFromRemoteResponse]{},+ ); err != nil {+ t.Fatalf("move data from remote: %v", err)+ }++ fileInfo, err := os.Stat(dataFileName)+ if err != nil {+ t.Fatalf("stat downloaded data file: %v", err)+ }+ if got := fileInfo.ModTime().Unix(); got != sourceModifiedTime.Unix() {+ t.Fatalf("downloaded modified time = %d, want %d", got, sourceModifiedTime.Unix())+ }+}diff --git a/weed/server/volume_grpc_tier_upload.go b/weed/server/volume_grpc_tier_upload.goindex 4f8ac028b0..dee062b586 100644--- a/weed/server/volume_grpc_tier_upload.go+++ b/weed/server/volume_grpc_tier_upload.go@@ -32,6 +32,10 @@ func (vs *VolumeServer) VolumeTierMoveDatToRemote(req *volume_server_pb.VolumeTiif !ok {return nil // already copied to remove. fmt.Errorf("volume %d is not on local disk", req.VolumeId)}+ _, modTime, err := diskFile.GetStat()+ if err != nil {+ return fmt.Errorf("stat data file %s: %v", diskFile.Name(), err)+ }// check valid storage backend typebackendStorage, found := backend.BackendStorages[req.DestinationBackendName]@@ -78,7 +82,7 @@ func (vs *VolumeServer) VolumeTierMoveDatToRemote(req *volume_server_pb.VolumeTiKey: key,Offset: 0,FileSize: uint64(size),- ModifiedTime: uint64(time.Now().Unix()),+ ModifiedTime: uint64(modTime.Unix()),Extension: ".dat",})