Skip to content

SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.

14.9 KB
Raw Blame
package shell

import (
	"context"
	"errors"
	"flag"
	"fmt"
	"io"
	"math"
	"sync"

	"github.com/seaweedfs/seaweedfs/weed/filer"
	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
	"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
	"github.com/seaweedfs/seaweedfs/weed/remote_storage"
	"github.com/seaweedfs/seaweedfs/weed/util"
)

func init() {
	Commands = append(Commands, &commandRemoteCache{})
}

type commandRemoteCache struct {
}

func (c *commandRemoteCache) Name() string {
	return "remote.cache"
}

func (c *commandRemoteCache) Help() string {
	return `comprehensive synchronization and caching between local and remote storage

	# assume a remote storage is configured to name "cloud1"
	remote.configure -name=cloud1 -type=s3 -s3.access_key=xxx -s3.secret_key=yyy
	# mount and pull one bucket
	remote.mount -dir=/xxx -remote=cloud1/bucket

	# comprehensive sync and cache: update metadata, cache content, and remove deleted files
	remote.cache -dir=/xxx                                # sync metadata, cache content, and remove deleted files (default)
	remote.cache -dir=/xxx -cacheContent=false            # sync metadata and cleanup only, no caching
	remote.cache -dir=/xxx -deleteLocalExtra=false        # skip removal of local files missing from remote
	remote.cache -dir=/xxx -concurrent=32                 # with custom file-level concurrency
	remote.cache -dir=/xxx -chunkConcurrency=16           # parallel chunk downloads per file (0 = server default 8)
	remote.cache -dir=/xxx -downloadConcurrency=10        # S3 multipart download concurrency per chunk (0 = server default 5)
	remote.cache -dir=/xxx -include=*.pdf                 # only sync PDF files
	remote.cache -dir=/xxx -exclude=*.tmp                 # exclude temporary files
	remote.cache -dir=/xxx -dryRun=true                   # show what would be done without making changes

	This command will:
	1. Synchronize metadata from remote storage
	2. Cache file content from remote by default
	3. Remove local files that no longer exist on remote by default (use -deleteLocalExtra=false to disable)

	This is designed to run regularly. So you can add it to some cronjob.

`
}

func (c *commandRemoteCache) HasTag(CommandTag) bool {
	return false
}

func (c *commandRemoteCache) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {

	remoteCacheCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)

	dir := remoteCacheCommand.String("dir", "", "a directory in filer")
	cache := remoteCacheCommand.Bool("cacheContent", true, "cache file content from remote")
	deleteLocalExtra := remoteCacheCommand.Bool("deleteLocalExtra", true, "delete local files that no longer exist on remote")
	concurrency := remoteCacheCommand.Int("concurrent", 16, "concurrent file operations")
	chunkConcurrency := remoteCacheCommand.Int("chunkConcurrency", 0, "parallel chunk downloads per file (0 = server default 8)")
	downloadConcurrency := remoteCacheCommand.Int("downloadConcurrency", 0, "S3 multipart download concurrency per chunk (0 = server default 5)")
	dryRun := remoteCacheCommand.Bool("dryRun", false, "show what would be done without making changes")
	fileFiler := newFileFilter(remoteCacheCommand)

	if err = remoteCacheCommand.Parse(args); err != nil {
		return err
	}

	if *dir == "" {
		return fmt.Errorf("need to specify -dir option")
	}
	if *chunkConcurrency < 0 || *chunkConcurrency > math.MaxInt32 {
		return fmt.Errorf("chunkConcurrency must be between 0 and %d", math.MaxInt32)
	}
	if *downloadConcurrency < 0 || *downloadConcurrency > math.MaxInt32 {
		return fmt.Errorf("downloadConcurrency must be between 0 and %d", math.MaxInt32)
	}

	mappings, localMountedDir, remoteStorageMountedLocation, remoteStorageConf, detectErr := detectMountInfo(commandEnv, writer, *dir)
	if detectErr != nil {
		jsonPrintln(writer, mappings)
		return detectErr
	}

	// perform comprehensive sync
	return c.doComprehensiveSync(commandEnv, writer, util.FullPath(localMountedDir), remoteStorageMountedLocation, util.FullPath(*dir), remoteStorageConf, *cache, *deleteLocalExtra, *concurrency, int32(*chunkConcurrency), int32(*downloadConcurrency), *dryRun, fileFiler)
}

func (c *commandRemoteCache) doComprehensiveSync(commandEnv *CommandEnv, writer io.Writer, localMountedDir util.FullPath, remoteMountedLocation *remote_pb.RemoteStorageLocation, dirToSync util.FullPath, remoteConf *remote_pb.RemoteConf, shouldCache bool, deleteLocalExtra bool, concurrency int, chunkConcurrency int32, downloadConcurrency int32, dryRun bool, fileFilter *FileFilter) error {

	// visit remote storage
	remoteStorage, err := remote_storage.GetRemoteStorage(remoteConf)
	if err != nil {
		return err
	}

	remote := filer.MapFullPathToRemoteStorageLocation(localMountedDir, remoteMountedLocation, dirToSync)

	// Step 1: Collect all remote files
	remoteFiles := make(map[string]*filer_pb.RemoteEntry)
	err = remoteStorage.Traverse(remote, func(remoteDir, name string, isDirectory bool, remoteEntry *filer_pb.RemoteEntry) error {
		localDir := filer.MapRemoteStorageLocationPathToFullPath(localMountedDir, remoteMountedLocation, remoteDir)
		fullPath := string(localDir.Child(name))
		remoteFiles[fullPath] = remoteEntry
		return nil
	})
	if err != nil {
		return fmt.Errorf("failed to traverse remote storage: %w", err)
	}

	fmt.Fprintf(writer, "Found %d files/directories in remote storage\n", len(remoteFiles))

	// Step 2: Collect all local files (only if we need to delete local extra files)
	localFiles := make(map[string]*filer_pb.Entry)
	if deleteLocalExtra {
		err = recursivelyTraverseDirectory(commandEnv, dirToSync, func(dir util.FullPath, entry *filer_pb.Entry) bool {
			if entry.RemoteEntry != nil { // only consider files that are part of remote mount
				fullPath := string(dir.Child(entry.Name))
				localFiles[fullPath] = entry
			}
			return true
		})
		if err != nil {
			return fmt.Errorf("failed to traverse local directory: %w", err)
		}
		fmt.Fprintf(writer, "Found %d files/directories in local storage\n", len(localFiles))
	} else {
		fmt.Fprintf(writer, "Skipping local file collection (deleteLocalExtra=false)\n")
	}

	// Step 3: Determine actions needed
	var filesToDelete []string
	var filesToUpdate []string
	var filesToCache []string

	// Find files to delete (exist locally but not remotely) - only if deleteLocalExtra is enabled
	if deleteLocalExtra {
		for localPath := range localFiles {
			if _, exists := remoteFiles[localPath]; !exists {
				filesToDelete = append(filesToDelete, localPath)
			}
		}
	}

	// Find files to update/cache (exist remotely)
	for remotePath, remoteEntry := range remoteFiles {
		if deleteLocalExtra {
			// When deleteLocalExtra is enabled, we have localFiles to compare with
			if localEntry, exists := localFiles[remotePath]; exists {
				// File exists locally, check if it needs updating
				if localEntry.RemoteEntry == nil ||
					localEntry.RemoteEntry.RemoteETag != remoteEntry.RemoteETag ||
					localEntry.RemoteEntry.RemoteMtime < remoteEntry.RemoteMtime ||
					(remoteEntry.RemoteContentEncoding != nil &&
						localEntry.RemoteEntry.GetRemoteContentEncoding() != remoteEntry.GetRemoteContentEncoding()) {
					filesToUpdate = append(filesToUpdate, remotePath)
				}
				// Check if it needs caching
				if shouldCache && shouldCacheToLocal(localEntry) && fileFilter.matches(localEntry) {
					filesToCache = append(filesToCache, remotePath)
				}
			} else {
				// File doesn't exist locally, needs to be created
				filesToUpdate = append(filesToUpdate, remotePath)
			}
		} else {
			// When deleteLocalExtra is disabled, we check each file individually
			// All remote files are candidates for update/creation
			filesToUpdate = append(filesToUpdate, remotePath)

			// For caching, we need to check if the local file exists and needs caching
			if shouldCache {
				// We need to look up the local file to check if it needs caching
				localDir, name := util.FullPath(remotePath).DirAndName()
				err := commandEnv.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
					lookupResp, lookupErr := client.LookupDirectoryEntry(context.Background(), &filer_pb.LookupDirectoryEntryRequest{
						Directory: localDir,
						Name:      name,
					})
					if lookupErr == nil {
						localEntry := lookupResp.Entry
						if shouldCacheToLocal(localEntry) && fileFilter.matches(localEntry) {
							filesToCache = append(filesToCache, remotePath)
						}
					}
					return nil // Don't propagate lookup errors here
				})
				if err != nil {
					// Log error but continue
					fmt.Fprintf(writer, "Warning: failed to lookup local file %s for caching check: %v\n", remotePath, err)
				}
			}
		}
	}

	fmt.Fprintf(writer, "Actions needed: %d files to delete, %d files to update, %d files to cache\n",
		len(filesToDelete), len(filesToUpdate), len(filesToCache))

	if dryRun {
		fmt.Fprintf(writer, "DRY RUN - showing what would be done:\n")
		for _, path := range filesToDelete {
			fmt.Fprintf(writer, "DELETE: %s\n", path)
		}
		for _, path := range filesToUpdate {
			fmt.Fprintf(writer, "UPDATE: %s\n", path)
		}
		for _, path := range filesToCache {
			fmt.Fprintf(writer, "CACHE: %s\n", path)
		}
		return nil
	}

	// Step 4: Execute actions
	return commandEnv.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
		ctx := context.Background()

		// Delete files that no longer exist on remote (only if deleteLocalExtra is enabled)
		if deleteLocalExtra {
			for _, pathToDelete := range filesToDelete {
				fmt.Fprintf(writer, "Deleting %s... ", pathToDelete)

				dir, name := util.FullPath(pathToDelete).DirAndName()
				err := filer_pb.DoRemove(ctx, client, dir, name, true, false, false, false, nil)
				if err != nil {
					fmt.Fprintf(writer, "failed: %v\n", err)
					return err
				}
				fmt.Fprintf(writer, "done\n")
			}
		}

		// Update metadata for files that exist on remote
		for _, pathToUpdate := range filesToUpdate {
			remoteEntry := remoteFiles[pathToUpdate]
			localDir, name := util.FullPath(pathToUpdate).DirAndName()

			fmt.Fprintf(writer, "Updating metadata for %s... ", pathToUpdate)

			// Check if file exists locally
			lookupResp, lookupErr := client.LookupDirectoryEntry(ctx, &filer_pb.LookupDirectoryEntryRequest{
				Directory: string(localDir),
				Name:      name,
			})

			if lookupErr != nil && lookupErr != filer_pb.ErrNotFound {
				fmt.Fprintf(writer, "failed to lookup: %v\n", lookupErr)
				continue
			}

			isDirectory := remoteEntry.RemoteSize == 0 && remoteEntry.RemoteMtime == 0
			if lookupErr == filer_pb.ErrNotFound {
				// Create new entry
				createErr := filer_pb.CreateEntry(ctx, client, &filer_pb.CreateEntryRequest{
					Directory: string(localDir),
					Entry: &filer_pb.Entry{
						Name:        name,
						IsDirectory: isDirectory,
						Attributes: &filer_pb.FuseAttributes{
							FileSize: uint64(remoteEntry.RemoteSize),
							Mtime:    remoteEntry.RemoteMtime,
							FileMode: remoteEntryFileMode(isDirectory),
						},
						Extended:    filer.MergeRemoteContentEncoding(remoteEntry, nil),
						RemoteEntry: remoteEntry,
					},
					OExcl: true,
				})
				if errors.Is(createErr, filer_pb.ErrEntryAlreadyExists) {
					fmt.Fprintf(writer, "skipped (created since it was looked up)\n")
					continue
				}
				if createErr != nil {
					fmt.Fprintf(writer, "failed to create: %v\n", createErr)
					continue
				}
			} else {
				// Update existing entry
				existingEntry := lookupResp.Entry
				if existingEntry.RemoteEntry == nil {
					// This is a local file, skip to avoid overwriting
					fmt.Fprintf(writer, "skipped (local file)\n")
					continue
				}

				stale, updateErr := doSaveRemoteEntry(client, string(localDir), existingEntry, remoteEntry)
				if stale {
					fmt.Fprintf(writer, "skipped (changed since it was looked up)\n")
					continue
				}
				if updateErr != nil {
					fmt.Fprintf(writer, "failed to update: %v\n", updateErr)
					continue
				}
			}
			fmt.Fprintf(writer, "done\n")
		}

		// Cache file content if requested
		if shouldCache && len(filesToCache) > 0 {
			fmt.Fprintf(writer, "Caching file content...\n")

			var wg sync.WaitGroup
			limitedConcurrentExecutor := util.NewLimitedConcurrentExecutor(concurrency)
			var executionErr error
			var execErrMu sync.Mutex

			for _, pathToCache := range filesToCache {
				wg.Add(1)
				pathToCacheCopy := pathToCache // Capture for closure
				limitedConcurrentExecutor.Execute(func() {
					defer wg.Done()

					// Get local entry (either from localFiles map or by lookup)
					var localEntry *filer_pb.Entry
					if deleteLocalExtra {
						localEntry = localFiles[pathToCacheCopy]
						if localEntry == nil {
							fmt.Fprintf(writer, "Warning: skipping cache for %s (local entry not found)\n", pathToCacheCopy)
							return
						}
					} else {
						// Look up the local entry since we don't have it in localFiles
						localDir, name := util.FullPath(pathToCacheCopy).DirAndName()
						lookupErr := commandEnv.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
							lookupResp, err := client.LookupDirectoryEntry(context.Background(), &filer_pb.LookupDirectoryEntryRequest{
								Directory: localDir,
								Name:      name,
							})
							if err == nil {
								localEntry = lookupResp.Entry
							}
							return err
						})
						if lookupErr != nil {
							fmt.Fprintf(writer, "Warning: failed to lookup local entry for caching %s: %v\n", pathToCacheCopy, lookupErr)
							return
						}
					}

					dir, _ := util.FullPath(pathToCacheCopy).DirAndName()

					fmt.Fprintf(writer, "Caching %s... ", pathToCacheCopy)

					if _, err := filer.CacheRemoteObjectToLocalCluster(commandEnv, util.FullPath(dir), localEntry, chunkConcurrency, downloadConcurrency); err != nil {
						fmt.Fprintf(writer, "failed: %v\n", err)
						execErrMu.Lock()
						if executionErr == nil {
							executionErr = err
						}
						execErrMu.Unlock()
						return
					}
					fmt.Fprintf(writer, "done\n")
				})
			}

			wg.Wait()
			if executionErr != nil {
				return executionErr
			}
		}

		return nil
	})
}

func recursivelyTraverseDirectory(filerClient filer_pb.FilerClient, dirPath util.FullPath, visitEntry func(dir util.FullPath, entry *filer_pb.Entry) bool) (err error) {

	err = filer_pb.ReadDirAllEntries(context.Background(), filerClient, dirPath, "", func(entry *filer_pb.Entry, isLast bool) error {
		if entry.IsDirectory {
			if !visitEntry(dirPath, entry) {
				return nil
			}
			subDir := dirPath.Child(entry.Name)
			if err := recursivelyTraverseDirectory(filerClient, subDir, visitEntry); err != nil {
				return err
			}
		} else {
			if !visitEntry(dirPath, entry) {
				return nil
			}
		}
		return nil
	})
	return
}

func shouldCacheToLocal(entry *filer_pb.Entry) bool {
	if entry.IsDirectory {
		return false
	}
	if entry.RemoteEntry == nil {
		return false
	}
	if entry.RemoteEntry.LastLocalSyncTsNs == 0 && entry.RemoteEntry.RemoteSize > 0 {
		return true
	}
	return false
}

func mayHaveCachedToLocal(entry *filer_pb.Entry) bool {
	if entry.IsDirectory {
		return false
	}
	if entry.RemoteEntry == nil {
		return false // should not uncache an entry that is not in remote
	}
	if entry.RemoteEntry.LastLocalSyncTsNs > 0 {
		return true
	}
	return false
}