Skip to content

SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.

6.2 KB
Raw Blame
package shell

import (
	"context"
	"flag"
	"fmt"
	"io"
	"path/filepath"
	"strings"
	"time"

	"github.com/seaweedfs/seaweedfs/weed/filer"
	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
	"github.com/seaweedfs/seaweedfs/weed/util"
	"google.golang.org/protobuf/proto"
)

func init() {
	Commands = append(Commands, &commandRemoteUncache{})
}

type commandRemoteUncache struct {
}

func (c *commandRemoteUncache) Name() string {
	return "remote.uncache"
}

func (c *commandRemoteUncache) Help() string {
	return `keep the metadata but remote cache the file content for mounted directories or files

	This is designed to run regularly. So you can add it to some cronjob.
	If a file is not synchronized with the remote copy, the file will be skipped to avoid loss of data.

	remote.uncache -dir=/xxx
	remote.uncache -dir=/xxx/some/sub/dir
	remote.uncache -dir=/xxx/some/sub/dir -include=*.pdf
	remote.uncache -dir=/xxx/some/sub/dir -exclude=*.txt
	remote.uncache -minSize=1024000    # uncache files larger than 100K
	remote.uncache -minAge=3600        # uncache files older than 1 hour (created time)
	remote.uncache -minCacheAge=3600   # uncache files older than 1 hour (cached time)

`
}

func (c *commandRemoteUncache) HasTag(CommandTag) bool {
	return false
}

func (c *commandRemoteUncache) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {

	remoteUncacheCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)

	dir := remoteUncacheCommand.String("dir", "", "a directory in filer")
	fileFiler := newFileFilter(remoteUncacheCommand)

	if err = remoteUncacheCommand.Parse(args); err != nil {
		return err
	}

	mappings, listErr := filer.ReadMountMappings(commandEnv.option.GrpcDialOption, commandEnv.FilerAddress())
	if listErr != nil {
		return listErr
	}
	if *dir != "" {
		var localMountedDir string
		for k := range mappings.Mappings {
			if strings.HasPrefix(*dir, k) {
				localMountedDir = k
			}
		}
		if localMountedDir == "" {
			jsonPrintln(writer, mappings)
			fmt.Fprintf(writer, "%s is not mounted\n", *dir)
			return nil
		}

		// pull content from remote
		if err = c.uncacheContentData(commandEnv, writer, util.FullPath(*dir), fileFiler); err != nil {
			return fmt.Errorf("uncache content data: %w", err)
		}
		return nil
	}

	for key, _ := range mappings.Mappings {
		if err := c.uncacheContentData(commandEnv, writer, util.FullPath(key), fileFiler); err != nil {
			return err
		}
	}

	return nil
}

func (c *commandRemoteUncache) uncacheContentData(commandEnv *CommandEnv, writer io.Writer, dirToCache util.FullPath, fileFilter *FileFilter) error {

	return recursivelyTraverseDirectory(commandEnv, dirToCache, func(dir util.FullPath, entry *filer_pb.Entry) bool {

		if !mayHaveCachedToLocal(entry) {
			return true // true means recursive traversal should continue
		}

		if !fileFilter.matches(entry) {
			return true
		}

		if entry.RemoteEntry.LastLocalSyncTsNs < entry.Attributes.Mtime*1e9+int64(entry.Attributes.MtimeNs) {
			return true // should not uncache an entry that is not synchronized with remote
		}

		fmt.Fprintf(writer, "Uncache %+v ... ", dir.Child(entry.Name))

		var stale bool
		err := commandEnv.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
			var updateErr error
			stale, updateErr = uncacheEntry(client, string(dir), entry)
			return updateErr
		})
		if stale {
			fmt.Fprintf(writer, "skipped, changed since it was read\n")
			return true
		}
		if err != nil {
			fmt.Fprintf(writer, "uncache %+v: %v\n", dir.Child(entry.Name), err)
			return false
		}
		fmt.Fprintf(writer, "Done\n")

		return true
	})
}

// uncacheEntry drops the local copy of entry only while the filer still holds
// it as read, so a write that landed after the read keeps its data.
func uncacheEntry(client filer_pb.SeaweedFilerClient, dir string, entry *filer_pb.Entry) (stale bool, err error) {
	readEntry := proto.Clone(entry).(*filer_pb.Entry)
	entry.RemoteEntry.LastLocalSyncTsNs = 0
	entry.Chunks = nil
	return updateEntryIfUnchanged(context.Background(), client, dir, entry, readEntry)
}

type FileFilter struct {
	include     *string
	exclude     *string
	minSize     *int64
	maxSize     *int64
	minAge      *int64
	maxAge      *int64
	minCacheAge *int64
	now         int64
}

func newFileFilter(remoteMountCommand *flag.FlagSet) (ff *FileFilter) {
	ff = &FileFilter{}
	ff.include = remoteMountCommand.String("include", "", "patterns of file names, e.g., *.pdf, *.html, ab?d.txt")
	ff.exclude = remoteMountCommand.String("exclude", "", "patterns of file names, e.g., *.pdf, *.html, ab?d.txt")
	ff.minSize = remoteMountCommand.Int64("minSize", -1, "minimum file size in bytes")
	ff.maxSize = remoteMountCommand.Int64("maxSize", -1, "maximum file size in bytes")
	ff.minAge = remoteMountCommand.Int64("minAge", -1, "minimum file age in seconds (created time)")
	ff.maxAge = remoteMountCommand.Int64("maxAge", -1, "maximum file age in seconds (created time)")
	ff.minCacheAge = remoteMountCommand.Int64("minCacheAge", -1, "minimum file cache age in seconds (last cached time)")
	ff.now = time.Now().Unix()
	return
}

// matchesName applies only the name-based include/exclude patterns,
// usable for remote entries where local attributes are not available.
func (ff *FileFilter) matchesName(name string) bool {
	if *ff.include != "" {
		if ok, _ := filepath.Match(*ff.include, name); !ok {
			return false
		}
	}
	if *ff.exclude != "" {
		if ok, _ := filepath.Match(*ff.exclude, name); ok {
			return false
		}
	}
	return true
}

func (ff *FileFilter) matches(entry *filer_pb.Entry) bool {
	if entry.Attributes == nil {
		return false
	}
	if !ff.matchesName(entry.Name) {
		return false
	}
	if *ff.minSize != -1 {
		if int64(entry.Attributes.FileSize) < *ff.minSize {
			return false
		}
	}
	if *ff.maxSize != -1 {
		if int64(entry.Attributes.FileSize) > *ff.maxSize {
			return false
		}
	}
	if *ff.minAge != -1 {
		if entry.Attributes.Crtime+*ff.minAge > ff.now {
			return false
		}
	}
	if *ff.maxAge != -1 {
		if entry.Attributes.Crtime+*ff.maxAge < ff.now {
			return false
		}
	}
	if *ff.minCacheAge != -1 {
		lastCachedTime := entry.Attributes.Crtime
		if entry.RemoteEntry != nil && entry.RemoteEntry.LastLocalSyncTsNs > 0 {
			lastCachedTime = entry.RemoteEntry.LastLocalSyncTsNs / 1e9
		}
		if lastCachedTime+*ff.minCacheAge > ff.now {
			return false
		}
	}
	return true
}