SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
package shell
import (
"context"
"flag"
"fmt"
"io"
"path/filepath"
"strings"
"time"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/util"
"google.golang.org/protobuf/proto"
)
func init() {
Commands = append(Commands, &commandRemoteUncache{})
}
type commandRemoteUncache struct {
}
func (c *commandRemoteUncache) Name() string {
return "remote.uncache"
}
func (c *commandRemoteUncache) Help() string {
return `keep the metadata but remote cache the file content for mounted directories or files
This is designed to run regularly. So you can add it to some cronjob.
If a file is not synchronized with the remote copy, the file will be skipped to avoid loss of data.
remote.uncache -dir=/xxx
remote.uncache -dir=/xxx/some/sub/dir
remote.uncache -dir=/xxx/some/sub/dir -include=*.pdf
remote.uncache -dir=/xxx/some/sub/dir -exclude=*.txt
remote.uncache -minSize=1024000 # uncache files larger than 100K
remote.uncache -minAge=3600 # uncache files older than 1 hour (created time)
remote.uncache -minCacheAge=3600 # uncache files older than 1 hour (cached time)
`
}
func (c *commandRemoteUncache) HasTag(CommandTag) bool {
return false
}
func (c *commandRemoteUncache) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
remoteUncacheCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
dir := remoteUncacheCommand.String("dir", "", "a directory in filer")
fileFiler := newFileFilter(remoteUncacheCommand)
if err = remoteUncacheCommand.Parse(args); err != nil {
return err
}
mappings, listErr := filer.ReadMountMappings(commandEnv.option.GrpcDialOption, commandEnv.FilerAddress())
if listErr != nil {
return listErr
}
if *dir != "" {
var localMountedDir string
for k := range mappings.Mappings {
if strings.HasPrefix(*dir, k) {
localMountedDir = k
}
}
if localMountedDir == "" {
jsonPrintln(writer, mappings)
fmt.Fprintf(writer, "%s is not mounted\n", *dir)
return nil
}
// pull content from remote
if err = c.uncacheContentData(commandEnv, writer, util.FullPath(*dir), fileFiler); err != nil {
return fmt.Errorf("uncache content data: %w", err)
}
return nil
}
for key, _ := range mappings.Mappings {
if err := c.uncacheContentData(commandEnv, writer, util.FullPath(key), fileFiler); err != nil {
return err
}
}
return nil
}
func (c *commandRemoteUncache) uncacheContentData(commandEnv *CommandEnv, writer io.Writer, dirToCache util.FullPath, fileFilter *FileFilter) error {
return recursivelyTraverseDirectory(commandEnv, dirToCache, func(dir util.FullPath, entry *filer_pb.Entry) bool {
if !mayHaveCachedToLocal(entry) {
return true // true means recursive traversal should continue
}
if !fileFilter.matches(entry) {
return true
}
if entry.RemoteEntry.LastLocalSyncTsNs < entry.Attributes.Mtime*1e9+int64(entry.Attributes.MtimeNs) {
return true // should not uncache an entry that is not synchronized with remote
}
fmt.Fprintf(writer, "Uncache %+v ... ", dir.Child(entry.Name))
var stale bool
err := commandEnv.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
var updateErr error
stale, updateErr = uncacheEntry(client, string(dir), entry)
return updateErr
})
if stale {
fmt.Fprintf(writer, "skipped, changed since it was read\n")
return true
}
if err != nil {
fmt.Fprintf(writer, "uncache %+v: %v\n", dir.Child(entry.Name), err)
return false
}
fmt.Fprintf(writer, "Done\n")
return true
})
}
// uncacheEntry drops the local copy of entry only while the filer still holds
// it as read, so a write that landed after the read keeps its data.
func uncacheEntry(client filer_pb.SeaweedFilerClient, dir string, entry *filer_pb.Entry) (stale bool, err error) {
readEntry := proto.Clone(entry).(*filer_pb.Entry)
entry.RemoteEntry.LastLocalSyncTsNs = 0
entry.Chunks = nil
return updateEntryIfUnchanged(context.Background(), client, dir, entry, readEntry)
}
type FileFilter struct {
include *string
exclude *string
minSize *int64
maxSize *int64
minAge *int64
maxAge *int64
minCacheAge *int64
now int64
}
func newFileFilter(remoteMountCommand *flag.FlagSet) (ff *FileFilter) {
ff = &FileFilter{}
ff.include = remoteMountCommand.String("include", "", "patterns of file names, e.g., *.pdf, *.html, ab?d.txt")
ff.exclude = remoteMountCommand.String("exclude", "", "patterns of file names, e.g., *.pdf, *.html, ab?d.txt")
ff.minSize = remoteMountCommand.Int64("minSize", -1, "minimum file size in bytes")
ff.maxSize = remoteMountCommand.Int64("maxSize", -1, "maximum file size in bytes")
ff.minAge = remoteMountCommand.Int64("minAge", -1, "minimum file age in seconds (created time)")
ff.maxAge = remoteMountCommand.Int64("maxAge", -1, "maximum file age in seconds (created time)")
ff.minCacheAge = remoteMountCommand.Int64("minCacheAge", -1, "minimum file cache age in seconds (last cached time)")
ff.now = time.Now().Unix()
return
}
// matchesName applies only the name-based include/exclude patterns,
// usable for remote entries where local attributes are not available.
func (ff *FileFilter) matchesName(name string) bool {
if *ff.include != "" {
if ok, _ := filepath.Match(*ff.include, name); !ok {
return false
}
}
if *ff.exclude != "" {
if ok, _ := filepath.Match(*ff.exclude, name); ok {
return false
}
}
return true
}
func (ff *FileFilter) matches(entry *filer_pb.Entry) bool {
if entry.Attributes == nil {
return false
}
if !ff.matchesName(entry.Name) {
return false
}
if *ff.minSize != -1 {
if int64(entry.Attributes.FileSize) < *ff.minSize {
return false
}
}
if *ff.maxSize != -1 {
if int64(entry.Attributes.FileSize) > *ff.maxSize {
return false
}
}
if *ff.minAge != -1 {
if entry.Attributes.Crtime+*ff.minAge > ff.now {
return false
}
}
if *ff.maxAge != -1 {
if entry.Attributes.Crtime+*ff.maxAge < ff.now {
return false
}
}
if *ff.minCacheAge != -1 {
lastCachedTime := entry.Attributes.Crtime
if entry.RemoteEntry != nil && entry.RemoteEntry.LastLocalSyncTsNs > 0 {
lastCachedTime = entry.RemoteEntry.LastLocalSyncTsNs / 1e9
}
if lastCachedTime+*ff.minCacheAge > ff.now {
return false
}
}
return true
}