"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
lib/transform: add transform library and --name-transform flag
nielash013c563293a0
diff --git a/cmd/bisync/bilib/names.go b/cmd/bisync/bilib/names.goindex d8951a0b5..cb266eb63 100644--- a/cmd/bisync/bilib/names.go+++ b/cmd/bisync/bilib/names.go@@ -5,8 +5,6 @@ import ("os""sort""strconv"- "strings"- "time")// Names comprises a set of file names@@ -85,81 +83,3 @@ func (am AliasMap) Alias(name1 string) string {}return name1}--// ParseGlobs determines whether a string contains {brackets}-// and returns the substring (including both brackets) for replacing-// substring is first opening bracket to last closing bracket ---// good for {{this}} but not {this}{this}-func ParseGlobs(s string) (hasGlobs bool, substring string) {- open := strings.Index(s, "{")- close := strings.LastIndex(s, "}")- if open >= 0 && close > open {- return true, s[open : close+1]- }- return false, ""-}--// TrimBrackets converts {{this}} to this-func TrimBrackets(s string) string {- return strings.Trim(s, "{}")-}--// TimeFormat converts a user-supplied string to a Go time constant, if possible-func TimeFormat(timeFormat string) string {- switch timeFormat {- case "Layout":- timeFormat = time.Layout- case "ANSIC":- timeFormat = time.ANSIC- case "UnixDate":- timeFormat = time.UnixDate- case "RubyDate":- timeFormat = time.RubyDate- case "RFC822":- timeFormat = time.RFC822- case "RFC822Z":- timeFormat = time.RFC822Z- case "RFC850":- timeFormat = time.RFC850- case "RFC1123":- timeFormat = time.RFC1123- case "RFC1123Z":- timeFormat = time.RFC1123Z- case "RFC3339":- timeFormat = time.RFC3339- case "RFC3339Nano":- timeFormat = time.RFC3339Nano- case "Kitchen":- timeFormat = time.Kitchen- case "Stamp":- timeFormat = time.Stamp- case "StampMilli":- timeFormat = time.StampMilli- case "StampMicro":- timeFormat = time.StampMicro- case "StampNano":- timeFormat = time.StampNano- case "DateTime":- // timeFormat = time.DateTime // missing in go1.19- timeFormat = "2006-01-02 15:04:05"- case "DateOnly":- // timeFormat = time.DateOnly // missing in go1.19- timeFormat = "2006-01-02"- case "TimeOnly":- // timeFormat = time.TimeOnly // missing in go1.19- timeFormat = "15:04:05"- case "MacFriendlyTime", "macfriendlytime", "mac":- timeFormat = "2006-01-02 0304PM" // not actually a Go constant -- but useful as macOS filenames can't have colons- }- return timeFormat-}--// AppyTimeGlobs converts "myfile-{DateOnly}.txt" to "myfile-2006-01-02.txt"-func AppyTimeGlobs(s string, t time.Time) string {- hasGlobs, substring := ParseGlobs(s)- if !hasGlobs {- return s- }- timeString := t.Local().Format(TimeFormat(TrimBrackets(substring)))- return strings.ReplaceAll(s, substring, timeString)-}diff --git a/cmd/bisync/resolve.go b/cmd/bisync/resolve.goindex d058978f2..b3e5d8048 100644--- a/cmd/bisync/resolve.go+++ b/cmd/bisync/resolve.go@@ -4,8 +4,6 @@ import ("context""fmt""math"- "mime"- "path""strings""time"@@ -13,6 +11,7 @@ import ("github.com/rclone/rclone/fs""github.com/rclone/rclone/fs/operations""github.com/rclone/rclone/lib/terminal"+ "github.com/rclone/rclone/lib/transform")// Prefer describes strategies for resolving sync conflicts@@ -97,8 +96,8 @@ func (b *bisyncRun) setResolveDefaults(ctx context.Context) error {}// replace glob variables, if anyt := time.Now() // capture static time here so it is the same for all files throughout this run- b.opt.ConflictSuffix1 = bilib.AppyTimeGlobs(b.opt.ConflictSuffix1, t)- b.opt.ConflictSuffix2 = bilib.AppyTimeGlobs(b.opt.ConflictSuffix2, t)+ b.opt.ConflictSuffix1 = transform.AppyTimeGlobs(b.opt.ConflictSuffix1, t)+ b.opt.ConflictSuffix2 = transform.AppyTimeGlobs(b.opt.ConflictSuffix2, t)// append dot (intentionally allow more than one)b.opt.ConflictSuffix1 = "." + b.opt.ConflictSuffix1@@ -130,6 +129,7 @@ type (path2 namePair})+type namePair struct {oldName stringnewName string@@ -240,24 +240,7 @@ func SuffixName(ctx context.Context, remote, suffix string) string {}ci := fs.GetConfig(ctx)if ci.SuffixKeepExtension {- var (- base = remote- exts = ""- first = true- ext = path.Ext(remote)- )- for ext != "" {- // Look second and subsequent extensions in mime types.- // If they aren't found then don't keep it as an extension.- if !first && mime.TypeByExtension(ext) == "" {- break- }- base = base[:len(base)-len(ext)]- exts = ext + exts- first = false- ext = path.Ext(base)- }- return base + suffix + exts+ return transform.SuffixKeepExtension(remote, suffix)}return remote + suffix}diff --git a/cmd/help.go b/cmd/help.goindex f65ff6d53..1b985daca 100644--- a/cmd/help.go+++ b/cmd/help.go@@ -191,7 +191,6 @@ func setupRootCommand(rootCmd *cobra.Command) {})cobra.OnInitialize(initConfig)-}// Traverse the tree of commands running fn on eachdiff --git a/docs/content/docs.md b/docs/content/docs.mdindex 4633429e0..056abcf09 100644--- a/docs/content/docs.md+++ b/docs/content/docs.md@@ -1970,6 +1970,13 @@ If the backend has a `--backend-upload-concurrency` setting (egnumber of transfers instead if it is larger than the value of`--multi-thread-streams` or `--multi-thread-streams` isn't set.+### --name-transform COMMAND[=XXXX] ###+`--name-transform` introduces path name transformations for+`rclone copy`, `rclone sync`, and `rclone move`. These transformations+enable modifications to source and destination file names by applying+prefixes, suffixes, and other alterations during transfer operations.+For detailed docs and examples, see [`convmv`](/commands/rclone_convmv/).+### --no-check-dest ###The `--no-check-dest` can be used with `move` or `copy` and it causesdiff --git a/fs/config.go b/fs/config.goindex 0d3f9ddc5..aaae5defc 100644--- a/fs/config.go+++ b/fs/config.go@@ -550,6 +550,11 @@ var ConfigOptionsInfo = Options{{Default: 0,Advanced: true,Groups: "Networking",+}, {+ Name: "name_transform",+ Default: []string{},+ Help: "Transform paths during the copy process.",+ Groups: "Copy",}}// ConfigInfo is filesystem config options@@ -661,6 +666,7 @@ type ConfigInfo struct {PartialSuffix string `config:"partial_suffix"`MetadataMapper SpaceSepList `config:"metadata_mapper"`MaxConnections int `config:"max_connections"`+ NameTransform []string `config:"name_transform"`}func init() {diff --git a/fs/march/march.go b/fs/march/march.goindex def481ac9..e3ffd58c7 100644--- a/fs/march/march.go+++ b/fs/march/march.go@@ -15,6 +15,7 @@ import ("github.com/rclone/rclone/fs/filter""github.com/rclone/rclone/fs/list""github.com/rclone/rclone/fs/walk"+ "github.com/rclone/rclone/lib/transform""golang.org/x/sync/errgroup""golang.org/x/text/unicode/norm")@@ -80,18 +81,22 @@ func (m *March) init(ctx context.Context) {}}-// srcKey turns a directory entry into a sort key using the defined transforms.-func (m *March) srcKey(entry fs.DirEntry) string {+// srcOrDstKey turns a directory entry into a sort key using the defined transforms.+func (m *March) srcOrDstKey(entry fs.DirEntry, isSrc bool) string {if entry == nil {return ""}name := path.Base(entry.Remote())+ _, isDirectory := entry.(fs.Directory)+ if isSrc {+ name = transform.Path(m.Ctx, name, isDirectory)+ }for _, transform := range m.transforms {name = transform(name)}// Suffix entries to make identically named files and// directories sort consistently with directories first.- if _, isDirectory := entry.(fs.Directory); isDirectory {+ if isDirectory {name += "D"} else {name += "F"@@ -99,9 +104,14 @@ func (m *March) srcKey(entry fs.DirEntry) string {return name}+// srcKey turns a directory entry into a sort key using the defined transforms.+func (m *March) srcKey(entry fs.DirEntry) string {+ return m.srcOrDstKey(entry, true)+}+// dstKey turns a directory entry into a sort key using the defined transforms.func (m *March) dstKey(entry fs.DirEntry) string {- return m.srcKey(entry) // FIXME actually do something different+ return m.srcOrDstKey(entry, false)}// makeListDir makes constructs a listing function for the given fs@@ -461,7 +471,6 @@ func (m *March) processJob(job listDirJob) ([]listDirJob, error) {noDst: true,})}-}, func(dst fs.DirEntry) {recurse := m.Callback.DstOnly(dst)if recurse && job.dstDepth > 0 {diff --git a/fs/operations/copy.go b/fs/operations/copy.goindex 8228a14c7..2d42fac07 100644--- a/fs/operations/copy.go+++ b/fs/operations/copy.go@@ -21,6 +21,7 @@ import ("github.com/rclone/rclone/fs/hash""github.com/rclone/rclone/lib/atexit""github.com/rclone/rclone/lib/pacer"+ "github.com/rclone/rclone/lib/transform")// State of the copy@@ -390,7 +391,7 @@ func Copy(ctx context.Context, f fs.Fs, dst fs.Object, remote string, src fs.Objf: f,dstFeatures: f.Features(),dst: dst,- remote: remote,+ remote: transform.Path(ctx, remote, false),src: src,ci: ci,tr: tr,@@ -399,7 +400,7 @@ func Copy(ctx context.Context, f fs.Fs, dst fs.Object, remote string, src fs.Obj}c.hashType, c.hashOption = CommonHash(ctx, f, src.Fs())if c.dst != nil {- c.remote = c.dst.Remote()+ c.remote = transform.Path(ctx, c.dst.Remote(), false)}// Are we using partials?//@@ -414,5 +415,5 @@ func Copy(ctx context.Context, f fs.Fs, dst fs.Object, remote string, src fs.Obj// CopyFile moves a single file possibly to a new namefunc CopyFile(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, dstFileName string, srcFileName string) (err error) {- return moveOrCopyFile(ctx, fdst, fsrc, dstFileName, srcFileName, true)+ return moveOrCopyFile(ctx, fdst, fsrc, dstFileName, srcFileName, true, false)}diff --git a/fs/operations/operations.go b/fs/operations/operations.goindex 6d3ca4f76..d8366c252 100644--- a/fs/operations/operations.go+++ b/fs/operations/operations.go@@ -39,6 +39,7 @@ import ("github.com/rclone/rclone/lib/pacer""github.com/rclone/rclone/lib/random""github.com/rclone/rclone/lib/readers"+ "github.com/rclone/rclone/lib/transform""golang.org/x/sync/errgroup""golang.org/x/text/unicode/norm")@@ -424,6 +425,8 @@ func MoveTransfer(ctx context.Context, fdst fs.Fs, dst fs.Object, remote string,// move - see Move for helpfunc move(ctx context.Context, fdst fs.Fs, dst fs.Object, remote string, src fs.Object, isTransfer bool) (newDst fs.Object, err error) {+ origRemote := remote // avoid double-transform on fallback to copy+ remote = transform.Path(ctx, remote, false)ci := fs.GetConfig(ctx)var tr *accounting.Transferif isTransfer {@@ -447,12 +450,14 @@ func move(ctx context.Context, fdst fs.Fs, dst fs.Object, remote string, src fs.if doMove := fdst.Features().Move; doMove != nil && (SameConfig(src.Fs(), fdst) || (SameRemoteType(src.Fs(), fdst) && (fdst.Features().ServerSideAcrossConfigs || ci.ServerSideAcrossConfigs))) {// Delete destination if it exists and is not the same file as src (could be same file while seemingly different if the remote is case insensitive)if dst != nil {- remote = dst.Remote()+ remote = transform.Path(ctx, dst.Remote(), false)if !SameObject(src, dst) {err = DeleteFile(ctx, dst)if err != nil {return newDst, err}+ } else if src.Remote() == remote {+ return newDst, nil} else if needsMoveCaseInsensitive(fdst, fdst, remote, src.Remote(), false) {doMove = func(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {return MoveCaseInsensitive(ctx, fdst, fdst, remote, src.Remote(), false, src)@@ -488,7 +493,7 @@ func move(ctx context.Context, fdst fs.Fs, dst fs.Object, remote string, src fs.}}// Move not found or didn't work so copy dst <- src- newDst, err = Copy(ctx, fdst, dst, remote, src)+ newDst, err = Copy(ctx, fdst, dst, origRemote, src)if err != nil {fs.Errorf(src, "Not deleting source as copy failed: %v", err)return newDst, err@@ -516,24 +521,7 @@ func SuffixName(ctx context.Context, remote string) string {return remote}if ci.SuffixKeepExtension {- var (- base = remote- exts = ""- first = true- ext = path.Ext(remote)- )- for ext != "" {- // Look second and subsequent extensions in mime types.- // If they aren't found then don't keep it as an extension.- if !first && mime.TypeByExtension(ext) == "" {- break- }- base = base[:len(base)-len(ext)]- exts = ext + exts- first = false- ext = path.Ext(base)- }- return base + ci.Suffix + exts+ return transform.SuffixKeepExtension(remote, ci.Suffix)}return remote + ci.Suffix}@@ -1994,12 +1982,12 @@ func MoveCaseInsensitive(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, dstFileNam}// moveOrCopyFile moves or copies a single file possibly to a new name-func moveOrCopyFile(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, dstFileName string, srcFileName string, cp bool) (err error) {+func moveOrCopyFile(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, dstFileName string, srcFileName string, cp bool, allowOverlap bool) (err error) {ci := fs.GetConfig(ctx)logger, usingLogger := GetLogger(ctx)dstFilePath := path.Join(fdst.Root(), dstFileName)srcFilePath := path.Join(fsrc.Root(), srcFileName)- if fdst.Name() == fsrc.Name() && dstFilePath == srcFilePath {+ if fdst.Name() == fsrc.Name() && dstFilePath == srcFilePath && !allowOverlap {fs.Debugf(fdst, "don't need to copy/move %s, it is already at target location", dstFileName)if usingLogger {srcObj, _ := fsrc.NewObject(ctx, srcFileName)@@ -2106,7 +2094,14 @@ func moveOrCopyFile(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, dstFileName str//// This is treated as a transfer.func MoveFile(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, dstFileName string, srcFileName string) (err error) {- return moveOrCopyFile(ctx, fdst, fsrc, dstFileName, srcFileName, false)+ return moveOrCopyFile(ctx, fdst, fsrc, dstFileName, srcFileName, false, false)+}++// TransformFile transforms a file in place using --name-transform+//+// This is treated as a transfer.+func TransformFile(ctx context.Context, fdst fs.Fs, srcFileName string) (err error) {+ return moveOrCopyFile(ctx, fdst, fdst, srcFileName, srcFileName, false, true)}// SetTier changes tier of object in remote@@ -2211,50 +2206,10 @@ func (l *ListFormat) SetOutput(output []func(entry *ListJSONItem) string) {// AddModTime adds file's Mod Time to outputfunc (l *ListFormat) AddModTime(timeFormat string) {- switch timeFormat {- case "":+ if timeFormat == "" {timeFormat = "2006-01-02 15:04:05"- case "Layout":- timeFormat = time.Layout- case "ANSIC":- timeFormat = time.ANSIC- case "UnixDate":- timeFormat = time.UnixDate- case "RubyDate":- timeFormat = time.RubyDate- case "RFC822":- timeFormat = time.RFC822- case "RFC822Z":- timeFormat = time.RFC822Z- case "RFC850":- timeFormat = time.RFC850- case "RFC1123":- timeFormat = time.RFC1123- case "RFC1123Z":- timeFormat = time.RFC1123Z- case "RFC3339":- timeFormat = time.RFC3339- case "RFC3339Nano":- timeFormat = time.RFC3339Nano- case "Kitchen":- timeFormat = time.Kitchen- case "Stamp":- timeFormat = time.Stamp- case "StampMilli":- timeFormat = time.StampMilli- case "StampMicro":- timeFormat = time.StampMicro- case "StampNano":- timeFormat = time.StampNano- case "DateTime":- // timeFormat = time.DateTime // missing in go1.19- timeFormat = "2006-01-02 15:04:05"- case "DateOnly":- // timeFormat = time.DateOnly // missing in go1.19- timeFormat = "2006-01-02"- case "TimeOnly":- // timeFormat = time.TimeOnly // missing in go1.19- timeFormat = "15:04:05"+ } else {+ timeFormat = transform.TimeFormat(timeFormat)}l.AppendOutput(func(entry *ListJSONItem) string {return entry.ModTime.When.Local().Format(timeFormat)diff --git a/fs/operations/rc.go b/fs/operations/rc.goindex 46bf41770..18119cc82 100644--- a/fs/operations/rc.go+++ b/fs/operations/rc.go@@ -62,7 +62,7 @@ func rcList(ctx context.Context, in rc.Params) (out rc.Params, err error) {if rc.NotErrParamNotFound(err) {return nil, err}- var list = []*ListJSONItem{}+ list := []*ListJSONItem{}err = ListJSON(ctx, f, remote, &opt, func(item *ListJSONItem) error {list = append(list, item)return nil@@ -193,7 +193,7 @@ func rcMoveOrCopyFile(ctx context.Context, in rc.Params, cp bool) (out rc.Paramsif err != nil {return nil, err}- return nil, moveOrCopyFile(ctx, dstFs, srcFs, dstRemote, srcRemote, cp)+ return nil, moveOrCopyFile(ctx, dstFs, srcFs, dstRemote, srcRemote, cp, false)}func init() {@@ -289,7 +289,6 @@ func rcSingleCommand(ctx context.Context, in rc.Params, name string, noRemote bovar request *http.Requestrequest, err := in.GetHTTPRequest()-if err != nil {return nil, err}@@ -629,12 +628,12 @@ func rcBackend(ctx context.Context, in rc.Params) (out rc.Params, err error) {if err != nil {return nil, err}- var opt = map[string]string{}+ opt := map[string]string{}err = in.GetStructMissingOK("opt", &opt)if err != nil {return nil, err}- var arg = []string{}+ arg := []string{}err = in.GetStructMissingOK("arg", &arg)if err != nil {return nil, err@@ -642,7 +641,6 @@ func rcBackend(ctx context.Context, in rc.Params) (out rc.Params, err error) {result, err := doCommand(ctx, command, arg, opt)if err != nil {return nil, fmt.Errorf("command %q failed: %w", command, err)-}out = make(rc.Params)out["result"] = result@@ -685,7 +683,6 @@ func rcDu(ctx context.Context, in rc.Params) (out rc.Params, err error) {dir, err := in.GetString("dir")if rc.IsErrParamNotFound(err) {dir = config.GetCacheDir()-} else if err != nil {return nil, err}diff --git a/fs/sync/sync.go b/fs/sync/sync.goindex 0e015769e..c7593f673 100644--- a/fs/sync/sync.go+++ b/fs/sync/sync.go@@ -20,6 +20,7 @@ import ("github.com/rclone/rclone/fs/march""github.com/rclone/rclone/fs/operations""github.com/rclone/rclone/lib/errcount"+ "github.com/rclone/rclone/lib/transform""golang.org/x/sync/errgroup")@@ -95,6 +96,7 @@ type syncCopyMove struct {setDirModTimes []setDirModTime // directories that need their modtime setsetDirModTimesMaxLevel int // max level of the directories to setmodifiedDirs map[string]struct{} // dirs with changed contents (if s.setDirModTimeAfter)+ allowOverlap bool // whether we allow src and dst to overlap (i.e. for convmv)}// For keeping track of delayed modtime sets@@ -126,8 +128,8 @@ func (strategy trackRenamesStrategy) leaf() bool {return (strategy & trackRenamesStrategyLeaf) != 0}-func newSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.DeleteMode, DoMove bool, deleteEmptySrcDirs bool, copyEmptySrcDirs bool) (*syncCopyMove, error) {- if (deleteMode != fs.DeleteModeOff || DoMove) && operations.OverlappingFilterCheck(ctx, fdst, fsrc) {+func newSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.DeleteMode, DoMove bool, deleteEmptySrcDirs bool, copyEmptySrcDirs bool, allowOverlap bool) (*syncCopyMove, error) {+ if (deleteMode != fs.DeleteModeOff || DoMove) && operations.OverlappingFilterCheck(ctx, fdst, fsrc) && !allowOverlap {return nil, fserrors.FatalError(fs.ErrorOverlapping)}ci := fs.GetConfig(ctx)@@ -161,6 +163,7 @@ func newSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.DeletesetDirModTime: (!ci.NoUpdateDirModTime && fsrc.Features().CanHaveEmptyDirectories) && (fdst.Features().WriteDirSetModTime || fdst.Features().MkdirMetadata != nil || fdst.Features().DirSetModTime != nil),setDirModTimeAfter: !ci.NoUpdateDirModTime && (!copyEmptySrcDirs || fsrc.Features().CanHaveEmptyDirectories && fdst.Features().DirModTimeUpdatesOnWrite),modifiedDirs: make(map[string]struct{}),+ allowOverlap: allowOverlap,}s.logger, s.usingLogger = operations.GetLogger(ctx)@@ -922,7 +925,7 @@ func (s *syncCopyMove) tryRename(src fs.Object) bool {//// dir is the start directory, "" for rootfunc (s *syncCopyMove) run() error {- if operations.Same(s.fdst, s.fsrc) {+ if operations.Same(s.fdst, s.fsrc) && !s.allowOverlap {fs.Errorf(s.fdst, "Nothing to do as source and destination are the same")return nil}@@ -1122,6 +1125,9 @@ func (s *syncCopyMove) copyDirMetadata(ctx context.Context, f fs.Fs, dst fs.DirenewDst, err = operations.SetDirModTime(ctx, f, dst, dir, src.ModTime(ctx))}}+ if transform.Transforming(ctx) && newDst != nil && src.Remote() != newDst.Remote() {+ s.markParentNotEmpty(src)+ }// If we need to set modtime after and we created a dir, then save it for laterif s.setDirModTime && s.setDirModTimeAfter && err == nil {if newDst != nil {@@ -1254,8 +1260,8 @@ func (s *syncCopyMove) SrcOnly(src fs.DirEntry) (recurse bool) {s.logger(s.ctx, operations.MissingOnDst, src, nil, fs.ErrorIsDir)// Create the directory and make sure the Metadata/ModTime is correct- s.copyDirMetadata(s.ctx, s.fdst, nil, x.Remote(), x)- s.markDirModified(x.Remote())+ s.copyDirMetadata(s.ctx, s.fdst, nil, transform.Path(s.ctx, x.Remote(), true), x)+ s.markDirModified(transform.Path(s.ctx, x.Remote(), true))return truedefault:panic("Bad object in DirEntries")@@ -1288,7 +1294,11 @@ func (s *syncCopyMove) Match(ctx context.Context, dst, src fs.DirEntry) (recurse}case fs.Directory:// Do the same thing to the entire contents of the directory- s.markParentNotEmpty(src)+ srcX = fs.NewOverrideDirectory(srcX, transform.Path(ctx, src.Remote(), true))+ src = srcX+ if !transform.Transforming(ctx) || src.Remote() != dst.Remote() {+ s.markParentNotEmpty(src)+ }dstX, ok := dst.(fs.Directory)if ok {s.logger(s.ctx, operations.Match, src, dst, fs.ErrorIsDir)@@ -1327,7 +1337,7 @@ func (s *syncCopyMove) Match(ctx context.Context, dst, src fs.DirEntry) (recurse// If DoMove is true then files will be moved instead of copied.//// dir is the start directory, "" for root-func runSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.DeleteMode, DoMove bool, deleteEmptySrcDirs bool, copyEmptySrcDirs bool) error {+func runSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.DeleteMode, DoMove bool, deleteEmptySrcDirs bool, copyEmptySrcDirs bool, allowOverlap bool) error {ci := fs.GetConfig(ctx)if deleteMode != fs.DeleteModeOff && DoMove {return fserrors.FatalError(errors.New("can't delete and move at the same time"))@@ -1338,7 +1348,7 @@ func runSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.Deletereturn fserrors.FatalError(errors.New("can't use --delete-before with --track-renames"))}// only delete stuff during in this pass- do, err := newSyncCopyMove(ctx, fdst, fsrc, fs.DeleteModeOnly, false, deleteEmptySrcDirs, copyEmptySrcDirs)+ do, err := newSyncCopyMove(ctx, fdst, fsrc, fs.DeleteModeOnly, false, deleteEmptySrcDirs, copyEmptySrcDirs, allowOverlap)if err != nil {return err}@@ -1349,7 +1359,7 @@ func runSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.Delete// Next pass does a copy onlydeleteMode = fs.DeleteModeOff}- do, err := newSyncCopyMove(ctx, fdst, fsrc, deleteMode, DoMove, deleteEmptySrcDirs, copyEmptySrcDirs)+ do, err := newSyncCopyMove(ctx, fdst, fsrc, deleteMode, DoMove, deleteEmptySrcDirs, copyEmptySrcDirs, allowOverlap)if err != nil {return err}@@ -1359,17 +1369,22 @@ func runSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.Delete// Sync fsrc into fdstfunc Sync(ctx context.Context, fdst, fsrc fs.Fs, copyEmptySrcDirs bool) error {ci := fs.GetConfig(ctx)- return runSyncCopyMove(ctx, fdst, fsrc, ci.DeleteMode, false, false, copyEmptySrcDirs)+ return runSyncCopyMove(ctx, fdst, fsrc, ci.DeleteMode, false, false, copyEmptySrcDirs, false)}// CopyDir copies fsrc into fdstfunc CopyDir(ctx context.Context, fdst, fsrc fs.Fs, copyEmptySrcDirs bool) error {- return runSyncCopyMove(ctx, fdst, fsrc, fs.DeleteModeOff, false, false, copyEmptySrcDirs)+ return runSyncCopyMove(ctx, fdst, fsrc, fs.DeleteModeOff, false, false, copyEmptySrcDirs, false)}// moveDir moves fsrc into fdstfunc moveDir(ctx context.Context, fdst, fsrc fs.Fs, deleteEmptySrcDirs bool, copyEmptySrcDirs bool) error {- return runSyncCopyMove(ctx, fdst, fsrc, fs.DeleteModeOff, true, deleteEmptySrcDirs, copyEmptySrcDirs)+ return runSyncCopyMove(ctx, fdst, fsrc, fs.DeleteModeOff, true, deleteEmptySrcDirs, copyEmptySrcDirs, false)+}++// Transform renames fdst in place+func Transform(ctx context.Context, fdst fs.Fs, deleteEmptySrcDirs bool, copyEmptySrcDirs bool) error {+ return runSyncCopyMove(ctx, fdst, fdst, fs.DeleteModeOff, true, deleteEmptySrcDirs, copyEmptySrcDirs, true)}// MoveDir moves fsrc into fdstdiff --git a/fs/sync/sync_test.go b/fs/sync/sync_test.goindex 4c6407190..3f765b0c0 100644--- a/fs/sync/sync_test.go+++ b/fs/sync/sync_test.go@@ -27,6 +27,7 @@ import ("github.com/rclone/rclone/fs/hash""github.com/rclone/rclone/fs/operations""github.com/rclone/rclone/fstest"+ "github.com/rclone/rclone/lib/transform""github.com/stretchr/testify/assert""github.com/stretchr/testify/require""golang.org/x/text/unicode/norm"@@ -2980,7 +2981,7 @@ func predictDstFromLogger(ctx context.Context) context.Context {if winner.Err != nil {errMsg = ";" + winner.Err.Error()}- operations.SyncFprintf(opt.JSON, "%s;%s;%v;%s%s\n", file.ModTime(ctx).Local().Format(timeFormat), checksum, file.Size(), file.Remote(), errMsg)+ operations.SyncFprintf(opt.JSON, "%s;%s;%v;%s%s\n", file.ModTime(ctx).Local().Format(timeFormat), checksum, file.Size(), transform.Path(ctx, file.Remote(), false), errMsg) // TODO: should the transform be handled in the sync instead of here?}}return operations.WithSyncLogger(ctx, opt)diff --git a/fs/sync/sync_transform_test.go b/fs/sync/sync_transform_test.gonew file mode 100644index 000000000..d45a80983--- /dev/null+++ b/fs/sync/sync_transform_test.go@@ -0,0 +1,483 @@+// Test transform++package sync++import (+ "cmp"+ "context"+ "fmt"+ "path"+ "slices"+ "strings"+ "testing"++ _ "github.com/rclone/rclone/backend/all"+ "github.com/rclone/rclone/fs"+ "github.com/rclone/rclone/fs/filter"+ "github.com/rclone/rclone/fs/operations"+ "github.com/rclone/rclone/fs/walk"+ "github.com/rclone/rclone/fstest"+ "github.com/rclone/rclone/lib/transform"+ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+ "golang.org/x/text/unicode/norm"+)++var debug = ``++func TestTransform(t *testing.T) {+ type args struct {+ TransformOpt []string+ TransformBackOpt []string+ Lossless bool // whether the TransformBackAlgo is always losslessly invertible+ }+ tests := []struct {+ name string+ args args+ }{+ {name: "NFC", args: args{+ TransformOpt: []string{"nfc"},+ TransformBackOpt: []string{"nfd"},+ Lossless: false,+ }},+ {name: "NFD", args: args{+ TransformOpt: []string{"nfd"},+ TransformBackOpt: []string{"nfc"},+ Lossless: false,+ }},+ {name: "base64", args: args{+ TransformOpt: []string{"base64encode"},+ TransformBackOpt: []string{"base64encode"},+ Lossless: false,+ }},+ {name: "prefix", args: args{+ TransformOpt: []string{"prefix=PREFIX"},+ TransformBackOpt: []string{"trimprefix=PREFIX"},+ Lossless: true,+ }},+ {name: "suffix", args: args{+ TransformOpt: []string{"suffix=SUFFIX"},+ TransformBackOpt: []string{"trimsuffix=SUFFIX"},+ Lossless: true,+ }},+ {name: "truncate", args: args{+ TransformOpt: []string{"truncate=10"},+ TransformBackOpt: []string{"truncate=10"},+ Lossless: false,+ }},+ {name: "encoder", args: args{+ TransformOpt: []string{"encoder=Colon,SquareBracket"},+ TransformBackOpt: []string{"decoder=Colon,SquareBracket"},+ Lossless: true,+ }},+ {name: "ISO-8859-1", args: args{+ TransformOpt: []string{"ISO-8859-1"},+ TransformBackOpt: []string{"ISO-8859-1"},+ Lossless: false,+ }},+ {name: "charmap", args: args{+ TransformOpt: []string{"all,charmap=ISO-8859-7"},+ TransformBackOpt: []string{"all,charmap=ISO-8859-7"},+ Lossless: false,+ }},+ {name: "lowercase", args: args{+ TransformOpt: []string{"all,lowercase"},+ TransformBackOpt: []string{"all,lowercase"},+ Lossless: false,+ }},+ {name: "ascii", args: args{+ TransformOpt: []string{"all,ascii"},+ TransformBackOpt: []string{"all,ascii"},+ Lossless: false,+ }},+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ r := fstest.NewRun(t)+ defer r.Finalise()++ ctx := context.Background()+ r.Mkdir(ctx, r.Flocal)+ r.Mkdir(ctx, r.Fremote)+ items := makeTestFiles(t, r, "dir1")+ deleteDSStore(t, r)+ r.CheckRemoteListing(t, items, nil)+ r.CheckLocalListing(t, items, nil)++ err := transform.SetOptions(ctx, tt.args.TransformOpt...)+ require.NoError(t, err)++ err = Sync(ctx, r.Fremote, r.Flocal, true)+ assert.NoError(t, err)+ compareNames(ctx, t, r, items)++ err = transform.SetOptions(ctx, tt.args.TransformBackOpt...)+ require.NoError(t, err)+ err = Sync(ctx, r.Fremote, r.Flocal, true)+ assert.NoError(t, err)+ compareNames(ctx, t, r, items)++ if tt.args.Lossless {+ deleteDSStore(t, r)+ r.CheckRemoteItems(t, items...)+ }+ })+ }+}++const alphabet = "abcdefg123456789"++var extras = []string{"apple", "banana", "appleappleapplebanana", "splitbananasplit"}++func makeTestFiles(t *testing.T, r *fstest.Run, dir string) []fstest.Item {+ t.Helper()+ n := 0+ // Create test files+ items := []fstest.Item{}+ for _, c := range alphabet {+ var out strings.Builder+ for i := rune(0); i < 7; i++ {+ out.WriteRune(c + i)+ }+ fileName := path.Join(dir, fmt.Sprintf("%04d-%s.txt", n, out.String()))+ fileName = strings.ToValidUTF8(fileName, "")+ fileName = strings.NewReplacer(":", "", "<", "", ">", "", "?", "").Replace(fileName) // remove characters illegal on windows++ if debug != "" {+ fileName = debug+ }++ item := r.WriteObject(context.Background(), fileName, fileName, t1)+ r.WriteFile(fileName, fileName, t1)+ items = append(items, item)+ n++++ if debug != "" {+ break+ }+ }++ for _, extra := range extras {+ item := r.WriteObject(context.Background(), extra, extra, t1)+ r.WriteFile(extra, extra, t1)+ items = append(items, item)+ }++ return items+}++func deleteDSStore(t *testing.T, r *fstest.Run) {+ ctxDSStore, fi := filter.AddConfig(context.Background())+ err := fi.AddRule(`+ *.DS_Store`)+ assert.NoError(t, err)+ err = fi.AddRule(`- **`)+ assert.NoError(t, err)+ err = operations.Delete(ctxDSStore, r.Fremote)+ assert.NoError(t, err)+}++func compareNames(ctx context.Context, t *testing.T, r *fstest.Run, items []fstest.Item) {+ var entries fs.DirEntries++ deleteDSStore(t, r)+ err := walk.ListR(context.Background(), r.Fremote, "", true, -1, walk.ListObjects, func(e fs.DirEntries) error {+ entries = append(entries, e...)+ return nil+ })+ assert.NoError(t, err)+ entries = slices.DeleteFunc(entries, func(E fs.DirEntry) bool { // remove those pesky .DS_Store files+ if strings.Contains(E.Remote(), ".DS_Store") {+ err := operations.DeleteFile(context.Background(), E.(fs.Object))+ assert.NoError(t, err)+ return true+ }+ return false+ })+ require.Equal(t, len(items), entries.Len())++ // sort by CONVERTED name+ slices.SortStableFunc(items, func(a, b fstest.Item) int {+ aConv := transform.Path(ctx, a.Path, false)+ bConv := transform.Path(ctx, b.Path, false)+ return cmp.Compare(aConv, bConv)+ })+ slices.SortStableFunc(entries, func(a, b fs.DirEntry) int {+ return cmp.Compare(a.Remote(), b.Remote())+ })++ for i, e := range entries {+ expect := transform.Path(ctx, items[i].Path, false)+ msg := fmt.Sprintf("expected %v, got %v", detectEncoding(expect), detectEncoding(e.Remote()))+ assert.Equal(t, expect, e.Remote(), msg)+ }+}++func detectEncoding(s string) string {+ if norm.NFC.IsNormalString(s) && norm.NFD.IsNormalString(s) {+ return "BOTH"+ }+ if !norm.NFC.IsNormalString(s) && norm.NFD.IsNormalString(s) {+ return "NFD"+ }+ if norm.NFC.IsNormalString(s) && !norm.NFD.IsNormalString(s) {+ return "NFC"+ }+ return "OTHER"+}++func TestTransformCopy(t *testing.T) {+ ctx := context.Background()+ r := fstest.NewRun(t)+ err := transform.SetOptions(ctx, "all,suffix_keep_extension=_somesuffix")+ require.NoError(t, err)+ file1 := r.WriteFile("sub dir/hello world.txt", "hello world", t1)++ r.Mkdir(ctx, r.Fremote)+ ctx = predictDstFromLogger(ctx)+ err = Sync(ctx, r.Fremote, r.Flocal, true)+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)+ require.NoError(t, err)++ r.CheckLocalItems(t, file1)+ r.CheckRemoteItems(t, fstest.NewItem("sub dir_somesuffix/hello world_somesuffix.txt", "hello world", t1))+}++func TestDoubleTransform(t *testing.T) {+ ctx := context.Background()+ r := fstest.NewRun(t)+ err := transform.SetOptions(ctx, "all,prefix=tac", "all,prefix=tic")+ require.NoError(t, err)+ file1 := r.WriteFile("toe/toe", "hello world", t1)++ r.Mkdir(ctx, r.Fremote)+ ctx = predictDstFromLogger(ctx)+ err = Sync(ctx, r.Fremote, r.Flocal, true)+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)+ require.NoError(t, err)++ r.CheckLocalItems(t, file1)+ r.CheckRemoteItems(t, fstest.NewItem("tictactoe/tictactoe", "hello world", t1))+}++func TestFileTag(t *testing.T) {+ ctx := context.Background()+ r := fstest.NewRun(t)+ err := transform.SetOptions(ctx, "file,prefix=tac", "file,prefix=tic")+ require.NoError(t, err)+ file1 := r.WriteFile("toe/toe/toe", "hello world", t1)++ r.Mkdir(ctx, r.Fremote)+ ctx = predictDstFromLogger(ctx)+ err = Sync(ctx, r.Fremote, r.Flocal, true)+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)+ require.NoError(t, err)++ r.CheckLocalItems(t, file1)+ r.CheckRemoteItems(t, fstest.NewItem("toe/toe/tictactoe", "hello world", t1))+}++func TestNoTag(t *testing.T) {+ ctx := context.Background()+ r := fstest.NewRun(t)+ err := transform.SetOptions(ctx, "prefix=tac", "prefix=tic")+ require.NoError(t, err)+ file1 := r.WriteFile("toe/toe/toe", "hello world", t1)++ r.Mkdir(ctx, r.Fremote)+ ctx = predictDstFromLogger(ctx)+ err = Sync(ctx, r.Fremote, r.Flocal, true)+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)+ require.NoError(t, err)++ r.CheckLocalItems(t, file1)+ r.CheckRemoteItems(t, fstest.NewItem("toe/toe/tictactoe", "hello world", t1))+}++func TestDirTag(t *testing.T) {+ ctx := context.Background()+ r := fstest.NewRun(t)+ err := transform.SetOptions(ctx, "dir,prefix=tac", "dir,prefix=tic")+ require.NoError(t, err)+ r.WriteFile("toe/toe/toe.txt", "hello world", t1)+ _, err = operations.MkdirModTime(ctx, r.Flocal, "empty_dir", t1)+ require.NoError(t, err)++ r.Mkdir(ctx, r.Fremote)+ ctx = predictDstFromLogger(ctx)+ err = Sync(ctx, r.Fremote, r.Flocal, true)+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)+ require.NoError(t, err)++ r.CheckLocalListing(t, []fstest.Item{fstest.NewItem("toe/toe/toe.txt", "hello world", t1)}, []string{"empty_dir", "toe", "toe/toe"})+ r.CheckRemoteListing(t, []fstest.Item{fstest.NewItem("tictactoe/tictactoe/toe.txt", "hello world", t1)}, []string{"tictacempty_dir", "tictactoe", "tictactoe/tictactoe"})+}++func TestAllTag(t *testing.T) {+ ctx := context.Background()+ r := fstest.NewRun(t)+ err := transform.SetOptions(ctx, "all,prefix=tac", "all,prefix=tic")+ require.NoError(t, err)+ r.WriteFile("toe/toe/toe.txt", "hello world", t1)+ _, err = operations.MkdirModTime(ctx, r.Flocal, "empty_dir", t1)+ require.NoError(t, err)++ r.Mkdir(ctx, r.Fremote)+ ctx = predictDstFromLogger(ctx)+ err = Sync(ctx, r.Fremote, r.Flocal, true)+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)+ require.NoError(t, err)++ r.CheckLocalListing(t, []fstest.Item{fstest.NewItem("toe/toe/toe.txt", "hello world", t1)}, []string{"empty_dir", "toe", "toe/toe"})+ r.CheckRemoteListing(t, []fstest.Item{fstest.NewItem("tictactoe/tictactoe/tictactoe.txt", "hello world", t1)}, []string{"tictacempty_dir", "tictactoe", "tictactoe/tictactoe"})+ err = operations.Check(ctx, &operations.CheckOpt{Fsrc: r.Flocal, Fdst: r.Fremote}) // should not error even though dst has transformed names+ assert.NoError(t, err)+}++func TestRunTwice(t *testing.T) {+ ctx := context.Background()+ r := fstest.NewRun(t)+ err := transform.SetOptions(ctx, "dir,prefix=tac", "dir,prefix=tic")+ require.NoError(t, err)+ file1 := r.WriteFile("toe/toe/toe.txt", "hello world", t1)++ r.Mkdir(ctx, r.Fremote)+ ctx = predictDstFromLogger(ctx)+ err = Sync(ctx, r.Fremote, r.Flocal, true)+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)+ require.NoError(t, err)++ r.CheckLocalItems(t, file1)+ r.CheckRemoteItems(t, fstest.NewItem("tictactoe/tictactoe/toe.txt", "hello world", t1))++ // result should not change second time, since src is unchanged+ ctx = predictDstFromLogger(ctx)+ err = Sync(ctx, r.Fremote, r.Flocal, true)+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)+ require.NoError(t, err)++ r.CheckLocalItems(t, file1)+ r.CheckRemoteItems(t, fstest.NewItem("tictactoe/tictactoe/toe.txt", "hello world", t1))+}++func TestSyntax(t *testing.T) {+ ctx := context.Background()+ err := transform.SetOptions(ctx, "prefix")+ assert.Error(t, err) // should error as required value is missing++ err = transform.SetOptions(ctx, "banana")+ assert.Error(t, err) // should error as unrecognized option++ err = transform.SetOptions(ctx, "=123")+ assert.Error(t, err) // should error as required key is missing++ err = transform.SetOptions(ctx, "prefix=123")+ assert.NoError(t, err) // should not error+}++func TestConflicting(t *testing.T) {+ ctx := context.Background()+ r := fstest.NewRun(t)+ err := transform.SetOptions(ctx, "prefix=tac", "trimprefix=tac")+ require.NoError(t, err)+ file1 := r.WriteFile("toe/toe/toe", "hello world", t1)++ r.Mkdir(ctx, r.Fremote)+ ctx = predictDstFromLogger(ctx)+ err = Sync(ctx, r.Fremote, r.Flocal, true)+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)+ require.NoError(t, err)++ // should result in no change as prefix and trimprefix cancel out+ r.CheckLocalItems(t, file1)+ r.CheckRemoteItems(t, fstest.NewItem("toe/toe/toe", "hello world", t1))+}++func TestMove(t *testing.T) {+ ctx := context.Background()+ r := fstest.NewRun(t)+ err := transform.SetOptions(ctx, "all,prefix=tac", "all,prefix=tic")+ require.NoError(t, err)+ r.WriteFile("toe/toe/toe.txt", "hello world", t1)+ _, err = operations.MkdirModTime(ctx, r.Flocal, "empty_dir", t1)+ require.NoError(t, err)++ r.Mkdir(ctx, r.Fremote)+ ctx = predictDstFromLogger(ctx)+ err = MoveDir(ctx, r.Fremote, r.Flocal, true, true)+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)+ require.NoError(t, err)++ r.CheckLocalListing(t, []fstest.Item{}, []string{})+ r.CheckRemoteListing(t, []fstest.Item{fstest.NewItem("tictactoe/tictactoe/tictactoe.txt", "hello world", t1)}, []string{"tictacempty_dir", "tictactoe", "tictactoe/tictactoe"})+}++func TestTransformFile(t *testing.T) {+ ctx := context.Background()+ r := fstest.NewRun(t)+ err := transform.SetOptions(ctx, "all,prefix=tac", "all,prefix=tic")+ require.NoError(t, err)+ r.WriteFile("toe/toe/toe.txt", "hello world", t1)+ _, err = operations.MkdirModTime(ctx, r.Flocal, "empty_dir", t1)+ require.NoError(t, err)++ r.Mkdir(ctx, r.Fremote)+ ctx = predictDstFromLogger(ctx)+ err = MoveDir(ctx, r.Fremote, r.Flocal, true, true)+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)+ require.NoError(t, err)++ r.CheckLocalListing(t, []fstest.Item{}, []string{})+ r.CheckRemoteListing(t, []fstest.Item{fstest.NewItem("tictactoe/tictactoe/tictactoe.txt", "hello world", t1)}, []string{"tictacempty_dir", "tictactoe", "tictactoe/tictactoe"})++ err = transform.SetOptions(ctx, "all,trimprefix=tic", "all,trimprefix=tac")+ require.NoError(t, err)+ err = operations.TransformFile(ctx, r.Fremote, "tictactoe/tictactoe/tictactoe.txt")+ require.NoError(t, err)+ r.CheckLocalListing(t, []fstest.Item{}, []string{})+ r.CheckRemoteListing(t, []fstest.Item{fstest.NewItem("toe/toe/toe.txt", "hello world", t1)}, []string{"tictacempty_dir", "tictactoe", "tictactoe/tictactoe", "toe", "toe/toe"})+}++func TestBase64(t *testing.T) {+ ctx := context.Background()+ r := fstest.NewRun(t)+ err := transform.SetOptions(ctx, "all,base64encode")+ require.NoError(t, err)+ file1 := r.WriteFile("toe/toe/toe.txt", "hello world", t1)++ r.Mkdir(ctx, r.Fremote)+ ctx = predictDstFromLogger(ctx)+ err = Sync(ctx, r.Fremote, r.Flocal, true)+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)+ require.NoError(t, err)++ r.CheckLocalItems(t, file1)+ r.CheckRemoteItems(t, fstest.NewItem("dG9l/dG9l/dG9lLnR4dA==", "hello world", t1))++ // round trip+ err = transform.SetOptions(ctx, "all,base64decode")+ require.NoError(t, err)+ ctx = predictDstFromLogger(ctx)+ err = Sync(ctx, r.Flocal, r.Fremote, true)+ testLoggerVsLsf(ctx, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t)+ require.NoError(t, err)++ r.CheckLocalItems(t, file1)+ r.CheckRemoteItems(t, fstest.NewItem("dG9l/dG9l/dG9lLnR4dA==", "hello world", t1))+}++func TestError(t *testing.T) {+ ctx := context.Background()+ r := fstest.NewRun(t)+ err := transform.SetOptions(ctx, "all,prefix=ta/c") // has illegal character+ require.NoError(t, err)+ file1 := r.WriteFile("toe/toe/toe", "hello world", t1)++ r.Mkdir(ctx, r.Fremote)+ // ctx = predictDstFromLogger(ctx)+ err = Sync(ctx, r.Fremote, r.Flocal, true)+ // testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)+ assert.Error(t, err)++ r.CheckLocalListing(t, []fstest.Item{file1}, []string{"toe", "toe/toe"})+ r.CheckRemoteListing(t, []fstest.Item{file1}, []string{"toe", "toe/toe"})+}diff --git a/lib/encoder/encoder.go b/lib/encoder/encoder.goindex b1ff541a3..c8f41d76c 100644--- a/lib/encoder/encoder.go+++ b/lib/encoder/encoder.go@@ -151,8 +151,8 @@ func init() {alias("Dot", EncodeDot)}-// validStrings returns all the valid MultiEncoder strings-func validStrings() string {+// ValidStrings returns all the valid MultiEncoder strings+func ValidStrings() string {var out []stringfor k := range nameToEncoding {out = append(out, k)@@ -192,7 +192,7 @@ func (mask *MultiEncoder) Set(in string) error {} else {i, err := strconv.ParseUint(part, 0, 0)if err != nil {- return fmt.Errorf("bad encoding %q: possible values are: %s", part, validStrings())+ return fmt.Errorf("bad encoding %q: possible values are: %s", part, ValidStrings())}out |= MultiEncoder(i)}@@ -313,8 +313,7 @@ func (mask MultiEncoder) Encode(in string) string {}if mask.Has(EncodeAsterisk) { // *switch r {- case '*',- '*':+ case '*', '*':return true}}@@ -346,64 +345,55 @@ func (mask MultiEncoder) Encode(in string) string {}if mask.Has(EncodeQuestion) { // ?switch r {- case '?',- '?':+ case '?', '?':return true}}if mask.Has(EncodeColon) { // :switch r {- case ':',- ':':+ case ':', ':':return true}}if mask.Has(EncodePipe) { // |switch r {- case '|',- '|':+ case '|', '|':return true}}if mask.Has(EncodeDoubleQuote) { // "switch r {- case '"',- '"':+ case '"', '"':return true}}if mask.Has(EncodeSingleQuote) { // 'switch r {- case '\'',- ''':+ case '\'', ''':return true}}if mask.Has(EncodeBackQuote) { // `switch r {- case '`',- '`':+ case '`', '`':return true}}if mask.Has(EncodeDollar) { // $switch r {- case '$',- '$':+ case '$', '$':return true}}if mask.Has(EncodeSlash) { // /switch r {- case '/',- '/':+ case '/', '/':return true}}if mask.Has(EncodeBackSlash) { // \switch r {- case '\\',- '\':+ case '\\', '\':return true}}@@ -416,15 +406,13 @@ func (mask MultiEncoder) Encode(in string) string {}if mask.Has(EncodeHash) { // #switch r {- case '#',- '#':+ case '#', '#':return true}}if mask.Has(EncodePercent) { // %switch r {- case '%',- '%':+ case '%', '%':return true}}@@ -1182,6 +1170,7 @@ func appendQuotedBytes(w io.Writer, s string) {_, _ = fmt.Fprintf(w, string(QuoteRune)+"%02X", b)}}+func appendUnquotedByte(w io.Writer, s string) bool {if len(s) < 2 {return false@@ -1202,12 +1191,15 @@ func (identity) Decode(in string) string { return in }func (i identity) FromStandardPath(s string) string {return FromStandardPath(i, s)}+func (i identity) FromStandardName(s string) string {return FromStandardName(i, s)}+func (i identity) ToStandardPath(s string) string {return ToStandardPath(i, s)}+func (i identity) ToStandardName(s string) string {return ToStandardName(i, s)}diff --git a/lib/transform/cmap.go b/lib/transform/cmap.gonew file mode 100644index 000000000..7f676e792--- /dev/null+++ b/lib/transform/cmap.go@@ -0,0 +1,71 @@+package transform++import (+ "fmt"+ "strings"+ "sync"++ "github.com/rclone/rclone/fs"+ "golang.org/x/text/encoding/charmap"+)++var (+ cmaps = map[int]*charmap.Charmap{}+ lock sync.Mutex+)++type cmapChoices struct{}++func (cmapChoices) Choices() []string {+ choices := make([]string, 1)+ i := 0+ for _, enc := range charmap.All {+ c, ok := enc.(*charmap.Charmap)+ if !ok {+ continue+ }+ name := strings.ReplaceAll(c.String(), " ", "-")+ if name == "" {+ name = fmt.Sprintf("unknown-%d", i)+ }+ lock.Lock()+ cmaps[i] = c+ lock.Unlock()+ choices = append(choices, name)+ i+++ }+ return choices+}++func (cmapChoices) Type() string {+ return "string"+}++func charmapByID(cm fs.Enum[cmapChoices]) *charmap.Charmap {+ lock.Lock()+ c, ok := cmaps[int(cm)]+ lock.Unlock()+ if ok {+ return c+ }+ return nil+}++func encodeWithReplacement(s string, cmap *charmap.Charmap) string {+ return strings.Map(func(r rune) rune {+ b, ok := cmap.EncodeRune(r)+ if !ok {+ return '_'+ }+ return cmap.DecodeByte(b)+ }, s)+}++func toASCII(s string) string {+ return strings.Map(func(r rune) rune {+ if r <= 127 {+ return r+ }+ return -1+ }, s)+}diff --git a/lib/transform/help.go b/lib/transform/help.gonew file mode 100644index 000000000..a92267844--- /dev/null+++ b/lib/transform/help.go@@ -0,0 +1,140 @@+package transform++import (+ "context"+ "fmt"+ "strings"++ "github.com/rclone/rclone/fs"+ "github.com/rclone/rclone/lib/encoder"+)++type commands struct {+ command string+ description string+}++type example struct {+ path string+ flags []string+}++var commandList = []commands{+ {command: "--name-transform prefix=XXXX", description: "Prepends XXXX to the file name."},+ {command: "--name-transform suffix=XXXX", description: "Appends XXXX to the file name after the extension."},+ {command: "--name-transform suffix_keep_extension=XXXX", description: "Appends XXXX to the file name while preserving the original file extension."},+ {command: "--name-transform trimprefix=XXXX", description: "Removes XXXX if it appears at the start of the file name."},+ {command: "--name-transform trimsuffix=XXXX", description: "Removes XXXX if it appears at the end of the file name."},+ {command: "--name-transform regex=/pattern/replacement/", description: "Applies a regex-based transformation."},+ {command: "--name-transform replace=old:new", description: "Replaces occurrences of old with new in the file name."},+ {command: "--name-transform date={YYYYMMDD}", description: "Appends or prefixes the specified date format."},+ {command: "--name-transform truncate=N", description: "Truncates the file name to a maximum of N characters."},+ {command: "--name-transform base64encode", description: "Encodes the file name in Base64."},+ {command: "--name-transform base64decode", description: "Decodes a Base64-encoded file name."},+ {command: "--name-transform encoder=ENCODING", description: "Converts the file name to the specified encoding (e.g., ISO-8859-1, Windows-1252, Macintosh)."},+ {command: "--name-transform decoder=ENCODING", description: "Decodes the file name from the specified encoding."},+ {command: "--name-transform charmap=MAP", description: "Applies a character mapping transformation."},+ {command: "--name-transform lowercase", description: "Converts the file name to lowercase."},+ {command: "--name-transform uppercase", description: "Converts the file name to UPPERCASE."},+ {command: "--name-transform titlecase", description: "Converts the file name to Title Case."},+ {command: "--name-transform ascii", description: "Strips non-ASCII characters."},+ {command: "--name-transform url", description: "URL-encodes the file name."},+ {command: "--name-transform nfc", description: "Converts the file name to NFC Unicode normalization form."},+ {command: "--name-transform nfd", description: "Converts the file name to NFD Unicode normalization form."},+ {command: "--name-transform nfkc", description: "Converts the file name to NFKC Unicode normalization form."},+ {command: "--name-transform nfkd", description: "Converts the file name to NFKD Unicode normalization form."},+ {command: "--name-transform command=/path/to/my/programfile names.", description: "Executes an external program to transform"},+}++var examples = []example{+ {"stories/The Quick Brown Fox!.txt", []string{"all,uppercase"}},+ {"stories/The Quick Brown Fox!.txt", []string{"all,replace=Fox:Turtle", "all,replace=Quick:Slow"}},+ {"stories/The Quick Brown Fox!.txt", []string{"all,base64encode"}},+ {"c3Rvcmllcw==/VGhlIFF1aWNrIEJyb3duIEZveCEudHh0", []string{"all,base64decode"}},+ {"stories/The Quick Brown 🦊 Fox Went to the Café!.txt", []string{"all,nfc"}},+ {"stories/The Quick Brown 🦊 Fox Went to the Café!.txt", []string{"all,nfd"}},+ {"stories/The Quick Brown 🦊 Fox!.txt", []string{"all,ascii"}},+ {"stories/The Quick Brown Fox!.txt", []string{"all,trimsuffix=.txt"}},+ {"stories/The Quick Brown Fox!.txt", []string{"all,prefix=OLD_"}},+ {"stories/The Quick Brown 🦊 Fox Went to the Café!.txt", []string{"all,charmap=ISO-8859-7"}},+ {"stories/The Quick Brown Fox: A Memoir [draft].txt", []string{"all,encoder=Colon,SquareBracket"}},+ {"stories/The Quick Brown 🦊 Fox Went to the Café!.txt", []string{"all,truncate=21"}},+ {"stories/The Quick Brown Fox!.txt", []string{"all,command=echo"}},+ {"stories/The Quick Brown Fox!", []string{"date=-{YYYYMMDD}"}},+ {"stories/The Quick Brown Fox!", []string{"date=-{macfriendlytime}"}},+ {"stories/The Quick Brown Fox!.txt", []string{"all,regex=[\\.\\w]/ab"}},+}++func (e example) command() string {+ s := fmt.Sprintf(`rclone convmv %q`, e.path)+ for _, f := range e.flags {+ s += fmt.Sprintf(" --name-transform %q", f)+ }+ return s+}++func (e example) output() string {+ ctx := context.Background()+ err := SetOptions(ctx, e.flags...)+ if err != nil {+ fs.Errorf(nil, "error generating help text: %v", err)+ }+ return Path(ctx, e.path, false)+}++// go run ./ convmv --help+func sprintExamples() string {+ s := "Examples: \n\n"+ for _, e := range examples {+ s += fmt.Sprintf("```\n%s\n", e.command())+ s += fmt.Sprintf("// Output: %s\n```\n\n", e.output())+ }+ return s+}++func commandTable() string {+ s := `| Command | Description |+|------|------|`+ for _, c := range commandList {+ s += fmt.Sprintf("\n| `%s` | %s |", c.command, c.description)+ }+ s += "\n\n\n"+ return s+}++var generatingHelpText bool++// SprintList returns the example help text as a string+func SprintList() string {+ var algos transformAlgo+ var charmaps fs.Enum[cmapChoices]+ generatingHelpText = true+ s := commandTable()+ s += fmt.Sprintln("Conversion modes: \n```")+ for _, v := range algos.Choices() {+ s += fmt.Sprintln(v + " ")+ }+ s += fmt.Sprintln("```")++ s += fmt.Sprintln("Char maps: \n```")+ for _, v := range charmaps.Choices() {+ s += fmt.Sprintln(v + " ")+ }+ s += fmt.Sprintln("```")++ s += fmt.Sprintln("Encoding masks: \n```")+ for _, v := range strings.Split(encoder.ValidStrings(), ",") {+ s += fmt.Sprintln(v + " ")+ }+ s += fmt.Sprintln("```")++ s += sprintExamples()++ generatingHelpText = false+ return s+}++// PrintList prints the example help text to stdout+func PrintList() {+ fmt.Println(SprintList())+}diff --git a/lib/transform/options.go b/lib/transform/options.gonew file mode 100644index 000000000..880fb6482--- /dev/null+++ b/lib/transform/options.go@@ -0,0 +1,248 @@+package transform++import (+ "context"+ "errors"+ "slices"+ "strings"+ "sync"++ "github.com/rclone/rclone/fs"+)++type transform struct {+ key transformAlgo // for example, "prefix"+ value string // for example, "some_prefix_"+ tag tag // file, dir, or all+}++// tag controls which part of the file path is affected (file, dir, all)+type tag int++// tag modes+const (+ file tag = iota // Only transform the leaf name of files (default)+ dir // Only transform name of directories - these may appear anywhere in the path+ all // Transform the entire path for files and directories+)++// Transforming returns true when transforms are in use+func Transforming(ctx context.Context) bool {+ ci := fs.GetConfig(ctx)+ return len(ci.NameTransform) > 0+}++// SetOptions sets the options in ctx from flags passed in.+// Any existing flags will be overwritten.+// s should be in the same format as cmd line flags, i.e. "all,prefix=XXX"+func SetOptions(ctx context.Context, s ...string) (err error) {+ ci := fs.GetConfig(ctx)+ ci.NameTransform = s+ _, err = getOptions(ctx)+ return err+}++// cache to minimize re-parsing+var (+ cachedNameTransform []string+ cachedOpt []transform+ cacheLock sync.Mutex+)++// getOptions sets the options from flags passed in.+func getOptions(ctx context.Context) (opt []transform, err error) {+ if !Transforming(ctx) {+ return opt, nil+ }++ ci := fs.GetConfig(ctx)++ // return cached opt if available+ if cachedNameTransform != nil && slices.Equal(ci.NameTransform, cachedNameTransform) {+ return cachedOpt, nil+ }++ for _, transform := range ci.NameTransform {+ t, err := parse(transform)+ if err != nil {+ return opt, err+ }+ opt = append(opt, t)+ }+ updateCache(ci.NameTransform, opt)+ return opt, nil+}++func updateCache(nt []string, o []transform) {+ cacheLock.Lock()+ cachedNameTransform = slices.Clone(nt)+ cachedOpt = o+ cacheLock.Unlock()+}++// parse a single instance of --name-transform+func parse(s string) (t transform, err error) {+ if s == "" {+ return t, nil+ }+ s = t.parseTag(s)+ err = t.parseKeyVal(s)+ return t, err+}++// parse the tag (file/dir/all), set the option accordingly, and return the trimmed string+//+// we don't worry about errors here because it will error anyway as an invalid key+func (t *transform) parseTag(s string) string {+ if strings.HasPrefix(s, "file,") {+ t.tag = file+ return strings.TrimPrefix(s, "file,")+ }+ if strings.HasPrefix(s, "dir,") {+ t.tag = dir+ return strings.TrimPrefix(s, "dir,")+ }+ if strings.HasPrefix(s, "all,") {+ t.tag = all+ return strings.TrimPrefix(s, "all,")+ }+ return s+}++// parse key and value (if any) by splitting on '=' sign+// (file/dir/all tag has already been trimmed)+func (t *transform) parseKeyVal(s string) (err error) {+ if !strings.ContainsRune(s, '=') {+ err = t.key.Set(s)+ if err != nil {+ return err+ }+ if t.requiresValue() {+ fs.Debugf(nil, "received %v", s)+ return errors.New("value is required for " + t.key.String())+ }+ return nil+ }+ split := strings.Split(s, "=")+ if len(split) != 2 {+ return errors.New("too many values")+ }+ if split[0] == "" {+ return errors.New("key cannot be blank")+ }+ err = t.key.Set(split[0])+ if err != nil {+ return err+ }+ t.value = split[1]+ return nil+}++// returns true if this particular algorithm requires a value+func (t *transform) requiresValue() bool {+ switch t.key {+ case ConvFindReplace:+ return true+ case ConvPrefix:+ return true+ case ConvSuffix:+ return true+ case ConvSuffixKeepExtension:+ return true+ case ConvTrimPrefix:+ return true+ case ConvTrimSuffix:+ return true+ case ConvIndex:+ return true+ case ConvDate:+ return true+ case ConvTruncate:+ return true+ case ConvEncoder:+ return true+ case ConvDecoder:+ return true+ case ConvRegex:+ return true+ case ConvCommand:+ return true+ }+ return false+}++// transformAlgo describes conversion setting+type transformAlgo = fs.Enum[transformChoices]++// Supported transform options+const (+ ConvNone transformAlgo = iota+ ConvToNFC+ ConvToNFD+ ConvToNFKC+ ConvToNFKD+ ConvFindReplace+ ConvPrefix+ ConvSuffix+ ConvSuffixKeepExtension+ ConvTrimPrefix+ ConvTrimSuffix+ ConvIndex+ ConvDate+ ConvTruncate+ ConvBase64Encode+ ConvBase64Decode+ ConvEncoder+ ConvDecoder+ ConvISO8859_1+ ConvWindows1252+ ConvMacintosh+ ConvCharmap+ ConvLowercase+ ConvUppercase+ ConvTitlecase+ ConvASCII+ ConvURL+ ConvRegex+ ConvCommand+)++type transformChoices struct{}++func (transformChoices) Choices() []string {+ return []string{+ ConvNone: "none",+ ConvToNFC: "nfc",+ ConvToNFD: "nfd",+ ConvToNFKC: "nfkc",+ ConvToNFKD: "nfkd",+ ConvFindReplace: "replace",+ ConvPrefix: "prefix",+ ConvSuffix: "suffix",+ ConvSuffixKeepExtension: "suffix_keep_extension",+ ConvTrimPrefix: "trimprefix",+ ConvTrimSuffix: "trimsuffix",+ ConvIndex: "index",+ ConvDate: "date",+ ConvTruncate: "truncate",+ ConvBase64Encode: "base64encode",+ ConvBase64Decode: "base64decode",+ ConvEncoder: "encoder",+ ConvDecoder: "decoder",+ ConvISO8859_1: "ISO-8859-1",+ ConvWindows1252: "Windows-1252",+ ConvMacintosh: "Macintosh",+ ConvCharmap: "charmap",+ ConvLowercase: "lowercase",+ ConvUppercase: "uppercase",+ ConvTitlecase: "titlecase",+ ConvASCII: "ascii",+ ConvURL: "url",+ ConvRegex: "regex",+ ConvCommand: "command",+ }+}++func (transformChoices) Type() string {+ return "string"+}diff --git a/lib/transform/transform.go b/lib/transform/transform.gonew file mode 100644index 000000000..0103ae87f--- /dev/null+++ b/lib/transform/transform.go@@ -0,0 +1,336 @@+// Package transform holds functions for path name transformations+package transform++import (+ "bytes"+ "context"+ "encoding/base64"+ "errors"+ "fmt"+ "mime"+ "net/url"+ "os/exec"+ "path"+ "regexp"+ "strconv"+ "strings"+ "time"+ "unicode/utf8"++ "github.com/rclone/rclone/fs"+ "github.com/rclone/rclone/fs/fserrors"+ "github.com/rclone/rclone/lib/encoder"+ "golang.org/x/text/encoding/charmap"+ "golang.org/x/text/unicode/norm"+)++// Path transforms a path s according to the --name-transform options in use+//+// If no transforms are in use, s is returned unchanged+func Path(ctx context.Context, s string, isDir bool) string {+ if !Transforming(ctx) {+ return s+ }++ old := s+ opt, err := getOptions(ctx)+ if err != nil {+ err = fs.CountError(ctx, err)+ fs.Errorf(s, "Failed to parse transform flags: %v", err)+ }+ for _, t := range opt {+ if isDir && t.tag == file {+ continue+ }+ baseOnly := !isDir && t.tag == file+ if t.tag == dir && !isDir {+ s, err = transformDir(s, t)+ } else {+ s, err = transformPath(s, t, baseOnly)+ }+ if err != nil {+ err = fs.CountError(ctx, fserrors.NoRetryError(err))+ fs.Errorf(s, "Failed to transform: %v", err)+ }+ }+ if old != s && !generatingHelpText {+ fs.Debugf(old, "transformed to: %v", s)+ }+ if strings.Count(old, "/") != strings.Count(s, "/") {+ err = fs.CountError(ctx, fserrors.NoRetryError(fmt.Errorf("number of path segments must match: %v (%v), %v (%v)", old, strings.Count(old, "/"), s, strings.Count(s, "/"))))+ fs.Errorf(old, "%v", err)+ return old+ }+ return s+}++// transformPath transforms a path string according to the chosen TransformAlgo.+// Each path segment is transformed separately, to preserve path separators.+// If baseOnly is true, only the base will be transformed (useful for renaming while walking a dir tree recursively.)+// for example, "some/nested/path" -> "some/nested/CONVERTEDPATH"+// otherwise, the entire is path is transformed.+func transformPath(s string, t transform, baseOnly bool) (string, error) {+ if s == "" || s == "/" || s == "\\" || s == "." {+ return "", nil+ }++ if baseOnly {+ transformedBase, err := transformPathSegment(path.Base(s), t)+ if err := validateSegment(transformedBase); err != nil {+ return "", err+ }+ return path.Join(path.Dir(s), transformedBase), err+ }++ segments := strings.Split(s, "/")+ transformedSegments := make([]string, len(segments))+ for _, seg := range segments {+ convSeg, err := transformPathSegment(seg, t)+ if err != nil {+ return "", err+ }+ if err := validateSegment(convSeg); err != nil {+ return "", err+ }+ transformedSegments = append(transformedSegments, convSeg)+ }+ return path.Join(transformedSegments...), nil+}++// transform all but the last path segment+func transformDir(s string, t transform) (string, error) {+ dirPath, err := transformPath(path.Dir(s), t, false)+ if err != nil {+ return "", err+ }+ return path.Join(dirPath, path.Base(s)), nil+}++// transformPathSegment transforms one path segment (or really any string) according to the chosen TransformAlgo.+// It assumes path separators have already been trimmed.+func transformPathSegment(s string, t transform) (string, error) {+ switch t.key {+ case ConvNone:+ return s, nil+ case ConvToNFC:+ return norm.NFC.String(s), nil+ case ConvToNFD:+ return norm.NFD.String(s), nil+ case ConvToNFKC:+ return norm.NFKC.String(s), nil+ case ConvToNFKD:+ return norm.NFKD.String(s), nil+ case ConvBase64Encode:+ return base64.URLEncoding.EncodeToString([]byte(s)), nil // URLEncoding to avoid slashes+ case ConvBase64Decode:+ if s == ".DS_Store" {+ return s, nil+ }+ b, err := base64.URLEncoding.DecodeString(s)+ if err != nil {+ fs.Errorf(s, "base64 error")+ }+ return string(b), err+ case ConvFindReplace:+ split := strings.Split(t.value, ":")+ if len(split) != 2 {+ return s, fmt.Errorf("wrong number of values: %v", t.value)+ }+ return strings.ReplaceAll(s, split[0], split[1]), nil+ case ConvPrefix:+ return t.value + s, nil+ case ConvSuffix:+ return s + t.value, nil+ case ConvSuffixKeepExtension:+ return SuffixKeepExtension(s, t.value), nil+ case ConvTrimPrefix:+ return strings.TrimPrefix(s, t.value), nil+ case ConvTrimSuffix:+ return strings.TrimSuffix(s, t.value), nil+ case ConvTruncate:+ max, err := strconv.Atoi(t.value)+ if err != nil {+ return s, err+ }+ if max <= 0 {+ return s, nil+ }+ if utf8.RuneCountInString(s) <= max {+ return s, nil+ }+ runes := []rune(s)+ return string(runes[:max]), nil+ case ConvEncoder:+ var enc encoder.MultiEncoder+ err := enc.Set(t.value)+ if err != nil {+ return s, err+ }+ return enc.Encode(s), nil+ case ConvDecoder:+ var enc encoder.MultiEncoder+ err := enc.Set(t.value)+ if err != nil {+ return s, err+ }+ return enc.Decode(s), nil+ case ConvISO8859_1:+ return encodeWithReplacement(s, charmap.ISO8859_1), nil+ case ConvWindows1252:+ return encodeWithReplacement(s, charmap.Windows1252), nil+ case ConvMacintosh:+ return encodeWithReplacement(s, charmap.Macintosh), nil+ case ConvCharmap:+ var cmapType fs.Enum[cmapChoices]+ err := cmapType.Set(t.value)+ if err != nil {+ return s, err+ }+ c := charmapByID(cmapType)+ return encodeWithReplacement(s, c), nil+ case ConvLowercase:+ return strings.ToLower(s), nil+ case ConvUppercase:+ return strings.ToUpper(s), nil+ case ConvTitlecase:+ return strings.ToTitle(s), nil+ case ConvASCII:+ return toASCII(s), nil+ case ConvURL:+ return url.QueryEscape(s), nil+ case ConvDate:+ return s + AppyTimeGlobs(t.value, time.Now()), nil+ case ConvRegex:+ split := strings.Split(t.value, "/")+ if len(split) != 2 {+ return s, fmt.Errorf("regex syntax error: %v", t.value)+ }+ re := regexp.MustCompile(split[0])+ return re.ReplaceAllString(s, split[1]), nil+ case ConvCommand:+ return mapper(s, t.value)+ default:+ return "", errors.New("this option is not yet implemented")+ }+}++// SuffixKeepExtension adds a suffix while keeping extension+//+// i.e. file.txt becomes file_somesuffix.txt not file.txt_somesuffix+func SuffixKeepExtension(remote string, suffix string) string {+ var (+ base = remote+ exts = ""+ first = true+ ext = path.Ext(remote)+ )+ for ext != "" {+ // Look second and subsequent extensions in mime types.+ // If they aren't found then don't keep it as an extension.+ if !first && mime.TypeByExtension(ext) == "" {+ break+ }+ base = base[:len(base)-len(ext)]+ exts = ext + exts+ first = false+ ext = path.Ext(base)+ }+ return base + suffix + exts+}++// forbid transformations that add/remove path separators+func validateSegment(s string) error {+ if strings.TrimSpace(s) == "" {+ return errors.New("transform cannot render path segments empty")+ }+ if strings.ContainsRune(s, '/') {+ return fmt.Errorf("transform cannot add path separators: %v", s)+ }+ return nil+}++// ParseGlobs determines whether a string contains {brackets}+// and returns the substring (including both brackets) for replacing+// substring is first opening bracket to last closing bracket --+// good for {{this}} but not {this}{this}+func ParseGlobs(s string) (hasGlobs bool, substring string) {+ open := strings.Index(s, "{")+ close := strings.LastIndex(s, "}")+ if open >= 0 && close > open {+ return true, s[open : close+1]+ }+ return false, ""+}++// TrimBrackets converts {{this}} to this+func TrimBrackets(s string) string {+ return strings.Trim(s, "{}")+}++// TimeFormat converts a user-supplied string to a Go time constant, if possible+func TimeFormat(timeFormat string) string {+ switch timeFormat {+ case "Layout":+ timeFormat = time.Layout+ case "ANSIC":+ timeFormat = time.ANSIC+ case "UnixDate":+ timeFormat = time.UnixDate+ case "RubyDate":+ timeFormat = time.RubyDate+ case "RFC822":+ timeFormat = time.RFC822+ case "RFC822Z":+ timeFormat = time.RFC822Z+ case "RFC850":+ timeFormat = time.RFC850+ case "RFC1123":+ timeFormat = time.RFC1123+ case "RFC1123Z":+ timeFormat = time.RFC1123Z+ case "RFC3339":+ timeFormat = time.RFC3339+ case "RFC3339Nano":+ timeFormat = time.RFC3339Nano+ case "Kitchen":+ timeFormat = time.Kitchen+ case "Stamp":+ timeFormat = time.Stamp+ case "StampMilli":+ timeFormat = time.StampMilli+ case "StampMicro":+ timeFormat = time.StampMicro+ case "StampNano":+ timeFormat = time.StampNano+ case "DateTime":+ timeFormat = time.DateTime+ case "DateOnly":+ timeFormat = time.DateOnly+ case "TimeOnly":+ timeFormat = time.TimeOnly+ case "MacFriendlyTime", "macfriendlytime", "mac":+ timeFormat = "2006-01-02 0304PM" // not actually a Go constant -- but useful as macOS filenames can't have colons+ case "YYYYMMDD":+ timeFormat = "20060102"+ }+ return timeFormat+}++// AppyTimeGlobs converts "myfile-{DateOnly}.txt" to "myfile-2006-01-02.txt"+func AppyTimeGlobs(s string, t time.Time) string {+ hasGlobs, substring := ParseGlobs(s)+ if !hasGlobs {+ return s+ }+ timeString := t.Local().Format(TimeFormat(TrimBrackets(substring)))+ return strings.ReplaceAll(s, substring, timeString)+}++func mapper(s string, command string) (string, error) {+ out, err := exec.Command(command, s).CombinedOutput()+ if err != nil {+ out = bytes.TrimSpace(out)+ return s, fmt.Errorf("%s: error running command %q: %v", out, command+" "+s, err)+ }+ return string(bytes.TrimSpace(out)), nil+}diff --git a/lib/transform/transform_test.go b/lib/transform/transform_test.gonew file mode 100644index 000000000..bd5e0c9a7--- /dev/null+++ b/lib/transform/transform_test.go@@ -0,0 +1,142 @@+package transform++import (+ "context"+ "testing"+ "time"++ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+)++// sync tests are in fs/sync/sync_transform_test.go to avoid import cycle issues++func newOptions(s ...string) (context.Context, error) {+ ctx := context.Background()+ err := SetOptions(ctx, s...)+ return ctx, err+}++func TestPath(t *testing.T) {+ for _, test := range []struct {+ path string+ want string+ }{+ {"", ""},+ {"toe/toe/toe", "tictactoe/tictactoe/tictactoe"},+ {"a/b/c", "tictaca/tictacb/tictacc"},+ } {+ ctx, err := newOptions("all,prefix=tac", "all,prefix=tic")+ require.NoError(t, err)++ got := Path(ctx, test.path, false)+ assert.Equal(t, test.want, got)+ }+}++func TestFileTagOnFile(t *testing.T) {+ for _, test := range []struct {+ path string+ want string+ }{+ {"a/b/c.txt", "a/b/1c.txt"},+ } {+ ctx, err := newOptions("file,prefix=1")+ require.NoError(t, err)++ got := Path(ctx, test.path, false)+ assert.Equal(t, test.want, got)+ }+}++func TestDirTagOnFile(t *testing.T) {+ for _, test := range []struct {+ path string+ want string+ }{+ {"a/b/c.txt", "1a/1b/c.txt"},+ } {+ ctx, err := newOptions("dir,prefix=1")+ require.NoError(t, err)++ got := Path(ctx, test.path, false)+ assert.Equal(t, test.want, got)+ }+}++func TestAllTag(t *testing.T) {+ for _, test := range []struct {+ path string+ want string+ }{+ {"a/b/c.txt", "1a/1b/1c.txt"},+ } {+ ctx, err := newOptions("all,prefix=1")+ require.NoError(t, err)++ got := Path(ctx, test.path, false)+ assert.Equal(t, test.want, got)+ }+}++func TestFileTagOnDir(t *testing.T) {+ for _, test := range []struct {+ path string+ want string+ }{+ {"a/b", "a/b"},+ } {+ ctx, err := newOptions("file,prefix=1")+ require.NoError(t, err)++ got := Path(ctx, test.path, true)+ assert.Equal(t, test.want, got)+ }+}++func TestDirTagOnDir(t *testing.T) {+ for _, test := range []struct {+ path string+ want string+ }{+ {"a/b", "1a/1b"},+ } {+ ctx, err := newOptions("dir,prefix=1")+ require.NoError(t, err)++ got := Path(ctx, test.path, true)+ assert.Equal(t, test.want, got)+ }+}++func TestVarious(t *testing.T) {+ for _, test := range []struct {+ path string+ want string+ flags []string+ }{+ {"stories/The Quick Brown Fox!.txt", "STORIES/THE QUICK BROWN FOX!.TXT", []string{"all,uppercase"}},+ {"stories/The Quick Brown Fox!.txt", "stories/The Slow Brown Turtle!.txt", []string{"all,replace=Fox:Turtle", "all,replace=Quick:Slow"}},+ {"stories/The Quick Brown Fox!.txt", "c3Rvcmllcw==/VGhlIFF1aWNrIEJyb3duIEZveCEudHh0", []string{"all,base64encode"}},+ {"c3Rvcmllcw==/VGhlIFF1aWNrIEJyb3duIEZveCEudHh0", "stories/The Quick Brown Fox!.txt", []string{"all,base64decode"}},+ {"stories/The Quick Brown 🦊 Fox Went to the Café!.txt", "stories/The Quick Brown 🦊 Fox Went to the Café!.txt", []string{"all,nfc"}},+ {"stories/The Quick Brown 🦊 Fox Went to the Café!.txt", "stories/The Quick Brown 🦊 Fox Went to the Café!.txt", []string{"all,nfd"}},+ {"stories/The Quick Brown 🦊 Fox!.txt", "stories/The Quick Brown Fox!.txt", []string{"all,ascii"}},+ {"stories/The Quick Brown 🦊 Fox!.txt", "stories/The+Quick+Brown+%F0%9F%A6%8A+Fox%21.txt", []string{"all,url"}},+ {"stories/The Quick Brown Fox!.txt", "stories/The Quick Brown Fox!", []string{"all,trimsuffix=.txt"}},+ {"stories/The Quick Brown Fox!.txt", "OLD_stories/OLD_The Quick Brown Fox!.txt", []string{"all,prefix=OLD_"}},+ {"stories/The Quick Brown 🦊 Fox Went to the Café!.txt", "stories/The Quick Brown _ Fox Went to the Caf_!.txt", []string{"all,charmap=ISO-8859-7"}},+ {"stories/The Quick Brown Fox: A Memoir [draft].txt", "stories/The Quick Brown Fox: A Memoir [draft].txt", []string{"all,encoder=Colon,SquareBracket"}},+ {"stories/The Quick Brown 🦊 Fox Went to the Café!.txt", "stories/The Quick Brown 🦊 Fox", []string{"all,truncate=21"}},+ {"stories/The Quick Brown Fox!.txt", "stories/The Quick Brown Fox!.txt", []string{"all,command=echo"}},+ {"stories/The Quick Brown Fox!.txt", "stories/The Quick Brown Fox!.txt-" + time.Now().Local().Format("20060102"), []string{"date=-{YYYYMMDD}"}},+ {"stories/The Quick Brown Fox!.txt", "stories/The Quick Brown Fox!.txt-" + time.Now().Local().Format("2006-01-02 0304PM"), []string{"date=-{macfriendlytime}"}},+ {"stories/The Quick Brown Fox!.txt", "ababababababab/ababab ababababab ababababab ababab!abababab", []string{"all,regex=[\\.\\w]/ab"}},+ } {+ ctx, err := newOptions(test.flags...)+ require.NoError(t, err)++ got := Path(ctx, test.path, false)+ assert.Equal(t, test.want, got)+ }+}