Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

lib/transform: add transform library and --name-transform flag

nielash013c563293a0
diff --git a/cmd/bisync/bilib/names.go b/cmd/bisync/bilib/names.go
index d8951a0b5..cb266eb63 100644
--- a/cmd/bisync/bilib/names.go
+++ b/cmd/bisync/bilib/names.go
@@ -5,8 +5,6 @@ import (
"os"
"sort"
"strconv"
- "strings"
- "time"
)
// Names comprises a set of file names
@@ -85,81 +83,3 @@ func (am AliasMap) Alias(name1 string) string {
}
return name1
}
-
-// ParseGlobs determines whether a string contains {brackets}
-// and returns the substring (including both brackets) for replacing
-// substring is first opening bracket to last closing bracket --
-// good for {{this}} but not {this}{this}
-func ParseGlobs(s string) (hasGlobs bool, substring string) {
- open := strings.Index(s, "{")
- close := strings.LastIndex(s, "}")
- if open >= 0 && close > open {
- return true, s[open : close+1]
- }
- return false, ""
-}
-
-// TrimBrackets converts {{this}} to this
-func TrimBrackets(s string) string {
- return strings.Trim(s, "{}")
-}
-
-// TimeFormat converts a user-supplied string to a Go time constant, if possible
-func TimeFormat(timeFormat string) string {
- switch timeFormat {
- case "Layout":
- timeFormat = time.Layout
- case "ANSIC":
- timeFormat = time.ANSIC
- case "UnixDate":
- timeFormat = time.UnixDate
- case "RubyDate":
- timeFormat = time.RubyDate
- case "RFC822":
- timeFormat = time.RFC822
- case "RFC822Z":
- timeFormat = time.RFC822Z
- case "RFC850":
- timeFormat = time.RFC850
- case "RFC1123":
- timeFormat = time.RFC1123
- case "RFC1123Z":
- timeFormat = time.RFC1123Z
- case "RFC3339":
- timeFormat = time.RFC3339
- case "RFC3339Nano":
- timeFormat = time.RFC3339Nano
- case "Kitchen":
- timeFormat = time.Kitchen
- case "Stamp":
- timeFormat = time.Stamp
- case "StampMilli":
- timeFormat = time.StampMilli
- case "StampMicro":
- timeFormat = time.StampMicro
- case "StampNano":
- timeFormat = time.StampNano
- case "DateTime":
- // timeFormat = time.DateTime // missing in go1.19
- timeFormat = "2006-01-02 15:04:05"
- case "DateOnly":
- // timeFormat = time.DateOnly // missing in go1.19
- timeFormat = "2006-01-02"
- case "TimeOnly":
- // timeFormat = time.TimeOnly // missing in go1.19
- timeFormat = "15:04:05"
- case "MacFriendlyTime", "macfriendlytime", "mac":
- timeFormat = "2006-01-02 0304PM" // not actually a Go constant -- but useful as macOS filenames can't have colons
- }
- return timeFormat
-}
-
-// AppyTimeGlobs converts "myfile-{DateOnly}.txt" to "myfile-2006-01-02.txt"
-func AppyTimeGlobs(s string, t time.Time) string {
- hasGlobs, substring := ParseGlobs(s)
- if !hasGlobs {
- return s
- }
- timeString := t.Local().Format(TimeFormat(TrimBrackets(substring)))
- return strings.ReplaceAll(s, substring, timeString)
-}
diff --git a/cmd/bisync/resolve.go b/cmd/bisync/resolve.go
index d058978f2..b3e5d8048 100644
--- a/cmd/bisync/resolve.go
+++ b/cmd/bisync/resolve.go
@@ -4,8 +4,6 @@ import (
"context"
"fmt"
"math"
- "mime"
- "path"
"strings"
"time"
@@ -13,6 +11,7 @@ import (
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/operations"
"github.com/rclone/rclone/lib/terminal"
+ "github.com/rclone/rclone/lib/transform"
)
// Prefer describes strategies for resolving sync conflicts
@@ -97,8 +96,8 @@ func (b *bisyncRun) setResolveDefaults(ctx context.Context) error {
}
// replace glob variables, if any
t := time.Now() // capture static time here so it is the same for all files throughout this run
- b.opt.ConflictSuffix1 = bilib.AppyTimeGlobs(b.opt.ConflictSuffix1, t)
- b.opt.ConflictSuffix2 = bilib.AppyTimeGlobs(b.opt.ConflictSuffix2, t)
+ b.opt.ConflictSuffix1 = transform.AppyTimeGlobs(b.opt.ConflictSuffix1, t)
+ b.opt.ConflictSuffix2 = transform.AppyTimeGlobs(b.opt.ConflictSuffix2, t)
// append dot (intentionally allow more than one)
b.opt.ConflictSuffix1 = "." + b.opt.ConflictSuffix1
@@ -130,6 +129,7 @@ type (
path2 namePair
}
)
+
type namePair struct {
oldName string
newName string
@@ -240,24 +240,7 @@ func SuffixName(ctx context.Context, remote, suffix string) string {
}
ci := fs.GetConfig(ctx)
if ci.SuffixKeepExtension {
- var (
- base = remote
- exts = ""
- first = true
- ext = path.Ext(remote)
- )
- for ext != "" {
- // Look second and subsequent extensions in mime types.
- // If they aren't found then don't keep it as an extension.
- if !first && mime.TypeByExtension(ext) == "" {
- break
- }
- base = base[:len(base)-len(ext)]
- exts = ext + exts
- first = false
- ext = path.Ext(base)
- }
- return base + suffix + exts
+ return transform.SuffixKeepExtension(remote, suffix)
}
return remote + suffix
}
diff --git a/cmd/help.go b/cmd/help.go
index f65ff6d53..1b985daca 100644
--- a/cmd/help.go
+++ b/cmd/help.go
@@ -191,7 +191,6 @@ func setupRootCommand(rootCmd *cobra.Command) {
})
cobra.OnInitialize(initConfig)
-
}
// Traverse the tree of commands running fn on each
diff --git a/docs/content/docs.md b/docs/content/docs.md
index 4633429e0..056abcf09 100644
--- a/docs/content/docs.md
+++ b/docs/content/docs.md
@@ -1970,6 +1970,13 @@ If the backend has a `--backend-upload-concurrency` setting (eg
number of transfers instead if it is larger than the value of
`--multi-thread-streams` or `--multi-thread-streams` isn't set.
+### --name-transform COMMAND[=XXXX] ###
+`--name-transform` introduces path name transformations for
+`rclone copy`, `rclone sync`, and `rclone move`. These transformations
+enable modifications to source and destination file names by applying
+prefixes, suffixes, and other alterations during transfer operations.
+For detailed docs and examples, see [`convmv`](/commands/rclone_convmv/).
+
### --no-check-dest ###
The `--no-check-dest` can be used with `move` or `copy` and it causes
diff --git a/fs/config.go b/fs/config.go
index 0d3f9ddc5..aaae5defc 100644
--- a/fs/config.go
+++ b/fs/config.go
@@ -550,6 +550,11 @@ var ConfigOptionsInfo = Options{{
Default: 0,
Advanced: true,
Groups: "Networking",
+}, {
+ Name: "name_transform",
+ Default: []string{},
+ Help: "Transform paths during the copy process.",
+ Groups: "Copy",
}}
// ConfigInfo is filesystem config options
@@ -661,6 +666,7 @@ type ConfigInfo struct {
PartialSuffix string `config:"partial_suffix"`
MetadataMapper SpaceSepList `config:"metadata_mapper"`
MaxConnections int `config:"max_connections"`
+ NameTransform []string `config:"name_transform"`
}
func init() {
diff --git a/fs/march/march.go b/fs/march/march.go
index def481ac9..e3ffd58c7 100644
--- a/fs/march/march.go
+++ b/fs/march/march.go
@@ -15,6 +15,7 @@ import (
"github.com/rclone/rclone/fs/filter"
"github.com/rclone/rclone/fs/list"
"github.com/rclone/rclone/fs/walk"
+ "github.com/rclone/rclone/lib/transform"
"golang.org/x/sync/errgroup"
"golang.org/x/text/unicode/norm"
)
@@ -80,18 +81,22 @@ func (m *March) init(ctx context.Context) {
}
}
-// srcKey turns a directory entry into a sort key using the defined transforms.
-func (m *March) srcKey(entry fs.DirEntry) string {
+// srcOrDstKey turns a directory entry into a sort key using the defined transforms.
+func (m *March) srcOrDstKey(entry fs.DirEntry, isSrc bool) string {
if entry == nil {
return ""
}
name := path.Base(entry.Remote())
+ _, isDirectory := entry.(fs.Directory)
+ if isSrc {
+ name = transform.Path(m.Ctx, name, isDirectory)
+ }
for _, transform := range m.transforms {
name = transform(name)
}
// Suffix entries to make identically named files and
// directories sort consistently with directories first.
- if _, isDirectory := entry.(fs.Directory); isDirectory {
+ if isDirectory {
name += "D"
} else {
name += "F"
@@ -99,9 +104,14 @@ func (m *March) srcKey(entry fs.DirEntry) string {
return name
}
+// srcKey turns a directory entry into a sort key using the defined transforms.
+func (m *March) srcKey(entry fs.DirEntry) string {
+ return m.srcOrDstKey(entry, true)
+}
+
// dstKey turns a directory entry into a sort key using the defined transforms.
func (m *March) dstKey(entry fs.DirEntry) string {
- return m.srcKey(entry) // FIXME actually do something different
+ return m.srcOrDstKey(entry, false)
}
// makeListDir makes constructs a listing function for the given fs
@@ -461,7 +471,6 @@ func (m *March) processJob(job listDirJob) ([]listDirJob, error) {
noDst: true,
})
}
-
}, func(dst fs.DirEntry) {
recurse := m.Callback.DstOnly(dst)
if recurse && job.dstDepth > 0 {
diff --git a/fs/operations/copy.go b/fs/operations/copy.go
index 8228a14c7..2d42fac07 100644
--- a/fs/operations/copy.go
+++ b/fs/operations/copy.go
@@ -21,6 +21,7 @@ import (
"github.com/rclone/rclone/fs/hash"
"github.com/rclone/rclone/lib/atexit"
"github.com/rclone/rclone/lib/pacer"
+ "github.com/rclone/rclone/lib/transform"
)
// State of the copy
@@ -390,7 +391,7 @@ func Copy(ctx context.Context, f fs.Fs, dst fs.Object, remote string, src fs.Obj
f: f,
dstFeatures: f.Features(),
dst: dst,
- remote: remote,
+ remote: transform.Path(ctx, remote, false),
src: src,
ci: ci,
tr: tr,
@@ -399,7 +400,7 @@ func Copy(ctx context.Context, f fs.Fs, dst fs.Object, remote string, src fs.Obj
}
c.hashType, c.hashOption = CommonHash(ctx, f, src.Fs())
if c.dst != nil {
- c.remote = c.dst.Remote()
+ c.remote = transform.Path(ctx, c.dst.Remote(), false)
}
// Are we using partials?
//
@@ -414,5 +415,5 @@ func Copy(ctx context.Context, f fs.Fs, dst fs.Object, remote string, src fs.Obj
// CopyFile moves a single file possibly to a new name
func CopyFile(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, dstFileName string, srcFileName string) (err error) {
- return moveOrCopyFile(ctx, fdst, fsrc, dstFileName, srcFileName, true)
+ return moveOrCopyFile(ctx, fdst, fsrc, dstFileName, srcFileName, true, false)
}
diff --git a/fs/operations/operations.go b/fs/operations/operations.go
index 6d3ca4f76..d8366c252 100644
--- a/fs/operations/operations.go
+++ b/fs/operations/operations.go
@@ -39,6 +39,7 @@ import (
"github.com/rclone/rclone/lib/pacer"
"github.com/rclone/rclone/lib/random"
"github.com/rclone/rclone/lib/readers"
+ "github.com/rclone/rclone/lib/transform"
"golang.org/x/sync/errgroup"
"golang.org/x/text/unicode/norm"
)
@@ -424,6 +425,8 @@ func MoveTransfer(ctx context.Context, fdst fs.Fs, dst fs.Object, remote string,
// move - see Move for help
func move(ctx context.Context, fdst fs.Fs, dst fs.Object, remote string, src fs.Object, isTransfer bool) (newDst fs.Object, err error) {
+ origRemote := remote // avoid double-transform on fallback to copy
+ remote = transform.Path(ctx, remote, false)
ci := fs.GetConfig(ctx)
var tr *accounting.Transfer
if isTransfer {
@@ -447,12 +450,14 @@ func move(ctx context.Context, fdst fs.Fs, dst fs.Object, remote string, src fs.
if doMove := fdst.Features().Move; doMove != nil && (SameConfig(src.Fs(), fdst) || (SameRemoteType(src.Fs(), fdst) && (fdst.Features().ServerSideAcrossConfigs || ci.ServerSideAcrossConfigs))) {
// Delete destination if it exists and is not the same file as src (could be same file while seemingly different if the remote is case insensitive)
if dst != nil {
- remote = dst.Remote()
+ remote = transform.Path(ctx, dst.Remote(), false)
if !SameObject(src, dst) {
err = DeleteFile(ctx, dst)
if err != nil {
return newDst, err
}
+ } else if src.Remote() == remote {
+ return newDst, nil
} else if needsMoveCaseInsensitive(fdst, fdst, remote, src.Remote(), false) {
doMove = func(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
return MoveCaseInsensitive(ctx, fdst, fdst, remote, src.Remote(), false, src)
@@ -488,7 +493,7 @@ func move(ctx context.Context, fdst fs.Fs, dst fs.Object, remote string, src fs.
}
}
// Move not found or didn't work so copy dst <- src
- newDst, err = Copy(ctx, fdst, dst, remote, src)
+ newDst, err = Copy(ctx, fdst, dst, origRemote, src)
if err != nil {
fs.Errorf(src, "Not deleting source as copy failed: %v", err)
return newDst, err
@@ -516,24 +521,7 @@ func SuffixName(ctx context.Context, remote string) string {
return remote
}
if ci.SuffixKeepExtension {
- var (
- base = remote
- exts = ""
- first = true
- ext = path.Ext(remote)
- )
- for ext != "" {
- // Look second and subsequent extensions in mime types.
- // If they aren't found then don't keep it as an extension.
- if !first && mime.TypeByExtension(ext) == "" {
- break
- }
- base = base[:len(base)-len(ext)]
- exts = ext + exts
- first = false
- ext = path.Ext(base)
- }
- return base + ci.Suffix + exts
+ return transform.SuffixKeepExtension(remote, ci.Suffix)
}
return remote + ci.Suffix
}
@@ -1994,12 +1982,12 @@ func MoveCaseInsensitive(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, dstFileNam
}
// moveOrCopyFile moves or copies a single file possibly to a new name
-func moveOrCopyFile(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, dstFileName string, srcFileName string, cp bool) (err error) {
+func moveOrCopyFile(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, dstFileName string, srcFileName string, cp bool, allowOverlap bool) (err error) {
ci := fs.GetConfig(ctx)
logger, usingLogger := GetLogger(ctx)
dstFilePath := path.Join(fdst.Root(), dstFileName)
srcFilePath := path.Join(fsrc.Root(), srcFileName)
- if fdst.Name() == fsrc.Name() && dstFilePath == srcFilePath {
+ if fdst.Name() == fsrc.Name() && dstFilePath == srcFilePath && !allowOverlap {
fs.Debugf(fdst, "don't need to copy/move %s, it is already at target location", dstFileName)
if usingLogger {
srcObj, _ := fsrc.NewObject(ctx, srcFileName)
@@ -2106,7 +2094,14 @@ func moveOrCopyFile(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, dstFileName str
//
// This is treated as a transfer.
func MoveFile(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, dstFileName string, srcFileName string) (err error) {
- return moveOrCopyFile(ctx, fdst, fsrc, dstFileName, srcFileName, false)
+ return moveOrCopyFile(ctx, fdst, fsrc, dstFileName, srcFileName, false, false)
+}
+
+// TransformFile transforms a file in place using --name-transform
+//
+// This is treated as a transfer.
+func TransformFile(ctx context.Context, fdst fs.Fs, srcFileName string) (err error) {
+ return moveOrCopyFile(ctx, fdst, fdst, srcFileName, srcFileName, false, true)
}
// SetTier changes tier of object in remote
@@ -2211,50 +2206,10 @@ func (l *ListFormat) SetOutput(output []func(entry *ListJSONItem) string) {
// AddModTime adds file's Mod Time to output
func (l *ListFormat) AddModTime(timeFormat string) {
- switch timeFormat {
- case "":
+ if timeFormat == "" {
timeFormat = "2006-01-02 15:04:05"
- case "Layout":
- timeFormat = time.Layout
- case "ANSIC":
- timeFormat = time.ANSIC
- case "UnixDate":
- timeFormat = time.UnixDate
- case "RubyDate":
- timeFormat = time.RubyDate
- case "RFC822":
- timeFormat = time.RFC822
- case "RFC822Z":
- timeFormat = time.RFC822Z
- case "RFC850":
- timeFormat = time.RFC850
- case "RFC1123":
- timeFormat = time.RFC1123
- case "RFC1123Z":
- timeFormat = time.RFC1123Z
- case "RFC3339":
- timeFormat = time.RFC3339
- case "RFC3339Nano":
- timeFormat = time.RFC3339Nano
- case "Kitchen":
- timeFormat = time.Kitchen
- case "Stamp":
- timeFormat = time.Stamp
- case "StampMilli":
- timeFormat = time.StampMilli
- case "StampMicro":
- timeFormat = time.StampMicro
- case "StampNano":
- timeFormat = time.StampNano
- case "DateTime":
- // timeFormat = time.DateTime // missing in go1.19
- timeFormat = "2006-01-02 15:04:05"
- case "DateOnly":
- // timeFormat = time.DateOnly // missing in go1.19
- timeFormat = "2006-01-02"
- case "TimeOnly":
- // timeFormat = time.TimeOnly // missing in go1.19
- timeFormat = "15:04:05"
+ } else {
+ timeFormat = transform.TimeFormat(timeFormat)
}
l.AppendOutput(func(entry *ListJSONItem) string {
return entry.ModTime.When.Local().Format(timeFormat)
diff --git a/fs/operations/rc.go b/fs/operations/rc.go
index 46bf41770..18119cc82 100644
--- a/fs/operations/rc.go
+++ b/fs/operations/rc.go
@@ -62,7 +62,7 @@ func rcList(ctx context.Context, in rc.Params) (out rc.Params, err error) {
if rc.NotErrParamNotFound(err) {
return nil, err
}
- var list = []*ListJSONItem{}
+ list := []*ListJSONItem{}
err = ListJSON(ctx, f, remote, &opt, func(item *ListJSONItem) error {
list = append(list, item)
return nil
@@ -193,7 +193,7 @@ func rcMoveOrCopyFile(ctx context.Context, in rc.Params, cp bool) (out rc.Params
if err != nil {
return nil, err
}
- return nil, moveOrCopyFile(ctx, dstFs, srcFs, dstRemote, srcRemote, cp)
+ return nil, moveOrCopyFile(ctx, dstFs, srcFs, dstRemote, srcRemote, cp, false)
}
func init() {
@@ -289,7 +289,6 @@ func rcSingleCommand(ctx context.Context, in rc.Params, name string, noRemote bo
var request *http.Request
request, err := in.GetHTTPRequest()
-
if err != nil {
return nil, err
}
@@ -629,12 +628,12 @@ func rcBackend(ctx context.Context, in rc.Params) (out rc.Params, err error) {
if err != nil {
return nil, err
}
- var opt = map[string]string{}
+ opt := map[string]string{}
err = in.GetStructMissingOK("opt", &opt)
if err != nil {
return nil, err
}
- var arg = []string{}
+ arg := []string{}
err = in.GetStructMissingOK("arg", &arg)
if err != nil {
return nil, err
@@ -642,7 +641,6 @@ func rcBackend(ctx context.Context, in rc.Params) (out rc.Params, err error) {
result, err := doCommand(ctx, command, arg, opt)
if err != nil {
return nil, fmt.Errorf("command %q failed: %w", command, err)
-
}
out = make(rc.Params)
out["result"] = result
@@ -685,7 +683,6 @@ func rcDu(ctx context.Context, in rc.Params) (out rc.Params, err error) {
dir, err := in.GetString("dir")
if rc.IsErrParamNotFound(err) {
dir = config.GetCacheDir()
-
} else if err != nil {
return nil, err
}
diff --git a/fs/sync/sync.go b/fs/sync/sync.go
index 0e015769e..c7593f673 100644
--- a/fs/sync/sync.go
+++ b/fs/sync/sync.go
@@ -20,6 +20,7 @@ import (
"github.com/rclone/rclone/fs/march"
"github.com/rclone/rclone/fs/operations"
"github.com/rclone/rclone/lib/errcount"
+ "github.com/rclone/rclone/lib/transform"
"golang.org/x/sync/errgroup"
)
@@ -95,6 +96,7 @@ type syncCopyMove struct {
setDirModTimes []setDirModTime // directories that need their modtime set
setDirModTimesMaxLevel int // max level of the directories to set
modifiedDirs map[string]struct{} // dirs with changed contents (if s.setDirModTimeAfter)
+ allowOverlap bool // whether we allow src and dst to overlap (i.e. for convmv)
}
// For keeping track of delayed modtime sets
@@ -126,8 +128,8 @@ func (strategy trackRenamesStrategy) leaf() bool {
return (strategy & trackRenamesStrategyLeaf) != 0
}
-func newSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.DeleteMode, DoMove bool, deleteEmptySrcDirs bool, copyEmptySrcDirs bool) (*syncCopyMove, error) {
- if (deleteMode != fs.DeleteModeOff || DoMove) && operations.OverlappingFilterCheck(ctx, fdst, fsrc) {
+func newSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.DeleteMode, DoMove bool, deleteEmptySrcDirs bool, copyEmptySrcDirs bool, allowOverlap bool) (*syncCopyMove, error) {
+ if (deleteMode != fs.DeleteModeOff || DoMove) && operations.OverlappingFilterCheck(ctx, fdst, fsrc) && !allowOverlap {
return nil, fserrors.FatalError(fs.ErrorOverlapping)
}
ci := fs.GetConfig(ctx)
@@ -161,6 +163,7 @@ func newSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.Delete
setDirModTime: (!ci.NoUpdateDirModTime && fsrc.Features().CanHaveEmptyDirectories) && (fdst.Features().WriteDirSetModTime || fdst.Features().MkdirMetadata != nil || fdst.Features().DirSetModTime != nil),
setDirModTimeAfter: !ci.NoUpdateDirModTime && (!copyEmptySrcDirs || fsrc.Features().CanHaveEmptyDirectories && fdst.Features().DirModTimeUpdatesOnWrite),
modifiedDirs: make(map[string]struct{}),
+ allowOverlap: allowOverlap,
}
s.logger, s.usingLogger = operations.GetLogger(ctx)
@@ -922,7 +925,7 @@ func (s *syncCopyMove) tryRename(src fs.Object) bool {
//
// dir is the start directory, "" for root
func (s *syncCopyMove) run() error {
- if operations.Same(s.fdst, s.fsrc) {
+ if operations.Same(s.fdst, s.fsrc) && !s.allowOverlap {
fs.Errorf(s.fdst, "Nothing to do as source and destination are the same")
return nil
}
@@ -1122,6 +1125,9 @@ func (s *syncCopyMove) copyDirMetadata(ctx context.Context, f fs.Fs, dst fs.Dire
newDst, err = operations.SetDirModTime(ctx, f, dst, dir, src.ModTime(ctx))
}
}
+ if transform.Transforming(ctx) && newDst != nil && src.Remote() != newDst.Remote() {
+ s.markParentNotEmpty(src)
+ }
// If we need to set modtime after and we created a dir, then save it for later
if s.setDirModTime && s.setDirModTimeAfter && err == nil {
if newDst != nil {
@@ -1254,8 +1260,8 @@ func (s *syncCopyMove) SrcOnly(src fs.DirEntry) (recurse bool) {
s.logger(s.ctx, operations.MissingOnDst, src, nil, fs.ErrorIsDir)
// Create the directory and make sure the Metadata/ModTime is correct
- s.copyDirMetadata(s.ctx, s.fdst, nil, x.Remote(), x)
- s.markDirModified(x.Remote())
+ s.copyDirMetadata(s.ctx, s.fdst, nil, transform.Path(s.ctx, x.Remote(), true), x)
+ s.markDirModified(transform.Path(s.ctx, x.Remote(), true))
return true
default:
panic("Bad object in DirEntries")
@@ -1288,7 +1294,11 @@ func (s *syncCopyMove) Match(ctx context.Context, dst, src fs.DirEntry) (recurse
}
case fs.Directory:
// Do the same thing to the entire contents of the directory
- s.markParentNotEmpty(src)
+ srcX = fs.NewOverrideDirectory(srcX, transform.Path(ctx, src.Remote(), true))
+ src = srcX
+ if !transform.Transforming(ctx) || src.Remote() != dst.Remote() {
+ s.markParentNotEmpty(src)
+ }
dstX, ok := dst.(fs.Directory)
if ok {
s.logger(s.ctx, operations.Match, src, dst, fs.ErrorIsDir)
@@ -1327,7 +1337,7 @@ func (s *syncCopyMove) Match(ctx context.Context, dst, src fs.DirEntry) (recurse
// If DoMove is true then files will be moved instead of copied.
//
// dir is the start directory, "" for root
-func runSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.DeleteMode, DoMove bool, deleteEmptySrcDirs bool, copyEmptySrcDirs bool) error {
+func runSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.DeleteMode, DoMove bool, deleteEmptySrcDirs bool, copyEmptySrcDirs bool, allowOverlap bool) error {
ci := fs.GetConfig(ctx)
if deleteMode != fs.DeleteModeOff && DoMove {
return fserrors.FatalError(errors.New("can't delete and move at the same time"))
@@ -1338,7 +1348,7 @@ func runSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.Delete
return fserrors.FatalError(errors.New("can't use --delete-before with --track-renames"))
}
// only delete stuff during in this pass
- do, err := newSyncCopyMove(ctx, fdst, fsrc, fs.DeleteModeOnly, false, deleteEmptySrcDirs, copyEmptySrcDirs)
+ do, err := newSyncCopyMove(ctx, fdst, fsrc, fs.DeleteModeOnly, false, deleteEmptySrcDirs, copyEmptySrcDirs, allowOverlap)
if err != nil {
return err
}
@@ -1349,7 +1359,7 @@ func runSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.Delete
// Next pass does a copy only
deleteMode = fs.DeleteModeOff
}
- do, err := newSyncCopyMove(ctx, fdst, fsrc, deleteMode, DoMove, deleteEmptySrcDirs, copyEmptySrcDirs)
+ do, err := newSyncCopyMove(ctx, fdst, fsrc, deleteMode, DoMove, deleteEmptySrcDirs, copyEmptySrcDirs, allowOverlap)
if err != nil {
return err
}
@@ -1359,17 +1369,22 @@ func runSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.Delete
// Sync fsrc into fdst
func Sync(ctx context.Context, fdst, fsrc fs.Fs, copyEmptySrcDirs bool) error {
ci := fs.GetConfig(ctx)
- return runSyncCopyMove(ctx, fdst, fsrc, ci.DeleteMode, false, false, copyEmptySrcDirs)
+ return runSyncCopyMove(ctx, fdst, fsrc, ci.DeleteMode, false, false, copyEmptySrcDirs, false)
}
// CopyDir copies fsrc into fdst
func CopyDir(ctx context.Context, fdst, fsrc fs.Fs, copyEmptySrcDirs bool) error {
- return runSyncCopyMove(ctx, fdst, fsrc, fs.DeleteModeOff, false, false, copyEmptySrcDirs)
+ return runSyncCopyMove(ctx, fdst, fsrc, fs.DeleteModeOff, false, false, copyEmptySrcDirs, false)
}
// moveDir moves fsrc into fdst
func moveDir(ctx context.Context, fdst, fsrc fs.Fs, deleteEmptySrcDirs bool, copyEmptySrcDirs bool) error {
- return runSyncCopyMove(ctx, fdst, fsrc, fs.DeleteModeOff, true, deleteEmptySrcDirs, copyEmptySrcDirs)
+ return runSyncCopyMove(ctx, fdst, fsrc, fs.DeleteModeOff, true, deleteEmptySrcDirs, copyEmptySrcDirs, false)
+}
+
+// Transform renames fdst in place
+func Transform(ctx context.Context, fdst fs.Fs, deleteEmptySrcDirs bool, copyEmptySrcDirs bool) error {
+ return runSyncCopyMove(ctx, fdst, fdst, fs.DeleteModeOff, true, deleteEmptySrcDirs, copyEmptySrcDirs, true)
}
// MoveDir moves fsrc into fdst
diff --git a/fs/sync/sync_test.go b/fs/sync/sync_test.go
index 4c6407190..3f765b0c0 100644
--- a/fs/sync/sync_test.go
+++ b/fs/sync/sync_test.go
@@ -27,6 +27,7 @@ import (
"github.com/rclone/rclone/fs/hash"
"github.com/rclone/rclone/fs/operations"
"github.com/rclone/rclone/fstest"
+ "github.com/rclone/rclone/lib/transform"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/text/unicode/norm"
@@ -2980,7 +2981,7 @@ func predictDstFromLogger(ctx context.Context) context.Context {
if winner.Err != nil {
errMsg = ";" + winner.Err.Error()
}
- operations.SyncFprintf(opt.JSON, "%s;%s;%v;%s%s\n", file.ModTime(ctx).Local().Format(timeFormat), checksum, file.Size(), file.Remote(), errMsg)
+ operations.SyncFprintf(opt.JSON, "%s;%s;%v;%s%s\n", file.ModTime(ctx).Local().Format(timeFormat), checksum, file.Size(), transform.Path(ctx, file.Remote(), false), errMsg) // TODO: should the transform be handled in the sync instead of here?
}
}
return operations.WithSyncLogger(ctx, opt)
diff --git a/fs/sync/sync_transform_test.go b/fs/sync/sync_transform_test.go
new file mode 100644
index 000000000..d45a80983
--- /dev/null
+++ b/fs/sync/sync_transform_test.go
@@ -0,0 +1,483 @@
+// Test transform
+
+package sync
+
+import (
+ "cmp"
+ "context"
+ "fmt"
+ "path"
+ "slices"
+ "strings"
+ "testing"
+
+ _ "github.com/rclone/rclone/backend/all"
+ "github.com/rclone/rclone/fs"
+ "github.com/rclone/rclone/fs/filter"
+ "github.com/rclone/rclone/fs/operations"
+ "github.com/rclone/rclone/fs/walk"
+ "github.com/rclone/rclone/fstest"
+ "github.com/rclone/rclone/lib/transform"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+ "golang.org/x/text/unicode/norm"
+)
+
+var debug = ``
+
+func TestTransform(t *testing.T) {
+ type args struct {
+ TransformOpt []string
+ TransformBackOpt []string
+ Lossless bool // whether the TransformBackAlgo is always losslessly invertible
+ }
+ tests := []struct {
+ name string
+ args args
+ }{
+ {name: "NFC", args: args{
+ TransformOpt: []string{"nfc"},
+ TransformBackOpt: []string{"nfd"},
+ Lossless: false,
+ }},
+ {name: "NFD", args: args{
+ TransformOpt: []string{"nfd"},
+ TransformBackOpt: []string{"nfc"},
+ Lossless: false,
+ }},
+ {name: "base64", args: args{
+ TransformOpt: []string{"base64encode"},
+ TransformBackOpt: []string{"base64encode"},
+ Lossless: false,
+ }},
+ {name: "prefix", args: args{
+ TransformOpt: []string{"prefix=PREFIX"},
+ TransformBackOpt: []string{"trimprefix=PREFIX"},
+ Lossless: true,
+ }},
+ {name: "suffix", args: args{
+ TransformOpt: []string{"suffix=SUFFIX"},
+ TransformBackOpt: []string{"trimsuffix=SUFFIX"},
+ Lossless: true,
+ }},
+ {name: "truncate", args: args{
+ TransformOpt: []string{"truncate=10"},
+ TransformBackOpt: []string{"truncate=10"},
+ Lossless: false,
+ }},
+ {name: "encoder", args: args{
+ TransformOpt: []string{"encoder=Colon,SquareBracket"},
+ TransformBackOpt: []string{"decoder=Colon,SquareBracket"},
+ Lossless: true,
+ }},
+ {name: "ISO-8859-1", args: args{
+ TransformOpt: []string{"ISO-8859-1"},
+ TransformBackOpt: []string{"ISO-8859-1"},
+ Lossless: false,
+ }},
+ {name: "charmap", args: args{
+ TransformOpt: []string{"all,charmap=ISO-8859-7"},
+ TransformBackOpt: []string{"all,charmap=ISO-8859-7"},
+ Lossless: false,
+ }},
+ {name: "lowercase", args: args{
+ TransformOpt: []string{"all,lowercase"},
+ TransformBackOpt: []string{"all,lowercase"},
+ Lossless: false,
+ }},
+ {name: "ascii", args: args{
+ TransformOpt: []string{"all,ascii"},
+ TransformBackOpt: []string{"all,ascii"},
+ Lossless: false,
+ }},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ r := fstest.NewRun(t)
+ defer r.Finalise()
+
+ ctx := context.Background()
+ r.Mkdir(ctx, r.Flocal)
+ r.Mkdir(ctx, r.Fremote)
+ items := makeTestFiles(t, r, "dir1")
+ deleteDSStore(t, r)
+ r.CheckRemoteListing(t, items, nil)
+ r.CheckLocalListing(t, items, nil)
+
+ err := transform.SetOptions(ctx, tt.args.TransformOpt...)
+ require.NoError(t, err)
+
+ err = Sync(ctx, r.Fremote, r.Flocal, true)
+ assert.NoError(t, err)
+ compareNames(ctx, t, r, items)
+
+ err = transform.SetOptions(ctx, tt.args.TransformBackOpt...)
+ require.NoError(t, err)
+ err = Sync(ctx, r.Fremote, r.Flocal, true)
+ assert.NoError(t, err)
+ compareNames(ctx, t, r, items)
+
+ if tt.args.Lossless {
+ deleteDSStore(t, r)
+ r.CheckRemoteItems(t, items...)
+ }
+ })
+ }
+}
+
+const alphabet = "abcdefg123456789"
+
+var extras = []string{"apple", "banana", "appleappleapplebanana", "splitbananasplit"}
+
+func makeTestFiles(t *testing.T, r *fstest.Run, dir string) []fstest.Item {
+ t.Helper()
+ n := 0
+ // Create test files
+ items := []fstest.Item{}
+ for _, c := range alphabet {
+ var out strings.Builder
+ for i := rune(0); i < 7; i++ {
+ out.WriteRune(c + i)
+ }
+ fileName := path.Join(dir, fmt.Sprintf("%04d-%s.txt", n, out.String()))
+ fileName = strings.ToValidUTF8(fileName, "")
+ fileName = strings.NewReplacer(":", "", "<", "", ">", "", "?", "").Replace(fileName) // remove characters illegal on windows
+
+ if debug != "" {
+ fileName = debug
+ }
+
+ item := r.WriteObject(context.Background(), fileName, fileName, t1)
+ r.WriteFile(fileName, fileName, t1)
+ items = append(items, item)
+ n++
+
+ if debug != "" {
+ break
+ }
+ }
+
+ for _, extra := range extras {
+ item := r.WriteObject(context.Background(), extra, extra, t1)
+ r.WriteFile(extra, extra, t1)
+ items = append(items, item)
+ }
+
+ return items
+}
+
+func deleteDSStore(t *testing.T, r *fstest.Run) {
+ ctxDSStore, fi := filter.AddConfig(context.Background())
+ err := fi.AddRule(`+ *.DS_Store`)
+ assert.NoError(t, err)
+ err = fi.AddRule(`- **`)
+ assert.NoError(t, err)
+ err = operations.Delete(ctxDSStore, r.Fremote)
+ assert.NoError(t, err)
+}
+
+func compareNames(ctx context.Context, t *testing.T, r *fstest.Run, items []fstest.Item) {
+ var entries fs.DirEntries
+
+ deleteDSStore(t, r)
+ err := walk.ListR(context.Background(), r.Fremote, "", true, -1, walk.ListObjects, func(e fs.DirEntries) error {
+ entries = append(entries, e...)
+ return nil
+ })
+ assert.NoError(t, err)
+ entries = slices.DeleteFunc(entries, func(E fs.DirEntry) bool { // remove those pesky .DS_Store files
+ if strings.Contains(E.Remote(), ".DS_Store") {
+ err := operations.DeleteFile(context.Background(), E.(fs.Object))
+ assert.NoError(t, err)
+ return true
+ }
+ return false
+ })
+ require.Equal(t, len(items), entries.Len())
+
+ // sort by CONVERTED name
+ slices.SortStableFunc(items, func(a, b fstest.Item) int {
+ aConv := transform.Path(ctx, a.Path, false)
+ bConv := transform.Path(ctx, b.Path, false)
+ return cmp.Compare(aConv, bConv)
+ })
+ slices.SortStableFunc(entries, func(a, b fs.DirEntry) int {
+ return cmp.Compare(a.Remote(), b.Remote())
+ })
+
+ for i, e := range entries {
+ expect := transform.Path(ctx, items[i].Path, false)
+ msg := fmt.Sprintf("expected %v, got %v", detectEncoding(expect), detectEncoding(e.Remote()))
+ assert.Equal(t, expect, e.Remote(), msg)
+ }
+}
+
+func detectEncoding(s string) string {
+ if norm.NFC.IsNormalString(s) && norm.NFD.IsNormalString(s) {
+ return "BOTH"
+ }
+ if !norm.NFC.IsNormalString(s) && norm.NFD.IsNormalString(s) {
+ return "NFD"
+ }
+ if norm.NFC.IsNormalString(s) && !norm.NFD.IsNormalString(s) {
+ return "NFC"
+ }
+ return "OTHER"
+}
+
+func TestTransformCopy(t *testing.T) {
+ ctx := context.Background()
+ r := fstest.NewRun(t)
+ err := transform.SetOptions(ctx, "all,suffix_keep_extension=_somesuffix")
+ require.NoError(t, err)
+ file1 := r.WriteFile("sub dir/hello world.txt", "hello world", t1)
+
+ r.Mkdir(ctx, r.Fremote)
+ ctx = predictDstFromLogger(ctx)
+ err = Sync(ctx, r.Fremote, r.Flocal, true)
+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
+ require.NoError(t, err)
+
+ r.CheckLocalItems(t, file1)
+ r.CheckRemoteItems(t, fstest.NewItem("sub dir_somesuffix/hello world_somesuffix.txt", "hello world", t1))
+}
+
+func TestDoubleTransform(t *testing.T) {
+ ctx := context.Background()
+ r := fstest.NewRun(t)
+ err := transform.SetOptions(ctx, "all,prefix=tac", "all,prefix=tic")
+ require.NoError(t, err)
+ file1 := r.WriteFile("toe/toe", "hello world", t1)
+
+ r.Mkdir(ctx, r.Fremote)
+ ctx = predictDstFromLogger(ctx)
+ err = Sync(ctx, r.Fremote, r.Flocal, true)
+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
+ require.NoError(t, err)
+
+ r.CheckLocalItems(t, file1)
+ r.CheckRemoteItems(t, fstest.NewItem("tictactoe/tictactoe", "hello world", t1))
+}
+
+func TestFileTag(t *testing.T) {
+ ctx := context.Background()
+ r := fstest.NewRun(t)
+ err := transform.SetOptions(ctx, "file,prefix=tac", "file,prefix=tic")
+ require.NoError(t, err)
+ file1 := r.WriteFile("toe/toe/toe", "hello world", t1)
+
+ r.Mkdir(ctx, r.Fremote)
+ ctx = predictDstFromLogger(ctx)
+ err = Sync(ctx, r.Fremote, r.Flocal, true)
+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
+ require.NoError(t, err)
+
+ r.CheckLocalItems(t, file1)
+ r.CheckRemoteItems(t, fstest.NewItem("toe/toe/tictactoe", "hello world", t1))
+}
+
+func TestNoTag(t *testing.T) {
+ ctx := context.Background()
+ r := fstest.NewRun(t)
+ err := transform.SetOptions(ctx, "prefix=tac", "prefix=tic")
+ require.NoError(t, err)
+ file1 := r.WriteFile("toe/toe/toe", "hello world", t1)
+
+ r.Mkdir(ctx, r.Fremote)
+ ctx = predictDstFromLogger(ctx)
+ err = Sync(ctx, r.Fremote, r.Flocal, true)
+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
+ require.NoError(t, err)
+
+ r.CheckLocalItems(t, file1)
+ r.CheckRemoteItems(t, fstest.NewItem("toe/toe/tictactoe", "hello world", t1))
+}
+
+func TestDirTag(t *testing.T) {
+ ctx := context.Background()
+ r := fstest.NewRun(t)
+ err := transform.SetOptions(ctx, "dir,prefix=tac", "dir,prefix=tic")
+ require.NoError(t, err)
+ r.WriteFile("toe/toe/toe.txt", "hello world", t1)
+ _, err = operations.MkdirModTime(ctx, r.Flocal, "empty_dir", t1)
+ require.NoError(t, err)
+
+ r.Mkdir(ctx, r.Fremote)
+ ctx = predictDstFromLogger(ctx)
+ err = Sync(ctx, r.Fremote, r.Flocal, true)
+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
+ require.NoError(t, err)
+
+ r.CheckLocalListing(t, []fstest.Item{fstest.NewItem("toe/toe/toe.txt", "hello world", t1)}, []string{"empty_dir", "toe", "toe/toe"})
+ r.CheckRemoteListing(t, []fstest.Item{fstest.NewItem("tictactoe/tictactoe/toe.txt", "hello world", t1)}, []string{"tictacempty_dir", "tictactoe", "tictactoe/tictactoe"})
+}
+
+func TestAllTag(t *testing.T) {
+ ctx := context.Background()
+ r := fstest.NewRun(t)
+ err := transform.SetOptions(ctx, "all,prefix=tac", "all,prefix=tic")
+ require.NoError(t, err)
+ r.WriteFile("toe/toe/toe.txt", "hello world", t1)
+ _, err = operations.MkdirModTime(ctx, r.Flocal, "empty_dir", t1)
+ require.NoError(t, err)
+
+ r.Mkdir(ctx, r.Fremote)
+ ctx = predictDstFromLogger(ctx)
+ err = Sync(ctx, r.Fremote, r.Flocal, true)
+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
+ require.NoError(t, err)
+
+ r.CheckLocalListing(t, []fstest.Item{fstest.NewItem("toe/toe/toe.txt", "hello world", t1)}, []string{"empty_dir", "toe", "toe/toe"})
+ r.CheckRemoteListing(t, []fstest.Item{fstest.NewItem("tictactoe/tictactoe/tictactoe.txt", "hello world", t1)}, []string{"tictacempty_dir", "tictactoe", "tictactoe/tictactoe"})
+ err = operations.Check(ctx, &operations.CheckOpt{Fsrc: r.Flocal, Fdst: r.Fremote}) // should not error even though dst has transformed names
+ assert.NoError(t, err)
+}
+
+func TestRunTwice(t *testing.T) {
+ ctx := context.Background()
+ r := fstest.NewRun(t)
+ err := transform.SetOptions(ctx, "dir,prefix=tac", "dir,prefix=tic")
+ require.NoError(t, err)
+ file1 := r.WriteFile("toe/toe/toe.txt", "hello world", t1)
+
+ r.Mkdir(ctx, r.Fremote)
+ ctx = predictDstFromLogger(ctx)
+ err = Sync(ctx, r.Fremote, r.Flocal, true)
+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
+ require.NoError(t, err)
+
+ r.CheckLocalItems(t, file1)
+ r.CheckRemoteItems(t, fstest.NewItem("tictactoe/tictactoe/toe.txt", "hello world", t1))
+
+ // result should not change second time, since src is unchanged
+ ctx = predictDstFromLogger(ctx)
+ err = Sync(ctx, r.Fremote, r.Flocal, true)
+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
+ require.NoError(t, err)
+
+ r.CheckLocalItems(t, file1)
+ r.CheckRemoteItems(t, fstest.NewItem("tictactoe/tictactoe/toe.txt", "hello world", t1))
+}
+
+func TestSyntax(t *testing.T) {
+ ctx := context.Background()
+ err := transform.SetOptions(ctx, "prefix")
+ assert.Error(t, err) // should error as required value is missing
+
+ err = transform.SetOptions(ctx, "banana")
+ assert.Error(t, err) // should error as unrecognized option
+
+ err = transform.SetOptions(ctx, "=123")
+ assert.Error(t, err) // should error as required key is missing
+
+ err = transform.SetOptions(ctx, "prefix=123")
+ assert.NoError(t, err) // should not error
+}
+
+func TestConflicting(t *testing.T) {
+ ctx := context.Background()
+ r := fstest.NewRun(t)
+ err := transform.SetOptions(ctx, "prefix=tac", "trimprefix=tac")
+ require.NoError(t, err)
+ file1 := r.WriteFile("toe/toe/toe", "hello world", t1)
+
+ r.Mkdir(ctx, r.Fremote)
+ ctx = predictDstFromLogger(ctx)
+ err = Sync(ctx, r.Fremote, r.Flocal, true)
+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
+ require.NoError(t, err)
+
+ // should result in no change as prefix and trimprefix cancel out
+ r.CheckLocalItems(t, file1)
+ r.CheckRemoteItems(t, fstest.NewItem("toe/toe/toe", "hello world", t1))
+}
+
+func TestMove(t *testing.T) {
+ ctx := context.Background()
+ r := fstest.NewRun(t)
+ err := transform.SetOptions(ctx, "all,prefix=tac", "all,prefix=tic")
+ require.NoError(t, err)
+ r.WriteFile("toe/toe/toe.txt", "hello world", t1)
+ _, err = operations.MkdirModTime(ctx, r.Flocal, "empty_dir", t1)
+ require.NoError(t, err)
+
+ r.Mkdir(ctx, r.Fremote)
+ ctx = predictDstFromLogger(ctx)
+ err = MoveDir(ctx, r.Fremote, r.Flocal, true, true)
+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
+ require.NoError(t, err)
+
+ r.CheckLocalListing(t, []fstest.Item{}, []string{})
+ r.CheckRemoteListing(t, []fstest.Item{fstest.NewItem("tictactoe/tictactoe/tictactoe.txt", "hello world", t1)}, []string{"tictacempty_dir", "tictactoe", "tictactoe/tictactoe"})
+}
+
+func TestTransformFile(t *testing.T) {
+ ctx := context.Background()
+ r := fstest.NewRun(t)
+ err := transform.SetOptions(ctx, "all,prefix=tac", "all,prefix=tic")
+ require.NoError(t, err)
+ r.WriteFile("toe/toe/toe.txt", "hello world", t1)
+ _, err = operations.MkdirModTime(ctx, r.Flocal, "empty_dir", t1)
+ require.NoError(t, err)
+
+ r.Mkdir(ctx, r.Fremote)
+ ctx = predictDstFromLogger(ctx)
+ err = MoveDir(ctx, r.Fremote, r.Flocal, true, true)
+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
+ require.NoError(t, err)
+
+ r.CheckLocalListing(t, []fstest.Item{}, []string{})
+ r.CheckRemoteListing(t, []fstest.Item{fstest.NewItem("tictactoe/tictactoe/tictactoe.txt", "hello world", t1)}, []string{"tictacempty_dir", "tictactoe", "tictactoe/tictactoe"})
+
+ err = transform.SetOptions(ctx, "all,trimprefix=tic", "all,trimprefix=tac")
+ require.NoError(t, err)
+ err = operations.TransformFile(ctx, r.Fremote, "tictactoe/tictactoe/tictactoe.txt")
+ require.NoError(t, err)
+ r.CheckLocalListing(t, []fstest.Item{}, []string{})
+ r.CheckRemoteListing(t, []fstest.Item{fstest.NewItem("toe/toe/toe.txt", "hello world", t1)}, []string{"tictacempty_dir", "tictactoe", "tictactoe/tictactoe", "toe", "toe/toe"})
+}
+
+func TestBase64(t *testing.T) {
+ ctx := context.Background()
+ r := fstest.NewRun(t)
+ err := transform.SetOptions(ctx, "all,base64encode")
+ require.NoError(t, err)
+ file1 := r.WriteFile("toe/toe/toe.txt", "hello world", t1)
+
+ r.Mkdir(ctx, r.Fremote)
+ ctx = predictDstFromLogger(ctx)
+ err = Sync(ctx, r.Fremote, r.Flocal, true)
+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
+ require.NoError(t, err)
+
+ r.CheckLocalItems(t, file1)
+ r.CheckRemoteItems(t, fstest.NewItem("dG9l/dG9l/dG9lLnR4dA==", "hello world", t1))
+
+ // round trip
+ err = transform.SetOptions(ctx, "all,base64decode")
+ require.NoError(t, err)
+ ctx = predictDstFromLogger(ctx)
+ err = Sync(ctx, r.Flocal, r.Fremote, true)
+ testLoggerVsLsf(ctx, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t)
+ require.NoError(t, err)
+
+ r.CheckLocalItems(t, file1)
+ r.CheckRemoteItems(t, fstest.NewItem("dG9l/dG9l/dG9lLnR4dA==", "hello world", t1))
+}
+
+func TestError(t *testing.T) {
+ ctx := context.Background()
+ r := fstest.NewRun(t)
+ err := transform.SetOptions(ctx, "all,prefix=ta/c") // has illegal character
+ require.NoError(t, err)
+ file1 := r.WriteFile("toe/toe/toe", "hello world", t1)
+
+ r.Mkdir(ctx, r.Fremote)
+ // ctx = predictDstFromLogger(ctx)
+ err = Sync(ctx, r.Fremote, r.Flocal, true)
+ // testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
+ assert.Error(t, err)
+
+ r.CheckLocalListing(t, []fstest.Item{file1}, []string{"toe", "toe/toe"})
+ r.CheckRemoteListing(t, []fstest.Item{file1}, []string{"toe", "toe/toe"})
+}
diff --git a/lib/encoder/encoder.go b/lib/encoder/encoder.go
index b1ff541a3..c8f41d76c 100644
--- a/lib/encoder/encoder.go
+++ b/lib/encoder/encoder.go
@@ -151,8 +151,8 @@ func init() {
alias("Dot", EncodeDot)
}
-// validStrings returns all the valid MultiEncoder strings
-func validStrings() string {
+// ValidStrings returns all the valid MultiEncoder strings
+func ValidStrings() string {
var out []string
for k := range nameToEncoding {
out = append(out, k)
@@ -192,7 +192,7 @@ func (mask *MultiEncoder) Set(in string) error {
} else {
i, err := strconv.ParseUint(part, 0, 0)
if err != nil {
- return fmt.Errorf("bad encoding %q: possible values are: %s", part, validStrings())
+ return fmt.Errorf("bad encoding %q: possible values are: %s", part, ValidStrings())
}
out |= MultiEncoder(i)
}
@@ -313,8 +313,7 @@ func (mask MultiEncoder) Encode(in string) string {
}
if mask.Has(EncodeAsterisk) { // *
switch r {
- case '*',
- '*':
+ case '*', '*':
return true
}
}
@@ -346,64 +345,55 @@ func (mask MultiEncoder) Encode(in string) string {
}
if mask.Has(EncodeQuestion) { // ?
switch r {
- case '?',
- '?':
+ case '?', '?':
return true
}
}
if mask.Has(EncodeColon) { // :
switch r {
- case ':',
- ':':
+ case ':', ':':
return true
}
}
if mask.Has(EncodePipe) { // |
switch r {
- case '|',
- '|':
+ case '|', '|':
return true
}
}
if mask.Has(EncodeDoubleQuote) { // "
switch r {
- case '"',
- '"':
+ case '"', '"':
return true
}
}
if mask.Has(EncodeSingleQuote) { // '
switch r {
- case '\'',
- ''':
+ case '\'', ''':
return true
}
}
if mask.Has(EncodeBackQuote) { // `
switch r {
- case '`',
- '`':
+ case '`', '`':
return true
}
}
if mask.Has(EncodeDollar) { // $
switch r {
- case '$',
- '$':
+ case '$', '$':
return true
}
}
if mask.Has(EncodeSlash) { // /
switch r {
- case '/',
- '/':
+ case '/', '/':
return true
}
}
if mask.Has(EncodeBackSlash) { // \
switch r {
- case '\\',
- '\':
+ case '\\', '\':
return true
}
}
@@ -416,15 +406,13 @@ func (mask MultiEncoder) Encode(in string) string {
}
if mask.Has(EncodeHash) { // #
switch r {
- case '#',
- '#':
+ case '#', '#':
return true
}
}
if mask.Has(EncodePercent) { // %
switch r {
- case '%',
- '%':
+ case '%', '%':
return true
}
}
@@ -1182,6 +1170,7 @@ func appendQuotedBytes(w io.Writer, s string) {
_, _ = fmt.Fprintf(w, string(QuoteRune)+"%02X", b)
}
}
+
func appendUnquotedByte(w io.Writer, s string) bool {
if len(s) < 2 {
return false
@@ -1202,12 +1191,15 @@ func (identity) Decode(in string) string { return in }
func (i identity) FromStandardPath(s string) string {
return FromStandardPath(i, s)
}
+
func (i identity) FromStandardName(s string) string {
return FromStandardName(i, s)
}
+
func (i identity) ToStandardPath(s string) string {
return ToStandardPath(i, s)
}
+
func (i identity) ToStandardName(s string) string {
return ToStandardName(i, s)
}
diff --git a/lib/transform/cmap.go b/lib/transform/cmap.go
new file mode 100644
index 000000000..7f676e792
--- /dev/null
+++ b/lib/transform/cmap.go
@@ -0,0 +1,71 @@
+package transform
+
+import (
+ "fmt"
+ "strings"
+ "sync"
+
+ "github.com/rclone/rclone/fs"
+ "golang.org/x/text/encoding/charmap"
+)
+
+var (
+ cmaps = map[int]*charmap.Charmap{}
+ lock sync.Mutex
+)
+
+type cmapChoices struct{}
+
+func (cmapChoices) Choices() []string {
+ choices := make([]string, 1)
+ i := 0
+ for _, enc := range charmap.All {
+ c, ok := enc.(*charmap.Charmap)
+ if !ok {
+ continue
+ }
+ name := strings.ReplaceAll(c.String(), " ", "-")
+ if name == "" {
+ name = fmt.Sprintf("unknown-%d", i)
+ }
+ lock.Lock()
+ cmaps[i] = c
+ lock.Unlock()
+ choices = append(choices, name)
+ i++
+ }
+ return choices
+}
+
+func (cmapChoices) Type() string {
+ return "string"
+}
+
+func charmapByID(cm fs.Enum[cmapChoices]) *charmap.Charmap {
+ lock.Lock()
+ c, ok := cmaps[int(cm)]
+ lock.Unlock()
+ if ok {
+ return c
+ }
+ return nil
+}
+
+func encodeWithReplacement(s string, cmap *charmap.Charmap) string {
+ return strings.Map(func(r rune) rune {
+ b, ok := cmap.EncodeRune(r)
+ if !ok {
+ return '_'
+ }
+ return cmap.DecodeByte(b)
+ }, s)
+}
+
+func toASCII(s string) string {
+ return strings.Map(func(r rune) rune {
+ if r <= 127 {
+ return r
+ }
+ return -1
+ }, s)
+}
diff --git a/lib/transform/help.go b/lib/transform/help.go
new file mode 100644
index 000000000..a92267844
--- /dev/null
+++ b/lib/transform/help.go
@@ -0,0 +1,140 @@
+package transform
+
+import (
+ "context"
+ "fmt"
+ "strings"
+
+ "github.com/rclone/rclone/fs"
+ "github.com/rclone/rclone/lib/encoder"
+)
+
+type commands struct {
+ command string
+ description string
+}
+
+type example struct {
+ path string
+ flags []string
+}
+
+var commandList = []commands{
+ {command: "--name-transform prefix=XXXX", description: "Prepends XXXX to the file name."},
+ {command: "--name-transform suffix=XXXX", description: "Appends XXXX to the file name after the extension."},
+ {command: "--name-transform suffix_keep_extension=XXXX", description: "Appends XXXX to the file name while preserving the original file extension."},
+ {command: "--name-transform trimprefix=XXXX", description: "Removes XXXX if it appears at the start of the file name."},
+ {command: "--name-transform trimsuffix=XXXX", description: "Removes XXXX if it appears at the end of the file name."},
+ {command: "--name-transform regex=/pattern/replacement/", description: "Applies a regex-based transformation."},
+ {command: "--name-transform replace=old:new", description: "Replaces occurrences of old with new in the file name."},
+ {command: "--name-transform date={YYYYMMDD}", description: "Appends or prefixes the specified date format."},
+ {command: "--name-transform truncate=N", description: "Truncates the file name to a maximum of N characters."},
+ {command: "--name-transform base64encode", description: "Encodes the file name in Base64."},
+ {command: "--name-transform base64decode", description: "Decodes a Base64-encoded file name."},
+ {command: "--name-transform encoder=ENCODING", description: "Converts the file name to the specified encoding (e.g., ISO-8859-1, Windows-1252, Macintosh)."},
+ {command: "--name-transform decoder=ENCODING", description: "Decodes the file name from the specified encoding."},
+ {command: "--name-transform charmap=MAP", description: "Applies a character mapping transformation."},
+ {command: "--name-transform lowercase", description: "Converts the file name to lowercase."},
+ {command: "--name-transform uppercase", description: "Converts the file name to UPPERCASE."},
+ {command: "--name-transform titlecase", description: "Converts the file name to Title Case."},
+ {command: "--name-transform ascii", description: "Strips non-ASCII characters."},
+ {command: "--name-transform url", description: "URL-encodes the file name."},
+ {command: "--name-transform nfc", description: "Converts the file name to NFC Unicode normalization form."},
+ {command: "--name-transform nfd", description: "Converts the file name to NFD Unicode normalization form."},
+ {command: "--name-transform nfkc", description: "Converts the file name to NFKC Unicode normalization form."},
+ {command: "--name-transform nfkd", description: "Converts the file name to NFKD Unicode normalization form."},
+ {command: "--name-transform command=/path/to/my/programfile names.", description: "Executes an external program to transform"},
+}
+
+var examples = []example{
+ {"stories/The Quick Brown Fox!.txt", []string{"all,uppercase"}},
+ {"stories/The Quick Brown Fox!.txt", []string{"all,replace=Fox:Turtle", "all,replace=Quick:Slow"}},
+ {"stories/The Quick Brown Fox!.txt", []string{"all,base64encode"}},
+ {"c3Rvcmllcw==/VGhlIFF1aWNrIEJyb3duIEZveCEudHh0", []string{"all,base64decode"}},
+ {"stories/The Quick Brown 🦊 Fox Went to the Café!.txt", []string{"all,nfc"}},
+ {"stories/The Quick Brown 🦊 Fox Went to the Café!.txt", []string{"all,nfd"}},
+ {"stories/The Quick Brown 🦊 Fox!.txt", []string{"all,ascii"}},
+ {"stories/The Quick Brown Fox!.txt", []string{"all,trimsuffix=.txt"}},
+ {"stories/The Quick Brown Fox!.txt", []string{"all,prefix=OLD_"}},
+ {"stories/The Quick Brown 🦊 Fox Went to the Café!.txt", []string{"all,charmap=ISO-8859-7"}},
+ {"stories/The Quick Brown Fox: A Memoir [draft].txt", []string{"all,encoder=Colon,SquareBracket"}},
+ {"stories/The Quick Brown 🦊 Fox Went to the Café!.txt", []string{"all,truncate=21"}},
+ {"stories/The Quick Brown Fox!.txt", []string{"all,command=echo"}},
+ {"stories/The Quick Brown Fox!", []string{"date=-{YYYYMMDD}"}},
+ {"stories/The Quick Brown Fox!", []string{"date=-{macfriendlytime}"}},
+ {"stories/The Quick Brown Fox!.txt", []string{"all,regex=[\\.\\w]/ab"}},
+}
+
+func (e example) command() string {
+ s := fmt.Sprintf(`rclone convmv %q`, e.path)
+ for _, f := range e.flags {
+ s += fmt.Sprintf(" --name-transform %q", f)
+ }
+ return s
+}
+
+func (e example) output() string {
+ ctx := context.Background()
+ err := SetOptions(ctx, e.flags...)
+ if err != nil {
+ fs.Errorf(nil, "error generating help text: %v", err)
+ }
+ return Path(ctx, e.path, false)
+}
+
+// go run ./ convmv --help
+func sprintExamples() string {
+ s := "Examples: \n\n"
+ for _, e := range examples {
+ s += fmt.Sprintf("```\n%s\n", e.command())
+ s += fmt.Sprintf("// Output: %s\n```\n\n", e.output())
+ }
+ return s
+}
+
+func commandTable() string {
+ s := `| Command | Description |
+|------|------|`
+ for _, c := range commandList {
+ s += fmt.Sprintf("\n| `%s` | %s |", c.command, c.description)
+ }
+ s += "\n\n\n"
+ return s
+}
+
+var generatingHelpText bool
+
+// SprintList returns the example help text as a string
+func SprintList() string {
+ var algos transformAlgo
+ var charmaps fs.Enum[cmapChoices]
+ generatingHelpText = true
+ s := commandTable()
+ s += fmt.Sprintln("Conversion modes: \n```")
+ for _, v := range algos.Choices() {
+ s += fmt.Sprintln(v + " ")
+ }
+ s += fmt.Sprintln("```")
+
+ s += fmt.Sprintln("Char maps: \n```")
+ for _, v := range charmaps.Choices() {
+ s += fmt.Sprintln(v + " ")
+ }
+ s += fmt.Sprintln("```")
+
+ s += fmt.Sprintln("Encoding masks: \n```")
+ for _, v := range strings.Split(encoder.ValidStrings(), ",") {
+ s += fmt.Sprintln(v + " ")
+ }
+ s += fmt.Sprintln("```")
+
+ s += sprintExamples()
+
+ generatingHelpText = false
+ return s
+}
+
+// PrintList prints the example help text to stdout
+func PrintList() {
+ fmt.Println(SprintList())
+}
diff --git a/lib/transform/options.go b/lib/transform/options.go
new file mode 100644
index 000000000..880fb6482
--- /dev/null
+++ b/lib/transform/options.go
@@ -0,0 +1,248 @@
+package transform
+
+import (
+ "context"
+ "errors"
+ "slices"
+ "strings"
+ "sync"
+
+ "github.com/rclone/rclone/fs"
+)
+
+type transform struct {
+ key transformAlgo // for example, "prefix"
+ value string // for example, "some_prefix_"
+ tag tag // file, dir, or all
+}
+
+// tag controls which part of the file path is affected (file, dir, all)
+type tag int
+
+// tag modes
+const (
+ file tag = iota // Only transform the leaf name of files (default)
+ dir // Only transform name of directories - these may appear anywhere in the path
+ all // Transform the entire path for files and directories
+)
+
+// Transforming returns true when transforms are in use
+func Transforming(ctx context.Context) bool {
+ ci := fs.GetConfig(ctx)
+ return len(ci.NameTransform) > 0
+}
+
+// SetOptions sets the options in ctx from flags passed in.
+// Any existing flags will be overwritten.
+// s should be in the same format as cmd line flags, i.e. "all,prefix=XXX"
+func SetOptions(ctx context.Context, s ...string) (err error) {
+ ci := fs.GetConfig(ctx)
+ ci.NameTransform = s
+ _, err = getOptions(ctx)
+ return err
+}
+
+// cache to minimize re-parsing
+var (
+ cachedNameTransform []string
+ cachedOpt []transform
+ cacheLock sync.Mutex
+)
+
+// getOptions sets the options from flags passed in.
+func getOptions(ctx context.Context) (opt []transform, err error) {
+ if !Transforming(ctx) {
+ return opt, nil
+ }
+
+ ci := fs.GetConfig(ctx)
+
+ // return cached opt if available
+ if cachedNameTransform != nil && slices.Equal(ci.NameTransform, cachedNameTransform) {
+ return cachedOpt, nil
+ }
+
+ for _, transform := range ci.NameTransform {
+ t, err := parse(transform)
+ if err != nil {
+ return opt, err
+ }
+ opt = append(opt, t)
+ }
+ updateCache(ci.NameTransform, opt)
+ return opt, nil
+}
+
+func updateCache(nt []string, o []transform) {
+ cacheLock.Lock()
+ cachedNameTransform = slices.Clone(nt)
+ cachedOpt = o
+ cacheLock.Unlock()
+}
+
+// parse a single instance of --name-transform
+func parse(s string) (t transform, err error) {
+ if s == "" {
+ return t, nil
+ }
+ s = t.parseTag(s)
+ err = t.parseKeyVal(s)
+ return t, err
+}
+
+// parse the tag (file/dir/all), set the option accordingly, and return the trimmed string
+//
+// we don't worry about errors here because it will error anyway as an invalid key
+func (t *transform) parseTag(s string) string {
+ if strings.HasPrefix(s, "file,") {
+ t.tag = file
+ return strings.TrimPrefix(s, "file,")
+ }
+ if strings.HasPrefix(s, "dir,") {
+ t.tag = dir
+ return strings.TrimPrefix(s, "dir,")
+ }
+ if strings.HasPrefix(s, "all,") {
+ t.tag = all
+ return strings.TrimPrefix(s, "all,")
+ }
+ return s
+}
+
+// parse key and value (if any) by splitting on '=' sign
+// (file/dir/all tag has already been trimmed)
+func (t *transform) parseKeyVal(s string) (err error) {
+ if !strings.ContainsRune(s, '=') {
+ err = t.key.Set(s)
+ if err != nil {
+ return err
+ }
+ if t.requiresValue() {
+ fs.Debugf(nil, "received %v", s)
+ return errors.New("value is required for " + t.key.String())
+ }
+ return nil
+ }
+ split := strings.Split(s, "=")
+ if len(split) != 2 {
+ return errors.New("too many values")
+ }
+ if split[0] == "" {
+ return errors.New("key cannot be blank")
+ }
+ err = t.key.Set(split[0])
+ if err != nil {
+ return err
+ }
+ t.value = split[1]
+ return nil
+}
+
+// returns true if this particular algorithm requires a value
+func (t *transform) requiresValue() bool {
+ switch t.key {
+ case ConvFindReplace:
+ return true
+ case ConvPrefix:
+ return true
+ case ConvSuffix:
+ return true
+ case ConvSuffixKeepExtension:
+ return true
+ case ConvTrimPrefix:
+ return true
+ case ConvTrimSuffix:
+ return true
+ case ConvIndex:
+ return true
+ case ConvDate:
+ return true
+ case ConvTruncate:
+ return true
+ case ConvEncoder:
+ return true
+ case ConvDecoder:
+ return true
+ case ConvRegex:
+ return true
+ case ConvCommand:
+ return true
+ }
+ return false
+}
+
+// transformAlgo describes conversion setting
+type transformAlgo = fs.Enum[transformChoices]
+
+// Supported transform options
+const (
+ ConvNone transformAlgo = iota
+ ConvToNFC
+ ConvToNFD
+ ConvToNFKC
+ ConvToNFKD
+ ConvFindReplace
+ ConvPrefix
+ ConvSuffix
+ ConvSuffixKeepExtension
+ ConvTrimPrefix
+ ConvTrimSuffix
+ ConvIndex
+ ConvDate
+ ConvTruncate
+ ConvBase64Encode
+ ConvBase64Decode
+ ConvEncoder
+ ConvDecoder
+ ConvISO8859_1
+ ConvWindows1252
+ ConvMacintosh
+ ConvCharmap
+ ConvLowercase
+ ConvUppercase
+ ConvTitlecase
+ ConvASCII
+ ConvURL
+ ConvRegex
+ ConvCommand
+)
+
+type transformChoices struct{}
+
+func (transformChoices) Choices() []string {
+ return []string{
+ ConvNone: "none",
+ ConvToNFC: "nfc",
+ ConvToNFD: "nfd",
+ ConvToNFKC: "nfkc",
+ ConvToNFKD: "nfkd",
+ ConvFindReplace: "replace",
+ ConvPrefix: "prefix",
+ ConvSuffix: "suffix",
+ ConvSuffixKeepExtension: "suffix_keep_extension",
+ ConvTrimPrefix: "trimprefix",
+ ConvTrimSuffix: "trimsuffix",
+ ConvIndex: "index",
+ ConvDate: "date",
+ ConvTruncate: "truncate",
+ ConvBase64Encode: "base64encode",
+ ConvBase64Decode: "base64decode",
+ ConvEncoder: "encoder",
+ ConvDecoder: "decoder",
+ ConvISO8859_1: "ISO-8859-1",
+ ConvWindows1252: "Windows-1252",
+ ConvMacintosh: "Macintosh",
+ ConvCharmap: "charmap",
+ ConvLowercase: "lowercase",
+ ConvUppercase: "uppercase",
+ ConvTitlecase: "titlecase",
+ ConvASCII: "ascii",
+ ConvURL: "url",
+ ConvRegex: "regex",
+ ConvCommand: "command",
+ }
+}
+
+func (transformChoices) Type() string {
+ return "string"
+}
diff --git a/lib/transform/transform.go b/lib/transform/transform.go
new file mode 100644
index 000000000..0103ae87f
--- /dev/null
+++ b/lib/transform/transform.go
@@ -0,0 +1,336 @@
+// Package transform holds functions for path name transformations
+package transform
+
+import (
+ "bytes"
+ "context"
+ "encoding/base64"
+ "errors"
+ "fmt"
+ "mime"
+ "net/url"
+ "os/exec"
+ "path"
+ "regexp"
+ "strconv"
+ "strings"
+ "time"
+ "unicode/utf8"
+
+ "github.com/rclone/rclone/fs"
+ "github.com/rclone/rclone/fs/fserrors"
+ "github.com/rclone/rclone/lib/encoder"
+ "golang.org/x/text/encoding/charmap"
+ "golang.org/x/text/unicode/norm"
+)
+
+// Path transforms a path s according to the --name-transform options in use
+//
+// If no transforms are in use, s is returned unchanged
+func Path(ctx context.Context, s string, isDir bool) string {
+ if !Transforming(ctx) {
+ return s
+ }
+
+ old := s
+ opt, err := getOptions(ctx)
+ if err != nil {
+ err = fs.CountError(ctx, err)
+ fs.Errorf(s, "Failed to parse transform flags: %v", err)
+ }
+ for _, t := range opt {
+ if isDir && t.tag == file {
+ continue
+ }
+ baseOnly := !isDir && t.tag == file
+ if t.tag == dir && !isDir {
+ s, err = transformDir(s, t)
+ } else {
+ s, err = transformPath(s, t, baseOnly)
+ }
+ if err != nil {
+ err = fs.CountError(ctx, fserrors.NoRetryError(err))
+ fs.Errorf(s, "Failed to transform: %v", err)
+ }
+ }
+ if old != s && !generatingHelpText {
+ fs.Debugf(old, "transformed to: %v", s)
+ }
+ if strings.Count(old, "/") != strings.Count(s, "/") {
+ err = fs.CountError(ctx, fserrors.NoRetryError(fmt.Errorf("number of path segments must match: %v (%v), %v (%v)", old, strings.Count(old, "/"), s, strings.Count(s, "/"))))
+ fs.Errorf(old, "%v", err)
+ return old
+ }
+ return s
+}
+
+// transformPath transforms a path string according to the chosen TransformAlgo.
+// Each path segment is transformed separately, to preserve path separators.
+// If baseOnly is true, only the base will be transformed (useful for renaming while walking a dir tree recursively.)
+// for example, "some/nested/path" -> "some/nested/CONVERTEDPATH"
+// otherwise, the entire is path is transformed.
+func transformPath(s string, t transform, baseOnly bool) (string, error) {
+ if s == "" || s == "/" || s == "\\" || s == "." {
+ return "", nil
+ }
+
+ if baseOnly {
+ transformedBase, err := transformPathSegment(path.Base(s), t)
+ if err := validateSegment(transformedBase); err != nil {
+ return "", err
+ }
+ return path.Join(path.Dir(s), transformedBase), err
+ }
+
+ segments := strings.Split(s, "/")
+ transformedSegments := make([]string, len(segments))
+ for _, seg := range segments {
+ convSeg, err := transformPathSegment(seg, t)
+ if err != nil {
+ return "", err
+ }
+ if err := validateSegment(convSeg); err != nil {
+ return "", err
+ }
+ transformedSegments = append(transformedSegments, convSeg)
+ }
+ return path.Join(transformedSegments...), nil
+}
+
+// transform all but the last path segment
+func transformDir(s string, t transform) (string, error) {
+ dirPath, err := transformPath(path.Dir(s), t, false)
+ if err != nil {
+ return "", err
+ }
+ return path.Join(dirPath, path.Base(s)), nil
+}
+
+// transformPathSegment transforms one path segment (or really any string) according to the chosen TransformAlgo.
+// It assumes path separators have already been trimmed.
+func transformPathSegment(s string, t transform) (string, error) {
+ switch t.key {
+ case ConvNone:
+ return s, nil
+ case ConvToNFC:
+ return norm.NFC.String(s), nil
+ case ConvToNFD:
+ return norm.NFD.String(s), nil
+ case ConvToNFKC:
+ return norm.NFKC.String(s), nil
+ case ConvToNFKD:
+ return norm.NFKD.String(s), nil
+ case ConvBase64Encode:
+ return base64.URLEncoding.EncodeToString([]byte(s)), nil // URLEncoding to avoid slashes
+ case ConvBase64Decode:
+ if s == ".DS_Store" {
+ return s, nil
+ }
+ b, err := base64.URLEncoding.DecodeString(s)
+ if err != nil {
+ fs.Errorf(s, "base64 error")
+ }
+ return string(b), err
+ case ConvFindReplace:
+ split := strings.Split(t.value, ":")
+ if len(split) != 2 {
+ return s, fmt.Errorf("wrong number of values: %v", t.value)
+ }
+ return strings.ReplaceAll(s, split[0], split[1]), nil
+ case ConvPrefix:
+ return t.value + s, nil
+ case ConvSuffix:
+ return s + t.value, nil
+ case ConvSuffixKeepExtension:
+ return SuffixKeepExtension(s, t.value), nil
+ case ConvTrimPrefix:
+ return strings.TrimPrefix(s, t.value), nil
+ case ConvTrimSuffix:
+ return strings.TrimSuffix(s, t.value), nil
+ case ConvTruncate:
+ max, err := strconv.Atoi(t.value)
+ if err != nil {
+ return s, err
+ }
+ if max <= 0 {
+ return s, nil
+ }
+ if utf8.RuneCountInString(s) <= max {
+ return s, nil
+ }
+ runes := []rune(s)
+ return string(runes[:max]), nil
+ case ConvEncoder:
+ var enc encoder.MultiEncoder
+ err := enc.Set(t.value)
+ if err != nil {
+ return s, err
+ }
+ return enc.Encode(s), nil
+ case ConvDecoder:
+ var enc encoder.MultiEncoder
+ err := enc.Set(t.value)
+ if err != nil {
+ return s, err
+ }
+ return enc.Decode(s), nil
+ case ConvISO8859_1:
+ return encodeWithReplacement(s, charmap.ISO8859_1), nil
+ case ConvWindows1252:
+ return encodeWithReplacement(s, charmap.Windows1252), nil
+ case ConvMacintosh:
+ return encodeWithReplacement(s, charmap.Macintosh), nil
+ case ConvCharmap:
+ var cmapType fs.Enum[cmapChoices]
+ err := cmapType.Set(t.value)
+ if err != nil {
+ return s, err
+ }
+ c := charmapByID(cmapType)
+ return encodeWithReplacement(s, c), nil
+ case ConvLowercase:
+ return strings.ToLower(s), nil
+ case ConvUppercase:
+ return strings.ToUpper(s), nil
+ case ConvTitlecase:
+ return strings.ToTitle(s), nil
+ case ConvASCII:
+ return toASCII(s), nil
+ case ConvURL:
+ return url.QueryEscape(s), nil
+ case ConvDate:
+ return s + AppyTimeGlobs(t.value, time.Now()), nil
+ case ConvRegex:
+ split := strings.Split(t.value, "/")
+ if len(split) != 2 {
+ return s, fmt.Errorf("regex syntax error: %v", t.value)
+ }
+ re := regexp.MustCompile(split[0])
+ return re.ReplaceAllString(s, split[1]), nil
+ case ConvCommand:
+ return mapper(s, t.value)
+ default:
+ return "", errors.New("this option is not yet implemented")
+ }
+}
+
+// SuffixKeepExtension adds a suffix while keeping extension
+//
+// i.e. file.txt becomes file_somesuffix.txt not file.txt_somesuffix
+func SuffixKeepExtension(remote string, suffix string) string {
+ var (
+ base = remote
+ exts = ""
+ first = true
+ ext = path.Ext(remote)
+ )
+ for ext != "" {
+ // Look second and subsequent extensions in mime types.
+ // If they aren't found then don't keep it as an extension.
+ if !first && mime.TypeByExtension(ext) == "" {
+ break
+ }
+ base = base[:len(base)-len(ext)]
+ exts = ext + exts
+ first = false
+ ext = path.Ext(base)
+ }
+ return base + suffix + exts
+}
+
+// forbid transformations that add/remove path separators
+func validateSegment(s string) error {
+ if strings.TrimSpace(s) == "" {
+ return errors.New("transform cannot render path segments empty")
+ }
+ if strings.ContainsRune(s, '/') {
+ return fmt.Errorf("transform cannot add path separators: %v", s)
+ }
+ return nil
+}
+
+// ParseGlobs determines whether a string contains {brackets}
+// and returns the substring (including both brackets) for replacing
+// substring is first opening bracket to last closing bracket --
+// good for {{this}} but not {this}{this}
+func ParseGlobs(s string) (hasGlobs bool, substring string) {
+ open := strings.Index(s, "{")
+ close := strings.LastIndex(s, "}")
+ if open >= 0 && close > open {
+ return true, s[open : close+1]
+ }
+ return false, ""
+}
+
+// TrimBrackets converts {{this}} to this
+func TrimBrackets(s string) string {
+ return strings.Trim(s, "{}")
+}
+
+// TimeFormat converts a user-supplied string to a Go time constant, if possible
+func TimeFormat(timeFormat string) string {
+ switch timeFormat {
+ case "Layout":
+ timeFormat = time.Layout
+ case "ANSIC":
+ timeFormat = time.ANSIC
+ case "UnixDate":
+ timeFormat = time.UnixDate
+ case "RubyDate":
+ timeFormat = time.RubyDate
+ case "RFC822":
+ timeFormat = time.RFC822
+ case "RFC822Z":
+ timeFormat = time.RFC822Z
+ case "RFC850":
+ timeFormat = time.RFC850
+ case "RFC1123":
+ timeFormat = time.RFC1123
+ case "RFC1123Z":
+ timeFormat = time.RFC1123Z
+ case "RFC3339":
+ timeFormat = time.RFC3339
+ case "RFC3339Nano":
+ timeFormat = time.RFC3339Nano
+ case "Kitchen":
+ timeFormat = time.Kitchen
+ case "Stamp":
+ timeFormat = time.Stamp
+ case "StampMilli":
+ timeFormat = time.StampMilli
+ case "StampMicro":
+ timeFormat = time.StampMicro
+ case "StampNano":
+ timeFormat = time.StampNano
+ case "DateTime":
+ timeFormat = time.DateTime
+ case "DateOnly":
+ timeFormat = time.DateOnly
+ case "TimeOnly":
+ timeFormat = time.TimeOnly
+ case "MacFriendlyTime", "macfriendlytime", "mac":
+ timeFormat = "2006-01-02 0304PM" // not actually a Go constant -- but useful as macOS filenames can't have colons
+ case "YYYYMMDD":
+ timeFormat = "20060102"
+ }
+ return timeFormat
+}
+
+// AppyTimeGlobs converts "myfile-{DateOnly}.txt" to "myfile-2006-01-02.txt"
+func AppyTimeGlobs(s string, t time.Time) string {
+ hasGlobs, substring := ParseGlobs(s)
+ if !hasGlobs {
+ return s
+ }
+ timeString := t.Local().Format(TimeFormat(TrimBrackets(substring)))
+ return strings.ReplaceAll(s, substring, timeString)
+}
+
+func mapper(s string, command string) (string, error) {
+ out, err := exec.Command(command, s).CombinedOutput()
+ if err != nil {
+ out = bytes.TrimSpace(out)
+ return s, fmt.Errorf("%s: error running command %q: %v", out, command+" "+s, err)
+ }
+ return string(bytes.TrimSpace(out)), nil
+}
diff --git a/lib/transform/transform_test.go b/lib/transform/transform_test.go
new file mode 100644
index 000000000..bd5e0c9a7
--- /dev/null
+++ b/lib/transform/transform_test.go
@@ -0,0 +1,142 @@
+package transform
+
+import (
+ "context"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+// sync tests are in fs/sync/sync_transform_test.go to avoid import cycle issues
+
+func newOptions(s ...string) (context.Context, error) {
+ ctx := context.Background()
+ err := SetOptions(ctx, s...)
+ return ctx, err
+}
+
+func TestPath(t *testing.T) {
+ for _, test := range []struct {
+ path string
+ want string
+ }{
+ {"", ""},
+ {"toe/toe/toe", "tictactoe/tictactoe/tictactoe"},
+ {"a/b/c", "tictaca/tictacb/tictacc"},
+ } {
+ ctx, err := newOptions("all,prefix=tac", "all,prefix=tic")
+ require.NoError(t, err)
+
+ got := Path(ctx, test.path, false)
+ assert.Equal(t, test.want, got)
+ }
+}
+
+func TestFileTagOnFile(t *testing.T) {
+ for _, test := range []struct {
+ path string
+ want string
+ }{
+ {"a/b/c.txt", "a/b/1c.txt"},
+ } {
+ ctx, err := newOptions("file,prefix=1")
+ require.NoError(t, err)
+
+ got := Path(ctx, test.path, false)
+ assert.Equal(t, test.want, got)
+ }
+}
+
+func TestDirTagOnFile(t *testing.T) {
+ for _, test := range []struct {
+ path string
+ want string
+ }{
+ {"a/b/c.txt", "1a/1b/c.txt"},
+ } {
+ ctx, err := newOptions("dir,prefix=1")
+ require.NoError(t, err)
+
+ got := Path(ctx, test.path, false)
+ assert.Equal(t, test.want, got)
+ }
+}
+
+func TestAllTag(t *testing.T) {
+ for _, test := range []struct {
+ path string
+ want string
+ }{
+ {"a/b/c.txt", "1a/1b/1c.txt"},
+ } {
+ ctx, err := newOptions("all,prefix=1")
+ require.NoError(t, err)
+
+ got := Path(ctx, test.path, false)
+ assert.Equal(t, test.want, got)
+ }
+}
+
+func TestFileTagOnDir(t *testing.T) {
+ for _, test := range []struct {
+ path string
+ want string
+ }{
+ {"a/b", "a/b"},
+ } {
+ ctx, err := newOptions("file,prefix=1")
+ require.NoError(t, err)
+
+ got := Path(ctx, test.path, true)
+ assert.Equal(t, test.want, got)
+ }
+}
+
+func TestDirTagOnDir(t *testing.T) {
+ for _, test := range []struct {
+ path string
+ want string
+ }{
+ {"a/b", "1a/1b"},
+ } {
+ ctx, err := newOptions("dir,prefix=1")
+ require.NoError(t, err)
+
+ got := Path(ctx, test.path, true)
+ assert.Equal(t, test.want, got)
+ }
+}
+
+func TestVarious(t *testing.T) {
+ for _, test := range []struct {
+ path string
+ want string
+ flags []string
+ }{
+ {"stories/The Quick Brown Fox!.txt", "STORIES/THE QUICK BROWN FOX!.TXT", []string{"all,uppercase"}},
+ {"stories/The Quick Brown Fox!.txt", "stories/The Slow Brown Turtle!.txt", []string{"all,replace=Fox:Turtle", "all,replace=Quick:Slow"}},
+ {"stories/The Quick Brown Fox!.txt", "c3Rvcmllcw==/VGhlIFF1aWNrIEJyb3duIEZveCEudHh0", []string{"all,base64encode"}},
+ {"c3Rvcmllcw==/VGhlIFF1aWNrIEJyb3duIEZveCEudHh0", "stories/The Quick Brown Fox!.txt", []string{"all,base64decode"}},
+ {"stories/The Quick Brown 🦊 Fox Went to the Café!.txt", "stories/The Quick Brown 🦊 Fox Went to the Café!.txt", []string{"all,nfc"}},
+ {"stories/The Quick Brown 🦊 Fox Went to the Café!.txt", "stories/The Quick Brown 🦊 Fox Went to the Café!.txt", []string{"all,nfd"}},
+ {"stories/The Quick Brown 🦊 Fox!.txt", "stories/The Quick Brown Fox!.txt", []string{"all,ascii"}},
+ {"stories/The Quick Brown 🦊 Fox!.txt", "stories/The+Quick+Brown+%F0%9F%A6%8A+Fox%21.txt", []string{"all,url"}},
+ {"stories/The Quick Brown Fox!.txt", "stories/The Quick Brown Fox!", []string{"all,trimsuffix=.txt"}},
+ {"stories/The Quick Brown Fox!.txt", "OLD_stories/OLD_The Quick Brown Fox!.txt", []string{"all,prefix=OLD_"}},
+ {"stories/The Quick Brown 🦊 Fox Went to the Café!.txt", "stories/The Quick Brown _ Fox Went to the Caf_!.txt", []string{"all,charmap=ISO-8859-7"}},
+ {"stories/The Quick Brown Fox: A Memoir [draft].txt", "stories/The Quick Brown Fox: A Memoir [draft].txt", []string{"all,encoder=Colon,SquareBracket"}},
+ {"stories/The Quick Brown 🦊 Fox Went to the Café!.txt", "stories/The Quick Brown 🦊 Fox", []string{"all,truncate=21"}},
+ {"stories/The Quick Brown Fox!.txt", "stories/The Quick Brown Fox!.txt", []string{"all,command=echo"}},
+ {"stories/The Quick Brown Fox!.txt", "stories/The Quick Brown Fox!.txt-" + time.Now().Local().Format("20060102"), []string{"date=-{YYYYMMDD}"}},
+ {"stories/The Quick Brown Fox!.txt", "stories/The Quick Brown Fox!.txt-" + time.Now().Local().Format("2006-01-02 0304PM"), []string{"date=-{macfriendlytime}"}},
+ {"stories/The Quick Brown Fox!.txt", "ababababababab/ababab ababababab ababababab ababab!abababab", []string{"all,regex=[\\.\\w]/ab"}},
+ } {
+ ctx, err := newOptions(test.flags...)
+ require.NoError(t, err)
+
+ got := Path(ctx, test.path, false)
+ assert.Equal(t, test.want, got)
+ }
+}