Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

index: add partial runs which only re-index changed directories

Nick Craig-Wood666d67f0eb8f
diff --git a/cmd/index/index.go b/cmd/index/index.go
index 72cb07ede..4ebe5644f 100644
--- a/cmd/index/index.go
+++ b/cmd/index/index.go
@@ -33,6 +33,10 @@ func init() {
flags.FVarP(cmdFlags, &opt.DirTime, "dir-time", "", "How to work out the time shown for a directory", "")
flags.BoolVarP(cmdFlags, &opt.NoModTime, "no-modtime", "", opt.NoModTime, "Don't show modification times in listings", "")
flags.BoolVarP(cmdFlags, &opt.Rewrite, "index-rewrite", "", opt.Rewrite, "Write every listing even if it is unchanged", "")
+ flags.StringArrayVarP(cmdFlags, &opt.Changed, "changed", "", nil, "Only re-index directories affected by this changed path (directories end in /)", "")
+ flags.StringArrayVarP(cmdFlags, &opt.ChangedFrom, "changed-from", "", nil, "Read changed paths from file, one per line (use - to read from stdin)", "")
+ flags.StringArrayVarP(cmdFlags, &opt.ChangedCombined, "changed-combined", "", nil, "Read changed paths from a sync --combined report (use - to read from stdin)", "")
+ flags.IntVarP(cmdFlags, &opt.ChangedMaxDirs, "changed-max-dirs", "", opt.ChangedMaxDirs, "Do a full index if a partial one would list more than this many directories (0 for no limit)", "")
flags.StringVarP(cmdFlags, &printTemplate, "print-template", "", "", "Print the built-in template for FORMAT and exit", "")
}
diff --git a/cmd/index/index.md b/cmd/index/index.md
index 5bb61cf5b..e5656a36f 100644
--- a/cmd/index/index.md
+++ b/cmd/index/index.md
@@ -85,6 +85,49 @@ nginx and Apache, and R2 custom domains with a URL rewrite rule. For
hosts that don't (plain S3 REST URLs, B2 friendly URLs) use
`--link-index` to make directory links `dir/index.html` instead.
+### Partial runs
+
+After a sync which changed a known set of files there is no need to
+walk the whole remote. Tell `rclone index` what changed and it re-indexes
+only the directories that could be affected:
+
+ rclone index r2:bucket --changed v1.76.0/ --changed version.txt
+ rclone index r2:bucket --changed-from changes.txt
+ rclone sync ./public r2:bucket --exclude index.html --combined - | rclone index r2:bucket --changed-combined -
+
+- `--changed PATH` names a changed file or directory relative to
+ `remote:path` and can be repeated. A directory, given with a trailing
+ slash or found to be one, is re-indexed completely.
+- `--changed-from FILE` reads one path per line exactly as written.
+- `--changed-combined FILE` reads the `--combined` report of
+ `rclone sync`: lines starting with `+`, `-`, `*` and `!` are changes and
+ `=` lines are ignored.
+- Both files accept `-` for standard input and all three flags can be
+ combined.
+
+A changed file re-indexes its directory and every directory above it,
+since their times and their listings can change too, and on bucket
+based storage a directory can cease to exist. Each of those is one
+directory listing. With `--dir-time newest` the directories above also
+need the times of their unchanged subdirectories, which are read from
+the modification times of those subdirectories' listings at the cost
+of one small request each. With `--use-server-modtime` that is the time
+the listing was written rather than the time of the newest file, so
+partial runs can drift by the indexing delay until the next full run.
+`--dir-time dir` and `none` need no such lookups.
+
+Directories which are neither named nor above a named path are left
+alone, as are directories deeper than `--max-depth`. An empty change
+list does nothing, and naming the root does a full run. There is no
+automatic cutover to a full run since that depends on the size of the
+whole remote: a partial run costs one listing per affected directory,
+a full run one request per thousand objects on S3-like storage or one
+per directory elsewhere. The number
+of directories a partial run will list is logged with `-v`, and
+`--changed-max-dirs` makes it fall back to a full run above that
+number. A good pattern is a partial run after each upload and a
+scheduled full run to catch anything missed.
+
### Site icon
The listings carry no icon or branding of their own. To give a site an
diff --git a/fs/operations/index.go b/fs/operations/index.go
index 9c7a10d52..87a431053 100644
--- a/fs/operations/index.go
+++ b/fs/operations/index.go
@@ -1,6 +1,7 @@
package operations
import (
+ "bufio"
"bytes"
"context"
"encoding/json"
@@ -12,6 +13,7 @@ import (
"path"
"sort"
"strings"
+ "sync"
texttemplate "text/template"
"time"
@@ -56,6 +58,13 @@ type IndexOpt struct {
DirTime IndexDirTime `json:"dirTime"` // how to work out directory times
NoModTime bool `json:"noModTime"` // don't show modification times
Rewrite bool `json:"rewrite"` // write every listing even if it is unchanged
+
+ // Partial runs only re-index the directories affected by these
+ // changed paths and their ancestors. Empty means a full run.
+ Changed []string `json:"changed"` // changed files or directories, directories with a trailing /
+ ChangedFrom []string `json:"changedFrom"` // files of changed paths, one per line
+ ChangedCombined []string `json:"changedCombined"` // files in the sync --combined report format
+ ChangedMaxDirs int `json:"changedMaxDirs"` // do a full run if a partial run would list more directories than this, 0 for no limit
}
// IndexOptDefault are the default options for Index
@@ -119,6 +128,8 @@ type index struct {
dirTime IndexDirTime
now time.Time
dirs map[string]*indexDir
+ unwalkedMu sync.Mutex
+ unwalked map[string]time.Time // newest times of directories which weren't walked
transfers errgroup.Group
ec *errcount.ErrCount
}
@@ -148,6 +159,7 @@ func newIndex(ctx context.Context, f fs.Fs, opt *IndexOpt) (*index, error) {
dirTime: opt.DirTime,
now: time.Now(),
dirs: map[string]*indexDir{},
+ unwalked: map[string]time.Time{},
ec: errcount.New(),
}
if ix.opt.NoModTime {
@@ -241,11 +253,18 @@ func newIndexTextTemplate(text string) (*texttemplate.Template, error) {
// run does the work of Index
func (ix *index) run(ctx context.Context) error {
ci := fs.GetConfig(ctx)
- err := walk.Walk(ctx, ix.f, "", true, ConfigMaxDepth(ctx, true), ix.walkFn(ctx))
+ changed, err := ix.changedPaths()
+ if err != nil {
+ return err
+ }
+ if changed == nil {
+ err = ix.walkAll(ctx)
+ } else {
+ err = ix.walkChanged(ctx, changed)
+ }
if err != nil {
return err
}
- ix.prune()
// Directories deepest first so that a directory's subdirectories
// are done before it.
@@ -256,6 +275,7 @@ func (ix *index) run(ctx context.Context) error {
sort.Slice(dirs, func(i, j int) bool {
return dirs[i].depth > dirs[j].depth
})
+ ix.prune(dirs)
// Read the modification times --checkers at a time as they may
// need a transaction each on some backends, and count them as
@@ -272,6 +292,17 @@ func (ix *index) run(ctx context.Context) error {
return nil
})
}
+ if ix.dirTime != IndexDirTimeNewest {
+ continue
+ }
+ for _, sub := range d.subdirs {
+ if ix.dirs[sub.Remote()] == nil {
+ g.Go(func() error {
+ ix.readUnwalkedTime(ctx, sub)
+ return nil
+ })
+ }
+ }
}
_ = g.Wait()
for _, d := range dirs {
@@ -293,9 +324,222 @@ func (ix *index) run(ctx context.Context) error {
return ix.ec.Err("index")
}
+// walkAll collects every directory for a full run
+func (ix *index) walkAll(ctx context.Context) error {
+ return walk.Walk(ctx, ix.f, "", true, ConfigMaxDepth(ctx, true), ix.walkFn(ctx))
+}
+
+// changedPaths returns the changed paths from the options, or nil
+// for a full run. Directories have a trailing slash.
+func (ix *index) changedPaths() (changed []string, err error) {
+ opt := ix.opt
+ if len(opt.Changed) == 0 && len(opt.ChangedFrom) == 0 && len(opt.ChangedCombined) == 0 {
+ return nil, nil
+ }
+ changed = []string{}
+ add := func(p string) {
+ isDir := strings.HasSuffix(p, "/")
+ p = strings.Trim(path.Clean("/"+p), "/")
+ if isDir {
+ p += "/"
+ }
+ changed = append(changed, p)
+ }
+ for _, p := range opt.Changed {
+ add(p)
+ }
+ for _, name := range opt.ChangedFrom {
+ err = forEachIndexLine(name, func(line string) error {
+ if line != "" {
+ add(line)
+ }
+ return nil
+ })
+ if err != nil {
+ return nil, err
+ }
+ }
+ for _, name := range opt.ChangedCombined {
+ err = forEachIndexLine(name, func(line string) error {
+ if line == "" {
+ return nil
+ }
+ if len(line) < 2 || line[1] != ' ' || !strings.ContainsRune("+-*!=", rune(line[0])) {
+ return fmt.Errorf("malformed line %q in combined report %q", line, name)
+ }
+ if line[0] != '=' {
+ add(line[2:])
+ }
+ return nil
+ })
+ if err != nil {
+ return nil, err
+ }
+ }
+ return changed, nil
+}
+
+// forEachIndexLine calls fn with every line of the file name, or of
+// stdin if name is "-", exactly as read
+func forEachIndexLine(name string, fn func(string) error) (err error) {
+ in := os.Stdin
+ if name != "-" {
+ in, err = os.Open(name)
+ if err != nil {
+ return err
+ }
+ defer fs.CheckClose(in, &err)
+ }
+ scanner := bufio.NewScanner(in)
+ for scanner.Scan() {
+ if err := fn(scanner.Text()); err != nil {
+ return err
+ }
+ }
+ return scanner.Err()
+}
+
+// walkChanged collects the directories a partial run needs: the
+// parent of every changed path and all of its ancestors, listed one
+// level deep, and every changed directory walked in full.
+//
+// Directories deeper than --max-depth are left alone as a full run
+// wouldn't walk them either.
+func (ix *index) walkChanged(ctx context.Context, changed []string) error {
+ if len(changed) == 0 {
+ fs.Infof(ix.f, "Nothing changed so nothing to index")
+ return nil
+ }
+ maxDepth := ConfigMaxDepth(ctx, true)
+ inDepth := func(dir string) bool {
+ return maxDepth < 0 || indexDepth(dir) < maxDepth
+ }
+ var subtrees []string
+ for _, p := range changed {
+ if p == "/" {
+ fs.Infof(ix.f, "Root changed so doing a full index")
+ return ix.walkAll(ctx)
+ }
+ if strings.HasSuffix(p, "/") {
+ subtrees = append(subtrees, strings.TrimSuffix(p, "/"))
+ }
+ }
+ subtrees = indexOuterDirs(subtrees)
+ // A changed directory's walk covers everything below it
+ inSubtree := func(dir string) bool {
+ for _, sub := range subtrees {
+ if dir == sub || strings.HasPrefix(dir, sub+"/") {
+ return true
+ }
+ }
+ return false
+ }
+ list := map[string]bool{}
+ for _, p := range changed {
+ for dir := indexParent(strings.TrimSuffix(p, "/")); ; dir = indexParent(dir) {
+ if inDepth(dir) && !inSubtree(dir) {
+ list[dir] = true
+ }
+ if dir == "" {
+ break
+ }
+ }
+ }
+ if ix.opt.ChangedMaxDirs > 0 && len(list)+len(subtrees) > ix.opt.ChangedMaxDirs {
+ fs.Infof(ix.f, "Partial index would list %d directories, more than %d, so doing a full index", len(list)+len(subtrees), ix.opt.ChangedMaxDirs)
+ return ix.walkAll(ctx)
+ }
+ fs.Infof(ix.f, "Partial index: listing %d directories and walking %d changed directories", len(list), len(subtrees))
+ walkFn := ix.walkFn(ctx)
+ for dir := range list {
+ err := walk.Walk(ctx, ix.f, dir, true, 1, walkFn)
+ if err != nil {
+ return err
+ }
+ }
+ // Changed paths which turn out to be directories are walked too
+ for _, p := range changed {
+ if d := ix.dirs[indexParent(p)]; d != nil && !strings.HasSuffix(p, "/") {
+ for _, sub := range d.subdirs {
+ if sub.Remote() == p {
+ subtrees = append(subtrees, p)
+ }
+ }
+ }
+ }
+ for _, dir := range indexOuterDirs(subtrees) {
+ if !inDepth(dir) {
+ continue
+ }
+ depth := maxDepth
+ if maxDepth >= 0 {
+ depth = maxDepth - indexDepth(dir)
+ }
+ err := walk.Walk(ctx, ix.f, dir, true, depth, walkFn)
+ if err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+// indexOuterDirs sorts dirs and drops any which is the same as, or
+// inside, another so that walking the result covers each once
+func indexOuterDirs(dirs []string) (outer []string) {
+ sort.Strings(dirs)
+ for _, dir := range dirs {
+ if n := len(outer); n > 0 && (dir == outer[n-1] || strings.HasPrefix(dir, outer[n-1]+"/")) {
+ continue
+ }
+ outer = append(outer, dir)
+ }
+ return outer
+}
+
+// indexDepth returns how many directories deep remote is, 0 for the root
+func indexDepth(remote string) int {
+ if remote == "" {
+ return 0
+ }
+ return strings.Count(remote, "/") + 1
+}
+
+// indexParent returns the parent directory of p, "" for the root
+func indexParent(p string) string {
+ dir := path.Dir(p)
+ if dir == "." {
+ return ""
+ }
+ return dir
+}
+
+// readUnwalkedTime finds the newest time of a directory which wasn't
+// walked from the modification time of its listing, which is set to
+// the directory's newest time when it is written
+func (ix *index) readUnwalkedTime(ctx context.Context, sub fs.Directory) {
+ o, err := ix.f.NewObject(ctx, path.Join(sub.Remote(), ix.outputs[0].name))
+ if err != nil {
+ if !errors.Is(err, fs.ErrorObjectNotFound) {
+ fs.Debugf(sub, "Failed to read listing modtime: %v", err)
+ }
+ return
+ }
+ tr := accounting.Stats(ctx).NewCheckingTransfer(o, "reading modtime")
+ t := o.ModTime(ctx)
+ tr.Done(ctx, nil)
+ ix.unwalkedMu.Lock()
+ ix.unwalked[sub.Remote()] = t
+ ix.unwalkedMu.Unlock()
+}
+
// walkFn returns the function which collects the directories
func (ix *index) walkFn(ctx context.Context) walk.Func {
return func(remote string, entries fs.DirEntries, err error) error {
+ if errors.Is(err, fs.ErrorDirNotFound) {
+ // A changed directory which no longer exists
+ ix.dirs[remote] = &indexDir{remote: remote, depth: indexDepth(remote), gone: true}
+ return nil
+ }
if err != nil {
return err
}
@@ -318,12 +562,10 @@ func (ix *index) walkFn(ctx context.Context) walk.Func {
}
d := &indexDir{
remote: remote,
+ depth: indexDepth(remote),
outputs: map[string]fs.Object{},
excluded: !included,
}
- if remote != "" {
- d.depth = strings.Count(remote, "/") + 1
- }
for _, entry := range entries {
switch x := entry.(type) {
case fs.Object:
@@ -358,29 +600,22 @@ func (ix *index) walkFn(ctx context.Context) walk.Func {
// On backends which can't have empty directories these will cease to
// exist once their outputs are deleted, so they are removed from their
// parent's listing, which may in turn leave the parent with nothing
-// but outputs.
-func (ix *index) prune() {
+// but outputs. dirs must be deepest first.
+func (ix *index) prune(dirs []*indexDir) {
if ix.f.Features().CanHaveEmptyDirectories {
return
}
- var pruneDir func(d *indexDir)
- pruneDir = func(d *indexDir) {
+ for _, d := range dirs {
kept := d.subdirs[:0]
for _, sub := range d.subdirs {
- if sd := ix.dirs[sub.Remote()]; sd != nil {
- pruneDir(sd)
- if sd.gone {
- continue
- }
+ if sd := ix.dirs[sub.Remote()]; sd != nil && sd.gone {
+ continue
}
kept = append(kept, sub)
}
d.subdirs = kept
d.gone = !d.hasContent && len(d.subdirs) == 0
}
- if root := ix.dirs[""]; root != nil {
- pruneDir(root)
- }
}
// setNewest sets d.newest from its files and subdirectories, which
@@ -392,12 +627,21 @@ func (ix *index) setNewest(ctx context.Context, d *indexDir) {
}
}
for _, sub := range d.subdirs {
- if sd := ix.dirs[sub.Remote()]; sd != nil && sd.newest.After(d.newest) {
- d.newest = sd.newest
+ if t := ix.subdirNewest(sub); t.After(d.newest) {
+ d.newest = t
}
}
}
+// subdirNewest returns the newest time of a subdirectory, which was
+// either walked or had its listing's modification time read
+func (ix *index) subdirNewest(sub fs.Directory) time.Time {
+ if sd := ix.dirs[sub.Remote()]; sd != nil {
+ return sd.newest
+ }
+ return ix.unwalked[sub.Remote()]
+}
+
// processDir brings the outputs of d up to date
func (ix *index) processDir(ctx context.Context, d *indexDir) {
if d.gone {
@@ -468,9 +712,7 @@ func (ix *index) render(ctx context.Context, d *indexDir) *serve.Directory {
case IndexDirTimeDir:
t = sub.ModTime(ctx)
case IndexDirTimeNewest:
- if sd := ix.dirs[sub.Remote()]; sd != nil {
- t = sd.newest
- }
+ t = ix.subdirNewest(sub)
}
// Not sub.Size() as some backends, eg local on macOS, report a
// directory size which changes when the listings are written
diff --git a/fs/operations/index_test.go b/fs/operations/index_test.go
index 275f200ab..cf8a1b117 100644
--- a/fs/operations/index_test.go
+++ b/fs/operations/index_test.go
@@ -3,6 +3,7 @@ package operations_test
import (
"context"
"encoding/json"
+ "fmt"
"mime"
"os"
"path/filepath"
@@ -387,3 +388,157 @@ func TestIndexNoHash(t *testing.T) {
assert.Equal(t, int64(1), transfers)
assert.Contains(t, r.read(t, "sub/index.html"), "file4.txt")
}
+
+func TestIndexChanged(t *testing.T) {
+ r := newIndexRun(t)
+ r.WriteObject(r.ctx, "other/notes.md", "notes", t1)
+ r.index(t)
+
+ // Add files in sub/deep and in other but only declare the one in sub/deep
+ r.WriteObject(r.ctx, "sub/deep/file4.txt", "hello4", t1)
+ r.WriteObject(r.ctx, "other/undeclared.txt", "hello", t1)
+ r.opt.Changed = []string{"sub/deep/file4.txt"}
+ transfers, _ := r.index(t)
+ assert.Equal(t, int64(1), transfers)
+ assert.Contains(t, r.read(t, "sub/deep/index.html"), "file4.txt")
+ assert.NotContains(t, r.read(t, "other/index.html"), "undeclared.txt")
+
+ // A new directory changes its parent's listing too
+ r.WriteObject(r.ctx, "sub/new/file5.txt", "hello5", t1)
+ r.opt.Changed = []string{"sub/new/file5.txt"}
+ transfers, _ = r.index(t)
+ assert.Equal(t, int64(2), transfers)
+ assert.Contains(t, r.read(t, "sub/index.html"), `<a href="new/">new/</a>`)
+ assert.Contains(t, r.read(t, "sub/new/index.html"), "file5.txt")
+
+ // A changed directory is walked whether or not it has a trailing slash
+ for i, changed := range []string{"other/", "other"} {
+ name := fmt.Sprintf("other/undeclared%d.txt", i)
+ r.WriteObject(r.ctx, name, "hello", t1)
+ r.opt.Changed = []string{changed}
+ transfers, _ = r.index(t)
+ assert.Equal(t, int64(1), transfers, changed)
+ assert.Contains(t, r.read(t, "other/index.html"), name[6:])
+ }
+
+ // An empty change list does nothing
+ r.WriteObject(r.ctx, "sub/file6.txt", "hello6", t1)
+ r.opt.Changed = nil
+ r.opt.ChangedFrom = []string{filepath.Join(t.TempDir(), "empty.txt")}
+ require.NoError(t, os.WriteFile(r.opt.ChangedFrom[0], nil, 0600))
+ transfers, _ = r.index(t)
+ assert.Equal(t, int64(0), transfers)
+ assert.NotContains(t, r.read(t, "sub/index.html"), "file6.txt")
+
+ // Naming the root does a full run
+ r.opt.ChangedFrom = nil
+ r.opt.Changed = []string{"/"}
+ transfers, _ = r.index(t)
+ assert.Equal(t, int64(1), transfers)
+ assert.Contains(t, r.read(t, "sub/index.html"), "file6.txt")
+
+ // So does exceeding --changed-max-dirs
+ r.WriteObject(r.ctx, "sub/file7.txt", "hello7", t1)
+ r.opt.Changed = []string{"other/notes.md"}
+ r.opt.ChangedMaxDirs = 1
+ transfers, _ = r.index(t)
+ assert.Equal(t, int64(1), transfers)
+ assert.Contains(t, r.read(t, "sub/index.html"), "file7.txt")
+ r.opt.ChangedMaxDirs = 0
+
+ // Paths are cleaned, and nested changed directories are walked once
+ r.WriteObject(r.ctx, "sub/deep/file8.txt", "hello8", t1)
+ r.opt.Changed = []string{"./sub/", "sub//deep/", "sub/deep/"}
+ transfers, _ = r.index(t)
+ assert.Equal(t, int64(1), transfers)
+ assert.Contains(t, r.read(t, "sub/deep/index.html"), "file8.txt")
+
+ // A partial run doesn't write listings a full run wouldn't
+ ctx, ci := fs.AddConfig(r.ctx)
+ ci.MaxDepth = 2
+ r.WriteObject(r.ctx, "sub/deep/file9.txt", "hello9", t1)
+ r.WriteObject(r.ctx, "sub/file10.txt", "hello10", t1)
+ r.opt.Changed = []string{"sub/deep/file9.txt", "sub/file10.txt"}
+ require.NoError(t, operations.Index(ctx, r.Fremote, &r.opt))
+ assert.NotContains(t, r.read(t, "sub/deep/index.html"), "file9.txt")
+ assert.Contains(t, r.read(t, "sub/index.html"), "file10.txt")
+}
+
+func TestIndexChangedDelete(t *testing.T) {
+ r := newIndexRun(t)
+ r.index(t)
+
+ // Delete the only file in sub/deep and declare it
+ o, err := r.Fremote.NewObject(r.ctx, "sub/deep/file3.txt")
+ require.NoError(t, err)
+ require.NoError(t, o.Remove(r.ctx))
+ r.opt.Changed = []string{"sub/deep/file3.txt"}
+ transfers, deletes := r.index(t)
+ if r.Fremote.Features().CanHaveEmptyDirectories {
+ assert.Equal(t, int64(1), transfers)
+ assert.Equal(t, int64(0), deletes)
+ assert.NotContains(t, r.read(t, "sub/deep/index.html"), "file3.txt")
+ } else {
+ assert.Equal(t, int64(1), transfers)
+ assert.Equal(t, int64(1), deletes)
+ assert.NotContains(t, r.read(t, "sub/index.html"), "deep/")
+ r.checkFiles(t, "file1.txt", "index.html", "sub/file2.txt", "sub/index.html")
+
+ // Declaring a directory which has gone is harmless
+ r.opt.Changed = []string{"sub/deep/"}
+ transfers, deletes = r.index(t)
+ assert.Equal(t, int64(0), transfers)
+ assert.Equal(t, int64(0), deletes)
+ }
+}
+
+func TestIndexChangedFiles(t *testing.T) {
+ r := newIndexRun(t)
+ r.WriteObject(r.ctx, "- odd/file.txt", "odd", t1)
+ r.index(t)
+ dir := t.TempDir()
+
+ // --changed-from takes paths exactly as written
+ r.WriteObject(r.ctx, "- odd/file8.txt", "hello8", t1)
+ r.WriteObject(r.ctx, "sub/file9.txt", "hello9", t1)
+ from := filepath.Join(dir, "from.txt")
+ require.NoError(t, os.WriteFile(from, []byte("- odd/file8.txt\n\n"), 0600))
+ r.opt.ChangedFrom = []string{from}
+ transfers, _ := r.index(t)
+ assert.Equal(t, int64(1), transfers)
+ assert.Contains(t, r.read(t, "- odd/index.html"), "file8.txt")
+ assert.NotContains(t, r.read(t, "sub/index.html"), "file9.txt")
+
+ // --changed-combined strips the prefixes and skips unchanged files
+ combined := filepath.Join(dir, "combined.txt")
+ require.NoError(t, os.WriteFile(combined, []byte("= file1.txt\n+ sub/file9.txt\n- sub/deep/gone.txt\n* - odd/file.txt\n! sub/deep/file3.txt\n"), 0600))
+ r.opt.ChangedFrom = nil
+ r.opt.ChangedCombined = []string{combined}
+ transfers, _ = r.index(t)
+ assert.Equal(t, int64(1), transfers)
+ assert.Contains(t, r.read(t, "sub/index.html"), "file9.txt")
+
+ // Malformed combined lines are an error
+ require.NoError(t, os.WriteFile(combined, []byte("sub/file9.txt\n"), 0600))
+ assert.Error(t, operations.Index(r.ctx, r.Fremote, &r.opt))
+}
+
+func TestIndexChangedDirTime(t *testing.T) {
+ r := newIndexRun(t)
+ r.opt.NoModTime = false
+ r.opt.DirTime = operations.IndexDirTimeNewest
+ r.opt.Outputs = []string{"index.json=json"}
+ r.WriteObject(r.ctx, "other/notes.md", "notes", t2)
+ r.index(t)
+ precision := r.Fremote.Precision()
+
+ // A partial run gets the time of the unchanged directory other
+ // from its listing, and the changed directory sub from the walk
+ r.WriteObject(r.ctx, "sub/deep/file4.txt", "hello4", fstest.Time("2030-01-01T00:00:00Z"))
+ r.opt.Changed = []string{"sub/deep/file4.txt"}
+ transfers, _ := r.index(t)
+ assert.Equal(t, int64(3), transfers)
+ modTimes := r.indexModTimes(t, "index.json")
+ fstest.AssertTimeEqualWithPrecision(t, "sub", fstest.Time("2030-01-01T00:00:00Z"), modTimes["sub"], precision)
+ fstest.AssertTimeEqualWithPrecision(t, "other", t2, modTimes["other"], precision)
+}
diff --git a/fs/operations/rc.go b/fs/operations/rc.go
index e0138a984..a2d3e0e09 100644
--- a/fs/operations/rc.go
+++ b/fs/operations/rc.go
@@ -712,6 +712,10 @@ func init() {
- dirTime - how to work out directory times: "newest", "dir" or "none"
- noModTime - don't show modification times
- rewrite - write every listing even if it is unchanged
+ - changed - list of changed paths for a partial run, directories ending in "/"
+ - changedFrom - list of files of changed paths, one per line
+ - changedCombined - list of files in the sync --combined report format
+ - changedMaxDirs - do a full run if a partial one would list more directories than this
See the [index](/commands/rclone_index/) command for more information on the above.
`,