Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

operations: add Index to write static directory listings into a remote

Nick Craig-Woodae9a50cacd89
diff --git a/.gitattributes b/.gitattributes
index 8aa2a2385..62df6b2ee 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -2,6 +2,9 @@
go.mod text eol=lf
go.sum text eol=lf
+# Embedded templates must render the same on every OS
+lib/http/templates/** text eol=lf
+
# Ignore generated files in GitHub language statistics and diffs
/MANUAL.* linguist-generated=true
/rclone.1 linguist-generated=true
diff --git a/fs/operations/index.go b/fs/operations/index.go
new file mode 100644
index 000000000..9c7a10d52
--- /dev/null
+++ b/fs/operations/index.go
@@ -0,0 +1,518 @@
+package operations
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ htmltemplate "html/template"
+ "io"
+ "os"
+ "path"
+ "sort"
+ "strings"
+ texttemplate "text/template"
+ "time"
+
+ "github.com/rclone/rclone/fs"
+ "github.com/rclone/rclone/fs/accounting"
+ "github.com/rclone/rclone/fs/filter"
+ "github.com/rclone/rclone/fs/hash"
+ "github.com/rclone/rclone/fs/object"
+ "github.com/rclone/rclone/fs/walk"
+ "github.com/rclone/rclone/lib/errcount"
+ libhttp "github.com/rclone/rclone/lib/http"
+ "github.com/rclone/rclone/lib/http/serve"
+ "golang.org/x/sync/errgroup"
+)
+
+type indexDirTimeChoices struct{}
+
+func (indexDirTimeChoices) Choices() []string {
+ return []string{
+ IndexDirTimeNewest: "newest",
+ IndexDirTimeDir: "dir",
+ IndexDirTimeNone: "none",
+ }
+}
+
+// IndexDirTime describes how Index works out the time shown for a directory
+type IndexDirTime = fs.Enum[indexDirTimeChoices]
+
+// IndexDirTime constants
+const (
+ IndexDirTimeNewest IndexDirTime = iota // the newest modification time of anything below the directory
+ IndexDirTimeDir // the directory's own modification time
+ IndexDirTimeNone // no time
+)
+
+// IndexOpt are the options for Index
+type IndexOpt struct {
+ Outputs []string `json:"output"` // listings to write in each directory as NAME=FORMAT
+ Rules filter.RulesOpt `json:"rules"` // which directories get listings
+ MaxDepth int `json:"maxDepth"` // only make listings this many directories deep, -1 for no limit
+ LinkIndex bool `json:"linkIndex"` // link to dir/NAME rather than dir/
+ DirTime IndexDirTime `json:"dirTime"` // how to work out directory times
+ NoModTime bool `json:"noModTime"` // don't show modification times
+ Rewrite bool `json:"rewrite"` // write every listing even if it is unchanged
+}
+
+// IndexOptDefault are the default options for Index
+var IndexOptDefault = IndexOpt{
+ Outputs: []string{"index.html=html"},
+ MaxDepth: -1,
+}
+
+// indexFormats are the built-in output formats
+var indexFormats = map[string]struct{ file, mimeType string }{
+ "html": {"templates/index.html", "text/html; charset=utf-8"},
+ "json": {"templates/index.json", "application/json"},
+ "caddy": {"templates/caddy.json", "application/json"},
+}
+
+// IndexTemplate returns the built-in template for format
+func IndexTemplate(format string) (string, error) {
+ f, ok := indexFormats[format]
+ if !ok {
+ return "", fmt.Errorf("unknown built-in format %q", format)
+ }
+ data, err := libhttp.Assets.ReadFile(f.file)
+ return string(data), err
+}
+
+// indexTemplate is implemented by both html/template and text/template
+type indexTemplate interface {
+ Execute(w io.Writer, data any) error
+}
+
+// indexOutput is one listing to write in each directory
+type indexOutput struct {
+ name string
+ mimeType string
+ tmpl indexTemplate
+}
+
+// indexDir is what Index knows about a directory
+type indexDir struct {
+ remote string
+ depth int
+ files []fs.Object // files to list, after filtering, excluding outputs
+ subdirs []fs.Directory // directories to list, after filtering
+ outputs map[string]fs.Object // existing outputs by leaf name
+ excluded bool // excluded by the index rules so nothing is written or deleted here
+ hasContent bool // there is something here other than outputs, before filtering
+ gone bool // contains nothing but outputs so will cease to exist
+ newest time.Time // the newest modification time of anything below here
+}
+
+// index holds the state of one run of Index
+type index struct {
+ f fs.Fs
+ opt *IndexOpt
+ fi *filter.Filter
+ includeDir func(string) (bool, error) // the normal filters applied to a directory
+ indexDir func(string) (bool, error) // the index rules applied to a directory
+ outputs []indexOutput
+ isOutput map[string]bool
+ htmlTemplate *htmltemplate.Template
+ dirTime IndexDirTime
+ now time.Time
+ dirs map[string]*indexDir
+ transfers errgroup.Group
+ ec *errcount.ErrCount
+}
+
+// Index writes a directory listing into every directory of f so that
+// it can be browsed when served as a static website.
+//
+// It works like sync: existing listings which are unchanged are left
+// alone, changed ones are rewritten, and listings in directories
+// which contain nothing else are deleted on backends which can't have
+// empty directories.
+func Index(ctx context.Context, f fs.Fs, opt *IndexOpt) error {
+ ix, err := newIndex(ctx, f, opt)
+ if err != nil {
+ return err
+ }
+ return ix.run(ctx)
+}
+
+// newIndex parses the options and templates
+func newIndex(ctx context.Context, f fs.Fs, opt *IndexOpt) (*index, error) {
+ ix := &index{
+ f: f,
+ opt: opt,
+ fi: filter.GetConfig(ctx),
+ isOutput: map[string]bool{},
+ dirTime: opt.DirTime,
+ now: time.Now(),
+ dirs: map[string]*indexDir{},
+ ec: errcount.New(),
+ }
+ if ix.opt.NoModTime {
+ ix.dirTime = IndexDirTimeNone
+ }
+ ix.includeDir = ix.fi.IncludeDirectory(ctx, f)
+
+ // The index rules are a second rule set applied to directory paths
+ indexFilter, err := filter.NewFilter(&filter.Options{
+ RulesOpt: opt.Rules,
+ MinAge: fs.DurationOff,
+ MaxAge: fs.DurationOff,
+ MinSize: -1,
+ MaxSize: -1,
+ })
+ if err != nil {
+ return nil, fmt.Errorf("index rules: %w", err)
+ }
+ ix.indexDir = indexFilter.IncludeDirectory(ctx, nil)
+
+ ix.htmlTemplate, err = libhttp.GetTemplate("")
+ if err != nil {
+ return nil, err
+ }
+ if len(opt.Outputs) == 0 {
+ return nil, errors.New("need at least one output")
+ }
+ for _, s := range opt.Outputs {
+ name, format, ok := strings.Cut(s, "=")
+ if !ok || name == "" || name == "." || name == ".." || format == "" || strings.Contains(name, "/") {
+ return nil, fmt.Errorf("bad output %q: want NAME=FORMAT", s)
+ }
+ out, err := ix.newOutput(name, format)
+ if err != nil {
+ return nil, fmt.Errorf("output %q: %w", s, err)
+ }
+ ix.outputs = append(ix.outputs, out)
+ ix.isOutput[name] = true
+ }
+ return ix, nil
+}
+
+// newOutput makes an output from a NAME=FORMAT pair, where format is
+// a built-in format or the path of a template file
+func (ix *index) newOutput(name, format string) (out indexOutput, err error) {
+ out.name = name
+ if builtin, ok := indexFormats[format]; ok {
+ out.mimeType = builtin.mimeType
+ if format == "html" {
+ out.tmpl = ix.htmlTemplate
+ return out, nil
+ }
+ data, err := libhttp.Assets.ReadFile(builtin.file)
+ if err != nil {
+ return out, err
+ }
+ out.tmpl, err = newIndexTextTemplate(string(data))
+ return out, err
+ }
+ out.mimeType = fs.MimeTypeFromName(name)
+ switch strings.ToLower(path.Ext(name)) {
+ case ".html", ".htm":
+ out.tmpl, err = libhttp.GetTemplate(format)
+ default:
+ var data []byte
+ data, err = os.ReadFile(format)
+ if err != nil {
+ return out, err
+ }
+ out.tmpl, err = newIndexTextTemplate(string(data))
+ }
+ return out, err
+}
+
+// newIndexTextTemplate parses a text/template for non-HTML outputs
+func newIndexTextTemplate(text string) (*texttemplate.Template, error) {
+ funcMap := texttemplate.FuncMap{
+ "json": func(v any) (string, error) {
+ data, err := json.Marshal(v)
+ return string(data), err
+ },
+ "afterEpoch": libhttp.AfterEpoch,
+ "contains": strings.Contains,
+ "hasPrefix": strings.HasPrefix,
+ "hasSuffix": strings.HasSuffix,
+ "trimSuffix": strings.TrimSuffix,
+ }
+ return texttemplate.New("index").Funcs(funcMap).Parse(text)
+}
+
+// run does the work of Index
+func (ix *index) run(ctx context.Context) error {
+ ci := fs.GetConfig(ctx)
+ err := walk.Walk(ctx, ix.f, "", true, ConfigMaxDepth(ctx, true), ix.walkFn(ctx))
+ if err != nil {
+ return err
+ }
+ ix.prune()
+
+ // Directories deepest first so that a directory's subdirectories
+ // are done before it.
+ dirs := make([]*indexDir, 0, len(ix.dirs))
+ for _, d := range ix.dirs {
+ dirs = append(dirs, d)
+ }
+ sort.Slice(dirs, func(i, j int) bool {
+ return dirs[i].depth > dirs[j].depth
+ })
+
+ // Read the modification times --checkers at a time as they may
+ // need a transaction each on some backends, and count them as
+ // checks so the progress is visible
+ if !ix.opt.NoModTime {
+ var g errgroup.Group
+ g.SetLimit(ci.Checkers)
+ for _, d := range dirs {
+ for _, o := range d.files {
+ g.Go(func() error {
+ tr := accounting.Stats(ctx).NewCheckingTransfer(o, "reading modtime")
+ o.ModTime(ctx)
+ tr.Done(ctx, nil)
+ return nil
+ })
+ }
+ }
+ _ = g.Wait()
+ for _, d := range dirs {
+ ix.setNewest(ctx, d)
+ }
+ }
+
+ ix.transfers.SetLimit(ci.Transfers)
+ var g errgroup.Group
+ g.SetLimit(ci.Checkers)
+ for _, d := range dirs {
+ g.Go(func() error {
+ ix.processDir(ctx, d)
+ return nil
+ })
+ }
+ _ = g.Wait()
+ _ = ix.transfers.Wait()
+ return ix.ec.Err("index")
+}
+
+// walkFn returns the function which collects the directories
+func (ix *index) walkFn(ctx context.Context) walk.Func {
+ return func(remote string, entries fs.DirEntries, err error) error {
+ if err != nil {
+ return err
+ }
+ if remote != "" {
+ // Directories excluded by the normal filters aren't listed
+ ok, err := ix.includeDir(remote)
+ if err != nil {
+ return err
+ }
+ if !ok {
+ return walk.ErrorSkipDir
+ }
+ }
+ // Directories excluded by the index rules are still walked
+ // as they may have included directories below them and
+ // their contents count towards their parent's time.
+ included, err := ix.indexDir(remote)
+ if err != nil {
+ return err
+ }
+ d := &indexDir{
+ remote: remote,
+ outputs: map[string]fs.Object{},
+ excluded: !included,
+ }
+ if remote != "" {
+ d.depth = strings.Count(remote, "/") + 1
+ }
+ for _, entry := range entries {
+ switch x := entry.(type) {
+ case fs.Object:
+ leaf := path.Base(x.Remote())
+ if !d.excluded && ix.isOutput[leaf] {
+ d.outputs[leaf] = x
+ continue
+ }
+ d.hasContent = true
+ if ix.fi.IncludeObject(ctx, x) {
+ d.files = append(d.files, x)
+ }
+ case fs.Directory:
+ ok, err := ix.includeDir(x.Remote())
+ if err != nil {
+ return err
+ }
+ if ok {
+ d.subdirs = append(d.subdirs, x)
+ } else {
+ d.hasContent = true
+ }
+ }
+ }
+ ix.dirs[remote] = d
+ return nil
+ }
+}
+
+// prune works out which directories contain nothing but outputs.
+//
+// On backends which can't have empty directories these will cease to
+// exist once their outputs are deleted, so they are removed from their
+// parent's listing, which may in turn leave the parent with nothing
+// but outputs.
+func (ix *index) prune() {
+ if ix.f.Features().CanHaveEmptyDirectories {
+ return
+ }
+ var pruneDir func(d *indexDir)
+ pruneDir = func(d *indexDir) {
+ kept := d.subdirs[:0]
+ for _, sub := range d.subdirs {
+ if sd := ix.dirs[sub.Remote()]; sd != nil {
+ pruneDir(sd)
+ if sd.gone {
+ continue
+ }
+ }
+ kept = append(kept, sub)
+ }
+ d.subdirs = kept
+ d.gone = !d.hasContent && len(d.subdirs) == 0
+ }
+ if root := ix.dirs[""]; root != nil {
+ pruneDir(root)
+ }
+}
+
+// setNewest sets d.newest from its files and subdirectories, which
+// must have been done already
+func (ix *index) setNewest(ctx context.Context, d *indexDir) {
+ for _, o := range d.files {
+ if t := o.ModTime(ctx); t.After(d.newest) {
+ d.newest = t
+ }
+ }
+ for _, sub := range d.subdirs {
+ if sd := ix.dirs[sub.Remote()]; sd != nil && sd.newest.After(d.newest) {
+ d.newest = sd.newest
+ }
+ }
+}
+
+// processDir brings the outputs of d up to date
+func (ix *index) processDir(ctx context.Context, d *indexDir) {
+ if d.gone {
+ for _, o := range d.outputs {
+ ix.transfers.Go(func() error {
+ err := DeleteFile(ctx, o)
+ if err != nil {
+ ix.ec.Add(err)
+ }
+ return nil
+ })
+ }
+ return
+ }
+ if d.excluded || (ix.opt.MaxDepth >= 0 && d.depth >= ix.opt.MaxDepth) {
+ return
+ }
+ dir := ix.render(ctx, d)
+ modTime := d.newest
+ if modTime.IsZero() {
+ modTime = ix.now
+ }
+ for _, out := range ix.outputs {
+ var buf bytes.Buffer
+ err := out.tmpl.Execute(&buf, dir)
+ if err != nil {
+ ix.ec.Add(err)
+ fs.Errorf(d.remote, "Failed to render %s: %v", out.name, fs.CountError(ctx, err))
+ continue
+ }
+ remote := path.Join(d.remote, out.name)
+ src := object.NewMemoryObject(remote, modTime, buf.Bytes()).WithMimeType(out.mimeType)
+ dst := d.outputs[out.name]
+ if dst != nil && !ix.opt.Rewrite {
+ equal, err := ix.equal(ctx, src, dst)
+ if err != nil {
+ ix.ec.Add(err)
+ fs.Errorf(dst, "Failed to check listing: %v", fs.CountError(ctx, err))
+ continue
+ }
+ if equal {
+ fs.Debugf(dst, "Unchanged skipping")
+ continue
+ }
+ }
+ ix.transfers.Go(func() error {
+ _, err := Copy(ctx, ix.f, dst, remote, src)
+ if err != nil {
+ ix.ec.Add(err)
+ fs.Errorf(remote, "Failed to write listing: %v", fs.CountError(ctx, err))
+ }
+ return nil
+ })
+ }
+}
+
+// render makes the listing for d
+func (ix *index) render(ctx context.Context, d *indexDir) *serve.Directory {
+ dir := serve.NewDirectory(d.remote, ix.htmlTemplate)
+ dir.Static = true
+ dir.DisableZip = true
+ if ix.opt.LinkIndex {
+ dir.SetLinkIndex(ix.outputs[0].name)
+ }
+ for _, sub := range d.subdirs {
+ var t time.Time
+ switch ix.dirTime {
+ case IndexDirTimeDir:
+ t = sub.ModTime(ctx)
+ case IndexDirTimeNewest:
+ if sd := ix.dirs[sub.Remote()]; sd != nil {
+ t = sd.newest
+ }
+ }
+ // Not sub.Size() as some backends, eg local on macOS, report a
+ // directory size which changes when the listings are written
+ dir.AddHTMLEntry(sub.Remote(), true, -1, t.UTC())
+ }
+ for _, o := range d.files {
+ var t time.Time
+ if !ix.opt.NoModTime {
+ t = o.ModTime(ctx)
+ }
+ dir.AddHTMLEntry(o.Remote(), false, o.Size(), t.UTC())
+ }
+ dir.ProcessQueryParams("", "")
+ return dir
+}
+
+// equal reports whether the existing listing dst has the same
+// contents as src, using the size and a hash where the backend has
+// one, and otherwise by reading dst
+func (ix *index) equal(ctx context.Context, src *object.MemoryObject, dst fs.Object) (bool, error) {
+ tr := accounting.Stats(ctx).NewCheckingTransfer(dst, "checking")
+ defer tr.Done(ctx, nil)
+ if src.Size() != dst.Size() {
+ return false, nil
+ }
+ equal, ht, err := CheckHashes(ctx, src, dst)
+ if err != nil {
+ return false, err
+ }
+ if ht != hash.None {
+ return equal, nil
+ }
+ in, err := Open(ctx, dst)
+ if err != nil {
+ return false, err
+ }
+ data, err := io.ReadAll(in)
+ if closeErr := in.Close(); err == nil {
+ err = closeErr
+ }
+ if err != nil {
+ return false, err
+ }
+ return bytes.Equal(data, src.Content()), nil
+}
diff --git a/fs/operations/index_test.go b/fs/operations/index_test.go
new file mode 100644
index 000000000..275f200ab
--- /dev/null
+++ b/fs/operations/index_test.go
@@ -0,0 +1,389 @@
+package operations_test
+
+import (
+ "context"
+ "encoding/json"
+ "mime"
+ "os"
+ "path/filepath"
+ "sort"
+ "testing"
+ "time"
+
+ "github.com/rclone/rclone/fs"
+ "github.com/rclone/rclone/fs/accounting"
+ "github.com/rclone/rclone/fs/config/obscure"
+ "github.com/rclone/rclone/fs/filter"
+ "github.com/rclone/rclone/fs/operations"
+ "github.com/rclone/rclone/fs/rc"
+ "github.com/rclone/rclone/fs/walk"
+ "github.com/rclone/rclone/fstest"
+ "github.com/rclone/rclone/fstest/fstests"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+// indexRun holds a test remote with a small tree of files in it
+type indexRun struct {
+ *fstest.Run
+ ctx context.Context
+ opt operations.IndexOpt
+}
+
+// newIndexRun makes a remote with file1.txt, sub/file2.txt and
+// sub/deep/file3.txt in it
+func newIndexRun(t *testing.T) *indexRun {
+ opt := operations.IndexOptDefault
+ opt.NoModTime = true
+ r := &indexRun{Run: fstest.NewRun(t), ctx: context.Background(), opt: opt}
+ r.writeFiles()
+ return r
+}
+
+// writeFiles writes the test files to the remote
+func (r *indexRun) writeFiles() {
+ r.WriteObject(r.ctx, "file1.txt", "hello", t1)
+ r.WriteObject(r.ctx, "sub/file2.txt", "hello2", t2)
+ r.WriteObject(r.ctx, "sub/deep/file3.txt", "hello3", t3)
+}
+
+// index runs Index and returns the number of transfers and deletes it made
+func (r *indexRun) index(t *testing.T) (transfers, deletes int64) {
+ stats := accounting.GlobalStats()
+ stats.ResetCounters()
+ require.NoError(t, operations.Index(r.ctx, r.Fremote, &r.opt))
+ return stats.GetTransfers(), stats.GetDeletes()
+}
+
+// read returns the contents of the object at remote
+func (r *indexRun) read(t *testing.T, remote string) string {
+ o, err := r.Fremote.NewObject(r.ctx, remote)
+ require.NoError(t, err, remote)
+ return fstests.ReadObject(r.ctx, t, o, -1)
+}
+
+// files returns the names of all the objects on the remote
+func (r *indexRun) files(t *testing.T) (names []string) {
+ err := walk.ListR(r.ctx, r.Fremote, "", true, -1, walk.ListObjects, func(entries fs.DirEntries) error {
+ for _, entry := range entries {
+ names = append(names, entry.Remote())
+ }
+ return nil
+ })
+ require.NoError(t, err)
+ sort.Strings(names)
+ return names
+}
+
+// checkMimeType checks the MIME type of the object at remote is want,
+// ignoring formatting differences, on backends which store it
+func (r *indexRun) checkMimeType(t *testing.T, remote, want string) {
+ o, err := r.Fremote.NewObject(r.ctx, remote)
+ require.NoError(t, err)
+ if _, ok := o.(fs.MimeTyper); !ok {
+ return
+ }
+ got := fs.MimeType(r.ctx, o)
+ wantType, wantParams, err := mime.ParseMediaType(want)
+ require.NoError(t, err)
+ gotType, gotParams, err := mime.ParseMediaType(got)
+ require.NoError(t, err, got)
+ assert.Equal(t, wantType, gotType)
+ assert.Equal(t, wantParams, gotParams)
+}
+
+// checkFiles checks the remote contains exactly the objects named
+func (r *indexRun) checkFiles(t *testing.T, want ...string) {
+ sort.Strings(want)
+ assert.Equal(t, want, r.files(t))
+}
+
+func TestIndex(t *testing.T) {
+ r := newIndexRun(t)
+ r.opt.Outputs = []string{"index.html=html", "index.json=json", "caddy.json=caddy"}
+
+ transfers, deletes := r.index(t)
+ assert.Equal(t, int64(9), transfers)
+ assert.Equal(t, int64(0), deletes)
+ r.checkFiles(t,
+ "file1.txt", "index.html", "index.json", "caddy.json",
+ "sub/file2.txt", "sub/index.html", "sub/index.json", "sub/caddy.json",
+ "sub/deep/file3.txt", "sub/deep/index.html", "sub/deep/index.json", "sub/deep/caddy.json",
+ )
+
+ // Check the formats at the root
+ assert.Equal(t, `[
+{"Path":"sub","Name":"sub","Size":-1,"MimeType":"inode/directory","ModTime":"","IsDir":true},
+{"Path":"file1.txt","Name":"file1.txt","Size":5,"MimeType":"text/plain; charset=utf-8","ModTime":"","IsDir":false}
+]
+`, r.read(t, "index.json"))
+ assert.Equal(t, `[{"name":"sub/","size":4096,"url":"./sub/","mod_time":"0001-01-01T00:00:00Z","mode":2147484141,"is_dir":true,"is_symlink":false},{"name":"file1.txt","size":5,"url":"./file1.txt","mod_time":"0001-01-01T00:00:00Z","mode":420,"is_dir":false,"is_symlink":false}]
+`, r.read(t, "caddy.json"))
+ html := r.read(t, "index.html")
+ assert.Contains(t, html, `<a href="sub/">sub/</a>`)
+ assert.Contains(t, html, `<a href="file1.txt">file1.txt</a>`)
+ assert.NotContains(t, html, "Go up")
+ assert.Contains(t, html, "sortBy")
+ assert.NotContains(t, html, "download=zip")
+
+ // The listings never show the outputs and link up below the root
+ html = r.read(t, "sub/index.html")
+ assert.Contains(t, html, `<a href="deep/">deep/</a>`)
+ assert.Contains(t, html, `<a href="file2.txt">file2.txt</a>`)
+ assert.NotContains(t, html, "index.html")
+ assert.Contains(t, html, "Go up")
+
+ // The MIME type is set where the backend supports it
+ r.checkMimeType(t, "index.html", "text/html; charset=utf-8")
+
+ // A second run changes nothing
+ transfers, deletes = r.index(t)
+ assert.Equal(t, int64(0), transfers)
+ assert.Equal(t, int64(0), deletes)
+
+ // Unless it is told to rewrite everything
+ r.opt.Rewrite = true
+ transfers, _ = r.index(t)
+ assert.Equal(t, int64(9), transfers)
+ r.opt.Rewrite = false
+
+ // Adding a file only rewrites the listing of its directory
+ r.WriteObject(r.ctx, "sub/deep/file4.txt", "hello4", t1)
+ transfers, deletes = r.index(t)
+ assert.Equal(t, int64(3), transfers)
+ assert.Equal(t, int64(0), deletes)
+ assert.Contains(t, r.read(t, "sub/deep/index.html"), "file4.txt")
+
+ // A run with --dry-run changes nothing
+ ctx, ci := fs.AddConfig(r.ctx)
+ ci.DryRun = true
+ r.WriteObject(ctx, "sub/deep/file5.txt", "hello5", t1)
+ require.NoError(t, operations.Index(ctx, r.Fremote, &r.opt))
+ assert.NotContains(t, r.read(t, "sub/deep/index.html"), "file5.txt")
+}
+
+func TestIndexDelete(t *testing.T) {
+ r := newIndexRun(t)
+ r.index(t)
+ canHaveEmptyDirectories := r.Fremote.Features().CanHaveEmptyDirectories
+
+ // Remove the only file in sub/deep, leaving just the listing
+ o, err := r.Fremote.NewObject(r.ctx, "sub/deep/file3.txt")
+ require.NoError(t, err)
+ require.NoError(t, o.Remove(r.ctx))
+
+ transfers, deletes := r.index(t)
+ if canHaveEmptyDirectories {
+ // The empty directory still exists so keeps its listing
+ assert.Equal(t, int64(1), transfers)
+ assert.Equal(t, int64(0), deletes)
+ r.checkFiles(t, "file1.txt", "index.html", "sub/file2.txt", "sub/index.html", "sub/deep/index.html")
+ assert.Contains(t, r.read(t, "sub/index.html"), "deep/")
+ assert.NotContains(t, r.read(t, "sub/deep/index.html"), "file3.txt")
+ } else {
+ // The directory only existed because of the listing, so it goes
+ assert.Equal(t, int64(1), transfers)
+ assert.Equal(t, int64(1), deletes)
+ r.checkFiles(t, "file1.txt", "index.html", "sub/file2.txt", "sub/index.html")
+ assert.NotContains(t, r.read(t, "sub/index.html"), "deep/")
+
+ // Removing sub/file2.txt cascades: sub is now only listings
+ o, err = r.Fremote.NewObject(r.ctx, "sub/file2.txt")
+ require.NoError(t, err)
+ require.NoError(t, o.Remove(r.ctx))
+ transfers, deletes = r.index(t)
+ assert.Equal(t, int64(1), transfers)
+ assert.Equal(t, int64(1), deletes)
+ r.checkFiles(t, "file1.txt", "index.html")
+ assert.NotContains(t, r.read(t, "index.html"), "sub/")
+ }
+}
+
+func TestIndexFilter(t *testing.T) {
+ r := newIndexRun(t)
+ r.WriteObject(r.ctx, "sub/deep/index.html", "hand written", t1)
+ r.WriteObject(r.ctx, "other/notes.md", "notes", t1)
+
+ // Excluded directories are not listed and not touched, and a
+ // directory whose files are all excluded still gets an empty listing
+ fi, err := filter.NewFilter(nil)
+ require.NoError(t, err)
+ require.NoError(t, fi.AddRule("- /sub/deep/**"))
+ require.NoError(t, fi.AddRule("- *.md"))
+ r.ctx = filter.ReplaceConfig(r.ctx, fi)
+
+ r.index(t)
+ r.checkFiles(t, "file1.txt", "index.html", "sub/file2.txt", "sub/index.html", "sub/deep/file3.txt", "sub/deep/index.html", "other/notes.md", "other/index.html")
+ assert.Equal(t, "hand written", r.read(t, "sub/deep/index.html"))
+ assert.NotContains(t, r.read(t, "sub/index.html"), "deep/")
+ other := r.read(t, "other/index.html")
+ assert.NotContains(t, other, "notes.md")
+ assert.Contains(t, r.read(t, "index.html"), `<a href="other/">other/</a>`)
+
+ // A second run changes nothing, in particular it doesn't delete
+ // other/index.html even though other/ has no visible content
+ transfers, deletes := r.index(t)
+ assert.Equal(t, int64(0), transfers)
+ assert.Equal(t, int64(0), deletes)
+}
+
+func TestIndexRules(t *testing.T) {
+ r := newIndexRun(t)
+ r.WriteObject(r.ctx, "sub/deep/index.html", "hand written", t1)
+
+ // A directory excluded by the index rules is still listed by
+ // its parent but nothing is written or deleted in it
+ r.opt.Rules.ExcludeRule = []string{"/sub/deep/**"}
+ r.index(t)
+ r.checkFiles(t, "file1.txt", "index.html", "sub/file2.txt", "sub/index.html", "sub/deep/file3.txt", "sub/deep/index.html")
+ assert.Equal(t, "hand written", r.read(t, "sub/deep/index.html"))
+ assert.Contains(t, r.read(t, "sub/index.html"), `<a href="deep/">deep/</a>`)
+
+ // Include rules imply excluding everything else
+ r.opt.Rules = filter.RulesOpt{IncludeRule: []string{"/sub/**"}}
+ o, err := r.Fremote.NewObject(r.ctx, "index.html")
+ require.NoError(t, err)
+ require.NoError(t, o.Remove(r.ctx))
+ transfers, _ := r.index(t)
+ assert.Equal(t, int64(1), transfers) // sub/deep/index.html
+ r.checkFiles(t, "file1.txt", "sub/file2.txt", "sub/index.html", "sub/deep/file3.txt", "sub/deep/index.html")
+ assert.NotEqual(t, "hand written", r.read(t, "sub/deep/index.html"))
+}
+
+func TestIndexMaxDepth(t *testing.T) {
+ r := newIndexRun(t)
+ r.opt.MaxDepth = 1
+ r.index(t)
+ r.checkFiles(t, "file1.txt", "index.html", "sub/file2.txt", "sub/deep/file3.txt")
+ assert.Contains(t, r.read(t, "index.html"), `<a href="sub/">sub/</a>`)
+}
+
+func TestIndexLinkIndex(t *testing.T) {
+ r := newIndexRun(t)
+ r.opt.LinkIndex = true
+ r.index(t)
+ assert.Contains(t, r.read(t, "index.html"), `<a href="sub/index.html">sub/</a>`)
+ html := r.read(t, "sub/index.html")
+ assert.Contains(t, html, `<a href="deep/index.html">deep/</a>`)
+ assert.Contains(t, html, `<a href="../index.html">`)
+}
+
+// indexModTimes returns the ModTime of each entry in the json listing at remote
+func (r *indexRun) indexModTimes(t *testing.T, remote string) map[string]time.Time {
+ var items []struct {
+ Name string
+ ModTime string
+ }
+ require.NoError(t, json.Unmarshal([]byte(r.read(t, remote)), &items))
+ modTimes := map[string]time.Time{}
+ for _, item := range items {
+ if item.ModTime == "" {
+ continue
+ }
+ modTime, err := time.Parse(time.RFC3339Nano, item.ModTime)
+ require.NoError(t, err)
+ modTimes[item.Name] = modTime
+ }
+ return modTimes
+}
+
+func TestIndexDirTime(t *testing.T) {
+ r := newIndexRun(t)
+ r.opt.NoModTime = false
+ r.opt.Outputs = []string{"index.json=json"}
+ precision := r.Fremote.Precision()
+
+ // newest: sub shows the newest time below it, and so does its listing
+ r.opt.DirTime = operations.IndexDirTimeNewest
+ r.index(t)
+ modTimes := r.indexModTimes(t, "index.json")
+ fstest.AssertTimeEqualWithPrecision(t, "file1.txt", t1, modTimes["file1.txt"], precision)
+ fstest.AssertTimeEqualWithPrecision(t, "sub", t3, modTimes["sub"], precision)
+ o, err := r.Fremote.NewObject(r.ctx, "index.json")
+ require.NoError(t, err)
+ fstest.AssertTimeEqualWithPrecision(t, "index.json", t3, o.ModTime(r.ctx), precision)
+
+ // none: no directory times
+ r.opt.DirTime = operations.IndexDirTimeNone
+ r.index(t)
+ modTimes = r.indexModTimes(t, "index.json")
+ fstest.AssertTimeEqualWithPrecision(t, "file1.txt", t1, modTimes["file1.txt"], precision)
+ _, found := modTimes["sub"]
+ assert.False(t, found)
+
+ // no modtime: no times at all
+ r.opt.NoModTime = true
+ r.index(t)
+ assert.Empty(t, r.indexModTimes(t, "index.json"))
+}
+
+func TestIndexTemplate(t *testing.T) {
+ r := newIndexRun(t)
+ tmpl := filepath.Join(t.TempDir(), "list.tmpl")
+ require.NoError(t, os.WriteFile(tmpl, []byte("{{.Path}}\n{{range .Entries}}{{.Leaf}} {{.MimeType}}\n{{end}}"), 0600))
+ r.opt.Outputs = []string{"list.txt=" + tmpl}
+ r.index(t)
+ assert.Equal(t, "/sub/\ndeep/ inode/directory\nfile2.txt text/plain; charset=utf-8\n", r.read(t, "sub/list.txt"))
+ r.checkMimeType(t, "sub/list.txt", "text/plain; charset=utf-8")
+
+ // Bad outputs
+ for _, output := range []string{"index.html", "=html", "a/b=html", "index.html=nosuchformat"} {
+ r.opt.Outputs = []string{output}
+ assert.Error(t, operations.Index(r.ctx, r.Fremote, &r.opt), output)
+ }
+}
+
+func TestIndexTemplateBuiltin(t *testing.T) {
+ for format := range map[string]bool{"html": true, "json": true, "caddy": true} {
+ text, err := operations.IndexTemplate(format)
+ require.NoError(t, err)
+ assert.NotEmpty(t, text)
+ }
+ _, err := operations.IndexTemplate("nosuchformat")
+ assert.Error(t, err)
+}
+
+func TestRcIndex(t *testing.T) {
+ r, call := rcNewRun(t, "operations/index")
+ r.WriteObject(context.Background(), "sub/a", "a", t1)
+ in := rc.Params{
+ "fs": r.FremoteName,
+ "opt": rc.Params{
+ "output": []string{"index.json=json"},
+ "noModTime": true,
+ "maxDepth": 1,
+ },
+ }
+ out, err := call.Fn(context.Background(), in)
+ require.NoError(t, err)
+ assert.Nil(t, out)
+ _, err = r.Fremote.NewObject(context.Background(), "index.json")
+ assert.NoError(t, err)
+ _, err = r.Fremote.NewObject(context.Background(), "sub/index.json")
+ assert.ErrorIs(t, err, fs.ErrorObjectNotFound)
+}
+
+// TestIndexNoHash checks the listings are read and compared on a
+// backend without hashes so that unchanged listings aren't rewritten
+func TestIndexNoHash(t *testing.T) {
+ if *fstest.RemoteName != "" {
+ t.Skip("Skipping test on non local remote")
+ }
+ opt := operations.IndexOptDefault
+ opt.NoModTime = true
+ r := &indexRun{Run: fstest.NewRun(t), ctx: context.Background(), opt: opt}
+ f, err := fs.NewFs(r.ctx, ":crypt,remote='"+r.Fremote.Root()+"',password="+obscure.MustObscure("potato")+":")
+ require.NoError(t, err)
+ require.Equal(t, 0, f.Hashes().Count())
+ r.Fremote = f
+ r.writeFiles()
+
+ transfers, _ := r.index(t)
+ assert.Equal(t, int64(3), transfers)
+ transfers, _ = r.index(t)
+ assert.Equal(t, int64(0), transfers)
+ r.WriteObject(r.ctx, "sub/file4.txt", "hello4", t1)
+ transfers, _ = r.index(t)
+ assert.Equal(t, int64(1), transfers)
+ assert.Contains(t, r.read(t, "sub/index.html"), "file4.txt")
+}
diff --git a/fs/operations/rc.go b/fs/operations/rc.go
index cdaa8f854..e0138a984 100644
--- a/fs/operations/rc.go
+++ b/fs/operations/rc.go
@@ -11,6 +11,7 @@ import (
"mime/multipart"
"net/http"
"path"
+ "slices"
"strings"
"time"
"unicode/utf8"
@@ -695,6 +696,44 @@ func rcDu(ctx context.Context, in rc.Params) (out rc.Params, err error) {
return out, nil
}
+func init() {
+ rc.Add(rc.Call{
+ Path: "operations/index",
+ Fn: rcIndex,
+ Title: "Write static directory listings into a remote",
+ Help: `This takes the following parameters:
+
+- fs - a remote name string e.g. "drive:"
+- opt - a dictionary of options to control the indexing (optional)
+ - output - list of listings to write in each directory as "NAME=FORMAT" (default ["index.html=html"])
+ - rules - which directories get listings, e.g. {"ExcludeRule": ["/private/**"]}
+ - maxDepth - only make listings this many directories deep (default -1 for no limit)
+ - linkIndex - link to "dir/NAME" rather than "dir/"
+ - dirTime - how to work out directory times: "newest", "dir" or "none"
+ - noModTime - don't show modification times
+ - rewrite - write every listing even if it is unchanged
+
+See the [index](/commands/rclone_index/) command for more information on the above.
+`,
+ })
+}
+
+// Index a remote
+func rcIndex(ctx context.Context, in rc.Params) (out rc.Params, err error) {
+ f, err := rc.GetFs(ctx, in)
+ if err != nil {
+ return nil, err
+ }
+ opt := IndexOptDefault
+ // Decoding into a slice reuses its backing array so don't share it with the defaults
+ opt.Outputs = slices.Clone(opt.Outputs)
+ err = in.GetStruct("opt", &opt)
+ if rc.NotErrParamNotFound(err) {
+ return nil, err
+ }
+ return nil, Index(ctx, f, &opt)
+}
+
func init() {
rc.Add(rc.Call{
Path: "operations/check",
diff --git a/lib/http/templates/caddy.json b/lib/http/templates/caddy.json
new file mode 100644
index 000000000..965f166cb
--- /dev/null
+++ b/lib/http/templates/caddy.json
@@ -0,0 +1 @@
+[{{range $i, $e := .Entries}}{{if $i}},{{end}}{"name":{{json $e.Leaf}},"size":{{if $e.IsDir}}4096{{else}}{{$e.Size}}{{end}},"url":{{if hasPrefix $e.URL "./"}}{{json $e.URL}}{{else}}{{json (print "./" $e.URL)}}{{end}},"mod_time":{{json $e.ModTime}},"mode":{{if $e.IsDir}}2147484141{{else}}420{{end}},"is_dir":{{$e.IsDir}},"is_symlink":false}{{end}}]
diff --git a/lib/http/templates/index.json b/lib/http/templates/index.json
new file mode 100644
index 000000000..f56da58a0
--- /dev/null
+++ b/lib/http/templates/index.json
@@ -0,0 +1,5 @@
+[
+{{- range $i, $e := .Entries}}{{if $i}},{{end}}
+{"Path":{{json (trimSuffix $e.Leaf "/")}},"Name":{{json (trimSuffix $e.Leaf "/")}},"Size":{{if $e.IsDir}}-1{{else}}{{$e.Size}}{{end}},"MimeType":{{json $e.MimeType}},"ModTime":{{if $e.ModTime | afterEpoch}}{{json $e.ModTime}}{{else}}""{{end}},"IsDir":{{$e.IsDir}}}
+{{- end}}
+]