"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
operations: add Index to write static directory listings into a remote
Nick Craig-Woodae9a50cacd89
diff --git a/.gitattributes b/.gitattributesindex 8aa2a2385..62df6b2ee 100644--- a/.gitattributes+++ b/.gitattributes@@ -2,6 +2,9 @@go.mod text eol=lfgo.sum text eol=lf+# Embedded templates must render the same on every OS+lib/http/templates/** text eol=lf+# Ignore generated files in GitHub language statistics and diffs/MANUAL.* linguist-generated=true/rclone.1 linguist-generated=truediff --git a/fs/operations/index.go b/fs/operations/index.gonew file mode 100644index 000000000..9c7a10d52--- /dev/null+++ b/fs/operations/index.go@@ -0,0 +1,518 @@+package operations++import (+ "bytes"+ "context"+ "encoding/json"+ "errors"+ "fmt"+ htmltemplate "html/template"+ "io"+ "os"+ "path"+ "sort"+ "strings"+ texttemplate "text/template"+ "time"++ "github.com/rclone/rclone/fs"+ "github.com/rclone/rclone/fs/accounting"+ "github.com/rclone/rclone/fs/filter"+ "github.com/rclone/rclone/fs/hash"+ "github.com/rclone/rclone/fs/object"+ "github.com/rclone/rclone/fs/walk"+ "github.com/rclone/rclone/lib/errcount"+ libhttp "github.com/rclone/rclone/lib/http"+ "github.com/rclone/rclone/lib/http/serve"+ "golang.org/x/sync/errgroup"+)++type indexDirTimeChoices struct{}++func (indexDirTimeChoices) Choices() []string {+ return []string{+ IndexDirTimeNewest: "newest",+ IndexDirTimeDir: "dir",+ IndexDirTimeNone: "none",+ }+}++// IndexDirTime describes how Index works out the time shown for a directory+type IndexDirTime = fs.Enum[indexDirTimeChoices]++// IndexDirTime constants+const (+ IndexDirTimeNewest IndexDirTime = iota // the newest modification time of anything below the directory+ IndexDirTimeDir // the directory's own modification time+ IndexDirTimeNone // no time+)++// IndexOpt are the options for Index+type IndexOpt struct {+ Outputs []string `json:"output"` // listings to write in each directory as NAME=FORMAT+ Rules filter.RulesOpt `json:"rules"` // which directories get listings+ MaxDepth int `json:"maxDepth"` // only make listings this many directories deep, -1 for no limit+ LinkIndex bool `json:"linkIndex"` // link to dir/NAME rather than dir/+ DirTime IndexDirTime `json:"dirTime"` // how to work out directory times+ NoModTime bool `json:"noModTime"` // don't show modification times+ Rewrite bool `json:"rewrite"` // write every listing even if it is unchanged+}++// IndexOptDefault are the default options for Index+var IndexOptDefault = IndexOpt{+ Outputs: []string{"index.html=html"},+ MaxDepth: -1,+}++// indexFormats are the built-in output formats+var indexFormats = map[string]struct{ file, mimeType string }{+ "html": {"templates/index.html", "text/html; charset=utf-8"},+ "json": {"templates/index.json", "application/json"},+ "caddy": {"templates/caddy.json", "application/json"},+}++// IndexTemplate returns the built-in template for format+func IndexTemplate(format string) (string, error) {+ f, ok := indexFormats[format]+ if !ok {+ return "", fmt.Errorf("unknown built-in format %q", format)+ }+ data, err := libhttp.Assets.ReadFile(f.file)+ return string(data), err+}++// indexTemplate is implemented by both html/template and text/template+type indexTemplate interface {+ Execute(w io.Writer, data any) error+}++// indexOutput is one listing to write in each directory+type indexOutput struct {+ name string+ mimeType string+ tmpl indexTemplate+}++// indexDir is what Index knows about a directory+type indexDir struct {+ remote string+ depth int+ files []fs.Object // files to list, after filtering, excluding outputs+ subdirs []fs.Directory // directories to list, after filtering+ outputs map[string]fs.Object // existing outputs by leaf name+ excluded bool // excluded by the index rules so nothing is written or deleted here+ hasContent bool // there is something here other than outputs, before filtering+ gone bool // contains nothing but outputs so will cease to exist+ newest time.Time // the newest modification time of anything below here+}++// index holds the state of one run of Index+type index struct {+ f fs.Fs+ opt *IndexOpt+ fi *filter.Filter+ includeDir func(string) (bool, error) // the normal filters applied to a directory+ indexDir func(string) (bool, error) // the index rules applied to a directory+ outputs []indexOutput+ isOutput map[string]bool+ htmlTemplate *htmltemplate.Template+ dirTime IndexDirTime+ now time.Time+ dirs map[string]*indexDir+ transfers errgroup.Group+ ec *errcount.ErrCount+}++// Index writes a directory listing into every directory of f so that+// it can be browsed when served as a static website.+//+// It works like sync: existing listings which are unchanged are left+// alone, changed ones are rewritten, and listings in directories+// which contain nothing else are deleted on backends which can't have+// empty directories.+func Index(ctx context.Context, f fs.Fs, opt *IndexOpt) error {+ ix, err := newIndex(ctx, f, opt)+ if err != nil {+ return err+ }+ return ix.run(ctx)+}++// newIndex parses the options and templates+func newIndex(ctx context.Context, f fs.Fs, opt *IndexOpt) (*index, error) {+ ix := &index{+ f: f,+ opt: opt,+ fi: filter.GetConfig(ctx),+ isOutput: map[string]bool{},+ dirTime: opt.DirTime,+ now: time.Now(),+ dirs: map[string]*indexDir{},+ ec: errcount.New(),+ }+ if ix.opt.NoModTime {+ ix.dirTime = IndexDirTimeNone+ }+ ix.includeDir = ix.fi.IncludeDirectory(ctx, f)++ // The index rules are a second rule set applied to directory paths+ indexFilter, err := filter.NewFilter(&filter.Options{+ RulesOpt: opt.Rules,+ MinAge: fs.DurationOff,+ MaxAge: fs.DurationOff,+ MinSize: -1,+ MaxSize: -1,+ })+ if err != nil {+ return nil, fmt.Errorf("index rules: %w", err)+ }+ ix.indexDir = indexFilter.IncludeDirectory(ctx, nil)++ ix.htmlTemplate, err = libhttp.GetTemplate("")+ if err != nil {+ return nil, err+ }+ if len(opt.Outputs) == 0 {+ return nil, errors.New("need at least one output")+ }+ for _, s := range opt.Outputs {+ name, format, ok := strings.Cut(s, "=")+ if !ok || name == "" || name == "." || name == ".." || format == "" || strings.Contains(name, "/") {+ return nil, fmt.Errorf("bad output %q: want NAME=FORMAT", s)+ }+ out, err := ix.newOutput(name, format)+ if err != nil {+ return nil, fmt.Errorf("output %q: %w", s, err)+ }+ ix.outputs = append(ix.outputs, out)+ ix.isOutput[name] = true+ }+ return ix, nil+}++// newOutput makes an output from a NAME=FORMAT pair, where format is+// a built-in format or the path of a template file+func (ix *index) newOutput(name, format string) (out indexOutput, err error) {+ out.name = name+ if builtin, ok := indexFormats[format]; ok {+ out.mimeType = builtin.mimeType+ if format == "html" {+ out.tmpl = ix.htmlTemplate+ return out, nil+ }+ data, err := libhttp.Assets.ReadFile(builtin.file)+ if err != nil {+ return out, err+ }+ out.tmpl, err = newIndexTextTemplate(string(data))+ return out, err+ }+ out.mimeType = fs.MimeTypeFromName(name)+ switch strings.ToLower(path.Ext(name)) {+ case ".html", ".htm":+ out.tmpl, err = libhttp.GetTemplate(format)+ default:+ var data []byte+ data, err = os.ReadFile(format)+ if err != nil {+ return out, err+ }+ out.tmpl, err = newIndexTextTemplate(string(data))+ }+ return out, err+}++// newIndexTextTemplate parses a text/template for non-HTML outputs+func newIndexTextTemplate(text string) (*texttemplate.Template, error) {+ funcMap := texttemplate.FuncMap{+ "json": func(v any) (string, error) {+ data, err := json.Marshal(v)+ return string(data), err+ },+ "afterEpoch": libhttp.AfterEpoch,+ "contains": strings.Contains,+ "hasPrefix": strings.HasPrefix,+ "hasSuffix": strings.HasSuffix,+ "trimSuffix": strings.TrimSuffix,+ }+ return texttemplate.New("index").Funcs(funcMap).Parse(text)+}++// run does the work of Index+func (ix *index) run(ctx context.Context) error {+ ci := fs.GetConfig(ctx)+ err := walk.Walk(ctx, ix.f, "", true, ConfigMaxDepth(ctx, true), ix.walkFn(ctx))+ if err != nil {+ return err+ }+ ix.prune()++ // Directories deepest first so that a directory's subdirectories+ // are done before it.+ dirs := make([]*indexDir, 0, len(ix.dirs))+ for _, d := range ix.dirs {+ dirs = append(dirs, d)+ }+ sort.Slice(dirs, func(i, j int) bool {+ return dirs[i].depth > dirs[j].depth+ })++ // Read the modification times --checkers at a time as they may+ // need a transaction each on some backends, and count them as+ // checks so the progress is visible+ if !ix.opt.NoModTime {+ var g errgroup.Group+ g.SetLimit(ci.Checkers)+ for _, d := range dirs {+ for _, o := range d.files {+ g.Go(func() error {+ tr := accounting.Stats(ctx).NewCheckingTransfer(o, "reading modtime")+ o.ModTime(ctx)+ tr.Done(ctx, nil)+ return nil+ })+ }+ }+ _ = g.Wait()+ for _, d := range dirs {+ ix.setNewest(ctx, d)+ }+ }++ ix.transfers.SetLimit(ci.Transfers)+ var g errgroup.Group+ g.SetLimit(ci.Checkers)+ for _, d := range dirs {+ g.Go(func() error {+ ix.processDir(ctx, d)+ return nil+ })+ }+ _ = g.Wait()+ _ = ix.transfers.Wait()+ return ix.ec.Err("index")+}++// walkFn returns the function which collects the directories+func (ix *index) walkFn(ctx context.Context) walk.Func {+ return func(remote string, entries fs.DirEntries, err error) error {+ if err != nil {+ return err+ }+ if remote != "" {+ // Directories excluded by the normal filters aren't listed+ ok, err := ix.includeDir(remote)+ if err != nil {+ return err+ }+ if !ok {+ return walk.ErrorSkipDir+ }+ }+ // Directories excluded by the index rules are still walked+ // as they may have included directories below them and+ // their contents count towards their parent's time.+ included, err := ix.indexDir(remote)+ if err != nil {+ return err+ }+ d := &indexDir{+ remote: remote,+ outputs: map[string]fs.Object{},+ excluded: !included,+ }+ if remote != "" {+ d.depth = strings.Count(remote, "/") + 1+ }+ for _, entry := range entries {+ switch x := entry.(type) {+ case fs.Object:+ leaf := path.Base(x.Remote())+ if !d.excluded && ix.isOutput[leaf] {+ d.outputs[leaf] = x+ continue+ }+ d.hasContent = true+ if ix.fi.IncludeObject(ctx, x) {+ d.files = append(d.files, x)+ }+ case fs.Directory:+ ok, err := ix.includeDir(x.Remote())+ if err != nil {+ return err+ }+ if ok {+ d.subdirs = append(d.subdirs, x)+ } else {+ d.hasContent = true+ }+ }+ }+ ix.dirs[remote] = d+ return nil+ }+}++// prune works out which directories contain nothing but outputs.+//+// On backends which can't have empty directories these will cease to+// exist once their outputs are deleted, so they are removed from their+// parent's listing, which may in turn leave the parent with nothing+// but outputs.+func (ix *index) prune() {+ if ix.f.Features().CanHaveEmptyDirectories {+ return+ }+ var pruneDir func(d *indexDir)+ pruneDir = func(d *indexDir) {+ kept := d.subdirs[:0]+ for _, sub := range d.subdirs {+ if sd := ix.dirs[sub.Remote()]; sd != nil {+ pruneDir(sd)+ if sd.gone {+ continue+ }+ }+ kept = append(kept, sub)+ }+ d.subdirs = kept+ d.gone = !d.hasContent && len(d.subdirs) == 0+ }+ if root := ix.dirs[""]; root != nil {+ pruneDir(root)+ }+}++// setNewest sets d.newest from its files and subdirectories, which+// must have been done already+func (ix *index) setNewest(ctx context.Context, d *indexDir) {+ for _, o := range d.files {+ if t := o.ModTime(ctx); t.After(d.newest) {+ d.newest = t+ }+ }+ for _, sub := range d.subdirs {+ if sd := ix.dirs[sub.Remote()]; sd != nil && sd.newest.After(d.newest) {+ d.newest = sd.newest+ }+ }+}++// processDir brings the outputs of d up to date+func (ix *index) processDir(ctx context.Context, d *indexDir) {+ if d.gone {+ for _, o := range d.outputs {+ ix.transfers.Go(func() error {+ err := DeleteFile(ctx, o)+ if err != nil {+ ix.ec.Add(err)+ }+ return nil+ })+ }+ return+ }+ if d.excluded || (ix.opt.MaxDepth >= 0 && d.depth >= ix.opt.MaxDepth) {+ return+ }+ dir := ix.render(ctx, d)+ modTime := d.newest+ if modTime.IsZero() {+ modTime = ix.now+ }+ for _, out := range ix.outputs {+ var buf bytes.Buffer+ err := out.tmpl.Execute(&buf, dir)+ if err != nil {+ ix.ec.Add(err)+ fs.Errorf(d.remote, "Failed to render %s: %v", out.name, fs.CountError(ctx, err))+ continue+ }+ remote := path.Join(d.remote, out.name)+ src := object.NewMemoryObject(remote, modTime, buf.Bytes()).WithMimeType(out.mimeType)+ dst := d.outputs[out.name]+ if dst != nil && !ix.opt.Rewrite {+ equal, err := ix.equal(ctx, src, dst)+ if err != nil {+ ix.ec.Add(err)+ fs.Errorf(dst, "Failed to check listing: %v", fs.CountError(ctx, err))+ continue+ }+ if equal {+ fs.Debugf(dst, "Unchanged skipping")+ continue+ }+ }+ ix.transfers.Go(func() error {+ _, err := Copy(ctx, ix.f, dst, remote, src)+ if err != nil {+ ix.ec.Add(err)+ fs.Errorf(remote, "Failed to write listing: %v", fs.CountError(ctx, err))+ }+ return nil+ })+ }+}++// render makes the listing for d+func (ix *index) render(ctx context.Context, d *indexDir) *serve.Directory {+ dir := serve.NewDirectory(d.remote, ix.htmlTemplate)+ dir.Static = true+ dir.DisableZip = true+ if ix.opt.LinkIndex {+ dir.SetLinkIndex(ix.outputs[0].name)+ }+ for _, sub := range d.subdirs {+ var t time.Time+ switch ix.dirTime {+ case IndexDirTimeDir:+ t = sub.ModTime(ctx)+ case IndexDirTimeNewest:+ if sd := ix.dirs[sub.Remote()]; sd != nil {+ t = sd.newest+ }+ }+ // Not sub.Size() as some backends, eg local on macOS, report a+ // directory size which changes when the listings are written+ dir.AddHTMLEntry(sub.Remote(), true, -1, t.UTC())+ }+ for _, o := range d.files {+ var t time.Time+ if !ix.opt.NoModTime {+ t = o.ModTime(ctx)+ }+ dir.AddHTMLEntry(o.Remote(), false, o.Size(), t.UTC())+ }+ dir.ProcessQueryParams("", "")+ return dir+}++// equal reports whether the existing listing dst has the same+// contents as src, using the size and a hash where the backend has+// one, and otherwise by reading dst+func (ix *index) equal(ctx context.Context, src *object.MemoryObject, dst fs.Object) (bool, error) {+ tr := accounting.Stats(ctx).NewCheckingTransfer(dst, "checking")+ defer tr.Done(ctx, nil)+ if src.Size() != dst.Size() {+ return false, nil+ }+ equal, ht, err := CheckHashes(ctx, src, dst)+ if err != nil {+ return false, err+ }+ if ht != hash.None {+ return equal, nil+ }+ in, err := Open(ctx, dst)+ if err != nil {+ return false, err+ }+ data, err := io.ReadAll(in)+ if closeErr := in.Close(); err == nil {+ err = closeErr+ }+ if err != nil {+ return false, err+ }+ return bytes.Equal(data, src.Content()), nil+}diff --git a/fs/operations/index_test.go b/fs/operations/index_test.gonew file mode 100644index 000000000..275f200ab--- /dev/null+++ b/fs/operations/index_test.go@@ -0,0 +1,389 @@+package operations_test++import (+ "context"+ "encoding/json"+ "mime"+ "os"+ "path/filepath"+ "sort"+ "testing"+ "time"++ "github.com/rclone/rclone/fs"+ "github.com/rclone/rclone/fs/accounting"+ "github.com/rclone/rclone/fs/config/obscure"+ "github.com/rclone/rclone/fs/filter"+ "github.com/rclone/rclone/fs/operations"+ "github.com/rclone/rclone/fs/rc"+ "github.com/rclone/rclone/fs/walk"+ "github.com/rclone/rclone/fstest"+ "github.com/rclone/rclone/fstest/fstests"+ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+)++// indexRun holds a test remote with a small tree of files in it+type indexRun struct {+ *fstest.Run+ ctx context.Context+ opt operations.IndexOpt+}++// newIndexRun makes a remote with file1.txt, sub/file2.txt and+// sub/deep/file3.txt in it+func newIndexRun(t *testing.T) *indexRun {+ opt := operations.IndexOptDefault+ opt.NoModTime = true+ r := &indexRun{Run: fstest.NewRun(t), ctx: context.Background(), opt: opt}+ r.writeFiles()+ return r+}++// writeFiles writes the test files to the remote+func (r *indexRun) writeFiles() {+ r.WriteObject(r.ctx, "file1.txt", "hello", t1)+ r.WriteObject(r.ctx, "sub/file2.txt", "hello2", t2)+ r.WriteObject(r.ctx, "sub/deep/file3.txt", "hello3", t3)+}++// index runs Index and returns the number of transfers and deletes it made+func (r *indexRun) index(t *testing.T) (transfers, deletes int64) {+ stats := accounting.GlobalStats()+ stats.ResetCounters()+ require.NoError(t, operations.Index(r.ctx, r.Fremote, &r.opt))+ return stats.GetTransfers(), stats.GetDeletes()+}++// read returns the contents of the object at remote+func (r *indexRun) read(t *testing.T, remote string) string {+ o, err := r.Fremote.NewObject(r.ctx, remote)+ require.NoError(t, err, remote)+ return fstests.ReadObject(r.ctx, t, o, -1)+}++// files returns the names of all the objects on the remote+func (r *indexRun) files(t *testing.T) (names []string) {+ err := walk.ListR(r.ctx, r.Fremote, "", true, -1, walk.ListObjects, func(entries fs.DirEntries) error {+ for _, entry := range entries {+ names = append(names, entry.Remote())+ }+ return nil+ })+ require.NoError(t, err)+ sort.Strings(names)+ return names+}++// checkMimeType checks the MIME type of the object at remote is want,+// ignoring formatting differences, on backends which store it+func (r *indexRun) checkMimeType(t *testing.T, remote, want string) {+ o, err := r.Fremote.NewObject(r.ctx, remote)+ require.NoError(t, err)+ if _, ok := o.(fs.MimeTyper); !ok {+ return+ }+ got := fs.MimeType(r.ctx, o)+ wantType, wantParams, err := mime.ParseMediaType(want)+ require.NoError(t, err)+ gotType, gotParams, err := mime.ParseMediaType(got)+ require.NoError(t, err, got)+ assert.Equal(t, wantType, gotType)+ assert.Equal(t, wantParams, gotParams)+}++// checkFiles checks the remote contains exactly the objects named+func (r *indexRun) checkFiles(t *testing.T, want ...string) {+ sort.Strings(want)+ assert.Equal(t, want, r.files(t))+}++func TestIndex(t *testing.T) {+ r := newIndexRun(t)+ r.opt.Outputs = []string{"index.html=html", "index.json=json", "caddy.json=caddy"}++ transfers, deletes := r.index(t)+ assert.Equal(t, int64(9), transfers)+ assert.Equal(t, int64(0), deletes)+ r.checkFiles(t,+ "file1.txt", "index.html", "index.json", "caddy.json",+ "sub/file2.txt", "sub/index.html", "sub/index.json", "sub/caddy.json",+ "sub/deep/file3.txt", "sub/deep/index.html", "sub/deep/index.json", "sub/deep/caddy.json",+ )++ // Check the formats at the root+ assert.Equal(t, `[+{"Path":"sub","Name":"sub","Size":-1,"MimeType":"inode/directory","ModTime":"","IsDir":true},+{"Path":"file1.txt","Name":"file1.txt","Size":5,"MimeType":"text/plain; charset=utf-8","ModTime":"","IsDir":false}+]+`, r.read(t, "index.json"))+ assert.Equal(t, `[{"name":"sub/","size":4096,"url":"./sub/","mod_time":"0001-01-01T00:00:00Z","mode":2147484141,"is_dir":true,"is_symlink":false},{"name":"file1.txt","size":5,"url":"./file1.txt","mod_time":"0001-01-01T00:00:00Z","mode":420,"is_dir":false,"is_symlink":false}]+`, r.read(t, "caddy.json"))+ html := r.read(t, "index.html")+ assert.Contains(t, html, `<a href="sub/">sub/</a>`)+ assert.Contains(t, html, `<a href="file1.txt">file1.txt</a>`)+ assert.NotContains(t, html, "Go up")+ assert.Contains(t, html, "sortBy")+ assert.NotContains(t, html, "download=zip")++ // The listings never show the outputs and link up below the root+ html = r.read(t, "sub/index.html")+ assert.Contains(t, html, `<a href="deep/">deep/</a>`)+ assert.Contains(t, html, `<a href="file2.txt">file2.txt</a>`)+ assert.NotContains(t, html, "index.html")+ assert.Contains(t, html, "Go up")++ // The MIME type is set where the backend supports it+ r.checkMimeType(t, "index.html", "text/html; charset=utf-8")++ // A second run changes nothing+ transfers, deletes = r.index(t)+ assert.Equal(t, int64(0), transfers)+ assert.Equal(t, int64(0), deletes)++ // Unless it is told to rewrite everything+ r.opt.Rewrite = true+ transfers, _ = r.index(t)+ assert.Equal(t, int64(9), transfers)+ r.opt.Rewrite = false++ // Adding a file only rewrites the listing of its directory+ r.WriteObject(r.ctx, "sub/deep/file4.txt", "hello4", t1)+ transfers, deletes = r.index(t)+ assert.Equal(t, int64(3), transfers)+ assert.Equal(t, int64(0), deletes)+ assert.Contains(t, r.read(t, "sub/deep/index.html"), "file4.txt")++ // A run with --dry-run changes nothing+ ctx, ci := fs.AddConfig(r.ctx)+ ci.DryRun = true+ r.WriteObject(ctx, "sub/deep/file5.txt", "hello5", t1)+ require.NoError(t, operations.Index(ctx, r.Fremote, &r.opt))+ assert.NotContains(t, r.read(t, "sub/deep/index.html"), "file5.txt")+}++func TestIndexDelete(t *testing.T) {+ r := newIndexRun(t)+ r.index(t)+ canHaveEmptyDirectories := r.Fremote.Features().CanHaveEmptyDirectories++ // Remove the only file in sub/deep, leaving just the listing+ o, err := r.Fremote.NewObject(r.ctx, "sub/deep/file3.txt")+ require.NoError(t, err)+ require.NoError(t, o.Remove(r.ctx))++ transfers, deletes := r.index(t)+ if canHaveEmptyDirectories {+ // The empty directory still exists so keeps its listing+ assert.Equal(t, int64(1), transfers)+ assert.Equal(t, int64(0), deletes)+ r.checkFiles(t, "file1.txt", "index.html", "sub/file2.txt", "sub/index.html", "sub/deep/index.html")+ assert.Contains(t, r.read(t, "sub/index.html"), "deep/")+ assert.NotContains(t, r.read(t, "sub/deep/index.html"), "file3.txt")+ } else {+ // The directory only existed because of the listing, so it goes+ assert.Equal(t, int64(1), transfers)+ assert.Equal(t, int64(1), deletes)+ r.checkFiles(t, "file1.txt", "index.html", "sub/file2.txt", "sub/index.html")+ assert.NotContains(t, r.read(t, "sub/index.html"), "deep/")++ // Removing sub/file2.txt cascades: sub is now only listings+ o, err = r.Fremote.NewObject(r.ctx, "sub/file2.txt")+ require.NoError(t, err)+ require.NoError(t, o.Remove(r.ctx))+ transfers, deletes = r.index(t)+ assert.Equal(t, int64(1), transfers)+ assert.Equal(t, int64(1), deletes)+ r.checkFiles(t, "file1.txt", "index.html")+ assert.NotContains(t, r.read(t, "index.html"), "sub/")+ }+}++func TestIndexFilter(t *testing.T) {+ r := newIndexRun(t)+ r.WriteObject(r.ctx, "sub/deep/index.html", "hand written", t1)+ r.WriteObject(r.ctx, "other/notes.md", "notes", t1)++ // Excluded directories are not listed and not touched, and a+ // directory whose files are all excluded still gets an empty listing+ fi, err := filter.NewFilter(nil)+ require.NoError(t, err)+ require.NoError(t, fi.AddRule("- /sub/deep/**"))+ require.NoError(t, fi.AddRule("- *.md"))+ r.ctx = filter.ReplaceConfig(r.ctx, fi)++ r.index(t)+ r.checkFiles(t, "file1.txt", "index.html", "sub/file2.txt", "sub/index.html", "sub/deep/file3.txt", "sub/deep/index.html", "other/notes.md", "other/index.html")+ assert.Equal(t, "hand written", r.read(t, "sub/deep/index.html"))+ assert.NotContains(t, r.read(t, "sub/index.html"), "deep/")+ other := r.read(t, "other/index.html")+ assert.NotContains(t, other, "notes.md")+ assert.Contains(t, r.read(t, "index.html"), `<a href="other/">other/</a>`)++ // A second run changes nothing, in particular it doesn't delete+ // other/index.html even though other/ has no visible content+ transfers, deletes := r.index(t)+ assert.Equal(t, int64(0), transfers)+ assert.Equal(t, int64(0), deletes)+}++func TestIndexRules(t *testing.T) {+ r := newIndexRun(t)+ r.WriteObject(r.ctx, "sub/deep/index.html", "hand written", t1)++ // A directory excluded by the index rules is still listed by+ // its parent but nothing is written or deleted in it+ r.opt.Rules.ExcludeRule = []string{"/sub/deep/**"}+ r.index(t)+ r.checkFiles(t, "file1.txt", "index.html", "sub/file2.txt", "sub/index.html", "sub/deep/file3.txt", "sub/deep/index.html")+ assert.Equal(t, "hand written", r.read(t, "sub/deep/index.html"))+ assert.Contains(t, r.read(t, "sub/index.html"), `<a href="deep/">deep/</a>`)++ // Include rules imply excluding everything else+ r.opt.Rules = filter.RulesOpt{IncludeRule: []string{"/sub/**"}}+ o, err := r.Fremote.NewObject(r.ctx, "index.html")+ require.NoError(t, err)+ require.NoError(t, o.Remove(r.ctx))+ transfers, _ := r.index(t)+ assert.Equal(t, int64(1), transfers) // sub/deep/index.html+ r.checkFiles(t, "file1.txt", "sub/file2.txt", "sub/index.html", "sub/deep/file3.txt", "sub/deep/index.html")+ assert.NotEqual(t, "hand written", r.read(t, "sub/deep/index.html"))+}++func TestIndexMaxDepth(t *testing.T) {+ r := newIndexRun(t)+ r.opt.MaxDepth = 1+ r.index(t)+ r.checkFiles(t, "file1.txt", "index.html", "sub/file2.txt", "sub/deep/file3.txt")+ assert.Contains(t, r.read(t, "index.html"), `<a href="sub/">sub/</a>`)+}++func TestIndexLinkIndex(t *testing.T) {+ r := newIndexRun(t)+ r.opt.LinkIndex = true+ r.index(t)+ assert.Contains(t, r.read(t, "index.html"), `<a href="sub/index.html">sub/</a>`)+ html := r.read(t, "sub/index.html")+ assert.Contains(t, html, `<a href="deep/index.html">deep/</a>`)+ assert.Contains(t, html, `<a href="../index.html">`)+}++// indexModTimes returns the ModTime of each entry in the json listing at remote+func (r *indexRun) indexModTimes(t *testing.T, remote string) map[string]time.Time {+ var items []struct {+ Name string+ ModTime string+ }+ require.NoError(t, json.Unmarshal([]byte(r.read(t, remote)), &items))+ modTimes := map[string]time.Time{}+ for _, item := range items {+ if item.ModTime == "" {+ continue+ }+ modTime, err := time.Parse(time.RFC3339Nano, item.ModTime)+ require.NoError(t, err)+ modTimes[item.Name] = modTime+ }+ return modTimes+}++func TestIndexDirTime(t *testing.T) {+ r := newIndexRun(t)+ r.opt.NoModTime = false+ r.opt.Outputs = []string{"index.json=json"}+ precision := r.Fremote.Precision()++ // newest: sub shows the newest time below it, and so does its listing+ r.opt.DirTime = operations.IndexDirTimeNewest+ r.index(t)+ modTimes := r.indexModTimes(t, "index.json")+ fstest.AssertTimeEqualWithPrecision(t, "file1.txt", t1, modTimes["file1.txt"], precision)+ fstest.AssertTimeEqualWithPrecision(t, "sub", t3, modTimes["sub"], precision)+ o, err := r.Fremote.NewObject(r.ctx, "index.json")+ require.NoError(t, err)+ fstest.AssertTimeEqualWithPrecision(t, "index.json", t3, o.ModTime(r.ctx), precision)++ // none: no directory times+ r.opt.DirTime = operations.IndexDirTimeNone+ r.index(t)+ modTimes = r.indexModTimes(t, "index.json")+ fstest.AssertTimeEqualWithPrecision(t, "file1.txt", t1, modTimes["file1.txt"], precision)+ _, found := modTimes["sub"]+ assert.False(t, found)++ // no modtime: no times at all+ r.opt.NoModTime = true+ r.index(t)+ assert.Empty(t, r.indexModTimes(t, "index.json"))+}++func TestIndexTemplate(t *testing.T) {+ r := newIndexRun(t)+ tmpl := filepath.Join(t.TempDir(), "list.tmpl")+ require.NoError(t, os.WriteFile(tmpl, []byte("{{.Path}}\n{{range .Entries}}{{.Leaf}} {{.MimeType}}\n{{end}}"), 0600))+ r.opt.Outputs = []string{"list.txt=" + tmpl}+ r.index(t)+ assert.Equal(t, "/sub/\ndeep/ inode/directory\nfile2.txt text/plain; charset=utf-8\n", r.read(t, "sub/list.txt"))+ r.checkMimeType(t, "sub/list.txt", "text/plain; charset=utf-8")++ // Bad outputs+ for _, output := range []string{"index.html", "=html", "a/b=html", "index.html=nosuchformat"} {+ r.opt.Outputs = []string{output}+ assert.Error(t, operations.Index(r.ctx, r.Fremote, &r.opt), output)+ }+}++func TestIndexTemplateBuiltin(t *testing.T) {+ for format := range map[string]bool{"html": true, "json": true, "caddy": true} {+ text, err := operations.IndexTemplate(format)+ require.NoError(t, err)+ assert.NotEmpty(t, text)+ }+ _, err := operations.IndexTemplate("nosuchformat")+ assert.Error(t, err)+}++func TestRcIndex(t *testing.T) {+ r, call := rcNewRun(t, "operations/index")+ r.WriteObject(context.Background(), "sub/a", "a", t1)+ in := rc.Params{+ "fs": r.FremoteName,+ "opt": rc.Params{+ "output": []string{"index.json=json"},+ "noModTime": true,+ "maxDepth": 1,+ },+ }+ out, err := call.Fn(context.Background(), in)+ require.NoError(t, err)+ assert.Nil(t, out)+ _, err = r.Fremote.NewObject(context.Background(), "index.json")+ assert.NoError(t, err)+ _, err = r.Fremote.NewObject(context.Background(), "sub/index.json")+ assert.ErrorIs(t, err, fs.ErrorObjectNotFound)+}++// TestIndexNoHash checks the listings are read and compared on a+// backend without hashes so that unchanged listings aren't rewritten+func TestIndexNoHash(t *testing.T) {+ if *fstest.RemoteName != "" {+ t.Skip("Skipping test on non local remote")+ }+ opt := operations.IndexOptDefault+ opt.NoModTime = true+ r := &indexRun{Run: fstest.NewRun(t), ctx: context.Background(), opt: opt}+ f, err := fs.NewFs(r.ctx, ":crypt,remote='"+r.Fremote.Root()+"',password="+obscure.MustObscure("potato")+":")+ require.NoError(t, err)+ require.Equal(t, 0, f.Hashes().Count())+ r.Fremote = f+ r.writeFiles()++ transfers, _ := r.index(t)+ assert.Equal(t, int64(3), transfers)+ transfers, _ = r.index(t)+ assert.Equal(t, int64(0), transfers)+ r.WriteObject(r.ctx, "sub/file4.txt", "hello4", t1)+ transfers, _ = r.index(t)+ assert.Equal(t, int64(1), transfers)+ assert.Contains(t, r.read(t, "sub/index.html"), "file4.txt")+}diff --git a/fs/operations/rc.go b/fs/operations/rc.goindex cdaa8f854..e0138a984 100644--- a/fs/operations/rc.go+++ b/fs/operations/rc.go@@ -11,6 +11,7 @@ import ("mime/multipart""net/http""path"+ "slices""strings""time""unicode/utf8"@@ -695,6 +696,44 @@ func rcDu(ctx context.Context, in rc.Params) (out rc.Params, err error) {return out, nil}+func init() {+ rc.Add(rc.Call{+ Path: "operations/index",+ Fn: rcIndex,+ Title: "Write static directory listings into a remote",+ Help: `This takes the following parameters:++- fs - a remote name string e.g. "drive:"+- opt - a dictionary of options to control the indexing (optional)+ - output - list of listings to write in each directory as "NAME=FORMAT" (default ["index.html=html"])+ - rules - which directories get listings, e.g. {"ExcludeRule": ["/private/**"]}+ - maxDepth - only make listings this many directories deep (default -1 for no limit)+ - linkIndex - link to "dir/NAME" rather than "dir/"+ - dirTime - how to work out directory times: "newest", "dir" or "none"+ - noModTime - don't show modification times+ - rewrite - write every listing even if it is unchanged++See the [index](/commands/rclone_index/) command for more information on the above.+`,+ })+}++// Index a remote+func rcIndex(ctx context.Context, in rc.Params) (out rc.Params, err error) {+ f, err := rc.GetFs(ctx, in)+ if err != nil {+ return nil, err+ }+ opt := IndexOptDefault+ // Decoding into a slice reuses its backing array so don't share it with the defaults+ opt.Outputs = slices.Clone(opt.Outputs)+ err = in.GetStruct("opt", &opt)+ if rc.NotErrParamNotFound(err) {+ return nil, err+ }+ return nil, Index(ctx, f, &opt)+}+func init() {rc.Add(rc.Call{Path: "operations/check",diff --git a/lib/http/templates/caddy.json b/lib/http/templates/caddy.jsonnew file mode 100644index 000000000..965f166cb--- /dev/null+++ b/lib/http/templates/caddy.json@@ -0,0 +1 @@+[{{range $i, $e := .Entries}}{{if $i}},{{end}}{"name":{{json $e.Leaf}},"size":{{if $e.IsDir}}4096{{else}}{{$e.Size}}{{end}},"url":{{if hasPrefix $e.URL "./"}}{{json $e.URL}}{{else}}{{json (print "./" $e.URL)}}{{end}},"mod_time":{{json $e.ModTime}},"mode":{{if $e.IsDir}}2147484141{{else}}420{{end}},"is_dir":{{$e.IsDir}},"is_symlink":false}{{end}}]diff --git a/lib/http/templates/index.json b/lib/http/templates/index.jsonnew file mode 100644index 000000000..f56da58a0--- /dev/null+++ b/lib/http/templates/index.json@@ -0,0 +1,5 @@+[+{{- range $i, $e := .Entries}}{{if $i}},{{end}}+{"Path":{{json (trimSuffix $e.Leaf "/")}},"Name":{{json (trimSuffix $e.Leaf "/")}},"Size":{{if $e.IsDir}}-1{{else}}{{$e.Size}}{{end}},"MimeType":{{json $e.MimeType}},"ModTime":{{if $e.ModTime | afterEpoch}}{{json $e.ModTime}}{{else}}""{{end}},"IsDir":{{$e.IsDir}}}+{{- end}}+]