Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

index: add a command to write static directory listings into a remote

Nick Craig-Wood1897074971b3
diff --git a/cmd/all/all.go b/cmd/all/all.go
index 912cde629..d17601dfe 100644
--- a/cmd/all/all.go
+++ b/cmd/all/all.go
@@ -33,6 +33,7 @@ import (
_ "github.com/rclone/rclone/cmd/gitannex"
_ "github.com/rclone/rclone/cmd/gui"
_ "github.com/rclone/rclone/cmd/hashsum"
+ _ "github.com/rclone/rclone/cmd/index"
_ "github.com/rclone/rclone/cmd/link"
_ "github.com/rclone/rclone/cmd/listremotes"
_ "github.com/rclone/rclone/cmd/ls"
diff --git a/cmd/index/index.go b/cmd/index/index.go
new file mode 100644
index 000000000..72cb07ede
--- /dev/null
+++ b/cmd/index/index.go
@@ -0,0 +1,66 @@
+// Package index provides the index command.
+package index
+
+import (
+ "context"
+ _ "embed"
+ "fmt"
+ "strings"
+
+ "github.com/rclone/rclone/cmd"
+ "github.com/rclone/rclone/fs/config/flags"
+ "github.com/rclone/rclone/fs/operations"
+ "github.com/spf13/cobra"
+)
+
+var (
+ opt = operations.IndexOptDefault
+ printTemplate string
+)
+
+func init() {
+ cmd.Root.AddCommand(commandDefinition)
+ cmdFlags := commandDefinition.Flags()
+ flags.StringArrayVarP(cmdFlags, &opt.Outputs, "output", "", opt.Outputs, "Listing to write in each directory as NAME=FORMAT", "")
+ flags.StringArrayVarP(cmdFlags, &opt.Rules.FilterRule, "index-filter", "", nil, "Add a rule for which directories get listings", "")
+ flags.StringArrayVarP(cmdFlags, &opt.Rules.FilterFrom, "index-filter-from", "", nil, "Read directory rules from a file (use - to read from stdin)", "")
+ flags.StringArrayVarP(cmdFlags, &opt.Rules.ExcludeRule, "index-exclude", "", nil, "Don't write listings in directories matching pattern", "")
+ flags.StringArrayVarP(cmdFlags, &opt.Rules.ExcludeFrom, "index-exclude-from", "", nil, "Read directory exclude patterns from file (use - to read from stdin)", "")
+ flags.StringArrayVarP(cmdFlags, &opt.Rules.IncludeRule, "index-include", "", nil, "Only write listings in directories matching pattern", "")
+ flags.StringArrayVarP(cmdFlags, &opt.Rules.IncludeFrom, "index-include-from", "", nil, "Read directory include patterns from file (use - to read from stdin)", "")
+ flags.IntVarP(cmdFlags, &opt.MaxDepth, "index-max-depth", "", opt.MaxDepth, "Only write listings this many directories deep (-1 for no limit)", "")
+ flags.BoolVarP(cmdFlags, &opt.LinkIndex, "link-index", "", opt.LinkIndex, "Link to dir/NAME rather than dir/ for hosts without index documents", "")
+ flags.FVarP(cmdFlags, &opt.DirTime, "dir-time", "", "How to work out the time shown for a directory", "")
+ flags.BoolVarP(cmdFlags, &opt.NoModTime, "no-modtime", "", opt.NoModTime, "Don't show modification times in listings", "")
+ flags.BoolVarP(cmdFlags, &opt.Rewrite, "index-rewrite", "", opt.Rewrite, "Write every listing even if it is unchanged", "")
+ flags.StringVarP(cmdFlags, &printTemplate, "print-template", "", "", "Print the built-in template for FORMAT and exit", "")
+}
+
+//go:embed index.md
+var indexHelp string
+
+var commandDefinition = &cobra.Command{
+ Use: "index remote:path",
+ Short: `Write static directory listings into a remote.`,
+ Long: strings.TrimSpace(indexHelp),
+ Annotations: map[string]string{
+ "versionIntroduced": "v1.76",
+ "groups": "Copy,Filter,Listing,Important",
+ },
+ RunE: func(command *cobra.Command, args []string) error {
+ if printTemplate != "" {
+ text, err := operations.IndexTemplate(printTemplate)
+ if err != nil {
+ return err
+ }
+ fmt.Print(text)
+ return nil
+ }
+ cmd.CheckArgs(1, 1, command, args)
+ fdst := cmd.NewFsDir(args)
+ cmd.Run(true, true, command, func() error {
+ return operations.Index(context.Background(), fdst, &opt)
+ })
+ return nil
+ },
+}
diff --git a/cmd/index/index.md b/cmd/index/index.md
new file mode 100644
index 000000000..5bb61cf5b
--- /dev/null
+++ b/cmd/index/index.md
@@ -0,0 +1,145 @@
+Write a directory listing, `index.html` by default, into every
+directory of `remote:path` so that it can be browsed when it is served
+as a static website, for example from an S3 website endpoint, a bucket
+behind a CDN, or a plain web server.
+
+This works like `rclone sync`: each run makes the minimum changes needed
+to bring the listings into line with the files. Listings which haven't
+changed aren't rewritten, so a run with no changes makes no uploads. The
+listings never show the listing files themselves.
+
+**Note** that `rclone index` treats every file with an output name
+(`index.html` by default) in the directories it indexes as its own, and
+will overwrite it or delete it without checking where it came from. To
+protect a hand-written index, exclude its directory with
+`--index-exclude`.
+
+### Outputs
+
+`--output NAME=FORMAT` says what to write in each directory and can be
+repeated. NAME is the file name, and FORMAT is one of the built-in
+formats below or the path of a Go template file.
+
+| Format | MIME type | Contents |
+|:-------|:----------|:---------|
+| `html` | `text/html; charset=utf-8` | The HTML listing from the `rclone serve http` template |
+| `json` | `application/json` | JSON in the format of `rclone lsjson` for the directory |
+| `caddy` | `application/json` | JSON in the format of Caddy's `file_server browse` |
+
+The default is `--output index.html=html`. The built-in formats are
+templates, so they can be printed with `--print-template FORMAT`, copied
+and changed. Outputs whose name ends in `.html` or `.htm` are rendered
+with Go's `html/template` and everything else with `text/template`, which
+has a `json` function for safe encoding. The template data is that of
+`rclone serve http` with `.Static` set (see
+[rclone serve http](/commands/rclone_serve_http/#template)). Outputs are
+uploaded with the MIME type of the built-in format, or that of the
+output name's extension for template outputs, since static hosts serve
+the stored type.
+
+### Filters
+
+The normal filters (`--include`, `--exclude`, `--filter-from`, `--max-age`, ...)
+control what appears in the listings. An excluded file isn't listed, and
+an excluded directory isn't listed or indexed.
+
+The `--index-include`, `--index-exclude`, `--index-filter` flags and their
+`-from` variants control which directories get listings. They take the same
+patterns as the normal filters, applied to directory paths, and
+`--index-include` implies excluding all other directories. A directory
+excluded here still appears in its parent's listing, but nothing is written
+or deleted inside it, so it can have its own hand-made index.
+`--index-max-depth` limits how deep listings are written.
+
+### Modification times
+
+Listings show the modification time of each file. When using the s3,
+oracleobjectstorage and swift backends rclone stores the modification time as
+metadata which isn't returned by the bucket listing, so **every object needs
+a HEAD request**, which can mean thousands of transactions per run. Avoid
+this with one of:
+
+- `--use-server-modtime` uses the time the object was uploaded instead,
+ which is what the web server sends as `Last-Modified`. Recommended for s3
+ unless the original modification times matter.
+- `--no-modtime` shows no times at all, which is the cheapest option.
+
+`--dir-time` sets the time shown for directories:
+
+- `newest` (default): the newest modification time of anything below the
+ directory. Note that this means one new file changes the listing of
+ every directory above it.
+- `dir`: the directory's own modification time. On most file systems
+ writing a listing into a directory changes its time, so this can take
+ more than one run to settle.
+- `none`: no time.
+
+The listing files are given the newest modification time below their
+directory, so the `Last-Modified` header of a listing is meaningful.
+
+### Links
+
+Directory links are `dir/` by default, which works on hosts that serve
+index documents: S3 website endpoints, GCS and Azure static websites,
+nginx and Apache, and R2 custom domains with a URL rewrite rule. For
+hosts that don't (plain S3 REST URLs, B2 friendly URLs) use
+`--link-index` to make directory links `dir/index.html` instead.
+
+### Site icon
+
+The listings carry no icon or branding of their own. To give a site an
+icon, upload a `favicon.ico` to the root of the remote, which browsers
+fetch from there for every page. Exclude it from the listings with
+`--exclude /favicon.ico` so it stays on the site but isn't shown.
+
+### Deleting listings
+
+On backends which can't have empty directories (S3, B2 and other
+bucket-based storage) a directory only exists while it has files in it.
+Once the last file in a directory has been removed its listing is
+deleted, since the directory only existed because of it. Backends which
+can have empty directories keep a listing for each empty directory.
+
+### Using with sync
+
+A sync from a local build deletes files on the destination which aren't
+in the source, which includes the listings. Either index the local build
+before syncing:
+
+ rclone index ./public
+ rclone sync ./public r2:bucket
+
+or exclude the listings from the sync so it neither uploads nor deletes
+them, then index the remote:
+
+ rclone sync ./public r2:bucket --exclude index.html
+ rclone index r2:bucket
+
+### Examples
+
+An S3 website bucket, using the upload times so no object needs a HEAD
+request:
+
+ rclone index s3:my-site --use-server-modtime
+
+HTML and rclone JSON listings, setting `Cache-Control` on them:
+
+ rclone index s3:bucket --output index.html=html --output index.json=json --header-upload "Cache-Control: max-age=300"
+
+Only list release files, and don't write listings under `/private`:
+
+ rclone index r2:bucket --include "*.{zip,deb,rpm}" --index-exclude "/private/**"
+
+A B2 bucket served from its friendly URLs:
+
+ rclone index b2:bucket --link-index
+
+See what would change:
+
+ rclone index r2:bucket --dry-run -v
+
+Listings which haven't changed aren't normally rewritten, so changing
+the headers or metadata set on them needs `--index-rewrite`, which
+writes every listing a run would render whether it changed or not:
+
+ rclone index r2:bucket --index-rewrite --header-upload "Cache-Control: max-age=3600"