"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
index: add a command to write static directory listings into a remote
Nick Craig-Wood1897074971b3
diff --git a/cmd/all/all.go b/cmd/all/all.goindex 912cde629..d17601dfe 100644--- a/cmd/all/all.go+++ b/cmd/all/all.go@@ -33,6 +33,7 @@ import (_ "github.com/rclone/rclone/cmd/gitannex"_ "github.com/rclone/rclone/cmd/gui"_ "github.com/rclone/rclone/cmd/hashsum"+ _ "github.com/rclone/rclone/cmd/index"_ "github.com/rclone/rclone/cmd/link"_ "github.com/rclone/rclone/cmd/listremotes"_ "github.com/rclone/rclone/cmd/ls"diff --git a/cmd/index/index.go b/cmd/index/index.gonew file mode 100644index 000000000..72cb07ede--- /dev/null+++ b/cmd/index/index.go@@ -0,0 +1,66 @@+// Package index provides the index command.+package index++import (+ "context"+ _ "embed"+ "fmt"+ "strings"++ "github.com/rclone/rclone/cmd"+ "github.com/rclone/rclone/fs/config/flags"+ "github.com/rclone/rclone/fs/operations"+ "github.com/spf13/cobra"+)++var (+ opt = operations.IndexOptDefault+ printTemplate string+)++func init() {+ cmd.Root.AddCommand(commandDefinition)+ cmdFlags := commandDefinition.Flags()+ flags.StringArrayVarP(cmdFlags, &opt.Outputs, "output", "", opt.Outputs, "Listing to write in each directory as NAME=FORMAT", "")+ flags.StringArrayVarP(cmdFlags, &opt.Rules.FilterRule, "index-filter", "", nil, "Add a rule for which directories get listings", "")+ flags.StringArrayVarP(cmdFlags, &opt.Rules.FilterFrom, "index-filter-from", "", nil, "Read directory rules from a file (use - to read from stdin)", "")+ flags.StringArrayVarP(cmdFlags, &opt.Rules.ExcludeRule, "index-exclude", "", nil, "Don't write listings in directories matching pattern", "")+ flags.StringArrayVarP(cmdFlags, &opt.Rules.ExcludeFrom, "index-exclude-from", "", nil, "Read directory exclude patterns from file (use - to read from stdin)", "")+ flags.StringArrayVarP(cmdFlags, &opt.Rules.IncludeRule, "index-include", "", nil, "Only write listings in directories matching pattern", "")+ flags.StringArrayVarP(cmdFlags, &opt.Rules.IncludeFrom, "index-include-from", "", nil, "Read directory include patterns from file (use - to read from stdin)", "")+ flags.IntVarP(cmdFlags, &opt.MaxDepth, "index-max-depth", "", opt.MaxDepth, "Only write listings this many directories deep (-1 for no limit)", "")+ flags.BoolVarP(cmdFlags, &opt.LinkIndex, "link-index", "", opt.LinkIndex, "Link to dir/NAME rather than dir/ for hosts without index documents", "")+ flags.FVarP(cmdFlags, &opt.DirTime, "dir-time", "", "How to work out the time shown for a directory", "")+ flags.BoolVarP(cmdFlags, &opt.NoModTime, "no-modtime", "", opt.NoModTime, "Don't show modification times in listings", "")+ flags.BoolVarP(cmdFlags, &opt.Rewrite, "index-rewrite", "", opt.Rewrite, "Write every listing even if it is unchanged", "")+ flags.StringVarP(cmdFlags, &printTemplate, "print-template", "", "", "Print the built-in template for FORMAT and exit", "")+}++//go:embed index.md+var indexHelp string++var commandDefinition = &cobra.Command{+ Use: "index remote:path",+ Short: `Write static directory listings into a remote.`,+ Long: strings.TrimSpace(indexHelp),+ Annotations: map[string]string{+ "versionIntroduced": "v1.76",+ "groups": "Copy,Filter,Listing,Important",+ },+ RunE: func(command *cobra.Command, args []string) error {+ if printTemplate != "" {+ text, err := operations.IndexTemplate(printTemplate)+ if err != nil {+ return err+ }+ fmt.Print(text)+ return nil+ }+ cmd.CheckArgs(1, 1, command, args)+ fdst := cmd.NewFsDir(args)+ cmd.Run(true, true, command, func() error {+ return operations.Index(context.Background(), fdst, &opt)+ })+ return nil+ },+}diff --git a/cmd/index/index.md b/cmd/index/index.mdnew file mode 100644index 000000000..5bb61cf5b--- /dev/null+++ b/cmd/index/index.md@@ -0,0 +1,145 @@+Write a directory listing, `index.html` by default, into every+directory of `remote:path` so that it can be browsed when it is served+as a static website, for example from an S3 website endpoint, a bucket+behind a CDN, or a plain web server.++This works like `rclone sync`: each run makes the minimum changes needed+to bring the listings into line with the files. Listings which haven't+changed aren't rewritten, so a run with no changes makes no uploads. The+listings never show the listing files themselves.++**Note** that `rclone index` treats every file with an output name+(`index.html` by default) in the directories it indexes as its own, and+will overwrite it or delete it without checking where it came from. To+protect a hand-written index, exclude its directory with+`--index-exclude`.++### Outputs++`--output NAME=FORMAT` says what to write in each directory and can be+repeated. NAME is the file name, and FORMAT is one of the built-in+formats below or the path of a Go template file.++| Format | MIME type | Contents |+|:-------|:----------|:---------|+| `html` | `text/html; charset=utf-8` | The HTML listing from the `rclone serve http` template |+| `json` | `application/json` | JSON in the format of `rclone lsjson` for the directory |+| `caddy` | `application/json` | JSON in the format of Caddy's `file_server browse` |++The default is `--output index.html=html`. The built-in formats are+templates, so they can be printed with `--print-template FORMAT`, copied+and changed. Outputs whose name ends in `.html` or `.htm` are rendered+with Go's `html/template` and everything else with `text/template`, which+has a `json` function for safe encoding. The template data is that of+`rclone serve http` with `.Static` set (see+[rclone serve http](/commands/rclone_serve_http/#template)). Outputs are+uploaded with the MIME type of the built-in format, or that of the+output name's extension for template outputs, since static hosts serve+the stored type.++### Filters++The normal filters (`--include`, `--exclude`, `--filter-from`, `--max-age`, ...)+control what appears in the listings. An excluded file isn't listed, and+an excluded directory isn't listed or indexed.++The `--index-include`, `--index-exclude`, `--index-filter` flags and their+`-from` variants control which directories get listings. They take the same+patterns as the normal filters, applied to directory paths, and+`--index-include` implies excluding all other directories. A directory+excluded here still appears in its parent's listing, but nothing is written+or deleted inside it, so it can have its own hand-made index.+`--index-max-depth` limits how deep listings are written.++### Modification times++Listings show the modification time of each file. When using the s3,+oracleobjectstorage and swift backends rclone stores the modification time as+metadata which isn't returned by the bucket listing, so **every object needs+a HEAD request**, which can mean thousands of transactions per run. Avoid+this with one of:++- `--use-server-modtime` uses the time the object was uploaded instead,+ which is what the web server sends as `Last-Modified`. Recommended for s3+ unless the original modification times matter.+- `--no-modtime` shows no times at all, which is the cheapest option.++`--dir-time` sets the time shown for directories:++- `newest` (default): the newest modification time of anything below the+ directory. Note that this means one new file changes the listing of+ every directory above it.+- `dir`: the directory's own modification time. On most file systems+ writing a listing into a directory changes its time, so this can take+ more than one run to settle.+- `none`: no time.++The listing files are given the newest modification time below their+directory, so the `Last-Modified` header of a listing is meaningful.++### Links++Directory links are `dir/` by default, which works on hosts that serve+index documents: S3 website endpoints, GCS and Azure static websites,+nginx and Apache, and R2 custom domains with a URL rewrite rule. For+hosts that don't (plain S3 REST URLs, B2 friendly URLs) use+`--link-index` to make directory links `dir/index.html` instead.++### Site icon++The listings carry no icon or branding of their own. To give a site an+icon, upload a `favicon.ico` to the root of the remote, which browsers+fetch from there for every page. Exclude it from the listings with+`--exclude /favicon.ico` so it stays on the site but isn't shown.++### Deleting listings++On backends which can't have empty directories (S3, B2 and other+bucket-based storage) a directory only exists while it has files in it.+Once the last file in a directory has been removed its listing is+deleted, since the directory only existed because of it. Backends which+can have empty directories keep a listing for each empty directory.++### Using with sync++A sync from a local build deletes files on the destination which aren't+in the source, which includes the listings. Either index the local build+before syncing:++ rclone index ./public+ rclone sync ./public r2:bucket++or exclude the listings from the sync so it neither uploads nor deletes+them, then index the remote:++ rclone sync ./public r2:bucket --exclude index.html+ rclone index r2:bucket++### Examples++An S3 website bucket, using the upload times so no object needs a HEAD+request:++ rclone index s3:my-site --use-server-modtime++HTML and rclone JSON listings, setting `Cache-Control` on them:++ rclone index s3:bucket --output index.html=html --output index.json=json --header-upload "Cache-Control: max-age=300"++Only list release files, and don't write listings under `/private`:++ rclone index r2:bucket --include "*.{zip,deb,rpm}" --index-exclude "/private/**"++A B2 bucket served from its friendly URLs:++ rclone index b2:bucket --link-index++See what would change:++ rclone index r2:bucket --dry-run -v++Listings which haven't changed aren't normally rewritten, so changing+the headers or metadata set on them needs `--index-rewrite`, which+writes every listing a run would render whether it changed or not:++ rclone index r2:bucket --index-rewrite --header-upload "Cache-Control: max-age=3600"