"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
http: add basic metadata and provide it via serve
Oleg Kunitsynecea0cd6f98b
diff --git a/backend/http/http.go b/backend/http/http.goindex 0a3cdb7b5..253bc04ab 100644--- a/backend/http/http.go+++ b/backend/http/http.go@@ -11,6 +11,7 @@ import ("io""mime""net/http"+ "net/textproto""net/url""path""strings"@@ -37,6 +38,10 @@ func init() {Description: "HTTP",NewFs: NewFs,CommandHelp: commandHelp,+ MetadataInfo: &fs.MetadataInfo{+ System: systemMetadataInfo,+ Help: `HTTP metadata keys are case insensitive and are always returned in lower case.`,+ },Options: []fs.Option{{Name: "url",Help: "URL of HTTP host to connect to.\n\nE.g. \"https://example.com\", or \"https://user:[email protected]\" to use a username and password.",@@ -98,6 +103,40 @@ sizes of any files, and some files that don't exist may be in the listing.`,fs.Register(fsi)}+// system metadata keys which this backend owns+var systemMetadataInfo = map[string]fs.MetadataHelp{+ "cache-control": {+ Help: "Cache-Control header",+ Type: "string",+ Example: "no-cache",+ },+ "content-disposition": {+ Help: "Content-Disposition header",+ Type: "string",+ Example: "inline",+ },+ "content-disposition-filename": {+ Help: "Filename retrieved from Content-Disposition header",+ Type: "string",+ Example: "file.txt",+ },+ "content-encoding": {+ Help: "Content-Encoding header",+ Type: "string",+ Example: "gzip",+ },+ "content-language": {+ Help: "Content-Language header",+ Type: "string",+ Example: "en-US",+ },+ "content-type": {+ Help: "Content-Type header",+ Type: "string",+ Example: "text/plain",+ },+}+// Options defines the configuration for this backendtype Options struct {Endpoint string `config:"url"`@@ -126,6 +165,13 @@ type Object struct {size int64modTime time.TimecontentType string++ // Metadata as pointers to strings as they often won't be present+ contentDisposition *string // Content-Disposition: header+ contentDispositionFilename *string // Filename retrieved from Content-Disposition: header+ cacheControl *string // Cache-Control: header+ contentEncoding *string // Content-Encoding: header+ contentLanguage *string // Content-Language: header}// statusError returns an error if the res contained an error@@ -277,6 +323,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, eci: ci,}f.features = (&fs.Features{+ ReadMetadata: true,CanHaveEmptyDirectories: true,}).Fill(ctx, f)@@ -429,6 +476,29 @@ func parse(base *url.URL, in io.Reader) (names []string, err error) {return names, nil}+// parseFilename extracts the filename from a Content-Disposition header+func parseFilename(contentDisposition string) (string, error) {+ // Normalize the contentDisposition to canonical MIME format+ mediaType, params, err := mime.ParseMediaType(contentDisposition)+ if err != nil {+ return "", fmt.Errorf("failed to parse contentDisposition: %v", err)+ }++ // Check if the contentDisposition is an attachment+ if strings.ToLower(mediaType) != "attachment" {+ return "", fmt.Errorf("not an attachment: %s", mediaType)+ }++ // Extract the filename from the parameters+ filename, ok := params["filename"]+ if !ok {+ return "", fmt.Errorf("filename not found in contentDisposition")+ }++ // Decode filename if it contains special encoding+ return textproto.TrimString(filename), nil+}+// Adds the configured headers to the request if anyfunc addHeaders(req *http.Request, opt *Options) {for i := 0; i < len(opt.Headers); i += 2 {@@ -577,6 +647,9 @@ func (o *Object) String() string {// Remote the name of the remote HTTP file, relative to the fs rootfunc (o *Object) Remote() string {+ if o.contentDispositionFilename != nil {+ return *o.contentDispositionFilename+ }return o.remote}@@ -634,6 +707,29 @@ func (o *Object) decodeMetadata(ctx context.Context, res *http.Response) error {o.modTime = to.contentType = res.Header.Get("Content-Type")o.size = rest.ParseSizeFromHeaders(res.Header)+ contentDisposition := res.Header.Get("Content-Disposition")+ if contentDisposition != "" {+ o.contentDisposition = &contentDisposition+ }+ if o.contentDisposition != nil {+ var filename string+ filename, err = parseFilename(*o.contentDisposition)+ if err == nil && filename != "" {+ o.contentDispositionFilename = &filename+ }+ }+ cacheControl := res.Header.Get("Cache-Control")+ if cacheControl != "" {+ o.cacheControl = &cacheControl+ }+ contentEncoding := res.Header.Get("Content-Encoding")+ if contentEncoding != "" {+ o.contentEncoding = &contentEncoding+ }+ contentLanguage := res.Header.Get("Content-Language")+ if contentLanguage != "" {+ o.contentLanguage = &contentLanguage+ }// If NoSlash is set then check ContentType to see if it is a directoryif o.fs.opt.NoSlash {@@ -772,6 +868,30 @@ func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[str}}+// Metadata returns metadata for an object+//+// It should return nil if there is no Metadata+func (o *Object) Metadata(ctx context.Context) (metadata fs.Metadata, err error) {+ metadata = make(fs.Metadata, 6)+ if o.contentType != "" {+ metadata["content-type"] = o.contentType+ }++ // Set system metadata+ setMetadata := func(k string, v *string) {+ if v == nil || *v == "" {+ return+ }+ metadata[k] = *v+ }+ setMetadata("content-disposition", o.contentDisposition)+ setMetadata("content-disposition-filename", o.contentDispositionFilename)+ setMetadata("cache-control", o.cacheControl)+ setMetadata("content-language", o.contentLanguage)+ setMetadata("content-encoding", o.contentEncoding)+ return metadata, nil+}+// Check the interfaces are satisfiedvar (_ fs.Fs = &Fs{}@@ -779,4 +899,5 @@ var (_ fs.Object = &Object{}_ fs.MimeTyper = &Object{}_ fs.Commander = &Fs{}+ _ fs.Metadataer = &Object{})diff --git a/backend/http/http_internal_test.go b/backend/http/http_internal_test.goindex 90a2bae78..2bc261eb2 100644--- a/backend/http/http_internal_test.go+++ b/backend/http/http_internal_test.go@@ -60,6 +60,17 @@ func prepareServer(t *testing.T) configmap.Simple {what := fmt.Sprintf("%s %s: Header ", r.Method, r.URL.Path)assert.Equal(t, headers[1], r.Header.Get(headers[0]), what+headers[0])assert.Equal(t, headers[3], r.Header.Get(headers[2]), what+headers[2])++ // Set the content disposition header for the fifth file+ // later we will check if it is set using the metadata method+ if r.URL.Path == "/five.txt.gz" {+ w.Header().Set("Content-Disposition", "attachment; filename=\"five.txt.gz\"")+ w.Header().Set("Content-Type", "text/plain; charset=utf-8")+ w.Header().Set("Cache-Control", "no-cache")+ w.Header().Set("Content-Language", "en-US")+ w.Header().Set("Content-Encoding", "gzip")+ }+fileServer.ServeHTTP(w, r)})@@ -102,27 +113,33 @@ func testListRoot(t *testing.T, f fs.Fs, noSlash bool) {sort.Sort(entries)- require.Equal(t, 4, len(entries))+ require.Equal(t, 5, len(entries))e := entries[0]- assert.Equal(t, "four", e.Remote())+ assert.Equal(t, "five.txt.gz", e.Remote())assert.Equal(t, int64(-1), e.Size())- _, ok := e.(fs.Directory)+ _, ok := e.(fs.Object)assert.True(t, ok)e = entries[1]+ assert.Equal(t, "four", e.Remote())+ assert.Equal(t, int64(-1), e.Size())+ _, ok = e.(fs.Directory)+ assert.True(t, ok)++ e = entries[2]assert.Equal(t, "one%.txt", e.Remote())assert.Equal(t, int64(5+lineEndSize), e.Size())_, ok = e.(*Object)assert.True(t, ok)- e = entries[2]+ e = entries[3]assert.Equal(t, "three", e.Remote())assert.Equal(t, int64(-1), e.Size())_, ok = e.(fs.Directory)assert.True(t, ok)- e = entries[3]+ e = entries[4]assert.Equal(t, "two.html", e.Remote())if noSlash {assert.Equal(t, int64(-1), e.Size())@@ -218,6 +235,23 @@ func TestNewObjectWithLeadingSlash(t *testing.T) {assert.Equal(t, fs.ErrorObjectNotFound, err)}+func TestNewObjectWithMetadata(t *testing.T) {+ f := prepare(t)+ o, err := f.NewObject(context.Background(), "/five.txt.gz")+ require.NoError(t, err)+ assert.Equal(t, "five.txt.gz", o.Remote())+ ho, ok := o.(*Object)+ assert.True(t, ok)+ metadata, err := ho.Metadata(context.Background())+ require.NoError(t, err)+ assert.Equal(t, "text/plain; charset=utf-8", metadata["content-type"])+ assert.Equal(t, "attachment; filename=\"five.txt.gz\"", metadata["content-disposition"])+ assert.Equal(t, "five.txt.gz", metadata["content-disposition-filename"])+ assert.Equal(t, "no-cache", metadata["cache-control"])+ assert.Equal(t, "en-US", metadata["content-language"])+ assert.Equal(t, "gzip", metadata["content-encoding"])+}+func TestOpen(t *testing.T) {m := prepareServer(t)diff --git a/backend/http/test/files/five.txt.gz b/backend/http/test/files/five.txt.gznew file mode 100644index 000000000..fa65c0084Binary files /dev/null and b/backend/http/test/files/five.txt.gz differdiff --git a/cmd/serve/serve.go b/cmd/serve/serve.goindex fcf3a445b..ab58e3e4c 100644--- a/cmd/serve/serve.go+++ b/cmd/serve/serve.go@@ -23,7 +23,15 @@ subcommand to specify the protocol, e.g.rclone serve http remote:` + "```" + `-Each subcommand has its own options which you can see in their help.`,+When the "--metadata" flag is enabled, the following metadata fields will be provided as headers:+- "content-disposition"+- "cache-control"+- "content-language"+- "content-encoding"+Note: The availability of these fields depends on whether the remote supports metadata.++Each subcommand has its own options which you can see in their help.+`,Annotations: map[string]string{"versionIntroduced": "v1.39",},diff --git a/docs/content/overview.md b/docs/content/overview.mdindex 68cf44895..28e8db9b4 100644--- a/docs/content/overview.md+++ b/docs/content/overview.md@@ -34,7 +34,7 @@ Here is an overview of the major features of each cloud storage system.| Google Photos | - | - | No | Yes | R | - || HDFS | - | R/W | No | No | - | - || HiDrive | HiDrive ¹² | R/W | No | No | - | - |-| HTTP | - | R | No | No | R | - |+| HTTP | - | R | No | No | R | R || iCloud Drive | - | R | No | No | - | - || Internet Archive | MD5, SHA1, CRC32 | R/W ¹¹ | No | No | - | RWU || Jottacloud | MD5 | R/W | Yes | No | R | RW |diff --git a/fs/object/object.go b/fs/object/object.goindex 9b4e25afe..07dd8a320 100644--- a/fs/object/object.go+++ b/fs/object/object.go@@ -191,11 +191,12 @@ var _ fs.Fs = MemoryFs// MemoryObject is an in memory objecttype MemoryObject struct {- remote string- modTime time.Time- content []byte- meta fs.Metadata- fs fs.Fs+ remote string+ modTime time.Time+ content []byte+ meta fs.Metadata+ fs fs.Fs+ mimeType string}// NewMemoryObject returns an in memory Object with the modTime and content passed in@@ -214,6 +215,12 @@ func (o *MemoryObject) WithMetadata(meta fs.Metadata) *MemoryObject {return o}+// WithMimeType adds mimeType to the MemoryObject+func (o *MemoryObject) WithMimeType(mimeType string) *MemoryObject {+ o.mimeType = mimeType+ return o+}+// Content returns the underlying bufferfunc (o *MemoryObject) Content() []byte {return o.content@@ -329,8 +336,14 @@ func (o *MemoryObject) Metadata(ctx context.Context) (fs.Metadata, error) {return o.meta, nil}+// MimeType on the object+func (o *MemoryObject) MimeType(ctx context.Context) string {+ return o.mimeType+}+// Check interfacesvar (_ fs.Object = (*MemoryObject)(nil)+ _ fs.MimeTyper = (*MemoryObject)(nil)_ fs.Metadataer = (*MemoryObject)(nil))diff --git a/fs/object/object_test.go b/fs/object/object_test.goindex be90c771e..d1597acc3 100644--- a/fs/object/object_test.go+++ b/fs/object/object_test.go@@ -87,6 +87,7 @@ func TestMemoryObject(t *testing.T) {content = content[:6] // make some extra capo := object.NewMemoryObject(remote, now, content)+ o.WithMimeType("text/plain; charset=utf-8")assert.Equal(t, content, o.Content())assert.Equal(t, object.MemoryFs, o.Fs())@@ -95,6 +96,7 @@ func TestMemoryObject(t *testing.T) {assert.Equal(t, now, o.ModTime(context.Background()))assert.Equal(t, int64(len(content)), o.Size())assert.Equal(t, true, o.Storable())+ assert.Equal(t, "text/plain; charset=utf-8", o.MimeType(context.Background()))Hash, err := o.Hash(context.Background(), hash.MD5)assert.NoError(t, err)diff --git a/lib/http/serve/serve.go b/lib/http/serve/serve.goindex 21f3c3322..244509b47 100644--- a/lib/http/serve/serve.go+++ b/lib/http/serve/serve.go@@ -39,6 +39,26 @@ func Object(w http.ResponseWriter, r *http.Request, o fs.Object) {modTime := o.ModTime(r.Context())w.Header().Set("Last-Modified", modTime.UTC().Format(http.TimeFormat))+ // Set metadata headers if present+ metadata, err := fs.GetMetadata(r.Context(), o)+ if err != nil {+ fs.Debugf(o, "Request get metadata error: %v", err)+ }+ if metadata != nil {+ if metadata["content-disposition"] != "" {+ w.Header().Set("Content-Disposition", metadata["content-disposition"])+ }+ if metadata["cache-control"] != "" {+ w.Header().Set("Cache-Control", metadata["cache-control"])+ }+ if metadata["content-language"] != "" {+ w.Header().Set("Content-Language", metadata["content-language"])+ }+ if metadata["content-encoding"] != "" {+ w.Header().Set("Content-Encoding", metadata["content-encoding"])+ }+ }+if r.Method == "HEAD" {return}diff --git a/lib/http/serve/serve_test.go b/lib/http/serve/serve_test.goindex 79daf773e..ed716cad9 100644--- a/lib/http/serve/serve_test.go+++ b/lib/http/serve/serve_test.go@@ -8,6 +8,8 @@ import ("testing""time"+ "github.com/rclone/rclone/fs"+ "github.com/rclone/rclone/fs/object""github.com/rclone/rclone/fstest/mockobject""github.com/stretchr/testify/assert")@@ -82,3 +84,23 @@ func TestObjectBadRange(t *testing.T) {body, _ := io.ReadAll(resp.Body)assert.Equal(t, "Bad Request\n", string(body))}++func TestObjectHEADMetadata(t *testing.T) {+ w := httptest.NewRecorder()+ r := httptest.NewRequest("HEAD", "http://example.com/aFile", nil)+ m := fs.Metadata{+ "content-disposition": "inline",+ "cache-control": "no-cache",+ "content-language": "en",+ "content-encoding": "gzip",+ }+ o := object.NewMemoryObject("aFile", time.Now(), []byte("")).+ WithMetadata(m).WithMimeType("text/plain; charset=utf-8")+ Object(w, r, o)+ resp := w.Result()+ assert.Equal(t, "text/plain; charset=utf-8", resp.Header.Get("Content-Type"))+ assert.Equal(t, "inline", resp.Header.Get("Content-Disposition"))+ assert.Equal(t, "no-cache", resp.Header.Get("Cache-Control"))+ assert.Equal(t, "en", resp.Header.Get("Content-Language"))+ assert.Equal(t, "gzip", resp.Header.Get("Content-Encoding"))+}