Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

http: add basic metadata and provide it via serve

Oleg Kunitsynecea0cd6f98b
diff --git a/backend/http/http.go b/backend/http/http.go
index 0a3cdb7b5..253bc04ab 100644
--- a/backend/http/http.go
+++ b/backend/http/http.go
@@ -11,6 +11,7 @@ import (
"io"
"mime"
"net/http"
+ "net/textproto"
"net/url"
"path"
"strings"
@@ -37,6 +38,10 @@ func init() {
Description: "HTTP",
NewFs: NewFs,
CommandHelp: commandHelp,
+ MetadataInfo: &fs.MetadataInfo{
+ System: systemMetadataInfo,
+ Help: `HTTP metadata keys are case insensitive and are always returned in lower case.`,
+ },
Options: []fs.Option{{
Name: "url",
Help: "URL of HTTP host to connect to.\n\nE.g. \"https://example.com\", or \"https://user:[email protected]\" to use a username and password.",
@@ -98,6 +103,40 @@ sizes of any files, and some files that don't exist may be in the listing.`,
fs.Register(fsi)
}
+// system metadata keys which this backend owns
+var systemMetadataInfo = map[string]fs.MetadataHelp{
+ "cache-control": {
+ Help: "Cache-Control header",
+ Type: "string",
+ Example: "no-cache",
+ },
+ "content-disposition": {
+ Help: "Content-Disposition header",
+ Type: "string",
+ Example: "inline",
+ },
+ "content-disposition-filename": {
+ Help: "Filename retrieved from Content-Disposition header",
+ Type: "string",
+ Example: "file.txt",
+ },
+ "content-encoding": {
+ Help: "Content-Encoding header",
+ Type: "string",
+ Example: "gzip",
+ },
+ "content-language": {
+ Help: "Content-Language header",
+ Type: "string",
+ Example: "en-US",
+ },
+ "content-type": {
+ Help: "Content-Type header",
+ Type: "string",
+ Example: "text/plain",
+ },
+}
+
// Options defines the configuration for this backend
type Options struct {
Endpoint string `config:"url"`
@@ -126,6 +165,13 @@ type Object struct {
size int64
modTime time.Time
contentType string
+
+ // Metadata as pointers to strings as they often won't be present
+ contentDisposition *string // Content-Disposition: header
+ contentDispositionFilename *string // Filename retrieved from Content-Disposition: header
+ cacheControl *string // Cache-Control: header
+ contentEncoding *string // Content-Encoding: header
+ contentLanguage *string // Content-Language: header
}
// statusError returns an error if the res contained an error
@@ -277,6 +323,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
ci: ci,
}
f.features = (&fs.Features{
+ ReadMetadata: true,
CanHaveEmptyDirectories: true,
}).Fill(ctx, f)
@@ -429,6 +476,29 @@ func parse(base *url.URL, in io.Reader) (names []string, err error) {
return names, nil
}
+// parseFilename extracts the filename from a Content-Disposition header
+func parseFilename(contentDisposition string) (string, error) {
+ // Normalize the contentDisposition to canonical MIME format
+ mediaType, params, err := mime.ParseMediaType(contentDisposition)
+ if err != nil {
+ return "", fmt.Errorf("failed to parse contentDisposition: %v", err)
+ }
+
+ // Check if the contentDisposition is an attachment
+ if strings.ToLower(mediaType) != "attachment" {
+ return "", fmt.Errorf("not an attachment: %s", mediaType)
+ }
+
+ // Extract the filename from the parameters
+ filename, ok := params["filename"]
+ if !ok {
+ return "", fmt.Errorf("filename not found in contentDisposition")
+ }
+
+ // Decode filename if it contains special encoding
+ return textproto.TrimString(filename), nil
+}
+
// Adds the configured headers to the request if any
func addHeaders(req *http.Request, opt *Options) {
for i := 0; i < len(opt.Headers); i += 2 {
@@ -577,6 +647,9 @@ func (o *Object) String() string {
// Remote the name of the remote HTTP file, relative to the fs root
func (o *Object) Remote() string {
+ if o.contentDispositionFilename != nil {
+ return *o.contentDispositionFilename
+ }
return o.remote
}
@@ -634,6 +707,29 @@ func (o *Object) decodeMetadata(ctx context.Context, res *http.Response) error {
o.modTime = t
o.contentType = res.Header.Get("Content-Type")
o.size = rest.ParseSizeFromHeaders(res.Header)
+ contentDisposition := res.Header.Get("Content-Disposition")
+ if contentDisposition != "" {
+ o.contentDisposition = &contentDisposition
+ }
+ if o.contentDisposition != nil {
+ var filename string
+ filename, err = parseFilename(*o.contentDisposition)
+ if err == nil && filename != "" {
+ o.contentDispositionFilename = &filename
+ }
+ }
+ cacheControl := res.Header.Get("Cache-Control")
+ if cacheControl != "" {
+ o.cacheControl = &cacheControl
+ }
+ contentEncoding := res.Header.Get("Content-Encoding")
+ if contentEncoding != "" {
+ o.contentEncoding = &contentEncoding
+ }
+ contentLanguage := res.Header.Get("Content-Language")
+ if contentLanguage != "" {
+ o.contentLanguage = &contentLanguage
+ }
// If NoSlash is set then check ContentType to see if it is a directory
if o.fs.opt.NoSlash {
@@ -772,6 +868,30 @@ func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[str
}
}
+// Metadata returns metadata for an object
+//
+// It should return nil if there is no Metadata
+func (o *Object) Metadata(ctx context.Context) (metadata fs.Metadata, err error) {
+ metadata = make(fs.Metadata, 6)
+ if o.contentType != "" {
+ metadata["content-type"] = o.contentType
+ }
+
+ // Set system metadata
+ setMetadata := func(k string, v *string) {
+ if v == nil || *v == "" {
+ return
+ }
+ metadata[k] = *v
+ }
+ setMetadata("content-disposition", o.contentDisposition)
+ setMetadata("content-disposition-filename", o.contentDispositionFilename)
+ setMetadata("cache-control", o.cacheControl)
+ setMetadata("content-language", o.contentLanguage)
+ setMetadata("content-encoding", o.contentEncoding)
+ return metadata, nil
+}
+
// Check the interfaces are satisfied
var (
_ fs.Fs = &Fs{}
@@ -779,4 +899,5 @@ var (
_ fs.Object = &Object{}
_ fs.MimeTyper = &Object{}
_ fs.Commander = &Fs{}
+ _ fs.Metadataer = &Object{}
)
diff --git a/backend/http/http_internal_test.go b/backend/http/http_internal_test.go
index 90a2bae78..2bc261eb2 100644
--- a/backend/http/http_internal_test.go
+++ b/backend/http/http_internal_test.go
@@ -60,6 +60,17 @@ func prepareServer(t *testing.T) configmap.Simple {
what := fmt.Sprintf("%s %s: Header ", r.Method, r.URL.Path)
assert.Equal(t, headers[1], r.Header.Get(headers[0]), what+headers[0])
assert.Equal(t, headers[3], r.Header.Get(headers[2]), what+headers[2])
+
+ // Set the content disposition header for the fifth file
+ // later we will check if it is set using the metadata method
+ if r.URL.Path == "/five.txt.gz" {
+ w.Header().Set("Content-Disposition", "attachment; filename=\"five.txt.gz\"")
+ w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+ w.Header().Set("Cache-Control", "no-cache")
+ w.Header().Set("Content-Language", "en-US")
+ w.Header().Set("Content-Encoding", "gzip")
+ }
+
fileServer.ServeHTTP(w, r)
})
@@ -102,27 +113,33 @@ func testListRoot(t *testing.T, f fs.Fs, noSlash bool) {
sort.Sort(entries)
- require.Equal(t, 4, len(entries))
+ require.Equal(t, 5, len(entries))
e := entries[0]
- assert.Equal(t, "four", e.Remote())
+ assert.Equal(t, "five.txt.gz", e.Remote())
assert.Equal(t, int64(-1), e.Size())
- _, ok := e.(fs.Directory)
+ _, ok := e.(fs.Object)
assert.True(t, ok)
e = entries[1]
+ assert.Equal(t, "four", e.Remote())
+ assert.Equal(t, int64(-1), e.Size())
+ _, ok = e.(fs.Directory)
+ assert.True(t, ok)
+
+ e = entries[2]
assert.Equal(t, "one%.txt", e.Remote())
assert.Equal(t, int64(5+lineEndSize), e.Size())
_, ok = e.(*Object)
assert.True(t, ok)
- e = entries[2]
+ e = entries[3]
assert.Equal(t, "three", e.Remote())
assert.Equal(t, int64(-1), e.Size())
_, ok = e.(fs.Directory)
assert.True(t, ok)
- e = entries[3]
+ e = entries[4]
assert.Equal(t, "two.html", e.Remote())
if noSlash {
assert.Equal(t, int64(-1), e.Size())
@@ -218,6 +235,23 @@ func TestNewObjectWithLeadingSlash(t *testing.T) {
assert.Equal(t, fs.ErrorObjectNotFound, err)
}
+func TestNewObjectWithMetadata(t *testing.T) {
+ f := prepare(t)
+ o, err := f.NewObject(context.Background(), "/five.txt.gz")
+ require.NoError(t, err)
+ assert.Equal(t, "five.txt.gz", o.Remote())
+ ho, ok := o.(*Object)
+ assert.True(t, ok)
+ metadata, err := ho.Metadata(context.Background())
+ require.NoError(t, err)
+ assert.Equal(t, "text/plain; charset=utf-8", metadata["content-type"])
+ assert.Equal(t, "attachment; filename=\"five.txt.gz\"", metadata["content-disposition"])
+ assert.Equal(t, "five.txt.gz", metadata["content-disposition-filename"])
+ assert.Equal(t, "no-cache", metadata["cache-control"])
+ assert.Equal(t, "en-US", metadata["content-language"])
+ assert.Equal(t, "gzip", metadata["content-encoding"])
+}
+
func TestOpen(t *testing.T) {
m := prepareServer(t)
diff --git a/backend/http/test/files/five.txt.gz b/backend/http/test/files/five.txt.gz
new file mode 100644
index 000000000..fa65c0084
Binary files /dev/null and b/backend/http/test/files/five.txt.gz differ
diff --git a/cmd/serve/serve.go b/cmd/serve/serve.go
index fcf3a445b..ab58e3e4c 100644
--- a/cmd/serve/serve.go
+++ b/cmd/serve/serve.go
@@ -23,7 +23,15 @@ subcommand to specify the protocol, e.g.
rclone serve http remote:
` + "```" + `
-Each subcommand has its own options which you can see in their help.`,
+When the "--metadata" flag is enabled, the following metadata fields will be provided as headers:
+- "content-disposition"
+- "cache-control"
+- "content-language"
+- "content-encoding"
+Note: The availability of these fields depends on whether the remote supports metadata.
+
+Each subcommand has its own options which you can see in their help.
+`,
Annotations: map[string]string{
"versionIntroduced": "v1.39",
},
diff --git a/docs/content/overview.md b/docs/content/overview.md
index 68cf44895..28e8db9b4 100644
--- a/docs/content/overview.md
+++ b/docs/content/overview.md
@@ -34,7 +34,7 @@ Here is an overview of the major features of each cloud storage system.
| Google Photos | - | - | No | Yes | R | - |
| HDFS | - | R/W | No | No | - | - |
| HiDrive | HiDrive ¹² | R/W | No | No | - | - |
-| HTTP | - | R | No | No | R | - |
+| HTTP | - | R | No | No | R | R |
| iCloud Drive | - | R | No | No | - | - |
| Internet Archive | MD5, SHA1, CRC32 | R/W ¹¹ | No | No | - | RWU |
| Jottacloud | MD5 | R/W | Yes | No | R | RW |
diff --git a/fs/object/object.go b/fs/object/object.go
index 9b4e25afe..07dd8a320 100644
--- a/fs/object/object.go
+++ b/fs/object/object.go
@@ -191,11 +191,12 @@ var _ fs.Fs = MemoryFs
// MemoryObject is an in memory object
type MemoryObject struct {
- remote string
- modTime time.Time
- content []byte
- meta fs.Metadata
- fs fs.Fs
+ remote string
+ modTime time.Time
+ content []byte
+ meta fs.Metadata
+ fs fs.Fs
+ mimeType string
}
// NewMemoryObject returns an in memory Object with the modTime and content passed in
@@ -214,6 +215,12 @@ func (o *MemoryObject) WithMetadata(meta fs.Metadata) *MemoryObject {
return o
}
+// WithMimeType adds mimeType to the MemoryObject
+func (o *MemoryObject) WithMimeType(mimeType string) *MemoryObject {
+ o.mimeType = mimeType
+ return o
+}
+
// Content returns the underlying buffer
func (o *MemoryObject) Content() []byte {
return o.content
@@ -329,8 +336,14 @@ func (o *MemoryObject) Metadata(ctx context.Context) (fs.Metadata, error) {
return o.meta, nil
}
+// MimeType on the object
+func (o *MemoryObject) MimeType(ctx context.Context) string {
+ return o.mimeType
+}
+
// Check interfaces
var (
_ fs.Object = (*MemoryObject)(nil)
+ _ fs.MimeTyper = (*MemoryObject)(nil)
_ fs.Metadataer = (*MemoryObject)(nil)
)
diff --git a/fs/object/object_test.go b/fs/object/object_test.go
index be90c771e..d1597acc3 100644
--- a/fs/object/object_test.go
+++ b/fs/object/object_test.go
@@ -87,6 +87,7 @@ func TestMemoryObject(t *testing.T) {
content = content[:6] // make some extra cap
o := object.NewMemoryObject(remote, now, content)
+ o.WithMimeType("text/plain; charset=utf-8")
assert.Equal(t, content, o.Content())
assert.Equal(t, object.MemoryFs, o.Fs())
@@ -95,6 +96,7 @@ func TestMemoryObject(t *testing.T) {
assert.Equal(t, now, o.ModTime(context.Background()))
assert.Equal(t, int64(len(content)), o.Size())
assert.Equal(t, true, o.Storable())
+ assert.Equal(t, "text/plain; charset=utf-8", o.MimeType(context.Background()))
Hash, err := o.Hash(context.Background(), hash.MD5)
assert.NoError(t, err)
diff --git a/lib/http/serve/serve.go b/lib/http/serve/serve.go
index 21f3c3322..244509b47 100644
--- a/lib/http/serve/serve.go
+++ b/lib/http/serve/serve.go
@@ -39,6 +39,26 @@ func Object(w http.ResponseWriter, r *http.Request, o fs.Object) {
modTime := o.ModTime(r.Context())
w.Header().Set("Last-Modified", modTime.UTC().Format(http.TimeFormat))
+ // Set metadata headers if present
+ metadata, err := fs.GetMetadata(r.Context(), o)
+ if err != nil {
+ fs.Debugf(o, "Request get metadata error: %v", err)
+ }
+ if metadata != nil {
+ if metadata["content-disposition"] != "" {
+ w.Header().Set("Content-Disposition", metadata["content-disposition"])
+ }
+ if metadata["cache-control"] != "" {
+ w.Header().Set("Cache-Control", metadata["cache-control"])
+ }
+ if metadata["content-language"] != "" {
+ w.Header().Set("Content-Language", metadata["content-language"])
+ }
+ if metadata["content-encoding"] != "" {
+ w.Header().Set("Content-Encoding", metadata["content-encoding"])
+ }
+ }
+
if r.Method == "HEAD" {
return
}
diff --git a/lib/http/serve/serve_test.go b/lib/http/serve/serve_test.go
index 79daf773e..ed716cad9 100644
--- a/lib/http/serve/serve_test.go
+++ b/lib/http/serve/serve_test.go
@@ -8,6 +8,8 @@ import (
"testing"
"time"
+ "github.com/rclone/rclone/fs"
+ "github.com/rclone/rclone/fs/object"
"github.com/rclone/rclone/fstest/mockobject"
"github.com/stretchr/testify/assert"
)
@@ -82,3 +84,23 @@ func TestObjectBadRange(t *testing.T) {
body, _ := io.ReadAll(resp.Body)
assert.Equal(t, "Bad Request\n", string(body))
}
+
+func TestObjectHEADMetadata(t *testing.T) {
+ w := httptest.NewRecorder()
+ r := httptest.NewRequest("HEAD", "http://example.com/aFile", nil)
+ m := fs.Metadata{
+ "content-disposition": "inline",
+ "cache-control": "no-cache",
+ "content-language": "en",
+ "content-encoding": "gzip",
+ }
+ o := object.NewMemoryObject("aFile", time.Now(), []byte("")).
+ WithMetadata(m).WithMimeType("text/plain; charset=utf-8")
+ Object(w, r, o)
+ resp := w.Result()
+ assert.Equal(t, "text/plain; charset=utf-8", resp.Header.Get("Content-Type"))
+ assert.Equal(t, "inline", resp.Header.Get("Content-Disposition"))
+ assert.Equal(t, "no-cache", resp.Header.Get("Cache-Control"))
+ assert.Equal(t, "en", resp.Header.Get("Content-Language"))
+ assert.Equal(t, "gzip", resp.Header.Get("Content-Encoding"))
+}