Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

http: add metadata defaults for --http-no-head - fixes #7988

Giridharf43a2146d267
diff --git a/backend/http/http.go b/backend/http/http.go
index b254a5021..a88eefcff 100644
--- a/backend/http/http.go
+++ b/backend/http/http.go
@@ -100,6 +100,30 @@ that directory listings are much quicker, but rclone won't have the times or
sizes of any files, and some files that don't exist may be in the listing.`,
Default: false,
Advanced: true,
+ }, {
+ Name: "no_head_size",
+ Help: `Set the reported file size for --http-no-head.
+
+When HEAD requests are disabled, rclone normally reports an unknown file size.
+Set this option to a non-negative value to report that size for every file,
+including 0. Leave it unset (the default, shown as "off") to keep reporting
+an unknown size.
+
+The configured size is authoritative. If it does not match the actual file
+size, transfers may fail their size check and retry unless --ignore-size is
+used. It can also make multi-thread transfers request incorrect ranges and
+make mounted files unreadable past the reported size.`,
+ Default: fs.SizeSuffix(-1),
+ Advanced: true,
+ }, {
+ Name: "no_head_time",
+ Help: `Set a default modification time for --http-no-head.
+
+When HEAD requests are disabled, rclone normally reports an unset modification
+time. Set this option to report the specified time for every file instead.
+Changing the configured time later makes every file appear modified to a sync.`,
+ Default: fs.Time{},
+ Advanced: true,
}, {
Name: "no_escape",
Help: "Do not escape URL metacharacters in path names.",
@@ -145,11 +169,13 @@ var systemMetadataInfo = map[string]fs.MetadataHelp{
// Options defines the configuration for this backend
type Options struct {
- Endpoint string `config:"url"`
- NoSlash bool `config:"no_slash"`
- NoHead bool `config:"no_head"`
- Headers fs.CommaSepList `config:"headers"`
- NoEscape bool `config:"no_escape"`
+ Endpoint string `config:"url"`
+ NoSlash bool `config:"no_slash"`
+ NoHead bool `config:"no_head"`
+ NoHeadSize fs.SizeSuffix `config:"no_head_size"`
+ NoHeadTime fs.Time `config:"no_head_time"`
+ Headers fs.CommaSepList `config:"headers"`
+ NoEscape bool `config:"no_escape"`
}
// Fs stores the interface to the remote HTTP files
@@ -322,7 +348,9 @@ func (f *Fs) httpConnection(ctx context.Context, opt *Options) (isFile bool, err
// the host specified in the config file.
func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
// Parse config into Options struct
- opt := new(Options)
+ opt := &Options{
+ NoHeadSize: fs.SizeSuffix(-1),
+ }
err := configstruct.Set(m, opt)
if err != nil {
return nil, err
@@ -717,8 +745,11 @@ func (o *Object) url() string {
// head sends a HEAD request to update info fields in the Object
func (o *Object) head(ctx context.Context) error {
if o.fs.opt.NoHead {
- o.size = -1
+ o.size = int64(o.fs.opt.NoHeadSize)
o.modTime = timeUnset
+ if o.fs.opt.NoHeadTime.IsSet() {
+ o.modTime = time.Time(o.fs.opt.NoHeadTime)
+ }
o.contentType = fs.MimeType(ctx, o)
return nil
}
diff --git a/backend/http/http_internal_test.go b/backend/http/http_internal_test.go
index fd48c7173..7faabc60e 100644
--- a/backend/http/http_internal_test.go
+++ b/backend/http/http_internal_test.go
@@ -314,6 +314,54 @@ func TestOpen(t *testing.T) {
}
}
+func TestNoHeadMetadataDefaults(t *testing.T) {
+ tests := []struct {
+ name string
+ size string
+ modTime string
+ wantSize int64
+ wantModTime time.Time
+ }{
+ {
+ name: "defaults",
+ wantSize: -1,
+ wantModTime: timeUnset,
+ }, {
+ name: "zero size",
+ size: "0B",
+ wantSize: 0,
+ wantModTime: timeUnset,
+ }, {
+ name: "configured metadata",
+ size: "100B",
+ modTime: "2024-08-05T07:36:07Z",
+ wantSize: 100,
+ wantModTime: time.Date(2024, 8, 5, 7, 36, 7, 0, time.UTC),
+ },
+ }
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ m := prepareServer(t)
+ m.Set("no_head", "true")
+ if test.size != "" {
+ m.Set("no_head_size", test.size)
+ }
+ if test.modTime != "" {
+ m.Set("no_head_time", test.modTime)
+ }
+
+ f, err := NewFs(context.Background(), remoteName, "", m)
+ require.NoError(t, err)
+ o, err := f.NewObject(context.Background(), "four/under four.txt")
+ require.NoError(t, err)
+
+ assert.Equal(t, test.wantSize, o.Size())
+ assert.Equal(t, test.wantModTime, o.ModTime(context.Background()))
+ })
+ }
+}
+
func TestMimeType(t *testing.T) {
f := prepare(t)