Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

copyurl: fix ignored --upload-headers and --download-headers

Andriy Senyshynbfa2fbb78edc
diff --git a/fs/operations/operations.go b/fs/operations/operations.go
index 1ada8f84f..d2220cc42 100644
--- a/fs/operations/operations.go
+++ b/fs/operations/operations.go
@@ -1821,8 +1821,12 @@ func RcatSize(ctx context.Context, fdst fs.Fs, dstFileName string, in io.ReadClo
return nil, err
}
+ var options []fs.OpenOption
+ for _, option := range fs.GetConfig(ctx).UploadHeaders {
+ options = append(options, option)
+ }
info := object.NewStaticObjectInfo(dstFileName, modTime, size, true, nil, fdst).WithMetadata(meta)
- obj, err = fdst.Put(ctx, in, info)
+ obj, err = fdst.Put(ctx, in, info, options...)
if err != nil {
fs.Errorf(dstFileName, "Post request put error: %v", err)
@@ -1847,7 +1851,14 @@ type copyURLFunc func(ctx context.Context, dstFileName string, in io.ReadCloser,
// copyURLFn copies the data from the url to the function supplied
func copyURLFn(ctx context.Context, dstFileName string, url string, autoFilename, dstFileNameFromHeader bool, fn copyURLFunc) (err error) {
client := fshttp.NewClient(ctx)
- resp, err := client.Get(url)
+ req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
+ if err != nil {
+ return err
+ }
+ for _, option := range fs.GetConfig(ctx).DownloadHeaders {
+ req.Header.Set(option.Key, option.Value)
+ }
+ resp, err := client.Do(req)
if err != nil {
return err
}
diff --git a/fs/operations/operations_test.go b/fs/operations/operations_test.go
index cb68b05ca..2ddb56a38 100644
--- a/fs/operations/operations_test.go
+++ b/fs/operations/operations_test.go
@@ -894,6 +894,29 @@ func TestCopyURL(t *testing.T) {
fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{file1, file2, fstest.NewItem(urlFileName, contents, t1), fstest.NewItem(headerFilename, contents, t1)}, nil, fs.ModTimeNotSupported)
}
+func TestCopyURLDownloadHeaders(t *testing.T) {
+ ctx := context.Background()
+ ctx, ci := fs.AddConfig(ctx)
+ r := fstest.NewRun(t)
+
+ contents := "file contents\n"
+ var gotHeader string
+ handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ gotHeader = r.Header.Get("X-Custom-Header")
+ _, err := w.Write([]byte(contents))
+ assert.NoError(t, err)
+ })
+ ts := httptest.NewServer(handler)
+ defer ts.Close()
+
+ ci.DownloadHeaders = []*fs.HTTPOption{{Key: "X-Custom-Header", Value: "test-value"}}
+
+ o, err := operations.CopyURL(ctx, r.Fremote, "file1", ts.URL, false, false, false)
+ require.NoError(t, err)
+ assert.Equal(t, int64(len(contents)), o.Size())
+ assert.Equal(t, "test-value", gotHeader, "DownloadHeaders should be sent in the HTTP request")
+}
+
func TestCopyURLToWriter(t *testing.T) {
ctx := context.Background()
contents := "file contents\n"
@@ -1665,6 +1688,45 @@ func TestRcatSizeMetadata(t *testing.T) {
}
}
+// putOptionSpy wraps an fs.Fs, capturing the options passed to Put.
+type putOptionSpy struct {
+ fs.Fs
+ gotOptions []fs.OpenOption
+}
+
+func (f *putOptionSpy) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
+ f.gotOptions = options
+ return f.Fs.Put(ctx, in, src, options...)
+}
+
+func TestRcatSizeUploadHeaders(t *testing.T) {
+ ctx := context.Background()
+ ctx, ci := fs.AddConfig(ctx)
+ r := fstest.NewRun(t)
+
+ ci.UploadHeaders = []*fs.HTTPOption{{Key: "X-Upload-Header", Value: "upload-value"}}
+
+ spy := &putOptionSpy{Fs: r.Fremote}
+
+ const body = "------------------------------------------------------------"
+ bodyReader := io.NopCloser(strings.NewReader(body))
+ obj, err := operations.RcatSize(ctx, spy, "potato1", bodyReader, int64(len(body)), t1, nil)
+ require.NoError(t, err)
+ assert.Equal(t, int64(len(body)), obj.Size())
+
+ // Verify the upload header was actually passed through to Put
+ require.NotEmpty(t, spy.gotOptions, "expected options to be passed to Put")
+ var found bool
+ for _, opt := range spy.gotOptions {
+ if httpOpt, ok := opt.(*fs.HTTPOption); ok {
+ if httpOpt.Key == "X-Upload-Header" && httpOpt.Value == "upload-value" {
+ found = true
+ }
+ }
+ }
+ assert.True(t, found, "X-Upload-Header not found in options passed to Put")
+}
+
func TestTouchDir(t *testing.T) {
ctx := context.Background()
r := fstest.NewRun(t)