"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
copyurl: fix ignored --upload-headers and --download-headers
Andriy Senyshynbfa2fbb78edc
diff --git a/fs/operations/operations.go b/fs/operations/operations.goindex 1ada8f84f..d2220cc42 100644--- a/fs/operations/operations.go+++ b/fs/operations/operations.go@@ -1821,8 +1821,12 @@ func RcatSize(ctx context.Context, fdst fs.Fs, dstFileName string, in io.ReadCloreturn nil, err}+ var options []fs.OpenOption+ for _, option := range fs.GetConfig(ctx).UploadHeaders {+ options = append(options, option)+ }info := object.NewStaticObjectInfo(dstFileName, modTime, size, true, nil, fdst).WithMetadata(meta)- obj, err = fdst.Put(ctx, in, info)+ obj, err = fdst.Put(ctx, in, info, options...)if err != nil {fs.Errorf(dstFileName, "Post request put error: %v", err)@@ -1847,7 +1851,14 @@ type copyURLFunc func(ctx context.Context, dstFileName string, in io.ReadCloser,// copyURLFn copies the data from the url to the function suppliedfunc copyURLFn(ctx context.Context, dstFileName string, url string, autoFilename, dstFileNameFromHeader bool, fn copyURLFunc) (err error) {client := fshttp.NewClient(ctx)- resp, err := client.Get(url)+ req, err := http.NewRequestWithContext(ctx, "GET", url, nil)+ if err != nil {+ return err+ }+ for _, option := range fs.GetConfig(ctx).DownloadHeaders {+ req.Header.Set(option.Key, option.Value)+ }+ resp, err := client.Do(req)if err != nil {return err}diff --git a/fs/operations/operations_test.go b/fs/operations/operations_test.goindex cb68b05ca..2ddb56a38 100644--- a/fs/operations/operations_test.go+++ b/fs/operations/operations_test.go@@ -894,6 +894,29 @@ func TestCopyURL(t *testing.T) {fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{file1, file2, fstest.NewItem(urlFileName, contents, t1), fstest.NewItem(headerFilename, contents, t1)}, nil, fs.ModTimeNotSupported)}+func TestCopyURLDownloadHeaders(t *testing.T) {+ ctx := context.Background()+ ctx, ci := fs.AddConfig(ctx)+ r := fstest.NewRun(t)++ contents := "file contents\n"+ var gotHeader string+ handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ gotHeader = r.Header.Get("X-Custom-Header")+ _, err := w.Write([]byte(contents))+ assert.NoError(t, err)+ })+ ts := httptest.NewServer(handler)+ defer ts.Close()++ ci.DownloadHeaders = []*fs.HTTPOption{{Key: "X-Custom-Header", Value: "test-value"}}++ o, err := operations.CopyURL(ctx, r.Fremote, "file1", ts.URL, false, false, false)+ require.NoError(t, err)+ assert.Equal(t, int64(len(contents)), o.Size())+ assert.Equal(t, "test-value", gotHeader, "DownloadHeaders should be sent in the HTTP request")+}+func TestCopyURLToWriter(t *testing.T) {ctx := context.Background()contents := "file contents\n"@@ -1665,6 +1688,45 @@ func TestRcatSizeMetadata(t *testing.T) {}}+// putOptionSpy wraps an fs.Fs, capturing the options passed to Put.+type putOptionSpy struct {+ fs.Fs+ gotOptions []fs.OpenOption+}++func (f *putOptionSpy) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {+ f.gotOptions = options+ return f.Fs.Put(ctx, in, src, options...)+}++func TestRcatSizeUploadHeaders(t *testing.T) {+ ctx := context.Background()+ ctx, ci := fs.AddConfig(ctx)+ r := fstest.NewRun(t)++ ci.UploadHeaders = []*fs.HTTPOption{{Key: "X-Upload-Header", Value: "upload-value"}}++ spy := &putOptionSpy{Fs: r.Fremote}++ const body = "------------------------------------------------------------"+ bodyReader := io.NopCloser(strings.NewReader(body))+ obj, err := operations.RcatSize(ctx, spy, "potato1", bodyReader, int64(len(body)), t1, nil)+ require.NoError(t, err)+ assert.Equal(t, int64(len(body)), obj.Size())++ // Verify the upload header was actually passed through to Put+ require.NotEmpty(t, spy.gotOptions, "expected options to be passed to Put")+ var found bool+ for _, opt := range spy.gotOptions {+ if httpOpt, ok := opt.(*fs.HTTPOption); ok {+ if httpOpt.Key == "X-Upload-Header" && httpOpt.Value == "upload-value" {+ found = true+ }+ }+ }+ assert.True(t, found, "X-Upload-Header not found in options passed to Put")+}+func TestTouchDir(t *testing.T) {ctx := context.Background()r := fstest.NewRun(t)