"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
fshttp: add --dump curl for dumping HTTP requests as curl commands
Nick Craig-Woodb715ad2a9751
diff --git a/docs/content/docs.md b/docs/content/docs.mdindex 35d78a872..f98bff7d9 100644--- a/docs/content/docs.md+++ b/docs/content/docs.md@@ -3284,6 +3284,10 @@ The available flags are:- `mapper` dumps the JSON blobs being sent to the program supplied with`--metadata-mapper` and received from it. It can be useful for debuggingthe metadata mapper interface.+- `curl` dumps the HTTP request as a `curl` command. Can be used with+ the other HTTP debugging flags (e.g. `requests`, `bodies`). By+ default the auth will be masked - use with `auth` to have the curl+ commands with authentication too.## Filteringdiff --git a/fs/dump.go b/fs/dump.goindex a461d660a..a2c974419 100644--- a/fs/dump.go+++ b/fs/dump.go@@ -14,6 +14,7 @@ const (DumpGoRoutinesDumpOpenFilesDumpMapper+ DumpCurl)type dumpChoices struct{}@@ -29,6 +30,7 @@ func (dumpChoices) Choices() []BitsChoicesInfo {{uint64(DumpGoRoutines), "goroutines"},{uint64(DumpOpenFiles), "openfiles"},{uint64(DumpMapper), "mapper"},+ {uint64(DumpCurl), "curl"},}}diff --git a/fs/fshttp/http.go b/fs/fshttp/http.goindex 65684f3db..626cdcab2 100644--- a/fs/fshttp/http.go+++ b/fs/fshttp/http.go@@ -15,6 +15,8 @@ import ("net/http/httputil""net/url""os"+ "slices"+ "strings""sync""time"@@ -24,6 +26,7 @@ import ("github.com/rclone/rclone/lib/structs""github.com/youmark/pkcs8""golang.org/x/net/publicsuffix"+ "moul.io/http2curl/v2")const (@@ -439,6 +442,18 @@ func cleanAuths(buf []byte) []byte {return buf}+// cleanCurl gets rid of Auth headers in a curl command+func cleanCurl(cmd *http2curl.CurlCommand) {+ for _, authBuf := range authBufs {+ auth := "'" + string(authBuf)+ for i, arg := range *cmd {+ if strings.HasPrefix(arg, auth) {+ (*cmd)[i] = auth + "XXXX'"+ }+ }+ }+}+var expireWindow = 30 * time.Secondfunc isCertificateExpired(cc *tls.Config) bool {@@ -492,6 +507,26 @@ func (t *Transport) RoundTrip(req *http.Request) (resp *http.Response, err errorfs.Debugf(nil, "%s", separatorReq)logMutex.Unlock()}+ // Dump curl request+ if t.dump&(fs.DumpCurl) != 0 {+ cmd, err := http2curl.GetCurlCommand(req)+ if err != nil {+ fs.Debugf(nil, "Failed to create curl command: %v", err)+ } else {+ // Patch -X HEAD into --head+ for i := range len(*cmd) - 1 {+ if (*cmd)[i] == "-X" && (*cmd)[i+1] == "'HEAD'" {+ (*cmd)[i] = "--head"+ *cmd = slices.Delete(*cmd, i+1, i+2)+ break+ }+ }+ if t.dump&fs.DumpAuth == 0 {+ cleanCurl(cmd)+ }+ fs.Debugf(nil, "HTTP REQUEST: %v", cmd)+ }+ }// Do round tripresp, err = t.Transport.RoundTrip(req)// Logf responsediff --git a/fs/fshttp/http_test.go b/fs/fshttp/http_test.goindex 18f191e73..084099c77 100644--- a/fs/fshttp/http_test.go+++ b/fs/fshttp/http_test.go@@ -19,6 +19,7 @@ import ("github.com/rclone/rclone/fs""github.com/stretchr/testify/assert""github.com/stretchr/testify/require"+ "moul.io/http2curl/v2")func TestCleanAuth(t *testing.T) {@@ -61,6 +62,32 @@ func TestCleanAuths(t *testing.T) {}}+func TestCleanCurl(t *testing.T) {+ for _, test := range []struct {+ in []string+ want []string+ }{{+ []string{""},+ []string{""},+ }, {+ []string{"floo"},+ []string{"floo"},+ }, {+ []string{"'Authorization: AAAAAAAAA'", "'Potato: Help'", ""},+ []string{"'Authorization: XXXX'", "'Potato: Help'", ""},+ }, {+ []string{"'X-Auth-Token: AAAAAAAAA'", "'Potato: Help'", ""},+ []string{"'X-Auth-Token: XXXX'", "'Potato: Help'", ""},+ }, {+ []string{"'X-Auth-Token: AAAAAAAAA'", "'Authorization: AAAAAAAAA'", "'Potato: Help'", ""},+ []string{"'X-Auth-Token: XXXX'", "'Authorization: XXXX'", "'Potato: Help'", ""},+ }} {+ in := http2curl.CurlCommand(test.in)+ cleanCurl(&in)+ assert.Equal(t, test.want, test.in, test.in)+ }+}+var certSerial = int64(0)// Create a test certificate and key pair that is valid for a specific