Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

fshttp: add --dump curl for dumping HTTP requests as curl commands

Nick Craig-Woodb715ad2a9751
diff --git a/docs/content/docs.md b/docs/content/docs.md
index 35d78a872..f98bff7d9 100644
--- a/docs/content/docs.md
+++ b/docs/content/docs.md
@@ -3284,6 +3284,10 @@ The available flags are:
- `mapper` dumps the JSON blobs being sent to the program supplied with
`--metadata-mapper` and received from it. It can be useful for debugging
the metadata mapper interface.
+- `curl` dumps the HTTP request as a `curl` command. Can be used with
+ the other HTTP debugging flags (e.g. `requests`, `bodies`). By
+ default the auth will be masked - use with `auth` to have the curl
+ commands with authentication too.
## Filtering
diff --git a/fs/dump.go b/fs/dump.go
index a461d660a..a2c974419 100644
--- a/fs/dump.go
+++ b/fs/dump.go
@@ -14,6 +14,7 @@ const (
DumpGoRoutines
DumpOpenFiles
DumpMapper
+ DumpCurl
)
type dumpChoices struct{}
@@ -29,6 +30,7 @@ func (dumpChoices) Choices() []BitsChoicesInfo {
{uint64(DumpGoRoutines), "goroutines"},
{uint64(DumpOpenFiles), "openfiles"},
{uint64(DumpMapper), "mapper"},
+ {uint64(DumpCurl), "curl"},
}
}
diff --git a/fs/fshttp/http.go b/fs/fshttp/http.go
index 65684f3db..626cdcab2 100644
--- a/fs/fshttp/http.go
+++ b/fs/fshttp/http.go
@@ -15,6 +15,8 @@ import (
"net/http/httputil"
"net/url"
"os"
+ "slices"
+ "strings"
"sync"
"time"
@@ -24,6 +26,7 @@ import (
"github.com/rclone/rclone/lib/structs"
"github.com/youmark/pkcs8"
"golang.org/x/net/publicsuffix"
+ "moul.io/http2curl/v2"
)
const (
@@ -439,6 +442,18 @@ func cleanAuths(buf []byte) []byte {
return buf
}
+// cleanCurl gets rid of Auth headers in a curl command
+func cleanCurl(cmd *http2curl.CurlCommand) {
+ for _, authBuf := range authBufs {
+ auth := "'" + string(authBuf)
+ for i, arg := range *cmd {
+ if strings.HasPrefix(arg, auth) {
+ (*cmd)[i] = auth + "XXXX'"
+ }
+ }
+ }
+}
+
var expireWindow = 30 * time.Second
func isCertificateExpired(cc *tls.Config) bool {
@@ -492,6 +507,26 @@ func (t *Transport) RoundTrip(req *http.Request) (resp *http.Response, err error
fs.Debugf(nil, "%s", separatorReq)
logMutex.Unlock()
}
+ // Dump curl request
+ if t.dump&(fs.DumpCurl) != 0 {
+ cmd, err := http2curl.GetCurlCommand(req)
+ if err != nil {
+ fs.Debugf(nil, "Failed to create curl command: %v", err)
+ } else {
+ // Patch -X HEAD into --head
+ for i := range len(*cmd) - 1 {
+ if (*cmd)[i] == "-X" && (*cmd)[i+1] == "'HEAD'" {
+ (*cmd)[i] = "--head"
+ *cmd = slices.Delete(*cmd, i+1, i+2)
+ break
+ }
+ }
+ if t.dump&fs.DumpAuth == 0 {
+ cleanCurl(cmd)
+ }
+ fs.Debugf(nil, "HTTP REQUEST: %v", cmd)
+ }
+ }
// Do round trip
resp, err = t.Transport.RoundTrip(req)
// Logf response
diff --git a/fs/fshttp/http_test.go b/fs/fshttp/http_test.go
index 18f191e73..084099c77 100644
--- a/fs/fshttp/http_test.go
+++ b/fs/fshttp/http_test.go
@@ -19,6 +19,7 @@ import (
"github.com/rclone/rclone/fs"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
+ "moul.io/http2curl/v2"
)
func TestCleanAuth(t *testing.T) {
@@ -61,6 +62,32 @@ func TestCleanAuths(t *testing.T) {
}
}
+func TestCleanCurl(t *testing.T) {
+ for _, test := range []struct {
+ in []string
+ want []string
+ }{{
+ []string{""},
+ []string{""},
+ }, {
+ []string{"floo"},
+ []string{"floo"},
+ }, {
+ []string{"'Authorization: AAAAAAAAA'", "'Potato: Help'", ""},
+ []string{"'Authorization: XXXX'", "'Potato: Help'", ""},
+ }, {
+ []string{"'X-Auth-Token: AAAAAAAAA'", "'Potato: Help'", ""},
+ []string{"'X-Auth-Token: XXXX'", "'Potato: Help'", ""},
+ }, {
+ []string{"'X-Auth-Token: AAAAAAAAA'", "'Authorization: AAAAAAAAA'", "'Potato: Help'", ""},
+ []string{"'X-Auth-Token: XXXX'", "'Authorization: XXXX'", "'Potato: Help'", ""},
+ }} {
+ in := http2curl.CurlCommand(test.in)
+ cleanCurl(&in)
+ assert.Equal(t, test.want, test.in, test.in)
+ }
+}
+
var certSerial = int64(0)
// Create a test certificate and key pair that is valid for a specific