"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
fs: tls: add --client-pass support for encrypted --client-key files
Nick Craig-Woodcfd0d2874220
diff --git a/bin/make-test-certs.sh b/bin/make-test-certs.shnew file mode 100755index 000000000..d7a90fed0--- /dev/null+++ b/bin/make-test-certs.sh@@ -0,0 +1,119 @@+#!/usr/bin/env bash+set -euo pipefail++# Create test TLS certificates for use with rclone.++OUT_DIR="${OUT_DIR:-./tls-test}"+CA_SUBJ="${CA_SUBJ:-/C=US/ST=Test/L=Test/O=Test Org/OU=Test Unit/CN=Test Root CA}"+SERVER_CN="${SERVER_CN:-localhost}"+CLIENT_CN="${CLIENT_CN:-Test Client}"+CLIENT_KEY_PASS="${CLIENT_KEY_PASS:-testpassword}"++CA_DAYS=${CA_DAYS:-3650}+SERVER_DAYS=${SERVER_DAYS:-825}+CLIENT_DAYS=${CLIENT_DAYS:-825}++mkdir -p "$OUT_DIR"+cd "$OUT_DIR"++# Create OpenSSL config++# CA extensions+cat > ca_openssl.cnf <<'EOF'+[ ca_ext ]+basicConstraints = critical, CA:true, pathlen:1+keyUsage = critical, keyCertSign, cRLSign+subjectKeyIdentifier = hash+authorityKeyIdentifier = keyid:always,issuer+EOF++# Server extensions (SAN includes localhost + loopback IP)+cat > server_openssl.cnf <<EOF+[ server_ext ]+basicConstraints = critical, CA:false+keyUsage = critical, digitalSignature, keyEncipherment+extendedKeyUsage = serverAuth+subjectKeyIdentifier = hash+authorityKeyIdentifier = keyid,issuer+subjectAltName = @alt_names++[ alt_names ]+DNS.1 = ${SERVER_CN}+IP.1 = 127.0.0.1+EOF++# Client extensions (for mTLS client auth)+cat > client_openssl.cnf <<'EOF'+[ client_ext ]+basicConstraints = critical, CA:false+keyUsage = critical, digitalSignature, keyEncipherment+extendedKeyUsage = clientAuth+subjectKeyIdentifier = hash+authorityKeyIdentifier = keyid,issuer+EOF++echo "Create CA key, CSR, and self-signed CA cert"+if [ ! -f ca.key.pem ]; then+ openssl genrsa -out ca.key.pem 4096+ chmod 600 ca.key.pem+fi++openssl req -new -key ca.key.pem -subj "$CA_SUBJ" -out ca.csr.pem++openssl x509 -req -in ca.csr.pem -signkey ca.key.pem \+ -sha256 -days "$CA_DAYS" \+ -extfile ca_openssl.cnf -extensions ca_ext \+ -out ca.cert.pem++echo "Create server key (NO PASSWORD) and cert signed by CA"+openssl genrsa -out server.key.pem 2048+chmod 600 server.key.pem++openssl req -new -key server.key.pem -subj "/CN=${SERVER_CN}" -out server.csr.pem++openssl x509 -req -in server.csr.pem \+ -CA ca.cert.pem -CAkey ca.key.pem -CAcreateserial \+ -out server.cert.pem -days "$SERVER_DAYS" -sha256 \+ -extfile server_openssl.cnf -extensions server_ext++echo "Create client key (PASSWORD-PROTECTED), CSR, and cert"+openssl genrsa -aes256 -passout pass:"$CLIENT_KEY_PASS" -out client.key.pem 2048+chmod 600 client.key.pem++openssl req -new -key client.key.pem -passin pass:"$CLIENT_KEY_PASS" \+ -subj "/CN=${CLIENT_CN}" -out client.csr.pem++openssl x509 -req -in client.csr.pem \+ -CA ca.cert.pem -CAkey ca.key.pem -CAcreateserial \+ -out client.cert.pem -days "$CLIENT_DAYS" -sha256 \+ -extfile client_openssl.cnf -extensions client_ext++echo "Verify chain"+openssl verify -CAfile ca.cert.pem server.cert.pem client.cert.pem++echo "Done"++echo+echo "Summary"+echo "-------"+printf "%-22s %s\n" \+ "CA key:" "ca.key.pem" \+ "CA cert:" "ca.cert.pem" \+ "Server key:" "server.key.pem (no password)" \+ "Server CSR:" "server.csr.pem" \+ "Server cert:" "server.cert.pem (SAN: ${SERVER_CN}, 127.0.0.1)" \+ "Client key:" "client.key.pem (encrypted)" \+ "Client CSR:" "client.csr.pem" \+ "Client cert:" "client.cert.pem" \+ "Client key password:" "$CLIENT_KEY_PASS"++echo+echo "Test rclone server"+echo+echo "rclone serve http -vv --addr :8080 --cert ${OUT_DIR}/server.cert.pem --key ${OUT_DIR}/server.key.pem --client-ca ${OUT_DIR}/ca.cert.pem ."++echo+echo "Test rclone client"+echo+echo "rclone lsf :http: --http-url 'https://localhost:8080' --ca-cert ${OUT_DIR}/ca.cert.pem --client-cert ${OUT_DIR}/client.cert.pem --client-key ${OUT_DIR}/client.key.pem --client-pass \$(rclone obscure $CLIENT_KEY_PASS)"+echodiff --git a/docs/content/docs.md b/docs/content/docs.mdindex ba9435781..80ee20042 100644--- a/docs/content/docs.md+++ b/docs/content/docs.md@@ -3000,6 +3000,20 @@ The `--client-key` flag is required too when using this.This loads the PEM encoded client side private key used for mutual TLSauthentication. Used in conjunction with `--client-cert`.+Supported types are:++- Unencrypted PKCS#1 ("BEGIN RSA PRIVATE KEY")+- Unencrypted PKCS#8 ("BEGIN PRIVATE KEY")+- Encrypted PKCS#8 ("BEGIN ENCRYPTED PRIVATE KEY")+- Legacy PEM encryption (e.g., DEK-Info headers), which are automatically detected.++### --client-pass string++This can be used to supply an optional password to decrypt the client key file.++**NB** the password should be obscured so it should be the output of+`rclone obscure YOURPASSWORD`.+### --no-check-certificate`--no-check-certificate` controls whether a client verifies thediff --git a/fs/config.go b/fs/config.goindex 04a18aafe..ce0a1df9e 100644--- a/fs/config.go+++ b/fs/config.go@@ -438,6 +438,12 @@ var ConfigOptionsInfo = Options{{Default: "",Help: "Client SSL private key (PEM) for mutual TLS auth",Groups: "Networking",+}, {+ Name: "client_pass",+ Default: "",+ Help: "Password for client SSL private key (PEM) for mutual TLS auth (obscured)",+ Groups: "Networking",+ IsPassword: true,}, {Name: "multi_thread_cutoff",Default: SizeSuffix(256 * 1024 * 1024),@@ -644,6 +650,7 @@ type ConfigInfo struct {CaCert []string `config:"ca_cert"` // Client Side CAClientCert string `config:"client_cert"` // Client Side CertClientKey string `config:"client_key"` // Client Side Key+ ClientPass string `config:"client_pass"` // Client Side Key Password (obscured)MultiThreadCutoff SizeSuffix `config:"multi_thread_cutoff"`MultiThreadStreams int `config:"multi_thread_streams"`MultiThreadSet bool `config:"multi_thread_set"` // whether MultiThreadStreams was set (set in fs/config/configflags)diff --git a/fs/fshttp/http.go b/fs/fshttp/http.goindex de9ca3cf8..65684f3db 100644--- a/fs/fshttp/http.go+++ b/fs/fshttp/http.go@@ -6,6 +6,8 @@ import ("context""crypto/tls""crypto/x509"+ "encoding/pem"+ "errors""fmt""net""net/http"@@ -18,7 +20,9 @@ import ("github.com/rclone/rclone/fs""github.com/rclone/rclone/fs/accounting"+ "github.com/rclone/rclone/fs/config/obscure""github.com/rclone/rclone/lib/structs"+ "github.com/youmark/pkcs8""golang.org/x/net/publicsuffix")@@ -48,6 +52,156 @@ func ResetTransport() {noTransport = new(sync.Once)}+// LoadKeyPair loads a TLS certificate and private key from PEM-encoded files,+// with extended support for encrypted private keys.+//+// This function is designed as a robust replacement for tls.X509KeyPair,+// providing the same core functionality but adding support for+// password-protected private keys.+//+// The certificate file (certFile) must contain one or more PEM-encoded+// certificates. The first certificate is treated as the leaf certificate, and+// any subsequent certificates are treated as its chain.+//+// The key file (keyFile) must contain a PEM-encoded private key. Supported+// formats are:+//+// - Unencrypted PKCS#1 ("BEGIN RSA PRIVATE KEY")+// - Unencrypted PKCS#8 ("BEGIN PRIVATE KEY")+// - Encrypted PKCS#8 ("BEGIN ENCRYPTED PRIVATE KEY")+// - Legacy PEM encryption (e.g., DEK-Info headers), which are automatically detected.+//+// The password parameter is used to decrypt the private key. If the+// key is not encrypted, this parameter is ignored and can be an empty+// string. The password should be an obscured string.+//+// On success, it returns a fully populated tls.Certificate struct, including the+// Leaf certificate field.+func LoadKeyPair(certFile, keyFile, password string) (cert tls.Certificate, err error) {+ certPEM, err := os.ReadFile(certFile)+ if err != nil {+ return cert, fmt.Errorf("read cert: %w", err)+ }+ keyPEM, err := os.ReadFile(keyFile)+ if err != nil {+ return cert, fmt.Errorf("read key: %w", err)+ }+ if password != "" {+ password, err = obscure.Reveal(password)+ if err != nil {+ return cert, fmt.Errorf("reveal key password: %w", err)+ }+ }++ // Fast path: unencrypted PKCS#1/PKCS#8+ cert, err = tls.X509KeyPair(certPEM, keyPEM)+ if err == nil {+ if len(cert.Certificate) == 0 {+ return cert, errors.New("no certificates parsed")+ }+ leaf, err := x509.ParseCertificate(cert.Certificate[0])+ if err != nil {+ return cert, fmt.Errorf("parse leaf: %w", err)+ }+ cert.Leaf = leaf+ return cert, nil+ }++ // Decrypt / parse key manually+ block, rest := pem.Decode(keyPEM)+ if block == nil {+ return cert, errors.New("no PEM block in key")+ }+ if len(rest) != 0 {+ fs.Debugf(nil, "Trailing data (%d bytes) in key PEM loaded from %q", len(rest), keyFile)+ }++ var privKey any+ switch {+ case block.Type == "ENCRYPTED PRIVATE KEY":+ if password == "" {+ return cert, errors.New("key is encrypted but no --client-pass provided")+ }+ privKey, err = pkcs8.ParsePKCS8PrivateKey(block.Bytes, []byte(password))+ if err != nil {+ return cert, fmt.Errorf("parse encrypted PKCS#8: %w", err)+ }++ case x509.IsEncryptedPEMBlock(block): //nolint:staticcheck // this is Legacy and insecure+ if password == "" {+ return cert, errors.New("key is encrypted but no --client-pass provided")+ }+ der, err := x509.DecryptPEMBlock(block, []byte(password)) //nolint:staticcheck // this is Legacy and insecure+ if err != nil {+ return cert, fmt.Errorf("decrypt PEM key: %w", err)+ }+ // Try PKCS#8, then RSA PKCS#1, then EC+ if k, kerr1 := x509.ParsePKCS8PrivateKey(der); kerr1 == nil {+ privKey = k+ } else if k, kerr2 := x509.ParsePKCS1PrivateKey(der); kerr2 == nil {+ privKey = k+ } else if k, kerr3 := x509.ParseECPrivateKey(der); kerr3 == nil {+ privKey = k+ } else {+ return cert, fmt.Errorf("parse decrypted key: pkcs8: %v, pkcs1: %v, ec: %v", kerr1, kerr2, kerr3)+ }++ default:+ // Unencrypted specific types+ switch block.Type {+ case "PRIVATE KEY":+ k, kerr := x509.ParsePKCS8PrivateKey(block.Bytes)+ if kerr != nil {+ return cert, fmt.Errorf("parse PKCS#8: %w", kerr)+ }+ privKey = k+ case "RSA PRIVATE KEY":+ k, kerr := x509.ParsePKCS1PrivateKey(block.Bytes)+ if kerr != nil {+ return cert, fmt.Errorf("parse PKCS#1 RSA: %w", kerr)+ }+ privKey = k+ case "EC PRIVATE KEY":+ k, kerr := x509.ParseECPrivateKey(block.Bytes)+ if kerr != nil {+ return cert, fmt.Errorf("parse EC: %w", kerr)+ }+ privKey = k+ default:+ return cert, fmt.Errorf("unsupported key type %q", block.Type)+ }+ }++ // Build cert chain from PEM+ var certDERs [][]byte+ for rest := certPEM; ; {+ var b *pem.Block+ b, rest = pem.Decode(rest)+ if b == nil {+ break+ }+ if b.Type == "CERTIFICATE" {+ certDERs = append(certDERs, b.Bytes)+ }+ }+ if len(certDERs) == 0 {+ return cert, fmt.Errorf("no CERTIFICATE blocks in %s", certFile)+ }++ cert = tls.Certificate{+ Certificate: certDERs,+ PrivateKey: privKey,+ }++ // Leaf is always the first certificate+ cert.Leaf, err = x509.ParseCertificate(cert.Certificate[0])+ if err != nil {+ return cert, fmt.Errorf("parse leaf: %w", err)+ }++ return cert, nil+}+// NewTransportCustom returns an http.RoundTripper with the correct timeouts.// The customize function is called if set to give the caller an opportunity to// customize any defaults in the Transport.@@ -85,17 +239,10 @@ func NewTransportCustom(ctx context.Context, customize func(*http.Transport)) *Tif ci.ClientCert == "" || ci.ClientKey == "" {fs.Fatalf(nil, "Both --client-cert and --client-key must be set")}- cert, err := tls.LoadX509KeyPair(ci.ClientCert, ci.ClientKey)+ cert, err := LoadKeyPair(ci.ClientCert, ci.ClientKey, ci.ClientPass)if err != nil {fs.Fatalf(nil, "Failed to load --client-cert/--client-key pair: %v", err)}- if cert.Leaf == nil {- // Leaf is always the first certificate- cert.Leaf, err = x509.ParseCertificate(cert.Certificate[0])- if err != nil {- fs.Fatalf(nil, "Failed to parse the certificate")- }- }t.TLSClientConfig.Certificates = []tls.Certificate{cert}}@@ -187,14 +334,12 @@ func NewClientWithUnixSocket(ctx context.Context, path string) *http.Client {// * Updates metricstype Transport struct {*http.Transport+ ci *fs.ConfigInfodump fs.DumpFlagsfilterRequest func(req *http.Request)userAgent stringheaders []*fs.HTTPOptionmetrics *Metrics- // Filename of the client cert in case we need to reload it- clientCert string- clientKey string// Mutex for serializing attempts at reloading the certificatesreloadMutex sync.Mutex}@@ -203,13 +348,12 @@ type Transport struct {// roundtrips including the body if logBody is set.func newTransport(ci *fs.ConfigInfo, transport *http.Transport) *Transport {return &Transport{- Transport: transport,- dump: ci.Dump,- userAgent: ci.UserAgent,- headers: ci.Headers,- metrics: DefaultMetrics,- clientCert: ci.ClientCert,- clientKey: ci.ClientKey,+ Transport: transport,+ ci: ci,+ dump: ci.Dump,+ userAgent: ci.UserAgent,+ headers: ci.Headers,+ metrics: DefaultMetrics,}}@@ -309,20 +453,10 @@ func (t *Transport) reloadCertificates() {if !isCertificateExpired(t.TLSClientConfig) {return}-- cert, err := tls.LoadX509KeyPair(t.clientCert, t.clientKey)+ cert, err := LoadKeyPair(t.ci.ClientCert, t.ci.ClientKey, t.ci.ClientPass)if err != nil {fs.Fatalf(nil, "Failed to load --client-cert/--client-key pair: %v", err)}- // Check if we need to parse the certificate again, we need it- // for checking the expiration date- if cert.Leaf == nil {- // Leaf is always the first certificate- cert.Leaf, err = x509.ParseCertificate(cert.Certificate[0])- if err != nil {- fs.Fatalf(nil, "Failed to parse the certificate")- }- }t.TLSClientConfig.Certificates = []tls.Certificate{cert}}