Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

fs: tls: add --client-pass support for encrypted --client-key files

Nick Craig-Woodcfd0d2874220
diff --git a/bin/make-test-certs.sh b/bin/make-test-certs.sh
new file mode 100755
index 000000000..d7a90fed0
--- /dev/null
+++ b/bin/make-test-certs.sh
@@ -0,0 +1,119 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Create test TLS certificates for use with rclone.
+
+OUT_DIR="${OUT_DIR:-./tls-test}"
+CA_SUBJ="${CA_SUBJ:-/C=US/ST=Test/L=Test/O=Test Org/OU=Test Unit/CN=Test Root CA}"
+SERVER_CN="${SERVER_CN:-localhost}"
+CLIENT_CN="${CLIENT_CN:-Test Client}"
+CLIENT_KEY_PASS="${CLIENT_KEY_PASS:-testpassword}"
+
+CA_DAYS=${CA_DAYS:-3650}
+SERVER_DAYS=${SERVER_DAYS:-825}
+CLIENT_DAYS=${CLIENT_DAYS:-825}
+
+mkdir -p "$OUT_DIR"
+cd "$OUT_DIR"
+
+# Create OpenSSL config
+
+# CA extensions
+cat > ca_openssl.cnf <<'EOF'
+[ ca_ext ]
+basicConstraints = critical, CA:true, pathlen:1
+keyUsage = critical, keyCertSign, cRLSign
+subjectKeyIdentifier = hash
+authorityKeyIdentifier = keyid:always,issuer
+EOF
+
+# Server extensions (SAN includes localhost + loopback IP)
+cat > server_openssl.cnf <<EOF
+[ server_ext ]
+basicConstraints = critical, CA:false
+keyUsage = critical, digitalSignature, keyEncipherment
+extendedKeyUsage = serverAuth
+subjectKeyIdentifier = hash
+authorityKeyIdentifier = keyid,issuer
+subjectAltName = @alt_names
+
+[ alt_names ]
+DNS.1 = ${SERVER_CN}
+IP.1 = 127.0.0.1
+EOF
+
+# Client extensions (for mTLS client auth)
+cat > client_openssl.cnf <<'EOF'
+[ client_ext ]
+basicConstraints = critical, CA:false
+keyUsage = critical, digitalSignature, keyEncipherment
+extendedKeyUsage = clientAuth
+subjectKeyIdentifier = hash
+authorityKeyIdentifier = keyid,issuer
+EOF
+
+echo "Create CA key, CSR, and self-signed CA cert"
+if [ ! -f ca.key.pem ]; then
+ openssl genrsa -out ca.key.pem 4096
+ chmod 600 ca.key.pem
+fi
+
+openssl req -new -key ca.key.pem -subj "$CA_SUBJ" -out ca.csr.pem
+
+openssl x509 -req -in ca.csr.pem -signkey ca.key.pem \
+ -sha256 -days "$CA_DAYS" \
+ -extfile ca_openssl.cnf -extensions ca_ext \
+ -out ca.cert.pem
+
+echo "Create server key (NO PASSWORD) and cert signed by CA"
+openssl genrsa -out server.key.pem 2048
+chmod 600 server.key.pem
+
+openssl req -new -key server.key.pem -subj "/CN=${SERVER_CN}" -out server.csr.pem
+
+openssl x509 -req -in server.csr.pem \
+ -CA ca.cert.pem -CAkey ca.key.pem -CAcreateserial \
+ -out server.cert.pem -days "$SERVER_DAYS" -sha256 \
+ -extfile server_openssl.cnf -extensions server_ext
+
+echo "Create client key (PASSWORD-PROTECTED), CSR, and cert"
+openssl genrsa -aes256 -passout pass:"$CLIENT_KEY_PASS" -out client.key.pem 2048
+chmod 600 client.key.pem
+
+openssl req -new -key client.key.pem -passin pass:"$CLIENT_KEY_PASS" \
+ -subj "/CN=${CLIENT_CN}" -out client.csr.pem
+
+openssl x509 -req -in client.csr.pem \
+ -CA ca.cert.pem -CAkey ca.key.pem -CAcreateserial \
+ -out client.cert.pem -days "$CLIENT_DAYS" -sha256 \
+ -extfile client_openssl.cnf -extensions client_ext
+
+echo "Verify chain"
+openssl verify -CAfile ca.cert.pem server.cert.pem client.cert.pem
+
+echo "Done"
+
+echo
+echo "Summary"
+echo "-------"
+printf "%-22s %s\n" \
+ "CA key:" "ca.key.pem" \
+ "CA cert:" "ca.cert.pem" \
+ "Server key:" "server.key.pem (no password)" \
+ "Server CSR:" "server.csr.pem" \
+ "Server cert:" "server.cert.pem (SAN: ${SERVER_CN}, 127.0.0.1)" \
+ "Client key:" "client.key.pem (encrypted)" \
+ "Client CSR:" "client.csr.pem" \
+ "Client cert:" "client.cert.pem" \
+ "Client key password:" "$CLIENT_KEY_PASS"
+
+echo
+echo "Test rclone server"
+echo
+echo "rclone serve http -vv --addr :8080 --cert ${OUT_DIR}/server.cert.pem --key ${OUT_DIR}/server.key.pem --client-ca ${OUT_DIR}/ca.cert.pem ."
+
+echo
+echo "Test rclone client"
+echo
+echo "rclone lsf :http: --http-url 'https://localhost:8080' --ca-cert ${OUT_DIR}/ca.cert.pem --client-cert ${OUT_DIR}/client.cert.pem --client-key ${OUT_DIR}/client.key.pem --client-pass \$(rclone obscure $CLIENT_KEY_PASS)"
+echo
diff --git a/docs/content/docs.md b/docs/content/docs.md
index ba9435781..80ee20042 100644
--- a/docs/content/docs.md
+++ b/docs/content/docs.md
@@ -3000,6 +3000,20 @@ The `--client-key` flag is required too when using this.
This loads the PEM encoded client side private key used for mutual TLS
authentication. Used in conjunction with `--client-cert`.
+Supported types are:
+
+- Unencrypted PKCS#1 ("BEGIN RSA PRIVATE KEY")
+- Unencrypted PKCS#8 ("BEGIN PRIVATE KEY")
+- Encrypted PKCS#8 ("BEGIN ENCRYPTED PRIVATE KEY")
+- Legacy PEM encryption (e.g., DEK-Info headers), which are automatically detected.
+
+### --client-pass string
+
+This can be used to supply an optional password to decrypt the client key file.
+
+**NB** the password should be obscured so it should be the output of
+`rclone obscure YOURPASSWORD`.
+
### --no-check-certificate
`--no-check-certificate` controls whether a client verifies the
diff --git a/fs/config.go b/fs/config.go
index 04a18aafe..ce0a1df9e 100644
--- a/fs/config.go
+++ b/fs/config.go
@@ -438,6 +438,12 @@ var ConfigOptionsInfo = Options{{
Default: "",
Help: "Client SSL private key (PEM) for mutual TLS auth",
Groups: "Networking",
+}, {
+ Name: "client_pass",
+ Default: "",
+ Help: "Password for client SSL private key (PEM) for mutual TLS auth (obscured)",
+ Groups: "Networking",
+ IsPassword: true,
}, {
Name: "multi_thread_cutoff",
Default: SizeSuffix(256 * 1024 * 1024),
@@ -644,6 +650,7 @@ type ConfigInfo struct {
CaCert []string `config:"ca_cert"` // Client Side CA
ClientCert string `config:"client_cert"` // Client Side Cert
ClientKey string `config:"client_key"` // Client Side Key
+ ClientPass string `config:"client_pass"` // Client Side Key Password (obscured)
MultiThreadCutoff SizeSuffix `config:"multi_thread_cutoff"`
MultiThreadStreams int `config:"multi_thread_streams"`
MultiThreadSet bool `config:"multi_thread_set"` // whether MultiThreadStreams was set (set in fs/config/configflags)
diff --git a/fs/fshttp/http.go b/fs/fshttp/http.go
index de9ca3cf8..65684f3db 100644
--- a/fs/fshttp/http.go
+++ b/fs/fshttp/http.go
@@ -6,6 +6,8 @@ import (
"context"
"crypto/tls"
"crypto/x509"
+ "encoding/pem"
+ "errors"
"fmt"
"net"
"net/http"
@@ -18,7 +20,9 @@ import (
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/accounting"
+ "github.com/rclone/rclone/fs/config/obscure"
"github.com/rclone/rclone/lib/structs"
+ "github.com/youmark/pkcs8"
"golang.org/x/net/publicsuffix"
)
@@ -48,6 +52,156 @@ func ResetTransport() {
noTransport = new(sync.Once)
}
+// LoadKeyPair loads a TLS certificate and private key from PEM-encoded files,
+// with extended support for encrypted private keys.
+//
+// This function is designed as a robust replacement for tls.X509KeyPair,
+// providing the same core functionality but adding support for
+// password-protected private keys.
+//
+// The certificate file (certFile) must contain one or more PEM-encoded
+// certificates. The first certificate is treated as the leaf certificate, and
+// any subsequent certificates are treated as its chain.
+//
+// The key file (keyFile) must contain a PEM-encoded private key. Supported
+// formats are:
+//
+// - Unencrypted PKCS#1 ("BEGIN RSA PRIVATE KEY")
+// - Unencrypted PKCS#8 ("BEGIN PRIVATE KEY")
+// - Encrypted PKCS#8 ("BEGIN ENCRYPTED PRIVATE KEY")
+// - Legacy PEM encryption (e.g., DEK-Info headers), which are automatically detected.
+//
+// The password parameter is used to decrypt the private key. If the
+// key is not encrypted, this parameter is ignored and can be an empty
+// string. The password should be an obscured string.
+//
+// On success, it returns a fully populated tls.Certificate struct, including the
+// Leaf certificate field.
+func LoadKeyPair(certFile, keyFile, password string) (cert tls.Certificate, err error) {
+ certPEM, err := os.ReadFile(certFile)
+ if err != nil {
+ return cert, fmt.Errorf("read cert: %w", err)
+ }
+ keyPEM, err := os.ReadFile(keyFile)
+ if err != nil {
+ return cert, fmt.Errorf("read key: %w", err)
+ }
+ if password != "" {
+ password, err = obscure.Reveal(password)
+ if err != nil {
+ return cert, fmt.Errorf("reveal key password: %w", err)
+ }
+ }
+
+ // Fast path: unencrypted PKCS#1/PKCS#8
+ cert, err = tls.X509KeyPair(certPEM, keyPEM)
+ if err == nil {
+ if len(cert.Certificate) == 0 {
+ return cert, errors.New("no certificates parsed")
+ }
+ leaf, err := x509.ParseCertificate(cert.Certificate[0])
+ if err != nil {
+ return cert, fmt.Errorf("parse leaf: %w", err)
+ }
+ cert.Leaf = leaf
+ return cert, nil
+ }
+
+ // Decrypt / parse key manually
+ block, rest := pem.Decode(keyPEM)
+ if block == nil {
+ return cert, errors.New("no PEM block in key")
+ }
+ if len(rest) != 0 {
+ fs.Debugf(nil, "Trailing data (%d bytes) in key PEM loaded from %q", len(rest), keyFile)
+ }
+
+ var privKey any
+ switch {
+ case block.Type == "ENCRYPTED PRIVATE KEY":
+ if password == "" {
+ return cert, errors.New("key is encrypted but no --client-pass provided")
+ }
+ privKey, err = pkcs8.ParsePKCS8PrivateKey(block.Bytes, []byte(password))
+ if err != nil {
+ return cert, fmt.Errorf("parse encrypted PKCS#8: %w", err)
+ }
+
+ case x509.IsEncryptedPEMBlock(block): //nolint:staticcheck // this is Legacy and insecure
+ if password == "" {
+ return cert, errors.New("key is encrypted but no --client-pass provided")
+ }
+ der, err := x509.DecryptPEMBlock(block, []byte(password)) //nolint:staticcheck // this is Legacy and insecure
+ if err != nil {
+ return cert, fmt.Errorf("decrypt PEM key: %w", err)
+ }
+ // Try PKCS#8, then RSA PKCS#1, then EC
+ if k, kerr1 := x509.ParsePKCS8PrivateKey(der); kerr1 == nil {
+ privKey = k
+ } else if k, kerr2 := x509.ParsePKCS1PrivateKey(der); kerr2 == nil {
+ privKey = k
+ } else if k, kerr3 := x509.ParseECPrivateKey(der); kerr3 == nil {
+ privKey = k
+ } else {
+ return cert, fmt.Errorf("parse decrypted key: pkcs8: %v, pkcs1: %v, ec: %v", kerr1, kerr2, kerr3)
+ }
+
+ default:
+ // Unencrypted specific types
+ switch block.Type {
+ case "PRIVATE KEY":
+ k, kerr := x509.ParsePKCS8PrivateKey(block.Bytes)
+ if kerr != nil {
+ return cert, fmt.Errorf("parse PKCS#8: %w", kerr)
+ }
+ privKey = k
+ case "RSA PRIVATE KEY":
+ k, kerr := x509.ParsePKCS1PrivateKey(block.Bytes)
+ if kerr != nil {
+ return cert, fmt.Errorf("parse PKCS#1 RSA: %w", kerr)
+ }
+ privKey = k
+ case "EC PRIVATE KEY":
+ k, kerr := x509.ParseECPrivateKey(block.Bytes)
+ if kerr != nil {
+ return cert, fmt.Errorf("parse EC: %w", kerr)
+ }
+ privKey = k
+ default:
+ return cert, fmt.Errorf("unsupported key type %q", block.Type)
+ }
+ }
+
+ // Build cert chain from PEM
+ var certDERs [][]byte
+ for rest := certPEM; ; {
+ var b *pem.Block
+ b, rest = pem.Decode(rest)
+ if b == nil {
+ break
+ }
+ if b.Type == "CERTIFICATE" {
+ certDERs = append(certDERs, b.Bytes)
+ }
+ }
+ if len(certDERs) == 0 {
+ return cert, fmt.Errorf("no CERTIFICATE blocks in %s", certFile)
+ }
+
+ cert = tls.Certificate{
+ Certificate: certDERs,
+ PrivateKey: privKey,
+ }
+
+ // Leaf is always the first certificate
+ cert.Leaf, err = x509.ParseCertificate(cert.Certificate[0])
+ if err != nil {
+ return cert, fmt.Errorf("parse leaf: %w", err)
+ }
+
+ return cert, nil
+}
+
// NewTransportCustom returns an http.RoundTripper with the correct timeouts.
// The customize function is called if set to give the caller an opportunity to
// customize any defaults in the Transport.
@@ -85,17 +239,10 @@ func NewTransportCustom(ctx context.Context, customize func(*http.Transport)) *T
if ci.ClientCert == "" || ci.ClientKey == "" {
fs.Fatalf(nil, "Both --client-cert and --client-key must be set")
}
- cert, err := tls.LoadX509KeyPair(ci.ClientCert, ci.ClientKey)
+ cert, err := LoadKeyPair(ci.ClientCert, ci.ClientKey, ci.ClientPass)
if err != nil {
fs.Fatalf(nil, "Failed to load --client-cert/--client-key pair: %v", err)
}
- if cert.Leaf == nil {
- // Leaf is always the first certificate
- cert.Leaf, err = x509.ParseCertificate(cert.Certificate[0])
- if err != nil {
- fs.Fatalf(nil, "Failed to parse the certificate")
- }
- }
t.TLSClientConfig.Certificates = []tls.Certificate{cert}
}
@@ -187,14 +334,12 @@ func NewClientWithUnixSocket(ctx context.Context, path string) *http.Client {
// * Updates metrics
type Transport struct {
*http.Transport
+ ci *fs.ConfigInfo
dump fs.DumpFlags
filterRequest func(req *http.Request)
userAgent string
headers []*fs.HTTPOption
metrics *Metrics
- // Filename of the client cert in case we need to reload it
- clientCert string
- clientKey string
// Mutex for serializing attempts at reloading the certificates
reloadMutex sync.Mutex
}
@@ -203,13 +348,12 @@ type Transport struct {
// roundtrips including the body if logBody is set.
func newTransport(ci *fs.ConfigInfo, transport *http.Transport) *Transport {
return &Transport{
- Transport: transport,
- dump: ci.Dump,
- userAgent: ci.UserAgent,
- headers: ci.Headers,
- metrics: DefaultMetrics,
- clientCert: ci.ClientCert,
- clientKey: ci.ClientKey,
+ Transport: transport,
+ ci: ci,
+ dump: ci.Dump,
+ userAgent: ci.UserAgent,
+ headers: ci.Headers,
+ metrics: DefaultMetrics,
}
}
@@ -309,20 +453,10 @@ func (t *Transport) reloadCertificates() {
if !isCertificateExpired(t.TLSClientConfig) {
return
}
-
- cert, err := tls.LoadX509KeyPair(t.clientCert, t.clientKey)
+ cert, err := LoadKeyPair(t.ci.ClientCert, t.ci.ClientKey, t.ci.ClientPass)
if err != nil {
fs.Fatalf(nil, "Failed to load --client-cert/--client-key pair: %v", err)
}
- // Check if we need to parse the certificate again, we need it
- // for checking the expiration date
- if cert.Leaf == nil {
- // Leaf is always the first certificate
- cert.Leaf, err = x509.ParseCertificate(cert.Certificate[0])
- if err != nil {
- fs.Fatalf(nil, "Failed to parse the certificate")
- }
- }
t.TLSClientConfig.Certificates = []tls.Certificate{cert}
}