Skip to content

armgit / fider

Public Imported from GitHub
0

Open platform to collect and prioritize feedback

Stop retrying Let's Encrypt for hosts that just failed

Matt Robertsfc3884e334e0
diff --git a/app/pkg/web/ssl.go b/app/pkg/web/ssl.go
index 59083aa28..e856e8e81 100644
--- a/app/pkg/web/ssl.go
+++ b/app/pkg/web/ssl.go
@@ -7,6 +7,8 @@ import (
"net"
"net/http"
"strings"
+ "sync"
+ "time"
"golang.org/x/crypto/acme"
"golang.org/x/net/idna"
@@ -43,6 +45,60 @@ func getDefaultTLSConfig(autoSSL bool) *tls.Config {
var errInvalidHostName = errors.New("autotls: invalid hostname")
+// certFailureCooldown is how long a host is skipped after a failed certificate acquisition.
+// Let's Encrypt allows roughly 5 failed validations per hostname per hour, so this keeps us
+// to at most 4, leaving headroom for the rest of the account.
+const certFailureCooldown = 15 * time.Minute
+
+// hostFailureCache records hosts whose certificate acquisition recently failed, so that we
+// stop asking Let's Encrypt for them until the cooldown elapses. Without it a single broken
+// custom domain retries on every TLS handshake, which rate limits the whole ACME account.
+type hostFailureCache struct {
+ mu sync.Mutex
+ failures map[string]time.Time
+}
+
+func newHostFailureCache() *hostFailureCache {
+ return &hostFailureCache{failures: make(map[string]time.Time)}
+}
+
+// normalizeHost matches how autocert canonicalizes the name it hands to the HostPolicy, so
+// that failures recorded from a raw ServerName are found again on the next handshake.
+func normalizeHost(host string) string {
+ return strings.TrimSuffix(strings.ToLower(host), ".")
+}
+
+func (c *hostFailureCache) onCooldown(host string) bool {
+ key := normalizeHost(host)
+
+ c.mu.Lock()
+ defer c.mu.Unlock()
+
+ failedAt, ok := c.failures[key]
+ if !ok {
+ return false
+ }
+
+ if time.Since(failedAt) >= certFailureCooldown {
+ delete(c.failures, key)
+ return false
+ }
+
+ return true
+}
+
+func (c *hostFailureCache) recordFailure(host string) {
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ c.failures[normalizeHost(host)] = time.Now()
+}
+
+func (c *hostFailureCache) clear(host string) {
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ delete(c.failures, normalizeHost(host))
+}
+
func isValidHostName(ctx context.Context, host string) error {
// In this context, host can only be custom domains, not a subdomain of fider.io
@@ -88,23 +144,25 @@ func isValidHostName(ctx context.Context, host string) error {
// CertificateManager is used to manage SSL certificates
type CertificateManager struct {
- ctx context.Context
- cert tls.Certificate
- leaf *x509.Certificate
- autotls autocert.Manager
+ ctx context.Context
+ cert tls.Certificate
+ leaf *x509.Certificate
+ autotls autocert.Manager
+ failures *hostFailureCache
}
// NewCertificateManager creates a new CertificateManager
func NewCertificateManager(ctx context.Context, certFile, keyFile string) (*CertificateManager, error) {
manager := &CertificateManager{
- ctx: ctx,
+ ctx: ctx,
+ failures: newHostFailureCache(),
autotls: autocert.Manager{
- Prompt: autocert.AcceptTOS,
- Cache: NewAutoCertCache(),
- Client: acmeClient(),
- HostPolicy: isValidHostName,
+ Prompt: autocert.AcceptTOS,
+ Cache: NewAutoCertCache(),
+ Client: acmeClient(),
},
}
+ manager.autotls.HostPolicy = manager.hostPolicy
if certFile != "" && keyFile != "" {
var err error
@@ -122,6 +180,16 @@ func NewCertificateManager(ctx context.Context, certFile, keyFile string) (*Cert
return manager, nil
}
+// hostPolicy is the autocert HostPolicy. autocert calls it before looking up or issuing a
+// certificate, but after serving TLS-ALPN challenge tokens, so refusing a host here stops us
+// from contacting Let's Encrypt without interfering with an in-flight challenge validation.
+func (m *CertificateManager) hostPolicy(ctx context.Context, host string) error {
+ if m.failures.onCooldown(host) {
+ return errors.Wrap(errInvalidHostName, "host %s is on cooldown after a recent certificate failure", host)
+ }
+ return isValidHostName(ctx, host)
+}
+
// GetCertificate decides which certificate to use
// It first tries to use loaded certificate for incoming request if it's compatible
// Otherwise fallsback to a automatically generated certificate by Let's Encrypt
@@ -167,8 +235,21 @@ func (m *CertificateManager) GetCertificate(hello *tls.ClientHelloInfo) (*tls.Ce
log.Debug(m.ctx, err.Error())
}
} else {
- log.Error(m.ctx, errors.Wrap(err, "failed to get certificate for %s", hello.ServerName))
+ // Put the host on cooldown so a domain that can't be issued doesn't retry on every
+ // handshake and get the ACME account rate limited.
+ m.failures.recordFailure(hello.ServerName)
+
+ failure := errors.Wrap(err, "failed to get certificate for %s", hello.ServerName)
+ // Same rationale as above: in multi-tenant mode a custom domain that fails ACME is
+ // the customer's problem to fix, not a Fider bug, so keep it out of error alarms.
+ if env.IsSingleHostMode() {
+ log.Error(m.ctx, failure)
+ } else {
+ log.Warn(m.ctx, failure.Error())
+ }
}
+ } else {
+ m.failures.clear(hello.ServerName)
}
return cert, err
diff --git a/app/pkg/web/ssl_test.go b/app/pkg/web/ssl_test.go
index 47022d3c2..c59842b7b 100644
--- a/app/pkg/web/ssl_test.go
+++ b/app/pkg/web/ssl_test.go
@@ -11,6 +11,7 @@ import (
"github.com/getfider/fider/app/models/query"
"github.com/getfider/fider/app/pkg/bus"
"github.com/getfider/fider/app/pkg/env"
+ "github.com/getfider/fider/app/pkg/errors"
"github.com/getfider/fider/app/services/blob/fs"
. "github.com/getfider/fider/app/pkg/assert"
@@ -152,6 +153,53 @@ func TestGetCertificate_ServerNameMatchesCertificate_ShouldReturnIt(t *testing.T
}
}
+func TestHostPolicy_WhenHostRecentlyFailed_ShouldSkipWithoutHittingLetsEncrypt(t *testing.T) {
+ RegisterT(t)
+ bus.Init(fs.Service{})
+ bus.AddHandler(mockGetTenantWithCorrectSubdomains)
+
+ manager, err := NewCertificateManager(context.Background(), "", "")
+ Expect(err).IsNil()
+
+ // a host with no recorded failure is handed over to isValidHostName, which rejects it
+ // because no tenant is registered with that cname
+ err = manager.hostPolicy(context.Background(), "feedback.heyworld.com")
+ Expect(errors.Cause(err)).Equals(errInvalidHostName)
+ Expect(err.Error()).ContainsSubstring("no tenant found with cname feedback.heyworld.com")
+
+ // once it fails, the host is skipped for the cooldown period
+ manager.failures.recordFailure("feedback.heyworld.com")
+ err = manager.hostPolicy(context.Background(), "feedback.heyworld.com")
+ Expect(errors.Cause(err)).Equals(errInvalidHostName)
+ Expect(err.Error()).ContainsSubstring("is on cooldown after a recent certificate failure")
+
+ // a successful handshake clears the cooldown
+ manager.failures.clear("feedback.heyworld.com")
+ err = manager.hostPolicy(context.Background(), "feedback.heyworld.com")
+ Expect(err.Error()).ContainsSubstring("no tenant found with cname feedback.heyworld.com")
+}
+
+func TestHostFailureCache(t *testing.T) {
+ RegisterT(t)
+
+ cache := newHostFailureCache()
+ Expect(cache.onCooldown("feedback.heyworld.com")).IsFalse()
+
+ cache.recordFailure("feedback.heyworld.com")
+ Expect(cache.onCooldown("feedback.heyworld.com")).IsTrue()
+ // autocert lowercases and strips the trailing dot before calling the host policy, so a
+ // failure recorded from a raw ServerName must still be found
+ Expect(cache.onCooldown("FEEDBACK.HeyWorld.com.")).IsTrue()
+ Expect(cache.onCooldown("ideas.app.com")).IsFalse()
+
+ cache.clear("feedback.heyworld.com")
+ Expect(cache.onCooldown("feedback.heyworld.com")).IsFalse()
+
+ // a failure older than the cooldown lets the host through again
+ cache.failures[normalizeHost("feedback.heyworld.com")] = time.Now().Add(-certFailureCooldown - time.Second)
+ Expect(cache.onCooldown("feedback.heyworld.com")).IsFalse()
+}
+
func TestGetCertificate_ServerNameDoesntMatchCertificate_ButEndsWithHostName_ShouldThrow(t *testing.T) {
RegisterT(t)
bus.Init(fs.Service{})