Open-source status page, incident response and uptime monitoring
TypeScript alertingincident-managementmcpmcp-servermonitoringmonitoring-as-codeobservabilityon-callopen-sourcesrestatus-pagestatuspagesynthetic-monitoringterraformtinybirdtursouptimeuptime-checkeruptime-monitor
checker: small improvment (#2955)
Thibault Le Ouay723329bda63c
diff --git a/apps/checker/checker/dns.go b/apps/checker/checker/dns.goindex b2383c2c8..a3f2f2311 100644--- a/apps/checker/checker/dns.go+++ b/apps/checker/checker/dns.go@@ -28,7 +28,7 @@ func Dns(ctx context.Context, host string) (*DnsResponse, error) {ips, err := resolver.LookupIP(ctx, "ip", host)if err != nil {- logger.Error().Err(err).Msg("DNS IP lookup failed")+ logger.Warn().Err(err).Msg("DNS IP lookup failed")return nil, fmt.Errorf("failed to lookup IPs: %w", err)}@@ -44,14 +44,14 @@ func Dns(ctx context.Context, host string) (*DnsResponse, error) {}CNAME, err := lookupCNAME(ctx, host)if err != nil {- logger.Error().Err(err).Msg("DNS CNAME record lookup failed")+ logger.Warn().Err(err).Msg("DNS CNAME record lookup failed")return nil, fmt.Errorf("failed to lookup CNAME record: %w", err)}MXRecords := lookupMX(ctx, host)NS, err := lookupNS(ctx, host)if err != nil {- logger.Error().Err(err).Msg("DNS NS record lookup failed")+ logger.Warn().Err(err).Msg("DNS NS record lookup failed")return nil, fmt.Errorf("failed to lookup NS record: %w", err)}TXT := lookupTXT(ctx, host)@@ -59,7 +59,7 @@ func Dns(ctx context.Context, host string) (*DnsResponse, error) {// MX and TXT tolerate lookup errors, but an expired deadline means those// records are unknown rather than absent — don't report that as a success.if err := ctx.Err(); err != nil {- logger.Error().Err(err).Msg("DNS lookup did not complete before the deadline")+ logger.Warn().Err(err).Msg("DNS lookup did not complete before the deadline")return nil, fmt.Errorf("DNS lookup for %s did not complete: %w", host, err)}diff --git a/apps/checker/checker/http.go b/apps/checker/checker/http.goindex aef56126a..3e9120375 100644--- a/apps/checker/checker/http.go+++ b/apps/checker/checker/http.go@@ -8,10 +8,12 @@ import ("errors""fmt""io"+ "net""net/http""net/http/httptrace""net/url""strings"+ "syscall""time""github.com/openstatushq/openstatus/apps/checker/request"@@ -56,6 +58,95 @@ func decodeBase64Body(body string) ([]byte, error) {return nil, fmt.Errorf("invalid base64 data url format")}+// defaultHTTPTimeout bounds a probe whose monitor has no timeout set: a zero+// timeout would otherwise let a dial or TLS handshake hang forever.+const defaultHTTPTimeout = 45 * time.Second++// NewHTTPClient returns a client with a transport of its own. Leaving Transport+// nil shares http.DefaultTransport: a probe could then be handed a pooled+// connection (no DNS/connect/TLS timing), and CloseIdleConnections would tear+// down the connections of every concurrent probe and of the Tinybird client.+// With keep-alives off, every probe is sent with "Connection: close".+func NewHTTPClient(timeout time.Duration, followRedirects bool) *http.Client {+ if timeout <= 0 {+ timeout = defaultHTTPTimeout+ }++ // Dial and TLS get the monitor's whole timeout: http.DefaultTransport caps+ // them at 30s and 10s, which cut a check short of what the user set.+ transport := &http.Transport{+ Proxy: http.ProxyFromEnvironment,+ DialContext: (&net.Dialer{Timeout: timeout}).DialContext,+ TLSHandshakeTimeout: timeout,+ ExpectContinueTimeout: time.Second,+ ForceAttemptHTTP2: true,+ DisableKeepAlives: true,+ }++ client := &http.Client{+ Timeout: timeout,+ Transport: transport,+ }++ if !followRedirects {+ client.CheckRedirect = func(req *http.Request, via []*http.Request) error {+ return http.ErrUseLastResponse+ }+ } else {+ // Explicitly limit the number of redirects to 10 (Go's default)+ client.CheckRedirect = func(req *http.Request, via []*http.Request) error {+ if len(via) >= 10 {+ return http.ErrUseLastResponse+ }++ return nil+ }+ }++ return client+}++// httpErrorMessage turns a failed request into the message stored for the+// customer: common transport failures get a short fixed message, and anything+// else loses the method and URL prefix. DNS messages leave out the resolver,+// which on fly is an internal address ([fdaa::3]:53).+func httpErrorMessage(err error, latency int64) string {+ var urlErr *url.Error+ if errors.As(err, &urlErr) && urlErr.Timeout() {+ return fmt.Sprintf("Timeout after %d ms", latency)+ }++ var dnsErr *net.DNSError+ if errors.As(err, &dnsErr) {+ return fmt.Sprintf("DNS lookup failed for %s: %s", dnsErr.Name, dnsErr.Err)+ }++ var certErr *tls.CertificateVerificationError+ if errors.As(err, &certErr) {+ return fmt.Sprintf("TLS certificate verification failed: %s", certErr.Err)+ }++ var opErr *net.OpError+ if errors.As(err, &opErr) && opErr.Op == "remote error" {+ return fmt.Sprintf("TLS handshake failed: %s", opErr.Err)+ }++ switch {+ case errors.Is(err, syscall.ECONNREFUSED):+ return "Connection refused"+ case errors.Is(err, syscall.ECONNRESET):+ return "Connection reset by peer"+ case errors.Is(err, io.EOF), errors.Is(err, io.ErrUnexpectedEOF):+ return "Connection closed by server before a response was received"+ }++ if urlErr != nil {+ return urlErr.Err.Error()+ }++ return err.Error()+}+// FIXME: This should only return the TCP Timing Data;func Http(ctx context.Context, client *http.Client, inputData request.HttpCheckerRequest) (Response, error) {logger := log.Ctx(ctx).With().Str("monitor", inputData.URL).Logger()@@ -126,14 +217,10 @@ func Http(ctx context.Context, client *http.Client, inputData request.HttpCheckelatency := time.Since(start).Milliseconds()if err != nil {- errorMsg := err.Error()-- var urlErr *url.Error- if errors.As(err, &urlErr) && urlErr.Timeout() {- errorMsg = fmt.Sprintf("Timeout after %d ms", latency)- }+ errorMsg := httpErrorMessage(err, latency)- logger.Error().Err(err).Msg("error while pinging")+ // A failing customer endpoint is expected, not a checker error.+ logger.Warn().Err(err).Msg("error while pinging")// Return Response with error field instead of returning a Go error// This ensures all failures (timeouts, connection refused, DNS failures, etc.)diff --git a/apps/checker/checker/http_internal_test.go b/apps/checker/checker/http_internal_test.gonew file mode 100644index 000000000..1edbe1f75--- /dev/null+++ b/apps/checker/checker/http_internal_test.go@@ -0,0 +1,107 @@+package checker++import (+ "crypto/tls"+ "crypto/x509"+ "errors"+ "io"+ "net"+ "net/http"+ "net/url"+ "os"+ "syscall"+ "testing"++ "github.com/stretchr/testify/assert"+)++type timeoutError struct{}++func (timeoutError) Error() string { return "i/o timeout" }+func (timeoutError) Timeout() bool { return true }+func (timeoutError) Temporary() bool { return true }++func urlError(err error) error {+ return &url.Error{Op: "Get", URL: "https://example.com", Err: err}+}++func TestHTTPErrorMessage(t *testing.T) {+ tests := []struct {+ name string+ err error+ want string+ }{+ {+ name: "timeout",+ err: urlError(timeoutError{}),+ want: "Timeout after 1200 ms",+ },+ {+ name: "dns not found hides the resolver address",+ err: urlError(&net.OpError{Op: "dial", Net: "tcp", Err: &net.DNSError{+ Err: "no such host", Name: "api.acme.dev", Server: "[fdaa::3]:53", IsNotFound: true,+ }}),+ want: "DNS lookup failed for api.acme.dev: no such host",+ },+ {+ name: "dns servfail",+ err: urlError(&net.OpError{Op: "dial", Net: "tcp", Err: &net.DNSError{+ Err: "server misbehaving", Name: "set.id", Server: "[fdaa::3]:53",+ }}),+ want: "DNS lookup failed for set.id: server misbehaving",+ },+ {+ name: "remote tls alert",+ err: urlError(&net.OpError{Op: "remote error", Err: errors.New("tls: handshake failure")}),+ want: "TLS handshake failed: tls: handshake failure",+ },+ {+ name: "certificate verification",+ err: urlError(&tls.CertificateVerificationError{Err: x509.UnknownAuthorityError{}}),+ want: "TLS certificate verification failed: x509: certificate signed by unknown authority",+ },+ {+ name: "connection refused",+ err: urlError(&net.OpError{Op: "dial", Net: "tcp", Err: &os.SyscallError{+ Syscall: "connect", Err: syscall.ECONNREFUSED,+ }}),+ want: "Connection refused",+ },+ {+ name: "connection reset",+ err: urlError(&net.OpError{Op: "read", Net: "tcp", Err: &os.SyscallError{+ Syscall: "read", Err: syscall.ECONNRESET,+ }}),+ want: "Connection reset by peer",+ },+ {+ name: "server closed the connection",+ err: urlError(io.EOF),+ want: "Connection closed by server before a response was received",+ },+ {+ name: "anything else drops the method and url prefix",+ err: urlError(errors.New("stopped after 10 redirects")),+ want: "stopped after 10 redirects",+ },+ }++ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ assert.Equal(t, tt.want, httpErrorMessage(tt.err, 1200))+ })+ }+}++// A monitor without a timeout must still be bounded: a zero timeout on the+// dialer and TLS handshake means no limit at all.+func TestNewHTTPClientZeroTimeoutFallsBack(t *testing.T) {+ client := NewHTTPClient(0, true)++ assert.Equal(t, defaultHTTPTimeout, client.Timeout)++ transport, ok := client.Transport.(*http.Transport)+ assert.True(t, ok)+ assert.Equal(t, defaultHTTPTimeout, transport.TLSHandshakeTimeout)+ assert.True(t, transport.DisableKeepAlives)+}diff --git a/apps/checker/checker/http_test.go b/apps/checker/checker/http_test.goindex b09c89fc4..9b4b4d03a 100644--- a/apps/checker/checker/http_test.go+++ b/apps/checker/checker/http_test.go@@ -4,8 +4,12 @@ import ("bytes""context""io"+ "net""net/http"+ "net/http/httptest"+ "sync/atomic""testing"+ "time""github.com/stretchr/testify/assert"@@ -117,3 +121,33 @@ func Test_ping(t *testing.T) {})}}++// Each probe must open its own connection: a pooled one would skip DNS,+// connect and TLS, and report those timings as zero.+func TestNewHTTPClientDoesNotReuseConnections(t *testing.T) {+ var newConns atomic.Int32++ server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {+ w.WriteHeader(http.StatusOK)+ }))+ server.Config.ConnState = func(_ net.Conn, state http.ConnState) {+ if state == http.StateNew {+ newConns.Add(1)+ }+ }+ server.Start()+ t.Cleanup(server.Close)++ client := checker.NewHTTPClient(5*time.Second, true)+ defer client.CloseIdleConnections()++ for range 2 {+ input := request.HttpCheckerRequest{URL: server.URL, Method: http.MethodGet}+ res, err := checker.Http(context.Background(), client, input)+ assert.NoError(t, err)+ assert.Equal(t, http.StatusOK, res.Status)+ assert.NotZero(t, res.Timing.ConnectStart)+ }++ assert.Equal(t, int32(2), newConns.Load())+}diff --git a/apps/checker/checker/update.go b/apps/checker/checker/update.goindex f3adbe19b..b1299cfea 100644--- a/apps/checker/checker/update.go+++ b/apps/checker/checker/update.go@@ -7,6 +7,7 @@ import ("fmt""os""strings"+ "sync""github.com/rs/zerolog/log""google.golang.org/api/option"@@ -26,11 +27,22 @@ type UpdateData struct {Latency int64 `json:"latency,omitempty"`}-func UpdateStatus(ctx context.Context, updateData UpdateData) error {+var (+ tasksClientMu sync.Mutex+ tasksClient *cloudtasks.Client+)++// cloudTasksClient builds the Cloud Tasks client once and shares it: it holds a+// gRPC connection and a token cache, too heavy to rebuild on every status+// change. A failed build is not cached, so the next call tries again.+func cloudTasksClient() (*cloudtasks.Client, error) {+ tasksClientMu.Lock()+ defer tasksClientMu.Unlock()++ if tasksClient != nil {+ return tasksClient, nil+ }- url := "https://openstatus-workflows.fly.dev/updateStatus"- basic := "Basic " + os.Getenv("CRON_SECRET")- payloadBuf := new(bytes.Buffer)c := os.Getenv("GCP_PRIVATE_KEY")c = strings.ReplaceAll(c, "\\n", "\n")opts := &auth.Options2LO{@@ -45,23 +57,45 @@ func UpdateStatus(ctx context.Context, updateData UpdateData) error {tp, err := auth.New2LOTokenProvider(opts)if err != nil {- log.Ctx(ctx).Error().Err(err).Msg("error while creating token provider")- return err+ return nil, fmt.Errorf("creating token provider: %w", err)}creds := auth.NewCredentials(&auth.CredentialsOptions{TokenProvider: tp,})- client, err := cloudtasks.NewClient(ctx, option.WithAuthCredentials(creds))+ // The client lives as long as the process, so it takes neither the+ // cancellation nor the values (logger, monitor) of the request that built it.+ client, err := cloudtasks.NewClient(context.Background(), option.WithAuthCredentials(creds))+ if err != nil {+ return nil, fmt.Errorf("creating cloud tasks client: %w", err)+ }++ tasksClient = client++ return tasksClient, nil+}++// UpdateStatus queues the status change that drives alerting. Callers do not+// act on the returned error, so every failure is logged here: a lost update is+// a missed alert.+func UpdateStatus(ctx context.Context, updateData UpdateData) error {+ // monitor_id is already on the context logger (handlers.withMonitorLogger).+ logger := log.Ctx(ctx).With().Str("status", updateData.Status).Logger()++ url := "https://openstatus-workflows.fly.dev/updateStatus"+ basic := "Basic " + os.Getenv("CRON_SECRET")+ payloadBuf := new(bytes.Buffer)++ client, err := cloudTasksClient()if err != nil {- log.Ctx(ctx).Error().Err(err).Msg("error while creating cloud tasks client")+ logger.Error().Err(err).Msg("error while updating status")+ return err}- defer client.Close()if err := json.NewEncoder(payloadBuf).Encode(updateData); err != nil {- log.Ctx(ctx).Error().Err(err).Msg("error while updating status")+ logger.Error().Err(err).Msg("error while updating status")return err}projectID := os.Getenv("GCP_PROJECT_ID")@@ -85,7 +119,7 @@ func UpdateStatus(ctx context.Context, updateData UpdateData) error {_, err = client.CreateTask(ctx, req)if err != nil {- log.Ctx(ctx).Error().Err(err).Msg("error while creating the cloud task")+ logger.Error().Err(err).Msg("error while creating the cloud task")return fmt.Errorf("cloudtasks.CreateTask: %w", err)}diff --git a/apps/checker/handlers/checker.go b/apps/checker/handlers/checker.goindex 74d6851da..936c05595 100644--- a/apps/checker/handlers/checker.go+++ b/apps/checker/handlers/checker.go@@ -73,25 +73,10 @@ func (h Handler) HTTPCheckerHandler(c *gin.Context) {return}- // We need a new client for each request to avoid connection reuse.- requestClient := &http.Client{- Timeout: time.Duration(req.Timeout) * time.Millisecond,- }- // Configure redirect policy based on FollowRedirects setting- if !req.FollowRedirects {- requestClient.CheckRedirect = func(req *http.Request, via []*http.Request) error {- return http.ErrUseLastResponse- }- } else {- // Explicitly limit the number of redirects to 10 (Go's default)- requestClient.CheckRedirect = func(req *http.Request, via []*http.Request) error {- if len(via) >= 10 {- return http.ErrUseLastResponse- }- return nil- }- }+ ctx = withMonitorLogger(ctx, req.MonitorID, req.WorkspaceID)++ requestClient := checker.NewHTTPClient(time.Duration(req.Timeout)*time.Millisecond, req.FollowRedirects)defer requestClient.CloseIdleConnections()// Might be a more efficient way to do it@@ -119,7 +104,8 @@ func (h Handler) HTTPCheckerHandler(c *gin.Context) {op := func() (struct{}, error) {called++- res, err := checker.Http(ctx, requestClient, req)+ attemptCtx := log.Ctx(ctx).With().Int("attempt", called).Logger().WithContext(ctx)+ res, err := checker.Http(attemptCtx, requestClient, req)if err != nil {return struct{}{}, fmt.Errorf("unable to ping: %w", err)diff --git a/apps/checker/handlers/dns.go b/apps/checker/handlers/dns.goindex 26a06789e..22efe01c8 100644--- a/apps/checker/handlers/dns.go+++ b/apps/checker/handlers/dns.go@@ -87,6 +87,8 @@ func (h Handler) DNSHandler(c *gin.Context) {return}+ ctx = withMonitorLogger(ctx, req.MonitorID, req.WorkspaceID)+workspaceId, err := strconv.ParseInt(req.WorkspaceID, 10, 64)if err != nil {c.JSON(http.StatusBadRequest, gin.H{"error": "invalid workspace id"})@@ -150,7 +152,6 @@ func (h Handler) DNSHandler(c *gin.Context) {latency = time.Now().UTC().UnixMilli() - startif err != nil {- log.Ctx(ctx).Error().Err(err).Msg("dns check failed")return nil, err}if len(req.RawAssertions) > 0 {@@ -276,6 +277,8 @@ func (h Handler) DNSHandlerRegion(c *gin.Context) {return}+ ctx = withMonitorLogger(ctx, req.MonitorID, req.WorkspaceID)+retry := defaultRetryif req.Retry != 0 {retry = int(req.Retry)@@ -322,7 +325,6 @@ func (h Handler) DNSHandlerRegion(c *gin.Context) {latency = time.Now().UTC().UnixMilli() - startif err != nil {- log.Ctx(ctx).Error().Err(err).Msg("dns check failed")return nil, err}if len(req.RawAssertions) > 0 {diff --git a/apps/checker/handlers/grpc.go b/apps/checker/handlers/grpc.goindex 7e82c8fcd..3faf3113a 100644--- a/apps/checker/handlers/grpc.go+++ b/apps/checker/handlers/grpc.go@@ -78,6 +78,8 @@ func (h Handler) GRPCHandler(c *gin.Context) {return}+ ctx = withMonitorLogger(ctx, req.MonitorID, req.WorkspaceID)+workspaceId, err := strconv.ParseInt(req.WorkspaceID, 10, 64)if err != nil {c.JSON(http.StatusBadRequest, gin.H{"error": "invalid request"})@@ -332,6 +334,8 @@ func (h Handler) GRPCHandlerRegion(c *gin.Context) {return}+ ctx = withMonitorLogger(ctx, req.MonitorID, req.WorkspaceID)+var response checker.GRPCResponsevar retry intdiff --git a/apps/checker/handlers/handler.go b/apps/checker/handlers/handler.goindex 7ee164b7a..e15a75f23 100644--- a/apps/checker/handlers/handler.go+++ b/apps/checker/handlers/handler.go@@ -1,9 +1,11 @@package handlersimport (- "net/http"+ "context""time"+ "github.com/rs/zerolog/log"+"github.com/openstatushq/openstatus/apps/checker/pkg/tinybird")@@ -23,6 +25,12 @@ type Handler struct {// Authorization could be handle by middleware-func NewHTTPClient() *http.Client {- return &http.Client{}+// withMonitorLogger tags every log line of a check with the monitor it belongs+// to: the probed URL alone does not say which workspace to look at.+func withMonitorLogger(ctx context.Context, monitorID, workspaceID string) context.Context {+ return log.Ctx(ctx).With().+ Str("monitor_id", monitorID).+ Str("workspace_id", workspaceID).+ Logger().+ WithContext(ctx)}diff --git a/apps/checker/handlers/icmp.go b/apps/checker/handlers/icmp.goindex 36213f731..da751988f 100644--- a/apps/checker/handlers/icmp.go+++ b/apps/checker/handlers/icmp.go@@ -70,6 +70,8 @@ func (h Handler) ICMPHandler(c *gin.Context) {return}+ ctx = withMonitorLogger(ctx, req.MonitorID, req.WorkspaceID)+workspaceId, err := strconv.ParseInt(req.WorkspaceID, 10, 64)if err != nil {c.JSON(http.StatusBadRequest, gin.H{"error": "invalid request"})@@ -300,6 +302,8 @@ func (h Handler) ICMPHandlerRegion(c *gin.Context) {return}+ ctx = withMonitorLogger(ctx, req.MonitorID, req.WorkspaceID)+var response checker.ICMPResponseop := func() (struct{}, error) {diff --git a/apps/checker/handlers/ping.go b/apps/checker/handlers/ping.goindex 9283d481d..56d806f28 100644--- a/apps/checker/handlers/ping.go+++ b/apps/checker/handlers/ping.go@@ -68,10 +68,10 @@ func (h Handler) PingRegionHandler(c *gin.Context) {}}- // We need a new client for each request to avoid connection reuse.- requestClient := &http.Client{- Timeout: 45 * time.Second,- }+ requestClient := checker.NewHTTPClient(45*time.Second, true)+ // Keep Go's default policy here: a redirect loop is an error, not the last+ // 3xx response.+ requestClient.CheckRedirect = nildefer requestClient.CloseIdleConnections()diff --git a/apps/checker/handlers/tcp.go b/apps/checker/handlers/tcp.goindex ed1a7cfce..ed4ad54d2 100644--- a/apps/checker/handlers/tcp.go+++ b/apps/checker/handlers/tcp.go@@ -65,6 +65,8 @@ func (h Handler) TCPHandler(c *gin.Context) {return}+ ctx = withMonitorLogger(ctx, req.MonitorID, req.WorkspaceID)+workspaceId, err := strconv.ParseInt(req.WorkspaceID, 10, 64)if err != nil {@@ -291,6 +293,8 @@ func (h Handler) TCPHandlerRegion(c *gin.Context) {return}+ ctx = withMonitorLogger(ctx, req.MonitorID, req.WorkspaceID)+var called intvar response checker.TCPResponsediff --git a/apps/checker/pkg/job/http_job.go b/apps/checker/pkg/job/http_job.goindex 0163233cc..ab81e5319 100644--- a/apps/checker/pkg/job/http_job.go+++ b/apps/checker/pkg/job/http_job.go@@ -4,7 +4,6 @@ import ("context""encoding/json""fmt"- "net/http""time""github.com/cenkalti/backoff/v5"@@ -74,24 +73,9 @@ func (jr jobRunner) HTTPJob(ctx context.Context, monitor *v1.HTTPMonitor, regionretry = 3}- requestClient := &http.Client{- Timeout: time.Duration(monitor.Timeout) * time.Millisecond,- }+ requestClient := checker.NewHTTPClient(time.Duration(monitor.Timeout)*time.Millisecond, monitor.FollowRedirects)defer requestClient.CloseIdleConnections()- if !monitor.FollowRedirects {- requestClient.CheckRedirect = func(req *http.Request, via []*http.Request) error {- return http.ErrUseLastResponse- }- } else {- requestClient.CheckRedirect = func(req *http.Request, via []*http.Request) error {- if len(via) >= 10 {- return http.ErrUseLastResponse- }- return nil- }- }-var degradedAfter int64if monitor.DegradedAt != nil {degradedAfter = *monitor.DegradedAt