Open-source status page, incident response and uptime monitoring
TypeScript alertingincident-managementmcpmcp-servermonitoringmonitoring-as-codeobservabilityon-callopen-sourcesrestatus-pagestatuspagesynthetic-monitoringterraformtinybirdtursouptimeuptime-checkeruptime-monitor
feat(checker): implement backoff retry (#570)
Arthur EICHELBERGER1f6680a6c9a9
diff --git a/apps/checker/cmd/main.go b/apps/checker/cmd/main.goindex 0697af97e..a38d02a33 100644--- a/apps/checker/cmd/main.go+++ b/apps/checker/cmd/main.go@@ -7,7 +7,6 @@ import ("net/http""os""os/signal"- "strconv""syscall""github.com/gin-gonic/gin"@@ -16,8 +15,16 @@ import ("github.com/openstatushq/openstatus/apps/checker/pkg/tinybird""github.com/openstatushq/openstatus/apps/checker/request""github.com/rs/zerolog/log"++ backoff "github.com/cenkalti/backoff/v4")+type statusCode int++func (s statusCode) IsSuccessful() bool {+ return s >= 200 && s < 300+}+func main() {ctx, cancel := context.WithCancel(context.Background())defer cancel()@@ -53,16 +60,6 @@ func main() {return}- switch i, err := strconv.Atoi(c.GetHeader("X-CloudTasks-TaskRetryCount")); {- case err != nil:- c.JSON(http.StatusBadRequest, gin.H{"error": "invalid retry count"})- return- case i > 1:- // Why would that be OK?- c.JSON(http.StatusOK, gin.H{"message": "ok"})- return- }-var req request.CheckerRequestif err := c.ShouldBindJSON(&req); err != nil {log.Ctx(ctx).Error().Err(err).Msg("failed to decode checker request")@@ -70,67 +67,64 @@ func main() {return}- response, err := checker.Ping(ctx, httpClient, req)- if err != nil {- response, err = checker.Ping(ctx, httpClient, req)+ op := func() error {+ res, err := checker.Ping(ctx, httpClient, req)if err != nil {- if err := tinybirdClient.SendEvent(ctx, checker.PingData{- URL: req.URL,- Region: flyRegion,- Message: err.Error(),- CronTimestamp: req.CronTimestamp,- Timestamp: req.CronTimestamp,- MonitorID: req.MonitorID,- WorkspaceID: req.WorkspaceID,- }); err != nil {- log.Ctx(ctx).Error().Err(err).Msg("failed to send event to tinybird")- }-- if req.Status == "active" {- checker.UpdateStatus(ctx, checker.UpdateData{- MonitorId: req.MonitorID,- Status: "error",- Message: err.Error(),- Region: flyRegion,- })- }-- c.JSON(http.StatusOK, gin.H{"message": "ok"})- return+ return fmt.Errorf("unable to ping: %w", err)}- }- if response.StatusCode < 200 || response.StatusCode >= 300 {- // Add one more retry- response, err = checker.Ping(ctx, httpClient, req)- if response.StatusCode < 200 || response.StatusCode >= 300 && req.Status == "active" {- // If the status code is not within the 200 range, we update the status to err+ statusCode := statusCode(res.StatusCode)+ if !statusCode.IsSuccessful() {+ // Q: Why here we do not check if the status was previously active?checker.UpdateStatus(ctx, checker.UpdateData{MonitorId: req.MonitorID,Status: "error",- StatusCode: response.StatusCode,+ StatusCode: res.StatusCode,+ Region: flyRegion,+ })+ } else if req.Status == "error" && statusCode.IsSuccessful() {+ // Q: Why here we check the data before updating the status in this scenario?+ checker.UpdateStatus(ctx, checker.UpdateData{+ MonitorId: req.MonitorID,+ Status: "active",Region: flyRegion,+ StatusCode: res.StatusCode,})}- }- // If the status was error and the status code is within the 200 range, we update the status to active- if req.Status == "error" && response.StatusCode >= 200 && response.StatusCode < 300 {- // If the status was error, we update it to active- checker.UpdateStatus(ctx, checker.UpdateData{- MonitorId: req.MonitorID,- Status: "active",- Region: flyRegion,- StatusCode: response.StatusCode,- })+ if err := tinybirdClient.SendEvent(ctx, res); err != nil {+ log.Ctx(ctx).Error().Err(err).Msg("failed to send event to tinybird")+ }++ return nil}- if err := tinybirdClient.SendEvent(ctx, response); err != nil {- log.Ctx(ctx).Error().Err(err).Msg("failed to send event to tinybird")+ if err := backoff.Retry(op, backoff.WithMaxRetries(backoff.NewExponentialBackOff(), 3)); err != nil {+ if err := tinybirdClient.SendEvent(ctx, checker.PingData{+ URL: req.URL,+ Region: flyRegion,+ Message: err.Error(),+ CronTimestamp: req.CronTimestamp,+ Timestamp: req.CronTimestamp,+ MonitorID: req.MonitorID,+ WorkspaceID: req.WorkspaceID,+ }); err != nil {+ log.Ctx(ctx).Error().Err(err).Msg("failed to send event to tinybird")+ }++ // If the status was previously active, we update it to error.+ // Q: Why not always updating the status? My idea is that the checker should be dumb and only check the status and return it.+ if req.Status == "active" {+ checker.UpdateStatus(ctx, checker.UpdateData{+ MonitorId: req.MonitorID,+ Status: "error",+ Message: err.Error(),+ Region: flyRegion,+ })+ }}c.JSON(http.StatusOK, gin.H{"message": "ok"})- return})router.GET("/ping", func(c *gin.Context) {diff --git a/apps/checker/go.mod b/apps/checker/go.modindex 800f66ca6..dede75a7f 100644--- a/apps/checker/go.mod+++ b/apps/checker/go.mod@@ -3,6 +3,7 @@ module github.com/openstatushq/openstatus/apps/checkergo 1.21.4require (+ github.com/cenkalti/backoff/v4 v4.2.1github.com/gin-gonic/gin v1.9.1github.com/rs/zerolog v1.31.0github.com/stretchr/testify v1.8.3diff --git a/apps/checker/go.sum b/apps/checker/go.sumindex 5006a1c0f..a28caf689 100644--- a/apps/checker/go.sum+++ b/apps/checker/go.sum@@ -1,6 +1,8 @@github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM=github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s=github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U=+github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM=+github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY=github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams=github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk=diff --git a/apps/checker/ping.go b/apps/checker/ping.goindex d1cc96670..4b449d718 100644--- a/apps/checker/ping.go+++ b/apps/checker/ping.go@@ -3,8 +3,8 @@ package checkerimport ("bytes""context"+ "errors""fmt"- "io""net/http""net/url""os"@@ -37,31 +37,25 @@ func Ping(ctx context.Context, client *http.Client, inputData request.CheckerReq}req.Header.Set("User-Agent", "OpenStatus/1.0")-- // Setting headersfor _, header := range inputData.Headers {- if header.Key != "" && header.Value != "" {- req.Header.Set(header.Key, header.Value)- }+ req.Header.Set(header.Key, header.Value)}start := time.Now()response, err := client.Do(req)latency := time.Since(start).Milliseconds()-if err != nil {- if urlErr, ok := err.(*url.Error); ok {- if urlErr.Timeout() {- return PingData{- Latency: latency,- MonitorID: inputData.MonitorID,- Region: region,- WorkspaceID: inputData.WorkspaceID,- Timestamp: time.Now().UTC().UnixMilli(),- URL: inputData.URL,- Message: fmt.Sprintf("Timeout after %d ms", latency),- }, nil- }+ var urlErr *url.Error+ if errors.As(err, &urlErr) && urlErr.Timeout() {+ return PingData{+ Latency: latency,+ MonitorID: inputData.MonitorID,+ Region: region,+ WorkspaceID: inputData.WorkspaceID,+ Timestamp: time.Now().UTC().UnixMilli(),+ URL: inputData.URL,+ Message: fmt.Sprintf("Timeout after %d ms", latency),+ }, nil}logger.Error().Err(err).Msg("error while pinging")@@ -69,11 +63,6 @@ func Ping(ctx context.Context, client *http.Client, inputData request.CheckerReq}defer response.Body.Close()- if _, err := io.ReadAll(response.Body); err != nil {- logger.Error().Err(err).Str("monitorURL", inputData.URL).Msg("error while reading body")- return PingData{}, fmt.Errorf("error while reading body from %s: %w", inputData.URL, err)- }-return PingData{Latency: latency,StatusCode: response.StatusCode,