Skip to content

Open-source status page, incident response and uptime monitoring

feat(checker): implement backoff retry (#570)

Arthur EICHELBERGER1f6680a6c9a9
diff --git a/apps/checker/cmd/main.go b/apps/checker/cmd/main.go
index 0697af97e..a38d02a33 100644
--- a/apps/checker/cmd/main.go
+++ b/apps/checker/cmd/main.go
@@ -7,7 +7,6 @@ import (
"net/http"
"os"
"os/signal"
- "strconv"
"syscall"
"github.com/gin-gonic/gin"
@@ -16,8 +15,16 @@ import (
"github.com/openstatushq/openstatus/apps/checker/pkg/tinybird"
"github.com/openstatushq/openstatus/apps/checker/request"
"github.com/rs/zerolog/log"
+
+ backoff "github.com/cenkalti/backoff/v4"
)
+type statusCode int
+
+func (s statusCode) IsSuccessful() bool {
+ return s >= 200 && s < 300
+}
+
func main() {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
@@ -53,16 +60,6 @@ func main() {
return
}
- switch i, err := strconv.Atoi(c.GetHeader("X-CloudTasks-TaskRetryCount")); {
- case err != nil:
- c.JSON(http.StatusBadRequest, gin.H{"error": "invalid retry count"})
- return
- case i > 1:
- // Why would that be OK?
- c.JSON(http.StatusOK, gin.H{"message": "ok"})
- return
- }
-
var req request.CheckerRequest
if err := c.ShouldBindJSON(&req); err != nil {
log.Ctx(ctx).Error().Err(err).Msg("failed to decode checker request")
@@ -70,67 +67,64 @@ func main() {
return
}
- response, err := checker.Ping(ctx, httpClient, req)
- if err != nil {
- response, err = checker.Ping(ctx, httpClient, req)
+ op := func() error {
+ res, err := checker.Ping(ctx, httpClient, req)
if err != nil {
- if err := tinybirdClient.SendEvent(ctx, checker.PingData{
- URL: req.URL,
- Region: flyRegion,
- Message: err.Error(),
- CronTimestamp: req.CronTimestamp,
- Timestamp: req.CronTimestamp,
- MonitorID: req.MonitorID,
- WorkspaceID: req.WorkspaceID,
- }); err != nil {
- log.Ctx(ctx).Error().Err(err).Msg("failed to send event to tinybird")
- }
-
- if req.Status == "active" {
- checker.UpdateStatus(ctx, checker.UpdateData{
- MonitorId: req.MonitorID,
- Status: "error",
- Message: err.Error(),
- Region: flyRegion,
- })
- }
-
- c.JSON(http.StatusOK, gin.H{"message": "ok"})
- return
+ return fmt.Errorf("unable to ping: %w", err)
}
- }
- if response.StatusCode < 200 || response.StatusCode >= 300 {
- // Add one more retry
- response, err = checker.Ping(ctx, httpClient, req)
- if response.StatusCode < 200 || response.StatusCode >= 300 && req.Status == "active" {
- // If the status code is not within the 200 range, we update the status to err
+ statusCode := statusCode(res.StatusCode)
+ if !statusCode.IsSuccessful() {
+ // Q: Why here we do not check if the status was previously active?
checker.UpdateStatus(ctx, checker.UpdateData{
MonitorId: req.MonitorID,
Status: "error",
- StatusCode: response.StatusCode,
+ StatusCode: res.StatusCode,
+ Region: flyRegion,
+ })
+ } else if req.Status == "error" && statusCode.IsSuccessful() {
+ // Q: Why here we check the data before updating the status in this scenario?
+ checker.UpdateStatus(ctx, checker.UpdateData{
+ MonitorId: req.MonitorID,
+ Status: "active",
Region: flyRegion,
+ StatusCode: res.StatusCode,
})
}
- }
- // If the status was error and the status code is within the 200 range, we update the status to active
- if req.Status == "error" && response.StatusCode >= 200 && response.StatusCode < 300 {
- // If the status was error, we update it to active
- checker.UpdateStatus(ctx, checker.UpdateData{
- MonitorId: req.MonitorID,
- Status: "active",
- Region: flyRegion,
- StatusCode: response.StatusCode,
- })
+ if err := tinybirdClient.SendEvent(ctx, res); err != nil {
+ log.Ctx(ctx).Error().Err(err).Msg("failed to send event to tinybird")
+ }
+
+ return nil
}
- if err := tinybirdClient.SendEvent(ctx, response); err != nil {
- log.Ctx(ctx).Error().Err(err).Msg("failed to send event to tinybird")
+ if err := backoff.Retry(op, backoff.WithMaxRetries(backoff.NewExponentialBackOff(), 3)); err != nil {
+ if err := tinybirdClient.SendEvent(ctx, checker.PingData{
+ URL: req.URL,
+ Region: flyRegion,
+ Message: err.Error(),
+ CronTimestamp: req.CronTimestamp,
+ Timestamp: req.CronTimestamp,
+ MonitorID: req.MonitorID,
+ WorkspaceID: req.WorkspaceID,
+ }); err != nil {
+ log.Ctx(ctx).Error().Err(err).Msg("failed to send event to tinybird")
+ }
+
+ // If the status was previously active, we update it to error.
+ // Q: Why not always updating the status? My idea is that the checker should be dumb and only check the status and return it.
+ if req.Status == "active" {
+ checker.UpdateStatus(ctx, checker.UpdateData{
+ MonitorId: req.MonitorID,
+ Status: "error",
+ Message: err.Error(),
+ Region: flyRegion,
+ })
+ }
}
c.JSON(http.StatusOK, gin.H{"message": "ok"})
- return
})
router.GET("/ping", func(c *gin.Context) {
diff --git a/apps/checker/go.mod b/apps/checker/go.mod
index 800f66ca6..dede75a7f 100644
--- a/apps/checker/go.mod
+++ b/apps/checker/go.mod
@@ -3,6 +3,7 @@ module github.com/openstatushq/openstatus/apps/checker
go 1.21.4
require (
+ github.com/cenkalti/backoff/v4 v4.2.1
github.com/gin-gonic/gin v1.9.1
github.com/rs/zerolog v1.31.0
github.com/stretchr/testify v1.8.3
diff --git a/apps/checker/go.sum b/apps/checker/go.sum
index 5006a1c0f..a28caf689 100644
--- a/apps/checker/go.sum
+++ b/apps/checker/go.sum
@@ -1,6 +1,8 @@
github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM=
github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s=
github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U=
+github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM=
+github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY=
github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams=
github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk=
diff --git a/apps/checker/ping.go b/apps/checker/ping.go
index d1cc96670..4b449d718 100644
--- a/apps/checker/ping.go
+++ b/apps/checker/ping.go
@@ -3,8 +3,8 @@ package checker
import (
"bytes"
"context"
+ "errors"
"fmt"
- "io"
"net/http"
"net/url"
"os"
@@ -37,31 +37,25 @@ func Ping(ctx context.Context, client *http.Client, inputData request.CheckerReq
}
req.Header.Set("User-Agent", "OpenStatus/1.0")
-
- // Setting headers
for _, header := range inputData.Headers {
- if header.Key != "" && header.Value != "" {
- req.Header.Set(header.Key, header.Value)
- }
+ req.Header.Set(header.Key, header.Value)
}
start := time.Now()
response, err := client.Do(req)
latency := time.Since(start).Milliseconds()
-
if err != nil {
- if urlErr, ok := err.(*url.Error); ok {
- if urlErr.Timeout() {
- return PingData{
- Latency: latency,
- MonitorID: inputData.MonitorID,
- Region: region,
- WorkspaceID: inputData.WorkspaceID,
- Timestamp: time.Now().UTC().UnixMilli(),
- URL: inputData.URL,
- Message: fmt.Sprintf("Timeout after %d ms", latency),
- }, nil
- }
+ var urlErr *url.Error
+ if errors.As(err, &urlErr) && urlErr.Timeout() {
+ return PingData{
+ Latency: latency,
+ MonitorID: inputData.MonitorID,
+ Region: region,
+ WorkspaceID: inputData.WorkspaceID,
+ Timestamp: time.Now().UTC().UnixMilli(),
+ URL: inputData.URL,
+ Message: fmt.Sprintf("Timeout after %d ms", latency),
+ }, nil
}
logger.Error().Err(err).Msg("error while pinging")
@@ -69,11 +63,6 @@ func Ping(ctx context.Context, client *http.Client, inputData request.CheckerReq
}
defer response.Body.Close()
- if _, err := io.ReadAll(response.Body); err != nil {
- logger.Error().Err(err).Str("monitorURL", inputData.URL).Msg("error while reading body")
- return PingData{}, fmt.Errorf("error while reading body from %s: %w", inputData.URL, err)
- }
-
return PingData{
Latency: latency,
StatusCode: response.StatusCode,