Open-source status page, incident response and uptime monitoring
TypeScript alertingincident-managementmcpmcp-servermonitoringmonitoring-as-codeobservabilityon-callopen-sourcesrestatus-pagestatuspagesynthetic-monitoringterraformtinybirdtursouptimeuptime-checkeruptime-monitor
package handlers
import (
"context"
"encoding/json"
"fmt"
"net/http"
"strconv"
"time"
"github.com/cenkalti/backoff/v5"
"github.com/gin-gonic/gin"
"github.com/google/uuid"
"github.com/openstatushq/openstatus/apps/checker/checker"
otelOS "github.com/openstatushq/openstatus/apps/checker/pkg/otel"
"github.com/openstatushq/openstatus/apps/checker/request"
"github.com/rs/zerolog/log"
)
// Only used for Tinybird.
type GRPCData struct {
ID string `json:"id"`
Timing string `json:"timing"`
ErrorMessage string `json:"errorMessage"`
Region string `json:"region"`
Trigger string `json:"trigger"`
URI string `json:"uri"`
Service string `json:"service,omitempty"`
ServingStatus string `json:"servingStatus,omitempty"`
RequestStatus string `json:"requestStatus,omitempty"`
RequestId int64 `json:"requestId,omitempty"`
WorkspaceID int64 `json:"workspaceId"`
MonitorID int64 `json:"monitorId"`
Timestamp int64 `json:"timestamp"`
Latency int64 `json:"latency"`
CronTimestamp int64 `json:"cronTimestamp"`
GRPCCode int64 `json:"grpcCode"`
Error uint8 `json:"error"`
}
func grpcCheck(req request.GRPCCheckerRequest) (checker.GRPCResult, error) {
return checker.CheckGRPC(
req.Timeout,
req.URI,
req.Service,
checker.ParseGRPCTLSMode(req.TLS),
req.Headers,
)
}
func (h Handler) GRPCHandler(c *gin.Context) {
ctx := c.Request.Context()
if !h.acceptCron(c) {
return
}
var req request.GRPCCheckerRequest
if err := c.ShouldBindJSON(&req); err != nil {
log.Ctx(ctx).Error().Err(err).Msg("failed to decode checker request")
c.JSON(http.StatusBadRequest, gin.H{"error": "invalid request"})
return
}
setCheckerEvent(c, req.URI, req.WorkspaceID, req.MonitorID, cronTrigger(req.Trigger), "grpc")
response, err := h.RunGRPCCheck(ctx, req)
if respondInvalid(c, err) {
return
}
returnData := c.Query("data")
if returnData == "true" {
c.JSON(http.StatusOK, response)
return
}
c.JSON(http.StatusOK, nil)
}
// RunGRPCCheck probes and records one cron gRPC check. A nil error means the
// result was recorded, whether the monitor passed or not.
func (h Handler) RunGRPCCheck(ctx context.Context, req request.GRPCCheckerRequest) (checker.GRPCResponse, error) {
dataSourceName := "grpc_response__v0"
ctx = withMonitorLogger(ctx, req.MonitorID, req.WorkspaceID)
var response checker.GRPCResponse
workspaceId, err := strconv.ParseInt(req.WorkspaceID, 10, 64)
if err != nil {
return response, InvalidRequestError{Reason: "invalid request"}
}
monitorId, err := strconv.ParseInt(req.MonitorID, 10, 64)
if err != nil {
return response, InvalidRequestError{Reason: "invalid request"}
}
trigger := cronTrigger(req.Trigger)
var recordErr error
var retry int
if req.Retry != 0 {
retry = int(req.Retry)
} else {
retry = 3
}
// CheckGRPC hands back what it learned alongside its error: whatever phases
// completed, plus the gRPC status code. That is what separates a DNS failure
// from a connect or TLS one, and a timeout from a refused connection. op()
// drops `res` when it returns an error to the retry loop, so keep the last
// attempt's result.
var lastResult checker.GRPCResult
op := func() (struct{}, error) {
res, err := grpcCheck(req)
lastResult = res
if err != nil {
// Only a call that never reached the server is worth repeating. A
// server answering NOT_SERVING will answer the same three more times.
return struct{}{}, fmt.Errorf("unable to check grpc %s", err)
}
timingAsString, err := json.Marshal(res.Timing)
if err != nil {
return struct{}{}, fmt.Errorf("error while parsing timing data %s: %w", req.URI, err)
}
id, err := uuid.NewV7()
if err != nil {
return struct{}{}, fmt.Errorf("error while generating uuid %w", err)
}
timestamp := time.Now().UTC().UnixMilli()
degraded := res.Healthy && req.DegradedAfter > 0 && res.Latency > req.DegradedAfter
requestStatus := "success"
switch {
case !res.Healthy:
requestStatus = "error"
case degraded:
requestStatus = "degraded"
}
errorFlag := uint8(0)
errorMessage := ""
if !res.Healthy {
errorFlag = 1
errorMessage = res.Message
}
data := GRPCData{
ID: id.String(),
WorkspaceID: workspaceId,
Timestamp: timestamp,
Error: errorFlag,
ErrorMessage: errorMessage,
Region: h.Region,
MonitorID: monitorId,
Timing: string(timingAsString),
Latency: res.Latency,
GRPCCode: res.GRPCCode,
ServingStatus: res.ServingStatus,
Service: req.Service,
CronTimestamp: req.CronTimestamp,
Trigger: trigger,
URI: req.URI,
RequestStatus: requestStatus,
}
response = checker.GRPCResponse{
Timestamp: timestamp,
Timing: res.Timing,
Latency: res.Latency,
GRPCCode: res.GRPCCode,
ServingStatus: res.ServingStatus,
Service: req.Service,
ErrorMessage: errorMessage,
Completed: res.Completed,
Error: errorFlag,
Region: h.Region,
JobType: "grpc",
}
switch {
case !res.Healthy && req.Status != "error":
checker.UpdateStatus(ctx, checker.UpdateData{
MonitorId: req.MonitorID,
Status: "error",
Message: res.Message,
Region: h.Region,
CronTimestamp: req.CronTimestamp,
Latency: res.Latency,
})
case degraded && req.Status != "degraded":
checker.UpdateStatus(ctx, checker.UpdateData{
MonitorId: req.MonitorID,
Status: "degraded",
Region: h.Region,
CronTimestamp: req.CronTimestamp,
Latency: res.Latency,
})
case res.Healthy && !degraded && req.Status != "active":
checker.UpdateStatus(ctx, checker.UpdateData{
MonitorId: req.MonitorID,
Status: "active",
Region: h.Region,
CronTimestamp: req.CronTimestamp,
Latency: res.Latency,
})
}
if err := h.TbClient.SendEvent(ctx, data, dataSourceName); err != nil {
log.Ctx(ctx).Error().Err(err).Msg("failed to send event to tinybird")
recordErr = err
}
return struct{}{}, nil
}
if _, err := backoff.Retry(ctx, op, backoff.WithBackOff(backoff.NewExponentialBackOff()), backoff.WithMaxTries(uint(retry)), backoff.WithMaxElapsedTime(retryBudget)); err != nil {
id, e := uuid.NewV7()
if e != nil {
log.Ctx(ctx).Error().Err(e).Msg("failed to send event to tinybird")
return response, e
}
// A marshal failure leaves the column empty rather than dropping the row:
// the rest of the failure record is still worth ingesting.
timingAsString, timingErr := json.Marshal(lastResult.Timing)
if timingErr != nil {
log.Ctx(ctx).Error().Err(timingErr).Msg("error while parsing timing data")
}
data := GRPCData{
ID: id.String(),
WorkspaceID: workspaceId,
CronTimestamp: req.CronTimestamp,
Timestamp: time.Now().UTC().UnixMilli(),
ErrorMessage: err.Error(),
Region: h.Region,
MonitorID: monitorId,
Error: 1,
Trigger: trigger,
URI: req.URI,
Service: req.Service,
Timing: string(timingAsString),
GRPCCode: lastResult.GRPCCode,
RequestStatus: "error",
}
if err := h.TbClient.SendEvent(ctx, data, dataSourceName); err != nil {
log.Ctx(ctx).Error().Err(err).Msg("failed to send event to tinybird")
recordErr = err
}
checker.UpdateStatus(ctx, checker.UpdateData{
MonitorId: req.MonitorID,
Status: "error",
Message: err.Error(),
Region: h.Region,
CronTimestamp: req.CronTimestamp,
})
// Only the success path inside op() fills these in, so a check that
// exhausted its retries would otherwise be returned under `?data=true`
// with an empty jobType and region — a shape no caller can parse.
response.JobType = "grpc"
response.Region = h.Region
response.ErrorMessage = err.Error()
response.Error = 1
response.Timing = lastResult.Timing
response.GRPCCode = lastResult.GRPCCode
}
if req.OtelConfig.Endpoint != "" {
otelOS.RecordGRPCMetrics(ctx, req, response, h.Region)
}
return response, recordErr
}
func (h Handler) GRPCHandlerRegion(c *gin.Context) {
ctx := c.Request.Context()
dataSourceName := "check_grpc_response__v0"
region := c.Param("region")
if region == "" {
c.String(http.StatusBadRequest, "region is required")
return
}
if c.GetHeader("Authorization") != fmt.Sprintf("Basic %s", h.Secret) {
c.JSON(http.StatusUnauthorized, gin.H{"error": "unauthorized"})
return
}
if h.CloudProvider == "fly" {
// if the request has been routed to a wrong region, we forward it to the correct one.
region := c.GetHeader("fly-prefer-region")
if region != "" && region != h.Region {
c.Header("fly-replay", fmt.Sprintf("region=%s", region))
c.String(http.StatusAccepted, "Forwarding request to %s", region)
return
}
}
var req request.GRPCCheckerRequest
if err := c.ShouldBindJSON(&req); err != nil {
log.Ctx(ctx).Error().Err(err).Msg("failed to decode checker request")
c.JSON(http.StatusBadRequest, gin.H{"error": "invalid request"})
return
}
ctx = withMonitorLogger(ctx, req.MonitorID, req.WorkspaceID)
var response checker.GRPCResponse
var retry int
if req.Retry != 0 {
retry = int(req.Retry)
} else {
retry = 3
}
// Same as GRPCHandler: op() discards `res` when it hands an error back to the
// retry loop, so keep what the last attempt learned.
var lastResult checker.GRPCResult
op := func() (struct{}, error) {
timestamp := time.Now().UTC().UnixMilli()
res, err := grpcCheck(req)
lastResult = res
if err != nil {
return struct{}{}, fmt.Errorf("unable to check grpc %s", err)
}
errorFlag := uint8(0)
errorMessage := ""
if !res.Healthy {
errorFlag = 1
errorMessage = res.Message
}
response = checker.GRPCResponse{
Timestamp: timestamp,
Timing: res.Timing,
Latency: res.Latency,
GRPCCode: res.GRPCCode,
ServingStatus: res.ServingStatus,
Service: req.Service,
ErrorMessage: errorMessage,
Completed: res.Completed,
Error: errorFlag,
Region: h.Region,
JobType: "grpc",
}
timingAsString, err := json.Marshal(res.Timing)
if err != nil {
return struct{}{}, fmt.Errorf("error while parsing timing data %s: %w", req.URI, err)
}
data := GRPCData{
CronTimestamp: req.CronTimestamp,
Timestamp: timestamp,
Error: errorFlag,
ErrorMessage: errorMessage,
Region: h.Region,
Timing: string(timingAsString),
Latency: res.Latency,
GRPCCode: res.GRPCCode,
ServingStatus: res.ServingStatus,
Service: req.Service,
RequestId: req.RequestId,
Trigger: "api",
URI: req.URI,
}
if req.RequestId != 0 {
if err := h.TbClient.SendEvent(ctx, data, dataSourceName); err != nil {
log.Ctx(ctx).Error().Err(err).Msg("failed to send event to tinybird")
}
}
return struct{}{}, nil
}
_, err := backoff.Retry(ctx, op, backoff.WithBackOff(backoff.NewExponentialBackOff()), backoff.WithMaxTries(uint(retry)))
if err != nil {
response.JobType = "grpc"
response.Region = h.Region
response.ErrorMessage = err.Error()
response.Error = 1
response.Timing = lastResult.Timing
response.GRPCCode = lastResult.GRPCCode
}
if req.OtelConfig.Endpoint != "" {
otelOS.RecordGRPCMetrics(ctx, req, response, region)
}
if err != nil {
c.JSON(http.StatusOK, gin.H{"message": "uri not reachable"})
return
}
c.JSON(http.StatusOK, response)
}