Skip to content

Open-source status page, incident response and uptime monitoring

11.4 KB
Raw Blame
package handlers

import (
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"strconv"
	"time"

	"github.com/cenkalti/backoff/v5"
	"github.com/gin-gonic/gin"
	"github.com/google/uuid"
	"github.com/openstatushq/openstatus/apps/checker/checker"
	otelOS "github.com/openstatushq/openstatus/apps/checker/pkg/otel"
	"github.com/openstatushq/openstatus/apps/checker/request"
	"github.com/rs/zerolog/log"
)

// Only used for Tinybird.
type GRPCData struct {
	ID            string `json:"id"`
	Timing        string `json:"timing"`
	ErrorMessage  string `json:"errorMessage"`
	Region        string `json:"region"`
	Trigger       string `json:"trigger"`
	URI           string `json:"uri"`
	Service       string `json:"service,omitempty"`
	ServingStatus string `json:"servingStatus,omitempty"`
	RequestStatus string `json:"requestStatus,omitempty"`

	RequestId     int64 `json:"requestId,omitempty"`
	WorkspaceID   int64 `json:"workspaceId"`
	MonitorID     int64 `json:"monitorId"`
	Timestamp     int64 `json:"timestamp"`
	Latency       int64 `json:"latency"`
	CronTimestamp int64 `json:"cronTimestamp"`
	GRPCCode      int64 `json:"grpcCode"`

	Error uint8 `json:"error"`
}

func grpcCheck(req request.GRPCCheckerRequest) (checker.GRPCResult, error) {
	return checker.CheckGRPC(
		req.Timeout,
		req.URI,
		req.Service,
		checker.ParseGRPCTLSMode(req.TLS),
		req.Headers,
	)
}

func (h Handler) GRPCHandler(c *gin.Context) {
	ctx := c.Request.Context()

	if !h.acceptCron(c) {
		return
	}

	var req request.GRPCCheckerRequest
	if err := c.ShouldBindJSON(&req); err != nil {
		log.Ctx(ctx).Error().Err(err).Msg("failed to decode checker request")
		c.JSON(http.StatusBadRequest, gin.H{"error": "invalid request"})

		return
	}

	setCheckerEvent(c, req.URI, req.WorkspaceID, req.MonitorID, cronTrigger(req.Trigger), "grpc")

	response, err := h.RunGRPCCheck(ctx, req)
	if respondInvalid(c, err) {
		return
	}

	returnData := c.Query("data")
	if returnData == "true" {
		c.JSON(http.StatusOK, response)

		return
	}

	c.JSON(http.StatusOK, nil)
}

// RunGRPCCheck probes and records one cron gRPC check. A nil error means the
// result was recorded, whether the monitor passed or not.
func (h Handler) RunGRPCCheck(ctx context.Context, req request.GRPCCheckerRequest) (checker.GRPCResponse, error) {
	dataSourceName := "grpc_response__v0"

	ctx = withMonitorLogger(ctx, req.MonitorID, req.WorkspaceID)

	var response checker.GRPCResponse

	workspaceId, err := strconv.ParseInt(req.WorkspaceID, 10, 64)
	if err != nil {
		return response, InvalidRequestError{Reason: "invalid request"}
	}

	monitorId, err := strconv.ParseInt(req.MonitorID, 10, 64)
	if err != nil {
		return response, InvalidRequestError{Reason: "invalid request"}
	}

	trigger := cronTrigger(req.Trigger)

	var recordErr error

	var retry int
	if req.Retry != 0 {
		retry = int(req.Retry)
	} else {
		retry = 3
	}

	// CheckGRPC hands back what it learned alongside its error: whatever phases
	// completed, plus the gRPC status code. That is what separates a DNS failure
	// from a connect or TLS one, and a timeout from a refused connection. op()
	// drops `res` when it returns an error to the retry loop, so keep the last
	// attempt's result.
	var lastResult checker.GRPCResult

	op := func() (struct{}, error) {
		res, err := grpcCheck(req)
		lastResult = res
		if err != nil {
			// Only a call that never reached the server is worth repeating. A
			// server answering NOT_SERVING will answer the same three more times.
			return struct{}{}, fmt.Errorf("unable to check grpc %s", err)
		}

		timingAsString, err := json.Marshal(res.Timing)
		if err != nil {
			return struct{}{}, fmt.Errorf("error while parsing timing data %s: %w", req.URI, err)
		}

		id, err := uuid.NewV7()
		if err != nil {
			return struct{}{}, fmt.Errorf("error while generating uuid %w", err)
		}

		timestamp := time.Now().UTC().UnixMilli()
		degraded := res.Healthy && req.DegradedAfter > 0 && res.Latency > req.DegradedAfter

		requestStatus := "success"
		switch {
		case !res.Healthy:
			requestStatus = "error"
		case degraded:
			requestStatus = "degraded"
		}

		errorFlag := uint8(0)
		errorMessage := ""
		if !res.Healthy {
			errorFlag = 1
			errorMessage = res.Message
		}

		data := GRPCData{
			ID:            id.String(),
			WorkspaceID:   workspaceId,
			Timestamp:     timestamp,
			Error:         errorFlag,
			ErrorMessage:  errorMessage,
			Region:        h.Region,
			MonitorID:     monitorId,
			Timing:        string(timingAsString),
			Latency:       res.Latency,
			GRPCCode:      res.GRPCCode,
			ServingStatus: res.ServingStatus,
			Service:       req.Service,
			CronTimestamp: req.CronTimestamp,
			Trigger:       trigger,
			URI:           req.URI,
			RequestStatus: requestStatus,
		}

		response = checker.GRPCResponse{
			Timestamp:     timestamp,
			Timing:        res.Timing,
			Latency:       res.Latency,
			GRPCCode:      res.GRPCCode,
			ServingStatus: res.ServingStatus,
			Service:       req.Service,
			ErrorMessage:  errorMessage,
			Completed:     res.Completed,
			Error:         errorFlag,
			Region:        h.Region,
			JobType:       "grpc",
		}

		switch {
		case !res.Healthy && req.Status != "error":
			checker.UpdateStatus(ctx, checker.UpdateData{
				MonitorId:     req.MonitorID,
				Status:        "error",
				Message:       res.Message,
				Region:        h.Region,
				CronTimestamp: req.CronTimestamp,
				Latency:       res.Latency,
			})
		case degraded && req.Status != "degraded":
			checker.UpdateStatus(ctx, checker.UpdateData{
				MonitorId:     req.MonitorID,
				Status:        "degraded",
				Region:        h.Region,
				CronTimestamp: req.CronTimestamp,
				Latency:       res.Latency,
			})
		case res.Healthy && !degraded && req.Status != "active":
			checker.UpdateStatus(ctx, checker.UpdateData{
				MonitorId:     req.MonitorID,
				Status:        "active",
				Region:        h.Region,
				CronTimestamp: req.CronTimestamp,
				Latency:       res.Latency,
			})
		}

		if err := h.TbClient.SendEvent(ctx, data, dataSourceName); err != nil {
			log.Ctx(ctx).Error().Err(err).Msg("failed to send event to tinybird")
			recordErr = err
		}

		return struct{}{}, nil
	}

	if _, err := backoff.Retry(ctx, op, backoff.WithBackOff(backoff.NewExponentialBackOff()), backoff.WithMaxTries(uint(retry)), backoff.WithMaxElapsedTime(retryBudget)); err != nil {
		id, e := uuid.NewV7()
		if e != nil {
			log.Ctx(ctx).Error().Err(e).Msg("failed to send event to tinybird")
			return response, e
		}
		// A marshal failure leaves the column empty rather than dropping the row:
		// the rest of the failure record is still worth ingesting.
		timingAsString, timingErr := json.Marshal(lastResult.Timing)
		if timingErr != nil {
			log.Ctx(ctx).Error().Err(timingErr).Msg("error while parsing timing data")
		}

		data := GRPCData{
			ID:            id.String(),
			WorkspaceID:   workspaceId,
			CronTimestamp: req.CronTimestamp,
			Timestamp:     time.Now().UTC().UnixMilli(),
			ErrorMessage:  err.Error(),
			Region:        h.Region,
			MonitorID:     monitorId,
			Error:         1,
			Trigger:       trigger,
			URI:           req.URI,
			Service:       req.Service,
			Timing:        string(timingAsString),
			GRPCCode:      lastResult.GRPCCode,
			RequestStatus: "error",
		}
		if err := h.TbClient.SendEvent(ctx, data, dataSourceName); err != nil {
			log.Ctx(ctx).Error().Err(err).Msg("failed to send event to tinybird")
			recordErr = err
		}
		checker.UpdateStatus(ctx, checker.UpdateData{
			MonitorId:     req.MonitorID,
			Status:        "error",
			Message:       err.Error(),
			Region:        h.Region,
			CronTimestamp: req.CronTimestamp,
		})

		// Only the success path inside op() fills these in, so a check that
		// exhausted its retries would otherwise be returned under `?data=true`
		// with an empty jobType and region — a shape no caller can parse.
		response.JobType = "grpc"
		response.Region = h.Region
		response.ErrorMessage = err.Error()
		response.Error = 1
		response.Timing = lastResult.Timing
		response.GRPCCode = lastResult.GRPCCode
	}

	if req.OtelConfig.Endpoint != "" {
		otelOS.RecordGRPCMetrics(ctx, req, response, h.Region)
	}

	return response, recordErr
}

func (h Handler) GRPCHandlerRegion(c *gin.Context) {
	ctx := c.Request.Context()
	dataSourceName := "check_grpc_response__v0"

	region := c.Param("region")
	if region == "" {
		c.String(http.StatusBadRequest, "region is required")

		return
	}

	if c.GetHeader("Authorization") != fmt.Sprintf("Basic %s", h.Secret) {
		c.JSON(http.StatusUnauthorized, gin.H{"error": "unauthorized"})

		return
	}

	if h.CloudProvider == "fly" {
		// if the request has been routed to a wrong region, we forward it to the correct one.
		region := c.GetHeader("fly-prefer-region")
		if region != "" && region != h.Region {
			c.Header("fly-replay", fmt.Sprintf("region=%s", region))
			c.String(http.StatusAccepted, "Forwarding request to %s", region)

			return
		}
	}

	var req request.GRPCCheckerRequest
	if err := c.ShouldBindJSON(&req); err != nil {
		log.Ctx(ctx).Error().Err(err).Msg("failed to decode checker request")
		c.JSON(http.StatusBadRequest, gin.H{"error": "invalid request"})

		return
	}

	ctx = withMonitorLogger(ctx, req.MonitorID, req.WorkspaceID)

	var response checker.GRPCResponse

	var retry int
	if req.Retry != 0 {
		retry = int(req.Retry)
	} else {
		retry = 3
	}

	// Same as GRPCHandler: op() discards `res` when it hands an error back to the
	// retry loop, so keep what the last attempt learned.
	var lastResult checker.GRPCResult

	op := func() (struct{}, error) {
		timestamp := time.Now().UTC().UnixMilli()
		res, err := grpcCheck(req)
		lastResult = res
		if err != nil {
			return struct{}{}, fmt.Errorf("unable to check grpc %s", err)
		}

		errorFlag := uint8(0)
		errorMessage := ""
		if !res.Healthy {
			errorFlag = 1
			errorMessage = res.Message
		}

		response = checker.GRPCResponse{
			Timestamp:     timestamp,
			Timing:        res.Timing,
			Latency:       res.Latency,
			GRPCCode:      res.GRPCCode,
			ServingStatus: res.ServingStatus,
			Service:       req.Service,
			ErrorMessage:  errorMessage,
			Completed:     res.Completed,
			Error:         errorFlag,
			Region:        h.Region,
			JobType:       "grpc",
		}

		timingAsString, err := json.Marshal(res.Timing)
		if err != nil {
			return struct{}{}, fmt.Errorf("error while parsing timing data %s: %w", req.URI, err)
		}

		data := GRPCData{
			CronTimestamp: req.CronTimestamp,
			Timestamp:     timestamp,
			Error:         errorFlag,
			ErrorMessage:  errorMessage,
			Region:        h.Region,
			Timing:        string(timingAsString),
			Latency:       res.Latency,
			GRPCCode:      res.GRPCCode,
			ServingStatus: res.ServingStatus,
			Service:       req.Service,
			RequestId:     req.RequestId,
			Trigger:       "api",
			URI:           req.URI,
		}

		if req.RequestId != 0 {
			if err := h.TbClient.SendEvent(ctx, data, dataSourceName); err != nil {
				log.Ctx(ctx).Error().Err(err).Msg("failed to send event to tinybird")
			}
		}

		return struct{}{}, nil
	}

	_, err := backoff.Retry(ctx, op, backoff.WithBackOff(backoff.NewExponentialBackOff()), backoff.WithMaxTries(uint(retry)))
	if err != nil {
		response.JobType = "grpc"
		response.Region = h.Region
		response.ErrorMessage = err.Error()
		response.Error = 1
		response.Timing = lastResult.Timing
		response.GRPCCode = lastResult.GRPCCode
	}

	if req.OtelConfig.Endpoint != "" {
		otelOS.RecordGRPCMetrics(ctx, req, response, region)
	}

	if err != nil {
		c.JSON(http.StatusOK, gin.H{"message": "uri not reachable"})

		return
	}

	c.JSON(http.StatusOK, response)
}