Open-source status page, incident response and uptime monitoring
TypeScript alertingincident-managementmcpmcp-servermonitoringmonitoring-as-codeobservabilityon-callopen-sourcesrestatus-pagestatuspagesynthetic-monitoringterraformtinybirdtursouptimeuptime-checkeruptime-monitor
package queue
import (
"context"
"encoding/json"
"errors"
"fmt"
"hash/fnv"
"time"
"cloud.google.com/go/pubsub/v2"
"golang.org/x/sync/errgroup"
"github.com/openstatushq/openstatus/apps/checker/checker"
"github.com/openstatushq/openstatus/apps/checker/handlers"
"github.com/openstatushq/openstatus/apps/checker/request"
)
type Runner interface {
RunHTTPCheck(ctx context.Context, req request.HttpCheckerRequest) (checker.Response, error)
RunTCPCheck(ctx context.Context, req request.TCPCheckerRequest) (checker.TCPResponse, error)
RunDNSCheck(ctx context.Context, req request.DNSCheckerRequest) (handlers.DNSResponse, error)
RunICMPCheck(ctx context.Context, req request.ICMPCheckerRequest) (checker.ICMPResponse, error)
RunGRPCCheck(ctx context.Context, req request.GRPCCheckerRequest) (checker.GRPCResponse, error)
}
type Outcome string
const (
OutcomeAck Outcome = "ack"
OutcomeNack Outcome = "nack"
OutcomePoison Outcome = "poison"
OutcomeDrain Outcome = "drain"
)
type Event struct {
Subscription string
MonitorID string
Periodicity string
JobType string
DeliveryAttempt int
Wait time.Duration
Duration time.Duration
Outcome Outcome
Err error
}
type Loop struct {
Subscription string
// MaxOutstanding caps the messages held at once, sleeping ones included.
MaxOutstanding int
// MaxConcurrent caps the checks running at once; zero means MaxOutstanding.
MaxConcurrent int
}
type Consumer struct {
Client *pubsub.Client
Runner Runner
Loops []Loop
// Drain is how long in-flight checks get to finish after shutdown starts
// before their message is nacked for redelivery.
Drain time.Duration
// Spread offsets each check past its cronTimestamp by a per-monitor delay
// in [0, Spread), so the checks of one cron tick do not all resolve and
// handshake in the same instant. Zero disables it.
Spread time.Duration
OnEvent func(ctx context.Context, e Event)
}
const (
maxOutstandingBytes = 64 << 20
maxExtension = 25 * time.Minute
)
// Subscription names the region's subscription for a periodicity class.
func Subscription(region string, thirtySeconds bool) string {
if thirtySeconds {
return fmt.Sprintf("checker-%s-30s", region)
}
return fmt.Sprintf("checker-%s", region)
}
// Run blocks until ctx is done and every loop has drained.
func (c *Consumer) Run(ctx context.Context) error {
g, gctx := errgroup.WithContext(ctx)
for _, loop := range c.Loops {
g.Go(func() error {
return c.receive(gctx, loop)
})
}
return g.Wait()
}
func (c *Consumer) receive(ctx context.Context, loop Loop) error {
if loop.MaxOutstanding <= 0 {
return fmt.Errorf("max outstanding for %s must be positive", loop.Subscription)
}
sub := c.Client.Subscriber(loop.Subscription)
sub.ReceiveSettings.MaxOutstandingMessages = loop.MaxOutstanding
sub.ReceiveSettings.MaxOutstandingBytes = maxOutstandingBytes
sub.ReceiveSettings.MaxExtension = maxExtension
sub.ReceiveSettings.ShutdownOptions = &pubsub.ShutdownOptions{
Timeout: c.Drain + 5*time.Second,
Behavior: pubsub.ShutdownBehaviorWaitForProcessing,
}
concurrent := loop.MaxConcurrent
if concurrent <= 0 {
concurrent = loop.MaxOutstanding
}
slots := make(chan struct{}, concurrent)
err := sub.Receive(ctx, func(msgCtx context.Context, msg *pubsub.Message) {
e := c.handle(msgCtx, loop.Subscription, slots, msg)
if c.OnEvent != nil {
c.OnEvent(msgCtx, e)
}
})
if err != nil {
return fmt.Errorf("receiving from %s: %w", loop.Subscription, err)
}
return nil
}
type envelope struct {
CronTimestamp int64 `json:"cronTimestamp"`
}
func (c *Consumer) handle(ctx context.Context, subscription string, slots chan struct{}, msg *pubsub.Message) Event {
start := time.Now()
e := Event{
Subscription: subscription,
MonitorID: msg.Attributes["monitorId"],
Periodicity: msg.Attributes["periodicity"],
JobType: msg.Attributes["jobType"],
}
if msg.DeliveryAttempt != nil {
e.DeliveryAttempt = *msg.DeliveryAttempt
}
finish := func(outcome Outcome, err error) Event {
switch outcome {
case OutcomeAck, OutcomePoison:
msg.Ack()
default:
msg.Nack()
}
e.Outcome = outcome
e.Err = err
e.Duration = max(0, time.Since(start)-e.Wait)
return e
}
var env envelope
if err := json.Unmarshal(msg.Data, &env); err != nil {
return finish(OutcomePoison, fmt.Errorf("decoding message: %w", err))
}
if env.CronTimestamp <= 0 {
return finish(OutcomePoison, errors.New("missing cronTimestamp"))
}
startAt := time.UnixMilli(env.CronTimestamp).Add(spread(e.MonitorID, c.Spread))
if wait := time.Until(startAt); wait > 0 {
e.Wait = wait
timer := time.NewTimer(wait)
select {
case <-timer.C:
case <-ctx.Done():
timer.Stop()
return finish(OutcomeDrain, ctx.Err())
}
}
select {
case slots <- struct{}{}:
case <-ctx.Done():
return finish(OutcomeDrain, ctx.Err())
}
// The check outlives a shutdown signal: Drain bounds how long it may run on.
checkCtx, cancelCheck := context.WithCancel(context.WithoutCancel(ctx))
defer cancelCheck()
done := make(chan error, 1)
go func() {
defer func() { <-slots }()
done <- c.dispatch(checkCtx, e.JobType, msg.Data)
}()
var err error
select {
case err = <-done:
case <-ctx.Done():
drain := time.NewTimer(c.Drain)
defer drain.Stop()
select {
case err = <-done:
case <-drain.C:
// Stop the check before the nack so a redelivery cannot run it
// alongside this one and record the result twice.
cancelCheck()
<-done
return finish(OutcomeDrain, errors.New("check still running at shutdown"))
}
}
var invalid handlers.InvalidRequestError
switch {
case err == nil:
return finish(OutcomeAck, nil)
case errors.Is(err, errUnknownJobType), errors.As(err, &invalid):
return finish(OutcomePoison, err)
default:
return finish(OutcomeNack, err)
}
}
// spread maps a monitor to a fixed delay in [0, window): the same monitor keeps
// the same offset every tick, so its checks stay one interval apart.
func spread(monitorID string, window time.Duration) time.Duration {
if window <= 0 {
return 0
}
h := fnv.New64a()
h.Write([]byte(monitorID))
return time.Duration(h.Sum64() % uint64(window))
}
var errUnknownJobType = errors.New("unknown job type")
func (c *Consumer) dispatch(ctx context.Context, jobType string, data []byte) error {
switch jobType {
case "http":
var req request.HttpCheckerRequest
if err := json.Unmarshal(data, &req); err != nil {
return handlers.InvalidRequestError{Reason: err.Error()}
}
_, err := c.Runner.RunHTTPCheck(ctx, req)
return err
case "tcp":
var req request.TCPCheckerRequest
if err := json.Unmarshal(data, &req); err != nil {
return handlers.InvalidRequestError{Reason: err.Error()}
}
_, err := c.Runner.RunTCPCheck(ctx, req)
return err
case "dns":
var req request.DNSCheckerRequest
if err := json.Unmarshal(data, &req); err != nil {
return handlers.InvalidRequestError{Reason: err.Error()}
}
_, err := c.Runner.RunDNSCheck(ctx, req)
return err
case "icmp":
var req request.ICMPCheckerRequest
if err := json.Unmarshal(data, &req); err != nil {
return handlers.InvalidRequestError{Reason: err.Error()}
}
_, err := c.Runner.RunICMPCheck(ctx, req)
return err
case "grpc":
var req request.GRPCCheckerRequest
if err := json.Unmarshal(data, &req); err != nil {
return handlers.InvalidRequestError{Reason: err.Error()}
}
_, err := c.Runner.RunGRPCCheck(ctx, req)
return err
default:
return fmt.Errorf("%w: %q", errUnknownJobType, jobType)
}
}