Skip to content

Open-source status page, incident response and uptime monitoring

7.4 KB
Raw Blame
package queue

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"hash/fnv"
	"time"

	"cloud.google.com/go/pubsub/v2"
	"golang.org/x/sync/errgroup"

	"github.com/openstatushq/openstatus/apps/checker/checker"
	"github.com/openstatushq/openstatus/apps/checker/handlers"
	"github.com/openstatushq/openstatus/apps/checker/request"
)

type Runner interface {
	RunHTTPCheck(ctx context.Context, req request.HttpCheckerRequest) (checker.Response, error)
	RunTCPCheck(ctx context.Context, req request.TCPCheckerRequest) (checker.TCPResponse, error)
	RunDNSCheck(ctx context.Context, req request.DNSCheckerRequest) (handlers.DNSResponse, error)
	RunICMPCheck(ctx context.Context, req request.ICMPCheckerRequest) (checker.ICMPResponse, error)
	RunGRPCCheck(ctx context.Context, req request.GRPCCheckerRequest) (checker.GRPCResponse, error)
}

type Outcome string

const (
	OutcomeAck    Outcome = "ack"
	OutcomeNack   Outcome = "nack"
	OutcomePoison Outcome = "poison"
	OutcomeDrain  Outcome = "drain"
)

type Event struct {
	Subscription    string
	MonitorID       string
	Periodicity     string
	JobType         string
	DeliveryAttempt int
	Wait            time.Duration
	Duration        time.Duration
	Outcome         Outcome
	Err             error
}

type Loop struct {
	Subscription string
	// MaxOutstanding caps the messages held at once, sleeping ones included.
	MaxOutstanding int
	// MaxConcurrent caps the checks running at once; zero means MaxOutstanding.
	MaxConcurrent int
}

type Consumer struct {
	Client *pubsub.Client
	Runner Runner
	Loops  []Loop
	// Drain is how long in-flight checks get to finish after shutdown starts
	// before their message is nacked for redelivery.
	Drain time.Duration
	// Spread offsets each check past its cronTimestamp by a per-monitor delay
	// in [0, Spread), so the checks of one cron tick do not all resolve and
	// handshake in the same instant. Zero disables it.
	Spread  time.Duration
	OnEvent func(ctx context.Context, e Event)
}

const (
	maxOutstandingBytes = 64 << 20
	maxExtension        = 25 * time.Minute
)

// Subscription names the region's subscription for a periodicity class.
func Subscription(region string, thirtySeconds bool) string {
	if thirtySeconds {
		return fmt.Sprintf("checker-%s-30s", region)
	}

	return fmt.Sprintf("checker-%s", region)
}

// Run blocks until ctx is done and every loop has drained.
func (c *Consumer) Run(ctx context.Context) error {
	g, gctx := errgroup.WithContext(ctx)
	for _, loop := range c.Loops {
		g.Go(func() error {
			return c.receive(gctx, loop)
		})
	}

	return g.Wait()
}

func (c *Consumer) receive(ctx context.Context, loop Loop) error {
	if loop.MaxOutstanding <= 0 {
		return fmt.Errorf("max outstanding for %s must be positive", loop.Subscription)
	}

	sub := c.Client.Subscriber(loop.Subscription)
	sub.ReceiveSettings.MaxOutstandingMessages = loop.MaxOutstanding
	sub.ReceiveSettings.MaxOutstandingBytes = maxOutstandingBytes
	sub.ReceiveSettings.MaxExtension = maxExtension
	sub.ReceiveSettings.ShutdownOptions = &pubsub.ShutdownOptions{
		Timeout:  c.Drain + 5*time.Second,
		Behavior: pubsub.ShutdownBehaviorWaitForProcessing,
	}

	concurrent := loop.MaxConcurrent
	if concurrent <= 0 {
		concurrent = loop.MaxOutstanding
	}
	slots := make(chan struct{}, concurrent)

	err := sub.Receive(ctx, func(msgCtx context.Context, msg *pubsub.Message) {
		e := c.handle(msgCtx, loop.Subscription, slots, msg)
		if c.OnEvent != nil {
			c.OnEvent(msgCtx, e)
		}
	})
	if err != nil {
		return fmt.Errorf("receiving from %s: %w", loop.Subscription, err)
	}

	return nil
}

type envelope struct {
	CronTimestamp int64 `json:"cronTimestamp"`
}

func (c *Consumer) handle(ctx context.Context, subscription string, slots chan struct{}, msg *pubsub.Message) Event {
	start := time.Now()
	e := Event{
		Subscription: subscription,
		MonitorID:    msg.Attributes["monitorId"],
		Periodicity:  msg.Attributes["periodicity"],
		JobType:      msg.Attributes["jobType"],
	}
	if msg.DeliveryAttempt != nil {
		e.DeliveryAttempt = *msg.DeliveryAttempt
	}
	finish := func(outcome Outcome, err error) Event {
		switch outcome {
		case OutcomeAck, OutcomePoison:
			msg.Ack()
		default:
			msg.Nack()
		}
		e.Outcome = outcome
		e.Err = err
		e.Duration = max(0, time.Since(start)-e.Wait)

		return e
	}

	var env envelope
	if err := json.Unmarshal(msg.Data, &env); err != nil {
		return finish(OutcomePoison, fmt.Errorf("decoding message: %w", err))
	}
	if env.CronTimestamp <= 0 {
		return finish(OutcomePoison, errors.New("missing cronTimestamp"))
	}

	startAt := time.UnixMilli(env.CronTimestamp).Add(spread(e.MonitorID, c.Spread))
	if wait := time.Until(startAt); wait > 0 {
		e.Wait = wait
		timer := time.NewTimer(wait)
		select {
		case <-timer.C:
		case <-ctx.Done():
			timer.Stop()
			return finish(OutcomeDrain, ctx.Err())
		}
	}

	select {
	case slots <- struct{}{}:
	case <-ctx.Done():
		return finish(OutcomeDrain, ctx.Err())
	}

	// The check outlives a shutdown signal: Drain bounds how long it may run on.
	checkCtx, cancelCheck := context.WithCancel(context.WithoutCancel(ctx))
	defer cancelCheck()
	done := make(chan error, 1)
	go func() {
		defer func() { <-slots }()
		done <- c.dispatch(checkCtx, e.JobType, msg.Data)
	}()

	var err error
	select {
	case err = <-done:
	case <-ctx.Done():
		drain := time.NewTimer(c.Drain)
		defer drain.Stop()
		select {
		case err = <-done:
		case <-drain.C:
			// Stop the check before the nack so a redelivery cannot run it
			// alongside this one and record the result twice.
			cancelCheck()
			<-done
			return finish(OutcomeDrain, errors.New("check still running at shutdown"))
		}
	}

	var invalid handlers.InvalidRequestError
	switch {
	case err == nil:
		return finish(OutcomeAck, nil)
	case errors.Is(err, errUnknownJobType), errors.As(err, &invalid):
		return finish(OutcomePoison, err)
	default:
		return finish(OutcomeNack, err)
	}
}

// spread maps a monitor to a fixed delay in [0, window): the same monitor keeps
// the same offset every tick, so its checks stay one interval apart.
func spread(monitorID string, window time.Duration) time.Duration {
	if window <= 0 {
		return 0
	}
	h := fnv.New64a()
	h.Write([]byte(monitorID))

	return time.Duration(h.Sum64() % uint64(window))
}

var errUnknownJobType = errors.New("unknown job type")

func (c *Consumer) dispatch(ctx context.Context, jobType string, data []byte) error {
	switch jobType {
	case "http":
		var req request.HttpCheckerRequest
		if err := json.Unmarshal(data, &req); err != nil {
			return handlers.InvalidRequestError{Reason: err.Error()}
		}
		_, err := c.Runner.RunHTTPCheck(ctx, req)
		return err
	case "tcp":
		var req request.TCPCheckerRequest
		if err := json.Unmarshal(data, &req); err != nil {
			return handlers.InvalidRequestError{Reason: err.Error()}
		}
		_, err := c.Runner.RunTCPCheck(ctx, req)
		return err
	case "dns":
		var req request.DNSCheckerRequest
		if err := json.Unmarshal(data, &req); err != nil {
			return handlers.InvalidRequestError{Reason: err.Error()}
		}
		_, err := c.Runner.RunDNSCheck(ctx, req)
		return err
	case "icmp":
		var req request.ICMPCheckerRequest
		if err := json.Unmarshal(data, &req); err != nil {
			return handlers.InvalidRequestError{Reason: err.Error()}
		}
		_, err := c.Runner.RunICMPCheck(ctx, req)
		return err
	case "grpc":
		var req request.GRPCCheckerRequest
		if err := json.Unmarshal(data, &req); err != nil {
			return handlers.InvalidRequestError{Reason: err.Error()}
		}
		_, err := c.Runner.RunGRPCCheck(ctx, req)
		return err
	default:
		return fmt.Errorf("%w: %q", errUnknownJobType, jobType)
	}
}