Skip to content

Open-source status page, incident response and uptime monitoring

6.8 KB
Raw Blame
package main

import (
	"bytes"
	"context"
	"errors"
	"fmt"
	"io"
	"net"
	"net/http"
	"net/http/httputil"
	"net/url"
	"os"
	"os/signal"
	"syscall"
	"time"

	"github.com/gin-gonic/gin"
	"github.com/rs/zerolog/log"
)

// retryTransport retries a request when the underlying dial fails.
// Safe because a dial failure means the request never reached the upstream,
// so there's no risk of duplicating side effects.
type retryTransport struct {
	base       http.RoundTripper
	maxRetries int
	backoff    time.Duration
}

func (rt *retryTransport) RoundTrip(req *http.Request) (*http.Response, error) {
	var body []byte
	if req.Body != nil && req.Body != http.NoBody {
		b, err := io.ReadAll(req.Body)
		req.Body.Close()
		if err != nil {
			return nil, err
		}
		body = b
	}

	var lastErr error
	for attempt := 0; attempt <= rt.maxRetries; attempt++ {
		if attempt > 0 {
			select {
			case <-req.Context().Done():
				return nil, req.Context().Err()
			case <-time.After(rt.backoff):
			}
		}

		if body != nil {
			req.Body = io.NopCloser(bytes.NewReader(body))
		}

		resp, err := rt.base.RoundTrip(req)
		if err == nil {
			return resp, nil
		}
		lastErr = err

		if !isDialError(err) {
			return nil, err
		}

		log.Warn().
			Err(err).
			Int("attempt", attempt+1).
			Str("url", req.URL.String()).
			Msg("retrying after dial failure")
	}
	return nil, lastErr
}

func isDialError(err error) bool {
	var opErr *net.OpError
	if errors.As(err, &opErr) {
		return opErr.Op == "dial"
	}
	return false
}

// The checker responds only after its whole retry loop, which can run for
// several monitor timeouts (up to 5m each). Matches the Cloud Tasks dispatch
// deadline set by the workflows cron.
const checkerResponseTimeout = 30 * time.Minute

// Tuned for Railway internal networking: short dial timeout so a dead pod
// fails fast instead of stalling for the default 30s, plus aggressive idle
// pruning so we don't reuse connections that the platform has silently torn down.
func newTransport() *http.Transport {
	return &http.Transport{
		DialContext: (&net.Dialer{
			Timeout:   5 * time.Second,
			KeepAlive: 30 * time.Second,
		}).DialContext,
		MaxIdleConns:          200,
		MaxIdleConnsPerHost:   50,
		IdleConnTimeout:       60 * time.Second,
		TLSHandshakeTimeout:   10 * time.Second,
		ExpectContinueTimeout: 1 * time.Second,
		ResponseHeaderTimeout: checkerResponseTimeout,
		ForceAttemptHTTP2:     true,
	}
}

var regionTargets = map[string]string{
	"europe-west4-drams3a":   "http://openstatus-checker-eu-west.railway.internal:8080",
	"us-east4-eqdc4a":        "http://openstatus-checker-us-east.railway.internal:8080",
	"us-west2":               "http://openstatus-checker-us-west.railway.internal:8080",
	"asia-southeast1-eqsg3a": "http://checker-southeast-asia.railway.internal:8080",
}

func buildProxies(transport http.RoundTripper) (map[string]*httputil.ReverseProxy, error) {
	proxies := make(map[string]*httputil.ReverseProxy, len(regionTargets))
	for region, target := range regionTargets {
		remote, err := url.Parse(target)
		if err != nil {
			return nil, fmt.Errorf("parse target for region %s: %w", region, err)
		}

		p := &httputil.ReverseProxy{
			Transport: transport,
			Rewrite: func(r *httputil.ProxyRequest) {
				r.SetURL(remote)
				r.Out.Host = remote.Host
			},
		}
		p.ErrorHandler = func(rw http.ResponseWriter, req *http.Request, err error) {
			log.Error().
				Err(err).
				Str("region", region).
				Str("target", remote.String()).
				Str("method", req.Method).
				Str("path", req.URL.Path).
				Msg("reverse proxy transport error")
			rw.WriteHeader(http.StatusBadGateway)
		}
		p.ModifyResponse = func(resp *http.Response) error {
			if resp.StatusCode >= 500 {
				log.Error().
					Str("region", region).
					Str("target", remote.String()).
					Str("method", resp.Request.Method).
					Str("path", resp.Request.URL.Path).
					Int("status", resp.StatusCode).
					Msg("upstream returned server error")
			} else if resp.StatusCode >= 400 {
				log.Warn().
					Str("region", region).
					Str("target", remote.String()).
					Str("method", resp.Request.Method).
					Str("path", resp.Request.URL.Path).
					Int("status", resp.StatusCode).
					Msg("upstream returned client error")
			}
			return nil
		}
		proxies[region] = p
	}
	return proxies, nil
}

func requestLogger() gin.HandlerFunc {
	return func(c *gin.Context) {
		start := time.Now()
		c.Next()

		status := c.Writer.Status()
		evt := log.Info()
		switch {
		case status >= 500:
			evt = log.Error()
		case status >= 400:
			evt = log.Warn()
		}

		evt.
			Str("method", c.Request.Method).
			Str("path", c.Request.URL.Path).
			Str("region", c.Request.Header.Get("railway-region")).
			Int("status", status).
			Dur("duration", time.Since(start)).
			Msg("request")
	}
}

func proxyHandler(proxies map[string]*httputil.ReverseProxy) gin.HandlerFunc {
	return func(c *gin.Context) {
		region := c.Request.Header.Get("railway-region")
		p, ok := proxies[region]
		if !ok {
			log.Warn().
				Str("region", region).
				Str("path", c.Request.URL.Path).
				Msg("unknown or missing railway-region header")
			c.JSON(http.StatusBadGateway, gin.H{"error": "unknown region", "region": region})
			return
		}
		p.ServeHTTP(c.Writer, c.Request)
	}
}

func main() {
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	done := make(chan os.Signal, 1)
	signal.Notify(done, os.Interrupt, syscall.SIGINT, syscall.SIGTERM)

	go func() {
		<-done
		cancel()
	}()

	cloudProvider := env("CLOUD_PROVIDER", "railway")
	region := env("RAILWAY_REPLICA_REGION", env("REGION", "local"))

	transport := &retryTransport{
		base:       newTransport(),
		maxRetries: 2,
		backoff:    200 * time.Millisecond,
	}
	proxies, err := buildProxies(transport)
	if err != nil {
		log.Fatal().Err(err).Msg("failed to build reverse proxies")
	}

	router := gin.New()
	router.Use(gin.Recovery(), requestLogger())
	router.NoRoute(proxyHandler(proxies))

	router.GET("/healthz", func(c *gin.Context) {
		c.JSON(http.StatusOK, gin.H{"message": "pong", "region": region, "provider": cloudProvider})
	})

	httpServer := &http.Server{
		Addr:              fmt.Sprintf("0.0.0.0:%s", env("PORT", "8080")),
		Handler:           router,
		ReadHeaderTimeout: 10 * time.Second,
		ReadTimeout:       60 * time.Second,
		WriteTimeout:      checkerResponseTimeout,
		IdleTimeout:       120 * time.Second,
	}

	go func() {
		if err := httpServer.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
			log.Ctx(ctx).Error().Err(err).Msg("failed to start http server")
			cancel()
		}
	}()

	<-ctx.Done()

	shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 30*time.Second)
	defer shutdownCancel()
	if err := httpServer.Shutdown(shutdownCtx); err != nil {
		log.Error().Err(err).Msg("failed to shutdown http server")
	}
}

func env(key, fallback string) string {
	if value, ok := os.LookupEnv(key); ok {
		return value
	}

	return fallback
}