Skip to content

Open-source status page, incident response and uptime monitoring

checker: exit non-zero when the pubsub consumer fails (#3008)

Thibault Le Ouay5270a3ed4456
diff --git a/apps/checker/cmd/server/main.go b/apps/checker/cmd/server/main.go
index bd75f3d81..ad76b6e3a 100644
--- a/apps/checker/cmd/server/main.go
+++ b/apps/checker/cmd/server/main.go
@@ -163,6 +163,16 @@ func Logger() gin.HandlerFunc {
}
func main() {
+ // Registered first so it runs after every other defer (log flush included).
+ // fly only restarts machines that exit non-zero, so a consumer failure must
+ // not end in a clean exit.
+ var consumerFailed bool
+ defer func() {
+ if consumerFailed {
+ os.Exit(1)
+ }
+ }()
+
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
@@ -292,9 +302,12 @@ func main() {
if env("PUBSUB_ENABLED", "false") == "true" {
go func() {
defer close(consumerDone)
- if err := runConsumer(ctx, h, region); err != nil {
+ err := runConsumer(ctx, h, region)
+ if err != nil {
log.Error().Err(err).Msg("pubsub consumer stopped")
}
+ // Receive returns nil on shutdown; stopping any other way is a failure.
+ consumerFailed = err != nil || ctx.Err() == nil
// A consumer that stopped while the process keeps serving /health
// would never be restarted, so take the whole process down.
cancel()