Open-source status page, incident response and uptime monitoring
TypeScript alertingincident-managementmcpmcp-servermonitoringmonitoring-as-codeobservabilityon-callopen-sourcesrestatus-pagestatuspagesynthetic-monitoringterraformtinybirdtursouptimeuptime-checkeruptime-monitor
checker: batch Tinybird events per data source (#2974)
Thibault Le Ouay739c89a6184e
diff --git a/apps/checker/cmd/server/main.go b/apps/checker/cmd/server/main.goindex a0374e72f..0a1cf617f 100644--- a/apps/checker/cmd/server/main.go+++ b/apps/checker/cmd/server/main.go@@ -238,7 +238,16 @@ func main() {defer httpClient.CloseIdleConnections()- tinybirdClient := tinybird.NewClient(httpClient, tinyBirdToken)+ tinybirdOptions := tinybird.DefaultBatcherOptions()+ tinybirdOptions.FlushInterval = envDuration("TINYBIRD_FLUSH_INTERVAL", tinybirdOptions.FlushInterval)+ tinybirdOptions.FlushJitter = envDuration("TINYBIRD_FLUSH_JITTER", tinybirdOptions.FlushJitter)+ tinybirdOptions.MaxBytes = envInt("TINYBIRD_MAX_BATCH_BYTES", tinybirdOptions.MaxBytes)+ // A zero budget would expire before the first request.+ if retryBudget := envDuration("TINYBIRD_RETRY_BUDGET", 0); retryBudget > 0 {+ tinybirdOptions.RetryBudget = retryBudget+ }+ tinybirdOptions.OnFlush = logTinybirdFlush+ tinybirdClient := tinybird.NewBatcher(httpClient, tinyBirdToken, tinybirdOptions)h := &handlers.Handler{Secret: cronSecret,@@ -294,6 +303,7 @@ func main() {}<-ctx.Done()+ shutdownStart := time.Now()// The consumer drains on its own from ctx; stop the listener alongside it.shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), consumerDrain)defer shutdownCancel()@@ -301,8 +311,20 @@ func main() {log.Ctx(ctx).Error().Err(err).Msg("failed to shutdown http server")}<-consumerDone++ // Checks that finished during the drain may still sit in a batch. The flush+ // gets whatever the drain left of the shutdown budget.+ flushCtx, flushCancel := context.WithDeadline(context.Background(), shutdownStart.Add(shutdownBudget))+ defer flushCancel()+ if err := tinybirdClient.Close(flushCtx); err != nil {+ log.Error().Err(err).Msg("failed to flush tinybird batches")+ }}+// shutdownBudget keeps the whole shutdown, final Tinybird flush included, under+// the 60s fly kill_timeout.+const shutdownBudget = 58 * time.Second+// consumerDrain stays under the 60s fly kill_timeout, so unfinished checks are// nacked for redelivery before the machine is killed.const consumerDrain = 50 * time.Second@@ -366,6 +388,41 @@ func logConsumerEvent(ctx context.Context, e queue.Event) {}}+func logTinybirdFlush(ctx context.Context, e tinybird.FlushEvent) {+ event := map[string]any{+ "timestamp": time.Now().Format(time.RFC3339),+ "datasource": e.DataSource,+ "rows": e.Rows,+ "bytes": e.Bytes,+ "attempts": e.Attempts,+ "ambiguous_attempts": e.AmbiguousAttempts,+ "status_code": e.StatusCode,+ "duration_ms": int(e.Duration.Milliseconds()),+ "oldest_wait_ms": int(e.OldestWait.Milliseconds()),+ "ratelimit_limit": e.RateLimitLimit,+ "ratelimit_remaining": e.RateLimitRemaining,+ }+ if e.Err != nil {+ event["error"] = map[string]any{+ "type": "TinybirdFlushError",+ "message": e.Err.Error(),+ }+ }++ if e.Attempts > 1 || shouldSample(event) {+ slog.LogAttrs(ctx, slog.LevelInfo, "tinybird flush", MapToAttrs(event)...)+ }+}++func envDuration(key string, fallback time.Duration) time.Duration {+ value, err := time.ParseDuration(env(key, ""))+ if err != nil || value < 0 {+ return fallback+ }++ return value+}+func envInt(key string, fallback int) int {value, err := strconv.Atoi(env(key, ""))if err != nil || value <= 0 {diff --git a/apps/checker/pkg/tinybird/batcher.go b/apps/checker/pkg/tinybird/batcher.gonew file mode 100644index 000000000..79b858dc4--- /dev/null+++ b/apps/checker/pkg/tinybird/batcher.go@@ -0,0 +1,322 @@+package tinybird++import (+ "bytes"+ "context"+ "encoding/json"+ "errors"+ "fmt"+ "io"+ "math/rand/v2"+ "net/http"+ "net/url"+ "strconv"+ "sync"+ "time"+)++var ErrBatcherClosed = errors.New("tinybird batcher closed")++type BatcherOptions struct {+ // A batch is flushed FlushInterval plus a random share of FlushJitter after+ // its first row, or as soon as it reaches MaxBytes.+ FlushInterval time.Duration+ FlushJitter time.Duration+ MaxBytes int+ // RetryBudget bounds one batch from the moment it is detached, including the+ // wait behind an earlier flush of the same data source, so a Tinybird outage+ // cannot queue batches indefinitely.+ RetryBudget time.Duration+ OnFlush func(context.Context, FlushEvent)+}++func DefaultBatcherOptions() BatcherOptions {+ return BatcherOptions{+ FlushInterval: 2 * time.Second,+ FlushJitter: time.Second,+ MaxBytes: 5 << 20,+ RetryBudget: 30 * time.Second,+ }+}++type FlushEvent struct {+ DataSource string+ Rows int+ Bytes int+ Attempts int+ // AmbiguousAttempts counts failed attempts Tinybird may still have ingested+ // (network errors and 5xx), so a later success can duplicate the rows.+ AmbiguousAttempts int+ StatusCode int+ Duration time.Duration+ OldestWait time.Duration+ RateLimitLimit int+ RateLimitRemaining int+ Err error+}++// Batcher implements Client by grouping rows per data source into one NDJSON+// request to the Events API. SendEvent returns once its row's batch is flushed.+type Batcher struct {+ httpClient *http.Client+ apiKey string+ baseURL string+ opts BatcherOptions++ // stop aborts in-flight flushes once Close gives up waiting.+ stop context.Context+ cancelStop context.CancelFunc++ mu sync.Mutex+ closed bool+ batches map[string]*batch+ senders map[string]*sync.Mutex+ flushes sync.WaitGroup+}++type batch struct {+ dataSource string+ body bytes.Buffer+ waiters []chan error+ startedAt time.Time+ deadline time.Time+ timer *time.Timer+}++func NewBatcher(httpClient *http.Client, apiKey string, opts BatcherOptions) *Batcher {+ stop, cancelStop := context.WithCancel(context.Background())++ return &Batcher{+ httpClient: httpClient,+ apiKey: apiKey,+ baseURL: getBaseURL(),+ opts: opts,+ stop: stop,+ cancelStop: cancelStop,+ batches: map[string]*batch{},+ senders: map[string]*sync.Mutex{},+ }+}++func (b *Batcher) SendEvent(ctx context.Context, event any, dataSourceName string) error {+ line, err := json.Marshal(event)+ if err != nil {+ return fmt.Errorf("unable to encode payload: %w", err)+ }++ done := make(chan error, 1)++ b.mu.Lock()+ if b.closed {+ b.mu.Unlock()+ return ErrBatcherClosed+ }+ current, ok := b.batches[dataSourceName]+ if !ok {+ current = &batch{dataSource: dataSourceName, startedAt: time.Now()}+ current.timer = time.AfterFunc(b.flushDelay(), func() { b.flushOnTimer(current) })+ b.batches[dataSourceName] = current+ }+ current.body.Write(line)+ current.body.WriteByte('\n')+ current.waiters = append(current.waiters, done)+ if current.body.Len() >= b.opts.MaxBytes {+ b.detachLocked(current)+ }+ b.mu.Unlock()++ // A cancelled caller's row stays in the batch and is still sent, so the+ // redelivered check can record it twice: delivery is at least once.+ select {+ case err := <-done:+ return err+ case <-ctx.Done():+ return ctx.Err()+ }+}++// Close flushes every pending batch and waits for in-flight flushes, bounded by+// ctx. When ctx ends first, in-flight flushes are aborted and their rows fail.+func (b *Batcher) Close(ctx context.Context) error {+ defer b.cancelStop()++ b.mu.Lock()+ b.closed = true+ for _, pending := range b.batches {+ b.detachLocked(pending)+ }+ b.mu.Unlock()++ done := make(chan struct{})+ go func() {+ b.flushes.Wait()+ close(done)+ }()++ select {+ case <-done:+ return nil+ case <-ctx.Done():+ b.cancelStop()+ <-done+ return ctx.Err()+ }+}++func (b *Batcher) flushDelay() time.Duration {+ if b.opts.FlushJitter <= 0 {+ return b.opts.FlushInterval+ }++ return b.opts.FlushInterval + rand.N(b.opts.FlushJitter)+}++func (b *Batcher) flushOnTimer(target *batch) {+ b.mu.Lock()+ defer b.mu.Unlock()+ if b.batches[target.dataSource] == target {+ b.detachLocked(target)+ }+}++// detachLocked hands the batch to a flush goroutine; new rows start a new batch.+func (b *Batcher) detachLocked(target *batch) {+ delete(b.batches, target.dataSource)+ target.timer.Stop()+ target.deadline = time.Now().Add(b.opts.RetryBudget)++ sender, ok := b.senders[target.dataSource]+ if !ok {+ sender = &sync.Mutex{}+ b.senders[target.dataSource] = sender+ }++ b.flushes.Add(1)+ go func() {+ defer b.flushes.Done()+ // One request per data source at a time, so retries don't pile up; the+ // batch deadline bounds the wait for the lock.+ sender.Lock()+ defer sender.Unlock()+ b.flush(target)+ }()+}++func (b *Batcher) flush(target *batch) {+ start := time.Now()+ event := FlushEvent{+ DataSource: target.dataSource,+ Rows: len(target.waiters),+ Bytes: target.body.Len(),+ RateLimitLimit: -1,+ RateLimitRemaining: -1,+ }++ ctx, cancel := context.WithDeadline(b.stop, target.deadline)+ defer cancel()++ event.Err = b.sendWithRetry(ctx, target, &event)+ event.Duration = time.Since(start)+ event.OldestWait = time.Since(target.startedAt)++ for _, waiter := range target.waiters {+ waiter <- event.Err+ }++ if b.opts.OnFlush != nil {+ b.opts.OnFlush(context.Background(), event)+ }+}++func (b *Batcher) sendWithRetry(ctx context.Context, target *batch, event *FlushEvent) error {+ if err := ctx.Err(); err != nil {+ return fmt.Errorf("flush not attempted: %w", err)+ }++ requestURL, err := url.Parse(b.baseURL)+ if err != nil {+ return fmt.Errorf("unable to parse url: %w", err)+ }+ q := requestURL.Query()+ q.Add("name", target.dataSource)+ requestURL.RawQuery = q.Encode()++ for {+ req, err := http.NewRequestWithContext(ctx, http.MethodPost, requestURL.String(), bytes.NewReader(target.body.Bytes()))+ if err != nil {+ return fmt.Errorf("unable to create request: %w", err)+ }+ req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", b.apiKey))++ event.Attempts+++ resp, err := b.httpClient.Do(req)+ var retryAfter time.Duration+ if err != nil {+ err = fmt.Errorf("unable to send request: %w", err)+ } else {+ event.StatusCode = resp.StatusCode+ event.RateLimitLimit = headerInt(resp.Header, "X-RateLimit-Limit")+ event.RateLimitRemaining = headerInt(resp.Header, "X-RateLimit-Remaining")+ retryAfter = parseRetryAfter(resp.Header.Get("Retry-After"))+ // Drain so the connection is reused by the next attempt.+ _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 64<<10))+ resp.Body.Close()++ if resp.StatusCode == http.StatusAccepted {+ return nil+ }+ err = fmt.Errorf("unexpected status code: %d", resp.StatusCode)+ }++ retryable := resp == nil || resp.StatusCode == http.StatusTooManyRequests || resp.StatusCode >= 500+ if !retryable {+ return err+ }+ if resp == nil || resp.StatusCode >= 500 {+ event.AmbiguousAttempts+++ }++ wait := retryAfter+ if wait <= 0 {+ wait = backoff(event.Attempts)+ }+ if time.Now().Add(wait).After(target.deadline) {+ return err+ }++ select {+ case <-time.After(wait):+ case <-ctx.Done():+ return err+ }+ }+}++func backoff(attempt int) time.Duration {+ base := min(250*time.Millisecond<<min(attempt-1, 5), 5*time.Second)++ return base/2 + rand.N(base)+}++func parseRetryAfter(value string) time.Duration {+ if value == "" {+ return 0+ }+ if seconds, err := strconv.Atoi(value); err == nil {+ return time.Duration(seconds) * time.Second+ }+ if at, err := http.ParseTime(value); err == nil {+ return time.Until(at)+ }++ return 0+}++func headerInt(header http.Header, key string) int {+ value, err := strconv.Atoi(header.Get(key))+ if err != nil {+ return -1+ }++ return value+}diff --git a/apps/checker/pkg/tinybird/batcher_test.go b/apps/checker/pkg/tinybird/batcher_test.gonew file mode 100644index 000000000..740cc91df--- /dev/null+++ b/apps/checker/pkg/tinybird/batcher_test.go@@ -0,0 +1,286 @@+package tinybird_test++import (+ "context"+ "errors"+ "io"+ "net/http"+ "strings"+ "sync"+ "sync/atomic"+ "testing"+ "time"++ "github.com/openstatushq/openstatus/apps/checker/pkg/tinybird"+ "github.com/stretchr/testify/require"+)++type recordedRequest struct {+ dataSource string+ lines []string+}++type fakeEventsAPI struct {+ mu sync.Mutex+ requests []recordedRequest+ respond func(attempt int) *http.Response+ attempts atomic.Int32+}++func (f *fakeEventsAPI) client() *http.Client {+ return (&interceptorHTTPClient{f: func(req *http.Request) (*http.Response, error) {+ body, _ := io.ReadAll(req.Body)+ f.mu.Lock()+ f.requests = append(f.requests, recordedRequest{+ dataSource: req.URL.Query().Get("name"),+ lines: strings.Split(strings.TrimSuffix(string(body), "\n"), "\n"),+ })+ f.mu.Unlock()++ attempt := int(f.attempts.Add(1))+ if f.respond != nil {+ if resp := f.respond(attempt); resp != nil {+ return resp, nil+ }+ return nil, errors.New("connection reset")+ }++ return status(http.StatusAccepted, nil), nil+ }}).GetHTTPClient()+}++func (f *fakeEventsAPI) recorded() []recordedRequest {+ f.mu.Lock()+ defer f.mu.Unlock()++ return append([]recordedRequest(nil), f.requests...)+}++func status(code int, header http.Header) *http.Response {+ if header == nil {+ header = http.Header{}+ }++ return &http.Response{StatusCode: code, Header: header, Body: io.NopCloser(strings.NewReader(""))}+}++func options() tinybird.BatcherOptions {+ return tinybird.BatcherOptions{+ FlushInterval: 20 * time.Millisecond,+ MaxBytes: 1 << 20,+ RetryBudget: time.Second,+ }+}++// eventLine is the NDJSON line sendAll writes per row.+const eventLine = `"event"` + "\n"++func sendAll(t *testing.T, b *tinybird.Batcher, dataSource string, n int) []error {+ t.Helper()+ errs := make([]error, n)+ var wg sync.WaitGroup+ for i := range n {+ wg.Go(func() {+ errs[i] = b.SendEvent(t.Context(), "event", dataSource)+ })+ }+ wg.Wait()++ return errs+}++func TestBatcher(t *testing.T) {+ t.Parallel()++ t.Run("it should send concurrent rows as one NDJSON request", func(t *testing.T) {+ t.Parallel()+ api := &fakeEventsAPI{}+ opts := options()+ // Only the 50th row can trigger the flush, whatever the scheduling.+ opts.FlushInterval = time.Hour+ opts.MaxBytes = 50 * len(eventLine)+ b := tinybird.NewBatcher(api.client(), "apiKey", opts)++ for _, err := range sendAll(t, b, "ping_response__v8", 50) {+ require.NoError(t, err)+ }++ requests := api.recorded()+ require.Len(t, requests, 1)+ require.Equal(t, "ping_response__v8", requests[0].dataSource)+ require.Len(t, requests[0].lines, 50)+ })++ t.Run("it should flush before the interval once the batch reaches MaxBytes", func(t *testing.T) {+ t.Parallel()+ api := &fakeEventsAPI{}+ opts := options()+ opts.FlushInterval = time.Hour+ opts.MaxBytes = 1+ b := tinybird.NewBatcher(api.client(), "apiKey", opts)++ require.NoError(t, b.SendEvent(t.Context(), "event", "test"))+ require.Len(t, api.recorded(), 1)+ })++ t.Run("it should keep data sources in separate requests", func(t *testing.T) {+ t.Parallel()+ api := &fakeEventsAPI{}+ b := tinybird.NewBatcher(api.client(), "apiKey", options())++ var wg sync.WaitGroup+ for _, ds := range []string{"ping_response__v8", "tcp_response__v0"} {+ wg.Go(func() { require.NoError(t, b.SendEvent(t.Context(), "event", ds)) })+ }+ wg.Wait()++ requests := api.recorded()+ require.Len(t, requests, 2)+ require.ElementsMatch(t, []string{"ping_response__v8", "tcp_response__v0"}, []string{requests[0].dataSource, requests[1].dataSource})+ })++ t.Run("it should honor Retry-After on 429 and then succeed", func(t *testing.T) {+ t.Parallel()+ api := &fakeEventsAPI{respond: func(attempt int) *http.Response {+ if attempt == 1 {+ return status(http.StatusTooManyRequests, http.Header{"Retry-After": {"1"}})+ }+ return status(http.StatusAccepted, nil)+ }}+ opts := options()+ opts.RetryBudget = 5 * time.Second+ events := make(chan tinybird.FlushEvent, 1)+ opts.OnFlush = func(_ context.Context, e tinybird.FlushEvent) { events <- e }+ b := tinybird.NewBatcher(api.client(), "apiKey", opts)++ start := time.Now()+ require.NoError(t, b.SendEvent(t.Context(), "event", "test"))+ require.GreaterOrEqual(t, time.Since(start), time.Second)+ require.Len(t, api.recorded(), 2)+ flushed := <-events+ require.Equal(t, 2, flushed.Attempts)+ require.Equal(t, 0, flushed.AmbiguousAttempts)+ })++ t.Run("it should return the error to every row once the retry budget is spent", func(t *testing.T) {+ t.Parallel()+ api := &fakeEventsAPI{respond: func(int) *http.Response { return status(http.StatusServiceUnavailable, nil) }}+ opts := options()+ opts.RetryBudget = 800 * time.Millisecond+ b := tinybird.NewBatcher(api.client(), "apiKey", opts)++ for _, err := range sendAll(t, b, "test", 5) {+ require.ErrorContains(t, err, "unexpected status code: 503")+ }+ require.Greater(t, len(api.recorded()), 1)+ })++ t.Run("it should retry network errors and count them as ambiguous", func(t *testing.T) {+ t.Parallel()+ api := &fakeEventsAPI{respond: func(attempt int) *http.Response {+ if attempt == 1 {+ return nil+ }+ return status(http.StatusAccepted, nil)+ }}+ opts := options()+ events := make(chan tinybird.FlushEvent, 1)+ opts.OnFlush = func(_ context.Context, e tinybird.FlushEvent) { events <- e }+ b := tinybird.NewBatcher(api.client(), "apiKey", opts)++ require.NoError(t, b.SendEvent(t.Context(), "event", "test"))+ require.Equal(t, 1, (<-events).AmbiguousAttempts)+ })++ t.Run("it should not retry a 400", func(t *testing.T) {+ t.Parallel()+ api := &fakeEventsAPI{respond: func(int) *http.Response { return status(http.StatusBadRequest, nil) }}+ b := tinybird.NewBatcher(api.client(), "apiKey", options())++ require.ErrorContains(t, b.SendEvent(t.Context(), "event", "test"), "unexpected status code: 400")+ require.Len(t, api.recorded(), 1)+ })++ t.Run("it should return when the caller's context is cancelled", func(t *testing.T) {+ t.Parallel()+ api := &fakeEventsAPI{}+ opts := options()+ opts.FlushInterval = time.Hour+ b := tinybird.NewBatcher(api.client(), "apiKey", opts)++ ctx, cancel := context.WithTimeout(t.Context(), 20*time.Millisecond)+ defer cancel()+ require.ErrorIs(t, b.SendEvent(ctx, "event", "test"), context.DeadlineExceeded)+ })++ t.Run("it should flush pending rows on Close and refuse new ones", func(t *testing.T) {+ t.Parallel()+ api := &fakeEventsAPI{}+ opts := options()+ opts.FlushInterval = time.Hour+ b := tinybird.NewBatcher(api.client(), "apiKey", opts)++ sent := make(chan error, 1)+ go func() { sent <- b.SendEvent(t.Context(), "event", "test") }()+ // The flush interval alone would never fire.+ require.Eventually(t, func() bool { return b.PendingRows("test") == 1 }, time.Second, time.Millisecond)+ require.NoError(t, b.Close(t.Context()))++ require.NoError(t, <-sent)+ require.Len(t, api.recorded(), 1)+ require.ErrorIs(t, b.SendEvent(t.Context(), "event", "test"), tinybird.ErrBatcherClosed)+ })++ t.Run("it should count the wait behind an earlier flush against the retry budget", func(t *testing.T) {+ t.Parallel()+ opts := options()+ opts.RetryBudget = 200 * time.Millisecond+ api := &fakeEventsAPI{respond: func(attempt int) *http.Response {+ if attempt == 1 {+ // A slow request holds the sender past the queued batch's deadline.+ time.Sleep(2 * opts.RetryBudget)+ }+ return status(http.StatusServiceUnavailable, nil)+ }}+ opts.FlushInterval = time.Hour+ opts.MaxBytes = 1+ b := tinybird.NewBatcher(api.client(), "apiKey", opts)++ first := make(chan error, 1)+ go func() { first <- b.SendEvent(t.Context(), "event", "test") }()+ require.Eventually(t, func() bool { return len(api.recorded()) > 0 }, time.Second, time.Millisecond)++ // Queued behind the first batch, which keeps its sender busy past this batch's budget.+ start := time.Now()+ err := b.SendEvent(t.Context(), "event", "test")+ require.Len(t, api.recorded(), 1)+ require.Less(t, time.Since(start), 3*opts.RetryBudget)+ require.ErrorIs(t, err, context.DeadlineExceeded)+ require.ErrorContains(t, err, "flush not attempted")+ require.ErrorContains(t, <-first, "unexpected status code: 503")+ })++ t.Run("it should abort in-flight retries once Close gives up", func(t *testing.T) {+ t.Parallel()+ api := &fakeEventsAPI{respond: func(int) *http.Response { return status(http.StatusServiceUnavailable, nil) }}+ opts := options()+ opts.FlushInterval = time.Hour+ opts.MaxBytes = 1+ opts.RetryBudget = time.Minute+ b := tinybird.NewBatcher(api.client(), "apiKey", opts)++ sent := make(chan error, 1)+ go func() { sent <- b.SendEvent(t.Context(), "event", "test") }()+ require.Eventually(t, func() bool { return len(api.recorded()) > 0 }, time.Second, time.Millisecond)++ ctx, cancel := context.WithTimeout(t.Context(), 50*time.Millisecond)+ defer cancel()+ require.ErrorIs(t, b.Close(ctx), context.DeadlineExceeded)+ select {+ case err := <-sent:+ require.ErrorContains(t, err, "unexpected status code: 503")+ default:+ t.Fatal("the row should fail as soon as Close returns")+ }+ })+}diff --git a/apps/checker/pkg/tinybird/export_test.go b/apps/checker/pkg/tinybird/export_test.gonew file mode 100644index 000000000..c1588afa1--- /dev/null+++ b/apps/checker/pkg/tinybird/export_test.go@@ -0,0 +1,12 @@+package tinybird++// PendingRows reports the rows buffered for a data source and not yet detached.+func (b *Batcher) PendingRows(dataSource string) int {+ b.mu.Lock()+ defer b.mu.Unlock()+ if pending, ok := b.batches[dataSource]; ok {+ return len(pending.waiters)+ }++ return 0+}