Skip to content

Open-source status page, incident response and uptime monitoring

checker: batch Tinybird events per data source (#2974)

Thibault Le Ouay739c89a6184e
diff --git a/apps/checker/cmd/server/main.go b/apps/checker/cmd/server/main.go
index a0374e72f..0a1cf617f 100644
--- a/apps/checker/cmd/server/main.go
+++ b/apps/checker/cmd/server/main.go
@@ -238,7 +238,16 @@ func main() {
defer httpClient.CloseIdleConnections()
- tinybirdClient := tinybird.NewClient(httpClient, tinyBirdToken)
+ tinybirdOptions := tinybird.DefaultBatcherOptions()
+ tinybirdOptions.FlushInterval = envDuration("TINYBIRD_FLUSH_INTERVAL", tinybirdOptions.FlushInterval)
+ tinybirdOptions.FlushJitter = envDuration("TINYBIRD_FLUSH_JITTER", tinybirdOptions.FlushJitter)
+ tinybirdOptions.MaxBytes = envInt("TINYBIRD_MAX_BATCH_BYTES", tinybirdOptions.MaxBytes)
+ // A zero budget would expire before the first request.
+ if retryBudget := envDuration("TINYBIRD_RETRY_BUDGET", 0); retryBudget > 0 {
+ tinybirdOptions.RetryBudget = retryBudget
+ }
+ tinybirdOptions.OnFlush = logTinybirdFlush
+ tinybirdClient := tinybird.NewBatcher(httpClient, tinyBirdToken, tinybirdOptions)
h := &handlers.Handler{
Secret: cronSecret,
@@ -294,6 +303,7 @@ func main() {
}
<-ctx.Done()
+ shutdownStart := time.Now()
// The consumer drains on its own from ctx; stop the listener alongside it.
shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), consumerDrain)
defer shutdownCancel()
@@ -301,8 +311,20 @@ func main() {
log.Ctx(ctx).Error().Err(err).Msg("failed to shutdown http server")
}
<-consumerDone
+
+ // Checks that finished during the drain may still sit in a batch. The flush
+ // gets whatever the drain left of the shutdown budget.
+ flushCtx, flushCancel := context.WithDeadline(context.Background(), shutdownStart.Add(shutdownBudget))
+ defer flushCancel()
+ if err := tinybirdClient.Close(flushCtx); err != nil {
+ log.Error().Err(err).Msg("failed to flush tinybird batches")
+ }
}
+// shutdownBudget keeps the whole shutdown, final Tinybird flush included, under
+// the 60s fly kill_timeout.
+const shutdownBudget = 58 * time.Second
+
// consumerDrain stays under the 60s fly kill_timeout, so unfinished checks are
// nacked for redelivery before the machine is killed.
const consumerDrain = 50 * time.Second
@@ -366,6 +388,41 @@ func logConsumerEvent(ctx context.Context, e queue.Event) {
}
}
+func logTinybirdFlush(ctx context.Context, e tinybird.FlushEvent) {
+ event := map[string]any{
+ "timestamp": time.Now().Format(time.RFC3339),
+ "datasource": e.DataSource,
+ "rows": e.Rows,
+ "bytes": e.Bytes,
+ "attempts": e.Attempts,
+ "ambiguous_attempts": e.AmbiguousAttempts,
+ "status_code": e.StatusCode,
+ "duration_ms": int(e.Duration.Milliseconds()),
+ "oldest_wait_ms": int(e.OldestWait.Milliseconds()),
+ "ratelimit_limit": e.RateLimitLimit,
+ "ratelimit_remaining": e.RateLimitRemaining,
+ }
+ if e.Err != nil {
+ event["error"] = map[string]any{
+ "type": "TinybirdFlushError",
+ "message": e.Err.Error(),
+ }
+ }
+
+ if e.Attempts > 1 || shouldSample(event) {
+ slog.LogAttrs(ctx, slog.LevelInfo, "tinybird flush", MapToAttrs(event)...)
+ }
+}
+
+func envDuration(key string, fallback time.Duration) time.Duration {
+ value, err := time.ParseDuration(env(key, ""))
+ if err != nil || value < 0 {
+ return fallback
+ }
+
+ return value
+}
+
func envInt(key string, fallback int) int {
value, err := strconv.Atoi(env(key, ""))
if err != nil || value <= 0 {
diff --git a/apps/checker/pkg/tinybird/batcher.go b/apps/checker/pkg/tinybird/batcher.go
new file mode 100644
index 000000000..79b858dc4
--- /dev/null
+++ b/apps/checker/pkg/tinybird/batcher.go
@@ -0,0 +1,322 @@
+package tinybird
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "math/rand/v2"
+ "net/http"
+ "net/url"
+ "strconv"
+ "sync"
+ "time"
+)
+
+var ErrBatcherClosed = errors.New("tinybird batcher closed")
+
+type BatcherOptions struct {
+ // A batch is flushed FlushInterval plus a random share of FlushJitter after
+ // its first row, or as soon as it reaches MaxBytes.
+ FlushInterval time.Duration
+ FlushJitter time.Duration
+ MaxBytes int
+ // RetryBudget bounds one batch from the moment it is detached, including the
+ // wait behind an earlier flush of the same data source, so a Tinybird outage
+ // cannot queue batches indefinitely.
+ RetryBudget time.Duration
+ OnFlush func(context.Context, FlushEvent)
+}
+
+func DefaultBatcherOptions() BatcherOptions {
+ return BatcherOptions{
+ FlushInterval: 2 * time.Second,
+ FlushJitter: time.Second,
+ MaxBytes: 5 << 20,
+ RetryBudget: 30 * time.Second,
+ }
+}
+
+type FlushEvent struct {
+ DataSource string
+ Rows int
+ Bytes int
+ Attempts int
+ // AmbiguousAttempts counts failed attempts Tinybird may still have ingested
+ // (network errors and 5xx), so a later success can duplicate the rows.
+ AmbiguousAttempts int
+ StatusCode int
+ Duration time.Duration
+ OldestWait time.Duration
+ RateLimitLimit int
+ RateLimitRemaining int
+ Err error
+}
+
+// Batcher implements Client by grouping rows per data source into one NDJSON
+// request to the Events API. SendEvent returns once its row's batch is flushed.
+type Batcher struct {
+ httpClient *http.Client
+ apiKey string
+ baseURL string
+ opts BatcherOptions
+
+ // stop aborts in-flight flushes once Close gives up waiting.
+ stop context.Context
+ cancelStop context.CancelFunc
+
+ mu sync.Mutex
+ closed bool
+ batches map[string]*batch
+ senders map[string]*sync.Mutex
+ flushes sync.WaitGroup
+}
+
+type batch struct {
+ dataSource string
+ body bytes.Buffer
+ waiters []chan error
+ startedAt time.Time
+ deadline time.Time
+ timer *time.Timer
+}
+
+func NewBatcher(httpClient *http.Client, apiKey string, opts BatcherOptions) *Batcher {
+ stop, cancelStop := context.WithCancel(context.Background())
+
+ return &Batcher{
+ httpClient: httpClient,
+ apiKey: apiKey,
+ baseURL: getBaseURL(),
+ opts: opts,
+ stop: stop,
+ cancelStop: cancelStop,
+ batches: map[string]*batch{},
+ senders: map[string]*sync.Mutex{},
+ }
+}
+
+func (b *Batcher) SendEvent(ctx context.Context, event any, dataSourceName string) error {
+ line, err := json.Marshal(event)
+ if err != nil {
+ return fmt.Errorf("unable to encode payload: %w", err)
+ }
+
+ done := make(chan error, 1)
+
+ b.mu.Lock()
+ if b.closed {
+ b.mu.Unlock()
+ return ErrBatcherClosed
+ }
+ current, ok := b.batches[dataSourceName]
+ if !ok {
+ current = &batch{dataSource: dataSourceName, startedAt: time.Now()}
+ current.timer = time.AfterFunc(b.flushDelay(), func() { b.flushOnTimer(current) })
+ b.batches[dataSourceName] = current
+ }
+ current.body.Write(line)
+ current.body.WriteByte('\n')
+ current.waiters = append(current.waiters, done)
+ if current.body.Len() >= b.opts.MaxBytes {
+ b.detachLocked(current)
+ }
+ b.mu.Unlock()
+
+ // A cancelled caller's row stays in the batch and is still sent, so the
+ // redelivered check can record it twice: delivery is at least once.
+ select {
+ case err := <-done:
+ return err
+ case <-ctx.Done():
+ return ctx.Err()
+ }
+}
+
+// Close flushes every pending batch and waits for in-flight flushes, bounded by
+// ctx. When ctx ends first, in-flight flushes are aborted and their rows fail.
+func (b *Batcher) Close(ctx context.Context) error {
+ defer b.cancelStop()
+
+ b.mu.Lock()
+ b.closed = true
+ for _, pending := range b.batches {
+ b.detachLocked(pending)
+ }
+ b.mu.Unlock()
+
+ done := make(chan struct{})
+ go func() {
+ b.flushes.Wait()
+ close(done)
+ }()
+
+ select {
+ case <-done:
+ return nil
+ case <-ctx.Done():
+ b.cancelStop()
+ <-done
+ return ctx.Err()
+ }
+}
+
+func (b *Batcher) flushDelay() time.Duration {
+ if b.opts.FlushJitter <= 0 {
+ return b.opts.FlushInterval
+ }
+
+ return b.opts.FlushInterval + rand.N(b.opts.FlushJitter)
+}
+
+func (b *Batcher) flushOnTimer(target *batch) {
+ b.mu.Lock()
+ defer b.mu.Unlock()
+ if b.batches[target.dataSource] == target {
+ b.detachLocked(target)
+ }
+}
+
+// detachLocked hands the batch to a flush goroutine; new rows start a new batch.
+func (b *Batcher) detachLocked(target *batch) {
+ delete(b.batches, target.dataSource)
+ target.timer.Stop()
+ target.deadline = time.Now().Add(b.opts.RetryBudget)
+
+ sender, ok := b.senders[target.dataSource]
+ if !ok {
+ sender = &sync.Mutex{}
+ b.senders[target.dataSource] = sender
+ }
+
+ b.flushes.Add(1)
+ go func() {
+ defer b.flushes.Done()
+ // One request per data source at a time, so retries don't pile up; the
+ // batch deadline bounds the wait for the lock.
+ sender.Lock()
+ defer sender.Unlock()
+ b.flush(target)
+ }()
+}
+
+func (b *Batcher) flush(target *batch) {
+ start := time.Now()
+ event := FlushEvent{
+ DataSource: target.dataSource,
+ Rows: len(target.waiters),
+ Bytes: target.body.Len(),
+ RateLimitLimit: -1,
+ RateLimitRemaining: -1,
+ }
+
+ ctx, cancel := context.WithDeadline(b.stop, target.deadline)
+ defer cancel()
+
+ event.Err = b.sendWithRetry(ctx, target, &event)
+ event.Duration = time.Since(start)
+ event.OldestWait = time.Since(target.startedAt)
+
+ for _, waiter := range target.waiters {
+ waiter <- event.Err
+ }
+
+ if b.opts.OnFlush != nil {
+ b.opts.OnFlush(context.Background(), event)
+ }
+}
+
+func (b *Batcher) sendWithRetry(ctx context.Context, target *batch, event *FlushEvent) error {
+ if err := ctx.Err(); err != nil {
+ return fmt.Errorf("flush not attempted: %w", err)
+ }
+
+ requestURL, err := url.Parse(b.baseURL)
+ if err != nil {
+ return fmt.Errorf("unable to parse url: %w", err)
+ }
+ q := requestURL.Query()
+ q.Add("name", target.dataSource)
+ requestURL.RawQuery = q.Encode()
+
+ for {
+ req, err := http.NewRequestWithContext(ctx, http.MethodPost, requestURL.String(), bytes.NewReader(target.body.Bytes()))
+ if err != nil {
+ return fmt.Errorf("unable to create request: %w", err)
+ }
+ req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", b.apiKey))
+
+ event.Attempts++
+ resp, err := b.httpClient.Do(req)
+ var retryAfter time.Duration
+ if err != nil {
+ err = fmt.Errorf("unable to send request: %w", err)
+ } else {
+ event.StatusCode = resp.StatusCode
+ event.RateLimitLimit = headerInt(resp.Header, "X-RateLimit-Limit")
+ event.RateLimitRemaining = headerInt(resp.Header, "X-RateLimit-Remaining")
+ retryAfter = parseRetryAfter(resp.Header.Get("Retry-After"))
+ // Drain so the connection is reused by the next attempt.
+ _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 64<<10))
+ resp.Body.Close()
+
+ if resp.StatusCode == http.StatusAccepted {
+ return nil
+ }
+ err = fmt.Errorf("unexpected status code: %d", resp.StatusCode)
+ }
+
+ retryable := resp == nil || resp.StatusCode == http.StatusTooManyRequests || resp.StatusCode >= 500
+ if !retryable {
+ return err
+ }
+ if resp == nil || resp.StatusCode >= 500 {
+ event.AmbiguousAttempts++
+ }
+
+ wait := retryAfter
+ if wait <= 0 {
+ wait = backoff(event.Attempts)
+ }
+ if time.Now().Add(wait).After(target.deadline) {
+ return err
+ }
+
+ select {
+ case <-time.After(wait):
+ case <-ctx.Done():
+ return err
+ }
+ }
+}
+
+func backoff(attempt int) time.Duration {
+ base := min(250*time.Millisecond<<min(attempt-1, 5), 5*time.Second)
+
+ return base/2 + rand.N(base)
+}
+
+func parseRetryAfter(value string) time.Duration {
+ if value == "" {
+ return 0
+ }
+ if seconds, err := strconv.Atoi(value); err == nil {
+ return time.Duration(seconds) * time.Second
+ }
+ if at, err := http.ParseTime(value); err == nil {
+ return time.Until(at)
+ }
+
+ return 0
+}
+
+func headerInt(header http.Header, key string) int {
+ value, err := strconv.Atoi(header.Get(key))
+ if err != nil {
+ return -1
+ }
+
+ return value
+}
diff --git a/apps/checker/pkg/tinybird/batcher_test.go b/apps/checker/pkg/tinybird/batcher_test.go
new file mode 100644
index 000000000..740cc91df
--- /dev/null
+++ b/apps/checker/pkg/tinybird/batcher_test.go
@@ -0,0 +1,286 @@
+package tinybird_test
+
+import (
+ "context"
+ "errors"
+ "io"
+ "net/http"
+ "strings"
+ "sync"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/openstatushq/openstatus/apps/checker/pkg/tinybird"
+ "github.com/stretchr/testify/require"
+)
+
+type recordedRequest struct {
+ dataSource string
+ lines []string
+}
+
+type fakeEventsAPI struct {
+ mu sync.Mutex
+ requests []recordedRequest
+ respond func(attempt int) *http.Response
+ attempts atomic.Int32
+}
+
+func (f *fakeEventsAPI) client() *http.Client {
+ return (&interceptorHTTPClient{f: func(req *http.Request) (*http.Response, error) {
+ body, _ := io.ReadAll(req.Body)
+ f.mu.Lock()
+ f.requests = append(f.requests, recordedRequest{
+ dataSource: req.URL.Query().Get("name"),
+ lines: strings.Split(strings.TrimSuffix(string(body), "\n"), "\n"),
+ })
+ f.mu.Unlock()
+
+ attempt := int(f.attempts.Add(1))
+ if f.respond != nil {
+ if resp := f.respond(attempt); resp != nil {
+ return resp, nil
+ }
+ return nil, errors.New("connection reset")
+ }
+
+ return status(http.StatusAccepted, nil), nil
+ }}).GetHTTPClient()
+}
+
+func (f *fakeEventsAPI) recorded() []recordedRequest {
+ f.mu.Lock()
+ defer f.mu.Unlock()
+
+ return append([]recordedRequest(nil), f.requests...)
+}
+
+func status(code int, header http.Header) *http.Response {
+ if header == nil {
+ header = http.Header{}
+ }
+
+ return &http.Response{StatusCode: code, Header: header, Body: io.NopCloser(strings.NewReader(""))}
+}
+
+func options() tinybird.BatcherOptions {
+ return tinybird.BatcherOptions{
+ FlushInterval: 20 * time.Millisecond,
+ MaxBytes: 1 << 20,
+ RetryBudget: time.Second,
+ }
+}
+
+// eventLine is the NDJSON line sendAll writes per row.
+const eventLine = `"event"` + "\n"
+
+func sendAll(t *testing.T, b *tinybird.Batcher, dataSource string, n int) []error {
+ t.Helper()
+ errs := make([]error, n)
+ var wg sync.WaitGroup
+ for i := range n {
+ wg.Go(func() {
+ errs[i] = b.SendEvent(t.Context(), "event", dataSource)
+ })
+ }
+ wg.Wait()
+
+ return errs
+}
+
+func TestBatcher(t *testing.T) {
+ t.Parallel()
+
+ t.Run("it should send concurrent rows as one NDJSON request", func(t *testing.T) {
+ t.Parallel()
+ api := &fakeEventsAPI{}
+ opts := options()
+ // Only the 50th row can trigger the flush, whatever the scheduling.
+ opts.FlushInterval = time.Hour
+ opts.MaxBytes = 50 * len(eventLine)
+ b := tinybird.NewBatcher(api.client(), "apiKey", opts)
+
+ for _, err := range sendAll(t, b, "ping_response__v8", 50) {
+ require.NoError(t, err)
+ }
+
+ requests := api.recorded()
+ require.Len(t, requests, 1)
+ require.Equal(t, "ping_response__v8", requests[0].dataSource)
+ require.Len(t, requests[0].lines, 50)
+ })
+
+ t.Run("it should flush before the interval once the batch reaches MaxBytes", func(t *testing.T) {
+ t.Parallel()
+ api := &fakeEventsAPI{}
+ opts := options()
+ opts.FlushInterval = time.Hour
+ opts.MaxBytes = 1
+ b := tinybird.NewBatcher(api.client(), "apiKey", opts)
+
+ require.NoError(t, b.SendEvent(t.Context(), "event", "test"))
+ require.Len(t, api.recorded(), 1)
+ })
+
+ t.Run("it should keep data sources in separate requests", func(t *testing.T) {
+ t.Parallel()
+ api := &fakeEventsAPI{}
+ b := tinybird.NewBatcher(api.client(), "apiKey", options())
+
+ var wg sync.WaitGroup
+ for _, ds := range []string{"ping_response__v8", "tcp_response__v0"} {
+ wg.Go(func() { require.NoError(t, b.SendEvent(t.Context(), "event", ds)) })
+ }
+ wg.Wait()
+
+ requests := api.recorded()
+ require.Len(t, requests, 2)
+ require.ElementsMatch(t, []string{"ping_response__v8", "tcp_response__v0"}, []string{requests[0].dataSource, requests[1].dataSource})
+ })
+
+ t.Run("it should honor Retry-After on 429 and then succeed", func(t *testing.T) {
+ t.Parallel()
+ api := &fakeEventsAPI{respond: func(attempt int) *http.Response {
+ if attempt == 1 {
+ return status(http.StatusTooManyRequests, http.Header{"Retry-After": {"1"}})
+ }
+ return status(http.StatusAccepted, nil)
+ }}
+ opts := options()
+ opts.RetryBudget = 5 * time.Second
+ events := make(chan tinybird.FlushEvent, 1)
+ opts.OnFlush = func(_ context.Context, e tinybird.FlushEvent) { events <- e }
+ b := tinybird.NewBatcher(api.client(), "apiKey", opts)
+
+ start := time.Now()
+ require.NoError(t, b.SendEvent(t.Context(), "event", "test"))
+ require.GreaterOrEqual(t, time.Since(start), time.Second)
+ require.Len(t, api.recorded(), 2)
+ flushed := <-events
+ require.Equal(t, 2, flushed.Attempts)
+ require.Equal(t, 0, flushed.AmbiguousAttempts)
+ })
+
+ t.Run("it should return the error to every row once the retry budget is spent", func(t *testing.T) {
+ t.Parallel()
+ api := &fakeEventsAPI{respond: func(int) *http.Response { return status(http.StatusServiceUnavailable, nil) }}
+ opts := options()
+ opts.RetryBudget = 800 * time.Millisecond
+ b := tinybird.NewBatcher(api.client(), "apiKey", opts)
+
+ for _, err := range sendAll(t, b, "test", 5) {
+ require.ErrorContains(t, err, "unexpected status code: 503")
+ }
+ require.Greater(t, len(api.recorded()), 1)
+ })
+
+ t.Run("it should retry network errors and count them as ambiguous", func(t *testing.T) {
+ t.Parallel()
+ api := &fakeEventsAPI{respond: func(attempt int) *http.Response {
+ if attempt == 1 {
+ return nil
+ }
+ return status(http.StatusAccepted, nil)
+ }}
+ opts := options()
+ events := make(chan tinybird.FlushEvent, 1)
+ opts.OnFlush = func(_ context.Context, e tinybird.FlushEvent) { events <- e }
+ b := tinybird.NewBatcher(api.client(), "apiKey", opts)
+
+ require.NoError(t, b.SendEvent(t.Context(), "event", "test"))
+ require.Equal(t, 1, (<-events).AmbiguousAttempts)
+ })
+
+ t.Run("it should not retry a 400", func(t *testing.T) {
+ t.Parallel()
+ api := &fakeEventsAPI{respond: func(int) *http.Response { return status(http.StatusBadRequest, nil) }}
+ b := tinybird.NewBatcher(api.client(), "apiKey", options())
+
+ require.ErrorContains(t, b.SendEvent(t.Context(), "event", "test"), "unexpected status code: 400")
+ require.Len(t, api.recorded(), 1)
+ })
+
+ t.Run("it should return when the caller's context is cancelled", func(t *testing.T) {
+ t.Parallel()
+ api := &fakeEventsAPI{}
+ opts := options()
+ opts.FlushInterval = time.Hour
+ b := tinybird.NewBatcher(api.client(), "apiKey", opts)
+
+ ctx, cancel := context.WithTimeout(t.Context(), 20*time.Millisecond)
+ defer cancel()
+ require.ErrorIs(t, b.SendEvent(ctx, "event", "test"), context.DeadlineExceeded)
+ })
+
+ t.Run("it should flush pending rows on Close and refuse new ones", func(t *testing.T) {
+ t.Parallel()
+ api := &fakeEventsAPI{}
+ opts := options()
+ opts.FlushInterval = time.Hour
+ b := tinybird.NewBatcher(api.client(), "apiKey", opts)
+
+ sent := make(chan error, 1)
+ go func() { sent <- b.SendEvent(t.Context(), "event", "test") }()
+ // The flush interval alone would never fire.
+ require.Eventually(t, func() bool { return b.PendingRows("test") == 1 }, time.Second, time.Millisecond)
+ require.NoError(t, b.Close(t.Context()))
+
+ require.NoError(t, <-sent)
+ require.Len(t, api.recorded(), 1)
+ require.ErrorIs(t, b.SendEvent(t.Context(), "event", "test"), tinybird.ErrBatcherClosed)
+ })
+
+ t.Run("it should count the wait behind an earlier flush against the retry budget", func(t *testing.T) {
+ t.Parallel()
+ opts := options()
+ opts.RetryBudget = 200 * time.Millisecond
+ api := &fakeEventsAPI{respond: func(attempt int) *http.Response {
+ if attempt == 1 {
+ // A slow request holds the sender past the queued batch's deadline.
+ time.Sleep(2 * opts.RetryBudget)
+ }
+ return status(http.StatusServiceUnavailable, nil)
+ }}
+ opts.FlushInterval = time.Hour
+ opts.MaxBytes = 1
+ b := tinybird.NewBatcher(api.client(), "apiKey", opts)
+
+ first := make(chan error, 1)
+ go func() { first <- b.SendEvent(t.Context(), "event", "test") }()
+ require.Eventually(t, func() bool { return len(api.recorded()) > 0 }, time.Second, time.Millisecond)
+
+ // Queued behind the first batch, which keeps its sender busy past this batch's budget.
+ start := time.Now()
+ err := b.SendEvent(t.Context(), "event", "test")
+ require.Len(t, api.recorded(), 1)
+ require.Less(t, time.Since(start), 3*opts.RetryBudget)
+ require.ErrorIs(t, err, context.DeadlineExceeded)
+ require.ErrorContains(t, err, "flush not attempted")
+ require.ErrorContains(t, <-first, "unexpected status code: 503")
+ })
+
+ t.Run("it should abort in-flight retries once Close gives up", func(t *testing.T) {
+ t.Parallel()
+ api := &fakeEventsAPI{respond: func(int) *http.Response { return status(http.StatusServiceUnavailable, nil) }}
+ opts := options()
+ opts.FlushInterval = time.Hour
+ opts.MaxBytes = 1
+ opts.RetryBudget = time.Minute
+ b := tinybird.NewBatcher(api.client(), "apiKey", opts)
+
+ sent := make(chan error, 1)
+ go func() { sent <- b.SendEvent(t.Context(), "event", "test") }()
+ require.Eventually(t, func() bool { return len(api.recorded()) > 0 }, time.Second, time.Millisecond)
+
+ ctx, cancel := context.WithTimeout(t.Context(), 50*time.Millisecond)
+ defer cancel()
+ require.ErrorIs(t, b.Close(ctx), context.DeadlineExceeded)
+ select {
+ case err := <-sent:
+ require.ErrorContains(t, err, "unexpected status code: 503")
+ default:
+ t.Fatal("the row should fail as soon as Close returns")
+ }
+ })
+}
diff --git a/apps/checker/pkg/tinybird/export_test.go b/apps/checker/pkg/tinybird/export_test.go
new file mode 100644
index 000000000..c1588afa1
--- /dev/null
+++ b/apps/checker/pkg/tinybird/export_test.go
@@ -0,0 +1,12 @@
+package tinybird
+
+// PendingRows reports the rows buffered for a data source and not yet detached.
+func (b *Batcher) PendingRows(dataSource string) int {
+ b.mu.Lock()
+ defer b.mu.Unlock()
+ if pending, ok := b.batches[dataSource]; ok {
+ return len(pending.waiters)
+ }
+
+ return 0
+}