Open-source status page, incident response and uptime monitoring
TypeScript alertingincident-managementmcpmcp-servermonitoringmonitoring-as-codeobservabilityon-callopen-sourcesrestatus-pagestatuspagesynthetic-monitoringterraformtinybirdtursouptimeuptime-checkeruptime-monitor
checker: bug timeout (#3018)
Thibault Le Ouay08484bca64d2
diff --git a/apps/checker/checker/http.go b/apps/checker/checker/http.goindex 3e9120375..facd2dd0e 100644--- a/apps/checker/checker/http.go+++ b/apps/checker/checker/http.go@@ -13,6 +13,7 @@ import ("net/http/httptrace""net/url""strings"+ "sync""syscall""time"@@ -33,6 +34,46 @@ type Timing struct {TransferDone int64 `json:"transferDone"`}+// clearIncompletePhases zeroes any phase whose start or done hook never+// fired (e.g. a timeout before the first byte). Downstream computes+// done - start, so a lone epoch timestamp would yield a huge negative duration.+func (t *Timing) clearIncompletePhases() {+ for _, phase := range [][2]*int64{+ {&t.DnsStart, &t.DnsDone},+ {&t.ConnectStart, &t.ConnectDone},+ {&t.TlsHandshakeStart, &t.TlsHandshakeDone},+ {&t.FirstByteStart, &t.FirstByteDone},+ {&t.TransferStart, &t.TransferDone},+ } {+ if *phase[0] == 0 || *phase[1] == 0 {+ *phase[0], *phase[1] = 0, 0+ }+ }+}++// httpTimer guards the phase struct: net/http can still run ConnectDone or+// TLSHandshakeDone on its dial goroutine after client.Do has timed out.+type httpTimer struct {+ mu sync.Mutex+ timing Timing+}++func (h *httpTimer) set(apply func(t *Timing)) {+ h.mu.Lock()+ defer h.mu.Unlock()+ apply(&h.timing)+}++// snapshot returns a copy with incomplete phases cleared, so a hook firing+// after it cannot put a lone timestamp back into the response.+func (h *httpTimer) snapshot() Timing {+ h.mu.Lock()+ defer h.mu.Unlock()+ timing := h.timing+ timing.clearIncompletePhases()+ return timing+}+type Response struct {Headers map[string]string `json:"headers,omitempty"`Body string `json:"body,omitempty"`@@ -191,21 +232,26 @@ func Http(ctx context.Context, client *http.Client, inputData request.HttpCheckereq.Header.Set("Content-Type", "application/json")}- timing := Timing{}+ timer := &httpTimer{}+ now := func() int64 { return time.Now().UTC().UnixMilli() }trace := &httptrace.ClientTrace{- DNSStart: func(_ httptrace.DNSStartInfo) { timing.DnsStart = time.Now().UTC().UnixMilli() },- DNSDone: func(_ httptrace.DNSDoneInfo) { timing.DnsDone = time.Now().UTC().UnixMilli() },- ConnectStart: func(_, _ string) { timing.ConnectStart = time.Now().UTC().UnixMilli() },- ConnectDone: func(_, _ string, _ error) { timing.ConnectDone = time.Now().UTC().UnixMilli() },- TLSHandshakeStart: func() { timing.TlsHandshakeStart = time.Now().UTC().UnixMilli() },- TLSHandshakeDone: func(_ tls.ConnectionState, _ error) { timing.TlsHandshakeDone = time.Now().UTC().UnixMilli() },+ DNSStart: func(_ httptrace.DNSStartInfo) { timer.set(func(t *Timing) { t.DnsStart = now() }) },+ DNSDone: func(_ httptrace.DNSDoneInfo) { timer.set(func(t *Timing) { t.DnsDone = now() }) },+ ConnectStart: func(_, _ string) { timer.set(func(t *Timing) { t.ConnectStart = now() }) },+ ConnectDone: func(_, _ string, _ error) { timer.set(func(t *Timing) { t.ConnectDone = now() }) },+ TLSHandshakeStart: func() { timer.set(func(t *Timing) { t.TlsHandshakeStart = now() }) },+ TLSHandshakeDone: func(_ tls.ConnectionState, _ error) {+ timer.set(func(t *Timing) { t.TlsHandshakeDone = now() })+ },GotConn: func(_ httptrace.GotConnInfo) {- timing.FirstByteStart = time.Now().UTC().UnixMilli()+ timer.set(func(t *Timing) { t.FirstByteStart = now() })},GotFirstResponseByte: func() {- timing.FirstByteDone = time.Now().UTC().UnixMilli()- timing.TransferStart = time.Now().UTC().UnixMilli()+ timer.set(func(t *Timing) {+ t.FirstByteDone = now()+ t.TransferStart = now()+ })},}@@ -218,6 +264,7 @@ func Http(ctx context.Context, client *http.Client, inputData request.HttpCheckeif err != nil {errorMsg := httpErrorMessage(err, latency)+ timing := timer.snapshot()// A failing customer endpoint is expected, not a checker error.logger.Warn().Err(err).Msg("error while pinging")@@ -240,7 +287,8 @@ func Http(ctx context.Context, client *http.Client, inputData request.HttpChecke// otherwise OOM these 512 MB machines (fleet-wide exit 137 waves).body, err := io.ReadAll(io.LimitReader(response.Body, maxResponseBodyBytes))- timing.TransferDone = time.Now().UTC().UnixMilli()+ timer.set(func(t *Timing) { t.TransferDone = now() })+ timing := timer.snapshot()if err != nil {return Response{diff --git a/apps/checker/checker/http_internal_test.go b/apps/checker/checker/http_internal_test.goindex 1edbe1f75..d94ca9f4f 100644--- a/apps/checker/checker/http_internal_test.go+++ b/apps/checker/checker/http_internal_test.go@@ -105,3 +105,47 @@ func TestNewHTTPClientZeroTimeoutFallsBack(t *testing.T) {assert.Equal(t, defaultHTTPTimeout, transport.TLSHandshakeTimeout)assert.True(t, transport.DisableKeepAlives)}++func TestTimingClearIncompletePhases(t *testing.T) {+ // Timeout after the TLS handshake: first byte never arrived.+ timing := Timing{+ DnsStart: 1000, DnsDone: 1001,+ ConnectStart: 1001, ConnectDone: 1003,+ TlsHandshakeStart: 1003, TlsHandshakeDone: 1017,+ FirstByteStart: 1017,+ }++ timing.clearIncompletePhases()++ assert.Equal(t, Timing{+ DnsStart: 1000, DnsDone: 1001,+ ConnectStart: 1001, ConnectDone: 1003,+ TlsHandshakeStart: 1003, TlsHandshakeDone: 1017,+ }, timing)+}++func TestTimingClearIncompletePhasesKeepsCompleteTiming(t *testing.T) {+ complete := Timing{+ DnsStart: 1000, DnsDone: 1001,+ ConnectStart: 1001, ConnectDone: 1003,+ TlsHandshakeStart: 1003, TlsHandshakeDone: 1017,+ FirstByteStart: 1017, FirstByteDone: 1040,+ TransferStart: 1040, TransferDone: 1042,+ }++ timing := complete+ timing.clearIncompletePhases()++ assert.Equal(t, complete, timing)+}++func TestHTTPTimerSnapshotIsDetached(t *testing.T) {+ timer := &httpTimer{}+ timer.set(func(t *Timing) { t.FirstByteStart = 1017 })++ snapshot := timer.snapshot()+ // A hook landing after the snapshot must not reach the returned copy.+ timer.set(func(t *Timing) { t.ConnectDone = 1003 })++ assert.Equal(t, Timing{}, snapshot)+}