The only tool you need to know what is happening and how to fix it.
Go alertingexception-trackerissue-trackinglogsmetricsmonitoringobservability-platformopen-telemetryotelsession-replaytelemetrytraces
feat: retry for webhook integrations (http -> https)
narrow-adventurec21d619ba9ab
diff --git a/backend/app/notifications/adapter_webhook.go b/backend/app/notifications/adapter_webhook.goindex c691fa74..c755692e 100644--- a/backend/app/notifications/adapter_webhook.go+++ b/backend/app/notifications/adapter_webhook.go@@ -10,9 +10,21 @@ import ("fmt""io""net/http"+ "net/url"+ "strings""time")+const (+ // Matches the outbox's per-attempt budget: receivers such as Google Apps Script answer only after running.+ webhookTimeout = 30 * time.Second+ webhookMaxRedirects = 3+)++var webhookClient = &http.Client{+ CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse },+}+type WebhookAdapter struct {URL string `json:"url"`Method string `json:"method,omitempty"`@@ -53,35 +65,79 @@ func (a *WebhookAdapter) Send(ctx context.Context, msg Message) error {method = "POST"}- req, err := http.NewRequestWithContext(ctx, method, a.URL, bytes.NewReader(body))+ ctx, cancel := context.WithTimeout(ctx, webhookTimeout)+ defer cancel()++ target := a.URL+ for redirects := 0; ; redirects++ {+ resp, err := a.deliver(ctx, method, target, body)+ if err != nil {+ return err+ }+ if resp.StatusCode >= 400 {+ return fmt.Errorf("webhook returned status %d", resp.StatusCode)+ }+ next := resendTarget(resp)+ if next == nil {+ return nil+ }+ if redirects == webhookMaxRedirects {+ return fmt.Errorf("webhook redirected more than %d times", webhookMaxRedirects)+ }+ target = next.String()+ }+}++func (a *WebhookAdapter) deliver(ctx context.Context, method, target string, body []byte) (*http.Response, error) {+ req, err := http.NewRequestWithContext(ctx, method, target, bytes.NewReader(body))if err != nil {- return fmt.Errorf("failed to create webhook request: %w", err)+ return nil, fmt.Errorf("failed to create webhook request: %w", err)}req.Header.Set("Content-Type", "application/json")- for k, v := range a.Headers {- req.Header.Set(k, v)- }+ if origin, err := url.Parse(a.URL); err == nil && sameSite(req.URL.Hostname(), origin.Hostname()) {+ for k, v := range a.Headers {+ req.Header.Set(k, v)+ }- if a.Secret != "" {- mac := hmac.New(sha256.New, []byte(a.Secret))- mac.Write(body)- sig := hex.EncodeToString(mac.Sum(nil))- req.Header.Set("X-Traceway-Signature", "sha256="+sig)+ if a.Secret != "" {+ mac := hmac.New(sha256.New, []byte(a.Secret))+ mac.Write(body)+ sig := hex.EncodeToString(mac.Sum(nil))+ req.Header.Set("X-Traceway-Signature", "sha256="+sig)+ }}- client := &http.Client{Timeout: 10 * time.Second}- resp, err := client.Do(req)+ resp, err := webhookClient.Do(req)if err != nil {- return fmt.Errorf("webhook request failed: %w", err)+ return nil, fmt.Errorf("webhook request failed: %w", err)}defer resp.Body.Close()io.Copy(io.Discard, resp.Body)- if resp.StatusCode >= 400 {- return fmt.Errorf("webhook returned status %d", resp.StatusCode)- }+ return resp, nil+}+func resendTarget(resp *http.Response) *url.URL {+ next, err := resp.Location()+ if err != nil {+ return nil+ }+ switch resp.StatusCode {+ case http.StatusMovedPermanently, http.StatusTemporaryRedirect, http.StatusPermanentRedirect:+ return next+ case http.StatusFound, http.StatusSeeOther:+ // Anything but an https upgrade is the receiver pointing at its response (Apps Script): the payload arrived.+ from := resp.Request.URL+ if from.Scheme == "http" && next.Scheme == "https" && sameSite(next.Hostname(), from.Hostname()) {+ return next+ }+ }return nil}++func sameSite(host, origin string) bool {+ host, origin = strings.ToLower(host), strings.ToLower(origin)+ return host == origin || strings.HasSuffix(host, "."+origin)+}diff --git a/backend/app/notifications/adapter_webhook_test.go b/backend/app/notifications/adapter_webhook_test.gonew file mode 100644index 00000000..31d18f2a--- /dev/null+++ b/backend/app/notifications/adapter_webhook_test.go@@ -0,0 +1,156 @@+package notifications++import (+ "context"+ "encoding/json"+ "net/http"+ "net/http/httptest"+ "strings"+ "sync/atomic"+ "testing"+)++type webhookHit struct {+ method string+ subject string+ custom string+ signature string+}++func webhookReceiver(hits chan<- webhookHit) http.HandlerFunc {+ return func(w http.ResponseWriter, r *http.Request) {+ var payload map[string]string+ _ = json.NewDecoder(r.Body).Decode(&payload)+ hits <- webhookHit{r.Method, payload["subject"], r.Header.Get("X-Custom"), r.Header.Get("X-Traceway-Signature")}+ }+}++func receivedHit(t *testing.T, hits <-chan webhookHit) webhookHit {+ t.Helper()+ select {+ case hit := <-hits:+ return hit+ default:+ t.Fatal("the receiver was never called")+ return webhookHit{}+ }+}++func redirectTo(code int, location string) http.HandlerFunc {+ return func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Location", location)+ w.WriteHeader(code)+ }+}++func testWebhook(url string) *WebhookAdapter {+ return &WebhookAdapter{URL: url, Headers: map[string]string{"X-Custom": "token"}, Secret: "s3cret"}+}++func TestWebhookResendsPayloadOnHTTPSUpgrade(t *testing.T) {+ for _, code := range []int{301, 302, 303, 307, 308} {+ hits := make(chan webhookHit, 1)+ receiver := httptest.NewTLSServer(webhookReceiver(hits))+ defer receiver.Close()+ plain := httptest.NewServer(redirectTo(code, receiver.URL+"/hook"))+ defer plain.Close()++ transport := webhookClient.Transport+ webhookClient.Transport = receiver.Client().Transport+ err := testWebhook(plain.URL+"/hook").Send(context.Background(), Message{Subject: "upgrade"})+ webhookClient.Transport = transport+ if err != nil {+ t.Fatalf("%d: %v", code, err)+ }++ hit := receivedHit(t, hits)+ if hit.method != http.MethodPost || hit.subject != "upgrade" {+ t.Errorf("%d: the payload must be re-sent as a POST, got %s with subject %q", code, hit.method, hit.subject)+ }+ if hit.custom != "token" || hit.signature == "" {+ t.Errorf("%d: headers and signature must follow an upgrade on the same host, got %+v", code, hit)+ }+ }+}++func TestWebhookTreatsRedirectToAResponseAsDelivered(t *testing.T) {+ var echoed atomic.Int32+ echo := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { echoed.Add(1) }))+ defer echo.Close()+ otherHost := strings.Replace(echo.URL, "127.0.0.1", "localhost", 1)++ cases := []struct {+ code int+ location string+ }{+ {http.StatusFound, otherHost + "/macros/echo"},+ {http.StatusSeeOther, otherHost + "/macros/echo"},+ {http.StatusFound, echo.URL + "/login"},+ }+ for _, tc := range cases {+ exec := httptest.NewServer(redirectTo(tc.code, tc.location))+ defer exec.Close()+ if err := testWebhook(exec.URL+"/exec").Send(context.Background(), Message{}); err != nil {+ t.Errorf("%d to %s: %v", tc.code, tc.location, err)+ }+ }+ if n := echoed.Load(); n != 0 {+ t.Errorf("the response behind a 302/303 must not be fetched, got %d requests", n)+ }+}++func TestWebhookStopsAfterMaxRedirects(t *testing.T) {+ var requests atomic.Int32+ var loop *httptest.Server+ loop = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ requests.Add(1)+ redirectTo(http.StatusTemporaryRedirect, loop.URL+r.URL.Path)(w, r)+ }))+ defer loop.Close()++ err := testWebhook(loop.URL+"/hook").Send(context.Background(), Message{})+ if err == nil || !strings.Contains(err.Error(), "redirected more than 3 times") {+ t.Fatalf("want the redirect cap error, got %v", err)+ }+ if n := requests.Load(); n != webhookMaxRedirects+1 {+ t.Errorf("want %d requests, got %d", webhookMaxRedirects+1, n)+ }+}++func TestWebhookKeepsHeadersOnTheChannelHost(t *testing.T) {+ hits := make(chan webhookHit, 1)+ receiver := httptest.NewServer(webhookReceiver(hits))+ defer receiver.Close()+ elsewhere := strings.Replace(receiver.URL, "127.0.0.1", "localhost", 1)+ moved := httptest.NewServer(redirectTo(http.StatusTemporaryRedirect, elsewhere+"/hook"))+ defer moved.Close()++ if err := testWebhook(moved.URL+"/hook").Send(context.Background(), Message{Subject: "moved"}); err != nil {+ t.Fatal(err)+ }+ hit := receivedHit(t, hits)+ if hit.subject != "moved" {+ t.Errorf("the payload must reach the new host, got %+v", hit)+ }+ if hit.custom != "" || hit.signature != "" {+ t.Errorf("headers and signature must not leave the channel's host, got %+v", hit)+ }+}++func TestWebhookSameSite(t *testing.T) {+ cases := []struct {+ host, origin string+ want bool+ }{+ {"example.com", "example.com", true},+ {"www.example.com", "example.com", true},+ {"EXAMPLE.com", "example.com", true},+ {"evilexample.com", "example.com", false},+ {"example.com", "www.example.com", false},+ }+ for _, tc := range cases {+ if got := sameSite(tc.host, tc.origin); got != tc.want {+ t.Errorf("sameSite(%q, %q) = %v, want %v", tc.host, tc.origin, got, tc.want)+ }+ }+}